Skip to main content

subc_daemon/
supervise.rs

1//! Identity policy updates read the running-protocol snapshot before taking the
2//! nonce locks. Nested identity locks are taken in this order: spawn nonces,
3//! reserved nonces, then `SupervisorHandle::implicit_protocol_none`.
4
5#[cfg(all(test, target_os = "linux"))]
6#[path = "supervise/audit_identity_tests.rs"]
7mod audit_identity_tests;
8mod resource_launch;
9pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
10
11use std::{
12    collections::{HashMap, HashSet, VecDeque},
13    error::Error,
14    fmt, io,
15    path::PathBuf,
16    process::{ExitStatus, Stdio},
17    sync::{Arc, Mutex, OnceLock},
18    time::{Duration, SystemTime, UNIX_EPOCH},
19};
20
21use cortexkit_log::Retention;
22use serde_json::Value;
23use sha2::{Digest, Sha256};
24use subc_control::{
25    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
26    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
27};
28use subc_protocol::{
29    manifest::{SelfSignalKind, SignalAnchor},
30    session::{
31        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
32        MODULE_CONTROL_OP_HEALTH_CHECK,
33    },
34    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
35};
36use tokio::{
37    process::{Child, Command},
38    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
39    task::JoinHandle,
40    time::{sleep, sleep_until, timeout, timeout_at, Instant},
41};
42use tracing::{debug, error, info, warn};
43
44use crate::{
45    child_roster::ChildRoster,
46    daemon_config::{
47        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
48    },
49    forwarding::{
50        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
51        ModuleDrainTarget, PendingModuleControlRpc,
52    },
53    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
54    registry::{ConnectionId, RegistrationEndReason, RegistryError},
55    stderr_tail::{
56        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
57        StderrTailSnapshot,
58    },
59    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
60    Frame, FrameSink, Registry,
61};
62
63#[cfg(not(windows))]
64use crate::provenance::spawned_file_identity;
65
66#[path = "supervise_swap.rs"]
67mod swap;
68
69#[path = "supervise_activity.rs"]
70mod activity;
71
72#[cfg(unix)]
73#[path = "supervise/admission.rs"]
74mod admission;
75mod admission_exit;
76#[cfg(all(test, target_os = "linux"))]
77mod async_spawn_tests;
78#[cfg(all(test, windows))]
79mod carried_job_tests;
80mod config_delivery;
81mod content_binding;
82mod program_binding;
83mod registration;
84pub(crate) mod script;
85/// Each launch nonce, mapped to its launch's module id and the launch.
86type NonceLaunches = HashMap<String, (String, std::sync::Weak<RegistrationSpawn>)>;
87pub(crate) use registration::NonceAttestation;
88#[cfg(all(test, unix))]
89pub(crate) use registration::RegistrationProof;
90pub(crate) use registration::RegistrationSpawn;
91pub(crate) use registration::REGISTRATION_CHANNEL_CLOSED;
92#[cfg(all(test, unix))]
93pub(crate) mod unix_channel_placement_tests;
94use program_binding::ProgramBindingPublication;
95mod config_exit_failure;
96#[cfg(all(test, any(target_os = "linux", windows)))]
97mod missing_path_tests;
98#[cfg(all(test, unix))]
99mod program_binding_tests;
100#[cfg(all(test, windows))]
101mod program_binding_windows_tests;
102#[cfg(all(test, windows))]
103mod windows_channel_tests;
104pub use config_exit_failure::ModuleStopReason;
105#[cfg(all(test, unix))]
106mod config_exit_hold_tests;
107pub(crate) mod containment;
108#[cfg(all(test, unix))]
109mod containment_tests;
110mod slow_retry;
111use slow_retry::{FailedRetry, FailureContext};
112mod host_health;
113use host_health::HostHealth;
114
115#[cfg(all(test, unix))]
116mod mark_readiness_tests;
117#[cfg(all(test, unix))]
118mod slow_retry_tests;
119
120/// Command-line flag used by supervised modules to find subc.
121///
122/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
123/// The path points at the TCP+key connection file; it is not an ambient signal and
124/// is never inherited by standalone children.
125pub const SUBC_ARG: &str = "--subc";
126
127const DEFAULT_MAX_RESTARTS: u32 = 3;
128const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
129const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
130/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
131/// to contain a real crash loop (which respawns in seconds) and short enough
132/// that unrelated crashes hours apart never accumulate into a permanent stop.
133const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
134/// How long a drain waits for already-dispatched requests to finalize before
135/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
136/// not RPC-scale: the original 2s value silently cut nearly every real tool
137/// call at the fence, making the wait-for-finalize design decorative for the
138/// workloads it existed for. Quiescence short-circuits, so an idle module
139/// restarts immediately regardless of this value; the budget is spent only
140/// when a genuine in-flight request is worth finishing. Per-module override:
141/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
142/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
143/// where a stuck request will never settle).
144pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
145pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
146const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
147const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
148/// How long a drain waits after one busy-gauge probe completes before it asks
149/// again.
150///
151/// A drain re-probes on this short interval of its own rather than on the
152/// module's health cadence. The default cadence (30 s) equals the default
153/// drain ceiling, so a cadence-paced drain that found the module busy at its
154/// first probe never asked a second time: it waited out the whole ceiling and
155/// forced GOODBYE even after the gauges had read zero. The interval is counted
156/// from the moment the previous probe finished, and probes run one at a time,
157/// so a slow module is never asked more often than it can answer. Periodic
158/// health checks outside a drain keep using the configured cadence.
159const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
160/// How long a restart waits for the exited process's output readers.
161///
162/// The restart does not depend on either output reader finishing. A stderr
163/// reader still running at this bound is left running, and its later output goes
164/// into the exited process's own section of the stderr ring (see
165/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
166/// bound no longer decides whether a crash's last lines are kept: under load
167/// the reader may simply not have been scheduled yet, and cutting it there
168/// discarded exactly the lines that explained the crash.
169///
170/// What the bound still decides is when the tail starts reporting
171/// `Incomplete`: a pipe open past it usually means a descendant of the exited
172/// process still holds it, and the tail cannot claim to be whole until that
173/// pipe closes. That is also the only case in which the wait costs the restart
174/// anything, because a pipe with no other holder reaches EOF when the process
175/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
176/// `Captured` at EOF with nothing lost.
177///
178/// The stdout reader carries no ring, only the capture file. It also continues
179/// to EOF after this bound: child exit closes the writer but does not mean the
180/// reader has consumed all buffered bytes. Late output from either pipe can
181/// therefore reach the shared capture file after the successor starts.
182const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
183/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
184pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
185const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
186/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
187/// dropped because the subscriber stopped draining its frames. The detail's
188/// `first_undelivered_cursor` names the first event it did not receive; the
189/// client resubscribes from the last cursor it did receive.
190pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
191
192#[cfg(target_os = "macos")]
193struct PrivacyExec {
194    content_check: Option<tokio::task::JoinHandle<Result<(), crate::config_gate::GateError>>>,
195    gated: bool,
196    admission_binding: Option<ProgramBindingPublication>,
197    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
198    deadline: tokio::time::Instant,
199    expected: Option<subc_os::FileIdentity>,
200    trampoline: Option<subc_os::FileIdentity>,
201    signature: Option<Arc<subc_os::privacy_identity::SuspendedSignature>>,
202    module_id: String,
203    #[cfg(test)]
204    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
205    #[cfg(test)]
206    image_probe: Option<fn(u32) -> Option<subc_os::Observation>>,
207    #[cfg(test)]
208    image_read_delay: Option<Duration>,
209}
210
211#[cfg(all(test, target_os = "macos"))]
212pub(crate) fn test_privacy_trampoline() -> PathBuf {
213    // The fixture bin lives in the profile folder the unit-test executable was
214    // built into. This honors custom CARGO_TARGET_DIR too.
215    let path = cortexkit_test_support::cargo_profile_bin("privacy-trampoline-fixture");
216    // Without the fixture every macOS spawn is refused, and the tests that
217    // spawn fail later as a module in state Failed, which names the wrong
218    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
219    assert!(
220        path.exists(),
221        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
222         --bins --features test-support` or `cargo test -p subc-daemon` first",
223        path.display()
224    );
225    path
226}
227
228#[cfg(target_os = "macos")]
229fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
230    use std::io::Read;
231    let mut probe = subc_os::child_spawn::spawn(
232        std::process::Command::new(path)
233            .args(["__disclaim-exec", "--probe-suspended"])
234            .stdin(Stdio::null())
235            .stdout(Stdio::piped())
236            .stderr(Stdio::piped()),
237    )
238    .map_err(|error| {
239        format!(
240            "privacy trampoline probe failed for {}: {error}",
241            path.display()
242        )
243    })?;
244    let deadline = std::time::Instant::now() + Duration::from_secs(5);
245    let status = loop {
246        match probe.try_wait() {
247            Ok(Some(status)) => break status,
248            Ok(None) if std::time::Instant::now() < deadline => {
249                std::thread::sleep(Duration::from_millis(5))
250            }
251            result => {
252                let _ = probe.kill();
253                let _ = probe.wait();
254                return Err(format!(
255                    "privacy trampoline probe failed or timed out for {}: {result:?}",
256                    path.display()
257                ));
258            }
259        }
260    };
261    let mut answer = String::new();
262    if let Some(stdout) = probe.stdout.take() {
263        let _ = stdout.take(256).read_to_string(&mut answer);
264    }
265    if status.success() && answer.trim() == subc_os::privacy_identity::SUSPENDED_TRAMPOLINE_PROBE {
266        return Ok(());
267    }
268    let mut diagnostic = String::new();
269    if let Some(stderr) = probe.stderr.take() {
270        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
271    }
272    let cause = diagnostic
273        .trim()
274        .strip_prefix("ck-subc: own privacy identity refused: ")
275        .unwrap_or("binary does not implement the privacy trampoline protocol");
276    Err(format!(
277        "{cause}: probe of {} exited {status}",
278        path.display()
279    ))
280}
281
282#[cfg(target_os = "macos")]
283fn privacy_command(
284    spec: &ModuleSpec,
285    binding: &SpawnProgramBinding,
286    roster: &ChildRoster,
287    gated: bool,
288) -> Result<
289    (
290        Command,
291        Option<PrivacyExec>,
292        subc_os::privacy_identity::ExecAcknowledgement,
293    ),
294    SuperviseError,
295> {
296    let failure = |cause: String| {
297        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
298        SuperviseError::Spawn {
299            program: spec.program.clone(),
300            source: io::Error::other(cause),
301            cgroup_path: None,
302        }
303    };
304    let trampoline = roster.privacy_trampoline().map_err(failure)?;
305    let expected = Some(binding.identity);
306    let trampoline_image = subc_os::file_identity(&trampoline);
307    let signature = gated
308        .then(|| {
309            subc_os::privacy_identity::SuspendedSignature::capture(&binding._file, &binding.path)
310                .map(Arc::new)
311        })
312        .transpose()
313        .map_err(|error| failure(error.to_string()))?;
314    if expected == trampoline_image {
315        return Err(failure(
316            "privacy identity module executable is missing or is the trampoline itself".to_string(),
317        ));
318    }
319    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
320        .map_err(|error| failure(error.to_string()))?;
321    let reader =
322        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
323    let mut command = Command::new(&trampoline);
324    command
325        .arg(if gated {
326            "__disclaim-exec-suspended"
327        } else {
328            "__disclaim-exec"
329        })
330        .arg(ack.fd().to_string())
331        .arg(&binding.path);
332    Ok((
333        command,
334        Some(PrivacyExec {
335            gated,
336            admission_binding: None,
337            content_check: None,
338            reader,
339            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
340            expected,
341            trampoline: trampoline_image,
342            signature,
343            module_id: spec.module_id.clone(),
344            #[cfg(test)]
345            image_probe: None,
346            #[cfg(test)]
347            image_read_delay: spec
348                .env
349                .iter()
350                .find(|(key, _)| key == "SUBC_TEST_PRIVACY_IMAGE_DELAY_MS")
351                .map(|(_, millis)| Duration::from_millis(millis.parse().unwrap())),
352            #[cfg(test)]
353            confirmation_pause: spec
354                .env
355                .iter()
356                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
357                .map(|(_, address)| (address.clone(), None)),
358        }),
359        ack,
360    ))
361}
362
363#[cfg(unix)]
364type SpawnRegistrationChannel = std::os::unix::net::UnixStream;
365#[cfg(windows)]
366type SpawnRegistrationChannel = subc_os::registration_channel::RegistrationPipe;
367#[cfg(test)]
368type CapturedRegistrationChannels =
369    Arc<Mutex<Option<Vec<(Arc<RegistrationSpawn>, CapturedRegistrationEndpoint)>>>>;
370#[cfg(test)]
371struct CapturedRegistrationEndpoint {
372    _channel: SpawnRegistrationChannel,
373}
374#[cfg(test)]
375impl fmt::Debug for CapturedRegistrationEndpoint {
376    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
377        f.write_str("CapturedRegistrationEndpoint")
378    }
379}
380
381struct SupervisedChild {
382    registration_spawn: Option<Arc<RegistrationSpawn>>,
383    /// Owned daemon endpoint, retained for the per-spawn registration service.
384    #[cfg(any(unix, windows))]
385    registration_channel: Option<SpawnRegistrationChannel>,
386    live_program_binding: Option<Arc<content_binding::LiveProgramBinding>>,
387    registered_config_connection: Option<ConnectionId>,
388    admission_base: Option<subc_config::Digest>,
389    registration_declaration: Option<String>,
390    _program_binding: Option<ProgramBindingPublication>,
391    subtree_target: crate::subtree_resources::Target,
392    child: containment::ProcessChild,
393    exec: crate::activity::ExecOwner,
394    #[cfg(windows)]
395    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
396    #[cfg(target_os = "macos")]
397    privacy_exec: Option<PrivacyExec>,
398    /// Set once this launch's exec acknowledgement confirms the module image.
399    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
400    /// `subc_os::privacy_identity`), which then replaces itself with the
401    /// module. The supervisor owns and can kill that pid from spawn, but
402    /// status readers report it only after this latch is set, so nothing
403    /// reports the trampoline's image as the module's.
404    #[cfg(target_os = "macos")]
405    report_ready: Arc<OnceLock<()>>,
406    /// Supervisor failure detail retained for the terminal record, including
407    /// image refusals and first-HELLO timeouts after a successful spawn.
408    spawn_failure: Option<String>,
409    /// Set only after a kill requested because launch admission failed was
410    /// accepted, never for an operator, health, or shutdown kill.
411    admission_kill_sent: bool,
412    /// The protocol this process was launched with. A reload can store a new
413    /// launch spec with a different protocol, but that takes effect only at the
414    /// next spawn, so this process keeps being handled by the protocol it
415    /// actually speaks.
416    protocol: ModuleProtocol,
417    /// This container's cgroup name: a bounded module/slot label followed by a
418    /// unique suffix (when cgroup placement is on). Carried replacements reuse
419    /// the name; ordinary launches each get a fresh container. A
420    /// retired process in a slot may still be draining when a later one is
421    /// spawned into that slot, so the suffix keeps the later process out of
422    /// the retired one's cgroup, which is the domain a kill applies to.
423    #[cfg(target_os = "linux")]
424    module_id: String,
425    #[cfg(target_os = "linux")]
426    cgroup_placement: Option<subc_cgroup::Placement>,
427    /// The job for an ordinary, non-carried launch. For carried
428    /// launches the actor owns it and the child holds only a weak container link.
429    ///
430    /// Dropping this handle is what reaps a surviving tree when no supervisor
431    /// code runs — a daemon crash — because the job carries
432    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
433    ///
434    /// That limit is not crash-only, and the difference is worth knowing: a
435    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
436    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
437    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
438    /// module at once. Before this change they survived that, saw EOF on the
439    /// control socket, and ran their own teardown; Unix keeps that path
440    /// deliberately, so a module can seal a WAL or close a capture rather than
441    /// be killed mid-write. So this trades graceful teardown on every Windows
442    /// daemon stop for containment on a crash, which is the right way round
443    /// today: orphaned GPU workers are a reported, recurring problem, and the
444    /// modules that write most heavily do not run on Windows.
445    ///
446    /// The fix is a real Windows stop path — the daemon draining before it
447    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
448    /// reaches only what the drain left behind, which is what it should reach.
449    #[cfg(windows)]
450    job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
451    stdout_pump: Option<JoinHandle<()>>,
452    stderr_pump: Option<StderrPump>,
453    stderr_ring: Arc<Mutex<StderrRing>>,
454    /// Conservative monotonic bound sampled before `command.spawn()`, not when
455    /// identity recording finishes or the spawn generation is published.
456    spawned_at: Instant,
457    config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
458    /// Ordinary spawns must send their first HELLO within this budget. Once
459    /// accepted, a later disconnect must not restart the registration clock.
460    registration_deadline: Option<Instant>,
461    registration_watch: Option<RegistrationWatch>,
462    spawned_at_ms: u64,
463    spawned_from: PathBuf,
464    launch_spec_digest: LaunchSpecDigest,
465    spawned_file_identity: Option<SpawnedFileIdentity>,
466    process_start_time: Option<u64>,
467    process_identity: Option<ProcessIdentity>,
468    pid: u32,
469    /// This process's entry in the daemon's child roster, released when the
470    /// process is reaped or this handle is dropped.
471    roster_guard: Option<crate::child_roster::RosterGuard>,
472}
473
474impl SupervisedChild {
475    fn retire_registration(&self) {
476        // All exit observers enter here before cleanup can await. The retired
477        // program-binding flag and the registration's Dead state make repeated
478        // observations harmless, even when another task observes the group
479        // leader retained unreaped to reserve its process-group id.
480        if let Some(spawn) = &self.registration_spawn {
481            spawn.retire();
482        } else if let Some(binding) = &self._program_binding {
483            binding.retire();
484        }
485        #[cfg(unix)]
486        if let Some(channel) = &self.registration_channel {
487            let _ = channel.shutdown(std::net::Shutdown::Both);
488        }
489        #[cfg(windows)]
490        if let Some(channel) = &self.registration_channel {
491            if let Ok(shutdown) = channel.shutdown_handle() {
492                shutdown.shutdown();
493            }
494        }
495    }
496    fn registered_since_spawn(&self) -> bool {
497        self.registration_watch
498            .as_ref()
499            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
500    }
501
502    fn awaiting_registration(&self) -> bool {
503        // Admission failure retires the channel before its kill becomes
504        // waitable. That closure must not replace the admission failure with
505        // a registration failure and spend crash restart budget.
506        let admission_failed = self.admission_kill_sent && self.spawn_failure.is_some();
507        !admission_failed && !self.registered_since_spawn()
508    }
509
510    fn id(&self) -> Option<u32> {
511        Some(self.pid)
512    }
513
514    fn process_identity(&self) -> Option<ProcessIdentity> {
515        self.process_identity
516    }
517
518    fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
519        let result = self.child.try_wait();
520        if !matches!(result, Ok(None)) {
521            self.retire_registration();
522            #[cfg(any(unix, windows))]
523            self.registration_channel.take();
524            if let Some(binding) = &self._program_binding {
525                binding.retire();
526            }
527        }
528        result
529    }
530
531    async fn wait(&mut self) -> io::Result<ExitStatus> {
532        #[cfg(target_os = "macos")]
533        self.confirm_privacy_exec().await;
534        // The roster entry is NOT released here. A daemon shutdown waits for the
535        // roster to empty and then exits the process, so releasing at the reap
536        // let it exit before the exit handler wrote this child's terminal record
537        // (the stderr drain and snapshot update sit in between), and the
538        // shutdown's own `daemon_shutdown` record was intermittently lost. The
539        // caller releases it after recording the exit (`release_roster`), and
540        // dropping the handle releases it too.
541        let result = self.child.wait().await;
542        self.retire_registration();
543        #[cfg(any(unix, windows))]
544        self.registration_channel.take();
545        // All plain and swap waits pass here. Revoke the reaped child's nonce
546        // before cgroup cleanup, stderr draining or exit observation can await.
547        if let Some(binding) = &self._program_binding {
548            binding.retire();
549        }
550        #[cfg(all(test, target_os = "linux"))]
551        if let Some(binding) = &self._program_binding {
552            binding.pause_after_wait_for_test().await;
553        }
554        #[cfg(target_os = "linux")]
555        if result.is_ok() && self.child.container().is_none() {
556            if let Some(placement) = self.cgroup_placement.as_ref() {
557                // Ordinary children still own a per-spawn cgroup: returning
558                // from wait must leave it empty and removed, as before carry.
559                // Keep cleanup ownership across cancellation until it finishes.
560                cleanup_reaped_cgroup(placement, &self.module_id).await;
561                self.cgroup_placement = None;
562            }
563        }
564        result
565    }
566
567    async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
568        self.retire_registration();
569        // Idempotent fallback for cleanup paths that did not call wait.
570        if let Some(binding) = self._program_binding.take() {
571            binding.retire();
572        }
573        if self.child.container().is_some() {
574            if !carry {
575                self.child.terminate_container().await;
576            }
577        } else {
578            #[cfg(target_os = "linux")]
579            if let Some(placement) = self.cgroup_placement.as_ref() {
580                cleanup_reaped_cgroup(placement, &self.module_id).await;
581                self.cgroup_placement = None;
582            }
583        }
584        self.child.reap().await?;
585        self.release_roster();
586        Ok(())
587    }
588
589    #[cfg(target_os = "macos")]
590    async fn confirm_privacy_exec(&mut self) {
591        if self
592            .privacy_exec
593            .as_ref()
594            .is_some_and(|pending| pending.content_check.is_some())
595        {
596            self.finish_privacy_content_check().await;
597            return;
598        }
599        let Some(pending) = &mut self.privacy_exec else {
600            return;
601        };
602        let result = tokio::time::timeout_at(pending.deadline, async {
603            let mut record = Vec::new();
604            loop {
605                let mut ready = pending.reader.readable().await?;
606                let read = ready.try_io(|reader| {
607                    use std::io::Read;
608                    let mut reader = reader.get_ref();
609                    let mut buffer = [0u8; 256];
610                    reader.read(&mut buffer).map(|count| (count, buffer))
611                });
612                match read {
613                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
614                    Ok(Ok((count, buffer))) => {
615                        if record.len() + count > 1024 {
616                            return Err(io::Error::other(
617                                "privacy exec refusal record is too long",
618                            ));
619                        }
620                        record.extend_from_slice(&buffer[..count]);
621                    }
622                    Ok(Err(error)) => return Err(error),
623                    Err(_) => continue,
624                }
625            }
626        })
627        .await;
628        #[cfg(test)]
629        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
630            use tokio::io::{AsyncReadExt, AsyncWriteExt};
631            if stream.is_none() {
632                *stream = Some(
633                    tokio::net::TcpStream::connect(address.as_str())
634                        .await
635                        .unwrap(),
636                );
637                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
638                stream
639                    .as_mut()
640                    .unwrap()
641                    .write_all(&self.pid.to_ne_bytes())
642                    .await
643                    .unwrap();
644            }
645            let mut reply = [0];
646            stream
647                .as_mut()
648                .unwrap()
649                .read_exact(&mut reply)
650                .await
651                .unwrap();
652            pending.confirmation_pause = None;
653            if &reply == b"F" {
654                Ok(Err(io::Error::other("injected late exec refusal")))
655            } else {
656                assert_eq!(&reply, b"X");
657                result
658            }
659        } else {
660            result
661        };
662        // Keep the reader in self across await: select cancellation must not
663        // discard the handshake or reset its original five-second deadline.
664        // Use try_wait so a child reaped during confirmation has its program
665        // binding retired before another command can use its nonce.
666        let waited = if matches!(&result, Ok(Ok(record)) if record.is_empty()) {
667            Some(self.try_wait())
668        } else {
669            None
670        };
671        let pending = self.privacy_exec.as_ref().expect("pending exec");
672        let cause = match result {
673            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
674            Ok(Err(error)) => Some(format!(
675                "privacy identity exec acknowledgement failed: {error}"
676            )),
677            Ok(Ok(record)) if !record.is_empty() => Some(
678                std::str::from_utf8(&record)
679                    .ok()
680                    .and_then(|record| {
681                        record
682                            .trim()
683                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
684                    })
685                    .filter(|cause| !cause.is_empty())
686                    .unwrap_or("invalid privacy exec refusal record")
687                    .to_string(),
688            ),
689            Ok(Ok(_)) if !pending.gated => match waited.expect("exec EOF checked child") {
690                Ok(Some(_)) => None,
691                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
692                Ok(None) => {
693                    #[cfg(test)]
694                    if let Some(delay) = pending.image_read_delay {
695                        tokio::time::sleep(delay).await;
696                    }
697                    let image = observe_spawned_image(self.pid);
698                    #[cfg(test)]
699                    let image = pending.image_probe.map_or(image, |probe| probe(self.pid));
700                    // An unstamped module can re-exec before we wake to read its
701                    // image. Only the trampoline image proves exec never happened;
702                    // retain the observed image for later orphan identification.
703                    if let Some(image) = image.filter(|image| {
704                        image.executable.is_some() && image.executable != pending.trampoline
705                    }) {
706                        match self
707                            .roster_guard
708                            .as_ref()
709                            .map(|guard| guard.confirm_executable(image))
710                            .transpose()
711                        {
712                            Err(error) => Some(format!(
713                                "could not durably record confirmed module image: {error}"
714                            )),
715                            Ok(_) => {
716                                let _ = self.report_ready.set(());
717                                None
718                            }
719                        }
720                    } else if image.is_none()
721                        || image.is_some_and(|image| image.executable.is_none())
722                    {
723                        // Empty EOF acknowledged exec. An image disappearing
724                        // during observation is an ordinary module exit.
725                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
726                            Ok(result) => {
727                                self.retire_registration();
728                                self.registration_channel.take();
729                                if let Some(binding) = &self._program_binding {
730                                    binding.retire();
731                                }
732                                result.err().map(|error| format!("privacy identity trampoline wait failed: {error}"))
733                            }
734                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
735                        }
736                    } else {
737                        Some("privacy identity trampoline did not exec the module".to_string())
738                    }
739                }
740            },
741            Ok(Ok(_)) => match waited.expect("exec EOF checked child") {
742                Ok(Some(_)) => {
743                    Some("privacy identity target exited before suspended admission".to_string())
744                }
745                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
746                Ok(None) => {
747                    // CLOEXEC closes the ack just before the kernel reports the
748                    // suspended target. EOF alone must never admit a running image.
749                    let stopped = tokio::time::timeout_at(pending.deadline, async {
750                        loop {
751                            if subc_os::privacy_identity::SuspendedSignature::is_suspended(
752                                self.pid,
753                            )? {
754                                return Ok::<_, io::Error>(());
755                            }
756                            tokio::time::sleep(Duration::from_millis(1)).await;
757                        }
758                    })
759                    .await;
760                    match stopped {
761                        Err(_) => {
762                            Some("privacy identity target was not suspended within 5s".to_string())
763                        }
764                        Ok(Err(error)) => Some(format!(
765                            "privacy identity suspended target unavailable: {error}"
766                        )),
767                        Ok(Ok(())) => {
768                            let image = observe_spawned_image(self.pid);
769                            if image.is_some_and(|image| {
770                                image.executable.is_some()
771                                    && image.executable != pending.trampoline
772                                    && image.executable == pending.expected
773                            }) {
774                                None
775                            } else {
776                                Some("privacy identity trampoline executable mismatch: expected the suspended module program".to_string())
777                            }
778                        }
779                    }
780                }
781            },
782        };
783        #[cfg(test)]
784        if let Some(binding) = &self._program_binding {
785            binding.pause_after_wait_for_test().await;
786        }
787        if cause.is_none() && pending.gated {
788            let publication = pending.admission_binding.clone();
789            let live = self.live_program_binding.clone();
790            let signature = Arc::clone(pending.signature.as_ref().expect("gated signature"));
791            let pid = self.pid;
792            let gated = pending.gated;
793            self.privacy_exec
794                .as_mut()
795                .expect("pending exec")
796                .content_check = Some(tokio::task::spawn_blocking(move || {
797                signature.verify(pid).map_err(|error| {
798                    crate::config_gate::GateError::new("program_refused", error)
799                })?;
800                if !gated {
801                    return Ok(());
802                }
803                match (publication, live) {
804                    (Some(publication), _) => publication.verify(),
805                    (None, Some(live)) => live.check(),
806                    _ => Err(crate::config_gate::GateError::new(
807                        "config_delivery_deferred",
808                        "live program witness is unavailable",
809                    )),
810                }
811            }));
812            self.finish_privacy_content_check().await;
813        } else {
814            self.finish_privacy_exec(cause);
815        }
816    }
817
818    #[cfg(target_os = "macos")]
819    async fn finish_privacy_content_check(&mut self) {
820        let result = self
821            .privacy_exec
822            .as_mut()
823            .expect("pending exec")
824            .content_check
825            .as_mut()
826            .expect("pending content check")
827            .await;
828        let cause = match result {
829            Ok(Ok(())) => None,
830            Ok(Err(error)) => Some(format!("{}: {}", error.code, error.detail)),
831            Err(error) => Some(format!(
832                "config_delivery_deferred: content check stopped: {error}"
833            )),
834        };
835        self.finish_privacy_exec(cause);
836    }
837
838    #[cfg(target_os = "macos")]
839    fn finish_privacy_exec(&mut self, mut cause: Option<String>) {
840        let mut pending = self.privacy_exec.take().expect("pending exec");
841        if cause.is_none() && pending.gated {
842            let admission = (|| -> io::Result<()> {
843                let image = observe_spawned_image(self.pid)
844                    .filter(|image| image.executable == pending.expected)
845                    .ok_or_else(|| io::Error::other("suspended executable mismatch"))?;
846                if let Some(guard) = &self.roster_guard {
847                    guard.confirm_executable(image)?;
848                }
849                pending
850                    .signature
851                    .as_ref()
852                    .expect("gated signature")
853                    .resume(self.pid)?;
854                let _ = self.report_ready.set(());
855                info!(module_id = %pending.module_id, pid = self.pid,
856                    "verified suspended module resumed with own privacy identity");
857                Ok(())
858            })();
859            if let Err(error) = admission {
860                cause = Some(format!("program_refused: {error}"));
861            }
862        }
863        if cause.is_some() {
864            if let Some(spawn) = &self.registration_spawn {
865                spawn.refuse();
866            }
867            if let Some(binding) = &mut pending.admission_binding {
868                binding.reject();
869            }
870        } else if let Some(spawn) = &self.registration_spawn {
871            spawn.ready();
872        }
873        // Exec confirmation describes launch success, not roster retirement.
874        // Keep even a reaped child recorded until its terminal history exists.
875        self.exec.0.finish(if cause.is_some() {
876            crate::activity::ExecState::Refused
877        } else {
878            crate::activity::ExecState::CommittedExec
879        });
880        if let Some(cause) = cause {
881            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
882            self.spawn_failure = Some(
883                if pending.gated
884                    && (cause.contains("executable mismatch")
885                        || cause.starts_with("program_refused:"))
886                {
887                    format!("program_refused: {cause}")
888                } else if pending.gated {
889                    format!("config_delivery_deferred: {cause}")
890                } else {
891                    cause
892                },
893            );
894            // No resume signal is sent on failure; kill the entire fresh group
895            // even if a faulty trampoline did not honor the suspended protocol.
896            let _ = self.start_admission_kill();
897        }
898    }
899
900    /// Releases the leader's roster entry after recording and finalizing its
901    /// exit. A carried container retains its separate roster entry and, for a
902    /// Unix process group, its unreaped first leader that reserves the group ID.
903    fn release_roster(&mut self) {
904        self.roster_guard = None;
905    }
906
907    /// Kill the child and, where containment is available, its process tree.
908    ///
909    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
910    /// helper process leaked the helper — the Synapse embedding module's CUDA
911    /// worker holds the GPU allocation, so the leak cost VRAM until the next
912    /// restart of something else. Terminating the job reaches grandchildren that
913    /// a tree walk cannot, including one whose parent has already exited and
914    /// been reparented away.
915    ///
916    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
917    /// direct-child kill still decides the outcome, so containment can never
918    /// change whether a module is reported as stopped.
919    fn start_kill(&mut self) -> io::Result<()> {
920        self.retire_registration();
921        if let Some(container) = self.child.container() {
922            container.hard_kill();
923            return self.kill_leader();
924        }
925        #[cfg(windows)]
926        if let Some(job) = &self.job {
927            if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
928                debug!(
929                    error = %error,
930                    "job termination failed; the direct-child kill still owns the outcome"
931                );
932            }
933        }
934        #[cfg(target_os = "linux")]
935        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
936        self.child.start_kill()
937    }
938
939    fn kill_leader(&mut self) -> io::Result<()> {
940        self.retire_registration();
941        self.child.start_kill()
942    }
943
944    fn start_admission_kill(&mut self) -> io::Result<()> {
945        self.retire_registration();
946        #[cfg(windows)]
947        let result = {
948            // Preserve the leader's distinctive status before ending the whole
949            // container/job; descendants must not wait for exit finalization.
950            let result = self
951                .spawned_file_identity
952                .as_ref()
953                .and_then(|identity| identity.windows_image.as_ref())
954                .ok_or_else(|| io::Error::other("admission child handle unavailable"))
955                .and_then(|image| {
956                    image.process().terminate_with_exit_code(
957                        admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE as u32,
958                    )
959                });
960            if let Some(container) = self.child.container() {
961                container.hard_kill();
962            } else if let Some(job) = &self.job {
963                if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
964                    debug!(%error, "job termination failed after admission leader termination");
965                }
966            }
967            result
968        };
969        #[cfg(not(windows))]
970        let result = self.start_kill().map(|()| true);
971        #[cfg(unix)]
972        let result = {
973            // A leader can exit before its helpers. Still end the fresh group
974            // even when the direct-child kill reports that the leader is gone.
975            let group_sent = self.child.container().is_none()
976                && rustix::process::Pid::from_raw(self.pid as i32).is_some_and(|group| {
977                    rustix::process::kill_process_group(group, rustix::process::Signal::KILL)
978                        .is_ok()
979                });
980            if group_sent {
981                Ok(true)
982            } else {
983                result
984            }
985        };
986        self.admission_kill_sent |= result?;
987        Ok(())
988    }
989
990    async fn drain_stderr(&mut self, module_id: &str) {
991        if let Some(pump) = self.stdout_pump.take() {
992            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
993        }
994
995        let Some(pump) = self.stderr_pump.take() else {
996            return;
997        };
998        settle_stderr_pump(
999            module_id,
1000            &self.stderr_ring,
1001            pump,
1002            STDERR_PUMP_DRAIN_TIMEOUT,
1003        )
1004        .await;
1005    }
1006}
1007
1008/// Bound the restart's wait, not the lifetime of the exited child's stdout
1009/// reader. Slow pipe reads or capture writes must not turn the bound into lost
1010/// output; dropping the handle lets the reader finish at EOF, like stderr.
1011async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
1012    match timeout(bound, &mut pump).await {
1013        Ok(Ok(())) => {}
1014        Ok(Err(error)) => {
1015            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
1016        }
1017        Err(_) => {
1018            drop(pump);
1019            warn!(
1020                module_id,
1021                waited = ?bound,
1022                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
1023            );
1024        }
1025    }
1026}
1027
1028/// The reader task for one process's stderr, with the ring generation its
1029/// lines are attributed to.
1030struct StderrPump {
1031    task: JoinHandle<()>,
1032    generation: u64,
1033}
1034
1035/// Retire an exited process's stderr reader and wait up to `bound` for it to
1036/// reach EOF. A reader still running at the bound is detached, not stopped: it
1037/// keeps filling the exited process's section of the ring until its pipe
1038/// closes, and the tail reads `Incomplete` until then. See
1039/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
1040async fn settle_stderr_pump(
1041    module_id: &str,
1042    ring: &Arc<Mutex<StderrRing>>,
1043    pump: StderrPump,
1044    bound: Duration,
1045) {
1046    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
1047    let StderrPump {
1048        mut task,
1049        generation,
1050    } = pump;
1051    lock().retire_pump(generation);
1052    match timeout(bound, &mut task).await {
1053        Ok(Ok(())) => {}
1054        Ok(Err(err)) => {
1055            let mut ring = lock();
1056            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
1057            ring.finish_pump(generation);
1058            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
1059        }
1060        Err(_) => {
1061            // Dropping the handle detaches the task; it ends at EOF on its pipe.
1062            drop(task);
1063            lock().mark_pump_late(
1064                generation,
1065                format!(
1066                    "stderr of the exited process had not reached EOF {bound:?} after it was \
1067                     retired (a descendant may still hold the pipe open); lines it still \
1068                     writes are kept in that process's section"
1069                ),
1070            );
1071            warn!(
1072                module_id,
1073                waited = ?bound,
1074                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
1075            );
1076        }
1077    }
1078}
1079
1080fn registration_release_events() -> &'static watch::Sender<u64> {
1081    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
1082    EVENTS.get_or_init(|| {
1083        let (sender, _receiver) = watch::channel(0);
1084        sender
1085    })
1086}
1087
1088pub(crate) fn notify_registration_release() {
1089    let events = registration_release_events();
1090    let next_generation = (*events.borrow()).wrapping_add(1);
1091    events.send_replace(next_generation);
1092}
1093
1094/// The installed companion that evaluates a script module.
1095#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1096#[non_exhaustive]
1097pub enum ScriptRuntime {
1098    Bun,
1099    Python,
1100}
1101
1102impl ScriptRuntime {
1103    pub const fn as_str(self) -> &'static str {
1104        match self {
1105            Self::Bun => "bun",
1106            Self::Python => "python",
1107        }
1108    }
1109}
1110
1111/// Script content and the runtime approved to evaluate it.
1112#[derive(Debug, Clone, PartialEq, Eq)]
1113#[non_exhaustive]
1114pub struct ScriptEntry {
1115    pub runtime: ScriptRuntime,
1116    pub path: PathBuf,
1117}
1118
1119impl ScriptEntry {
1120    pub fn new(runtime: ScriptRuntime, path: PathBuf) -> Self {
1121        Self { runtime, path }
1122    }
1123}
1124
1125/// How to launch one singleton module process.
1126#[derive(Debug, Clone, PartialEq, Eq)]
1127pub struct ModuleSpec {
1128    pub module_id: String,
1129    pub program: PathBuf,
1130    /// Bundled script content. The program is always the installed runtime runner.
1131    pub script: Option<ScriptEntry>,
1132    pub args: Vec<String>,
1133    pub env: Vec<(String, String)>,
1134    /// When true this is a reserved module: each spawn gets a fresh one-time launch
1135    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
1136    /// process can register this module_id (a security-boundary module like the
1137    /// credential vault must not be impersonable while it is down/restarting).
1138    pub reserved: bool,
1139    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
1140    /// Prefixes come from daemon config and must end in `:` before they reach the
1141    /// supervisor; the owner module's current spawn nonce authorizes claims under
1142    /// each prefix.
1143    pub reserved_prefixes: Vec<String>,
1144    /// The wire protocol this module speaks, as DECLARED in daemon config.
1145    ///
1146    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
1147    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
1148    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
1149    /// and NO launch nonce, and a clean exit the daemon did not request is
1150    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
1151    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
1152    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
1153    /// because a process ignores an environment variable it does not read.
1154    ///
1155    /// The argument is the part that cannot be "harmless to a process that
1156    /// ignores it": a stock binary exits on an unknown flag before it listens
1157    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
1158    /// first conformance run against this mode found it. The nonce is withheld
1159    /// because a process that will never present it gains nothing from holding
1160    /// it, and a secret in the environment of a process that does not need it is
1161    /// a leak surface for no benefit.
1162    pub protocol: ModuleProtocol,
1163    /// Whether two processes of this module may run at once, which is what a
1164    /// blue/green swap does for the length of its overlap. Declared in daemon
1165    /// config because the daemon must be able to answer it while the module is
1166    /// down, and so a module cannot talk itself into it after registering.
1167    pub overlap: ModuleOverlap,
1168    /// Keep module-started helpers in the same container across replacements.
1169    /// This is a daemon-config policy, fixed for the lifetime of a container;
1170    /// stopping the module or losing its supervisor still destroys the container.
1171    pub carry_containment: bool,
1172}
1173
1174/// Only the configured launch inputs enter this value; spawned connection paths,
1175/// nonces and inherited environment never do. Its bytes remain daemon-private.
1176#[derive(Clone, PartialEq, Eq)]
1177pub(crate) struct LaunchSpecDigest([u8; 32]);
1178
1179impl fmt::Debug for LaunchSpecDigest {
1180    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1181        f.write_str("LaunchSpecDigest([redacted])")
1182    }
1183}
1184
1185impl LaunchSpecDigest {
1186    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
1187        fn field(hash: &mut Sha256, bytes: &[u8]) {
1188            hash.update((bytes.len() as u64).to_le_bytes());
1189            hash.update(bytes);
1190        }
1191        let mut hash = Sha256::new();
1192        if let Some(script) = &spec.script {
1193            field(
1194                &mut hash,
1195                match script.runtime {
1196                    ScriptRuntime::Bun => b"bun-script",
1197                    ScriptRuntime::Python => b"python-script",
1198                },
1199            );
1200            field(&mut hash, script.path.as_os_str().as_encoded_bytes());
1201        }
1202        if let Some(program) = spec.program.to_str() {
1203            field(&mut hash, b"utf8-program");
1204            field(&mut hash, program.as_bytes());
1205        } else {
1206            // The daemon config is UTF-8, but in-process callers may provide
1207            // native paths: lossy conversion would collapse distinct programs.
1208            #[cfg(unix)]
1209            {
1210                use std::os::unix::ffi::OsStrExt;
1211                field(&mut hash, b"unix-program");
1212                field(&mut hash, spec.program.as_os_str().as_bytes());
1213            }
1214            #[cfg(windows)]
1215            {
1216                use std::os::windows::ffi::OsStrExt;
1217                field(&mut hash, b"windows-program");
1218                let bytes = spec
1219                    .program
1220                    .as_os_str()
1221                    .encode_wide()
1222                    .flat_map(u16::to_le_bytes)
1223                    .collect::<Vec<_>>();
1224                field(&mut hash, &bytes);
1225            }
1226            #[cfg(not(any(unix, windows)))]
1227            field(&mut hash, spec.program.to_string_lossy().as_bytes());
1228        }
1229        hash.update((spec.args.len() as u64).to_le_bytes());
1230        for arg in &spec.args {
1231            field(&mut hash, arg.as_bytes());
1232        }
1233        let mut env = spec.env.iter().collect::<Vec<_>>();
1234        env.sort_unstable();
1235        hash.update((env.len() as u64).to_le_bytes());
1236        for (key, value) in env {
1237            field(&mut hash, key.as_bytes());
1238            field(&mut hash, value.as_bytes());
1239        }
1240        field(
1241            &mut hash,
1242            match spec.protocol {
1243                ModuleProtocol::Subc => b"subc",
1244                ModuleProtocol::None => b"none",
1245            },
1246        );
1247        Self(hash.finalize().into())
1248    }
1249}
1250
1251/// Whether a module tolerates a second process of itself running alongside.
1252///
1253/// Most modules are single-writer on their store (a WAL, a capture log, a
1254/// resident index behind a writer barrier), and two processes on one store
1255/// corrupt it. So a swap, which overlaps the old and new process by design,
1256/// is refused unless the module's config opts in with `overlap: "safe"`.
1257#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1258pub enum ModuleOverlap {
1259    /// Never run two processes of this module at once. The default.
1260    #[default]
1261    Exclusive,
1262    /// The module has said a second process of itself is harmless for the
1263    /// length of a swap.
1264    ///
1265    /// Declare it only if a second instance can run for a few seconds without
1266    /// touching ANY single-writer store: every database, WAL, index, projector
1267    /// and scheduled job the module owns. A lease on part of that state is not
1268    /// enough. broca's session lease guards WAL appends while its run index, its
1269    /// store projector and its archive fold timer (which unlinks live WAL files)
1270    /// stay single-writer, so broca is exclusive despite holding a lease. The
1271    /// refusal only fires after this has been decided, so the decision is the
1272    /// check.
1273    Safe,
1274}
1275
1276impl ModuleOverlap {
1277    pub fn as_str(self) -> &'static str {
1278        match self {
1279            Self::Exclusive => "exclusive",
1280            Self::Safe => "safe",
1281        }
1282    }
1283}
1284
1285/// Environment variable telling a spawned module which case it was started
1286/// for, before it sends HELLO. Only a swap candidate carries it, as
1287/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
1288///
1289/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
1290/// longer because nobody waits on it, while a plain restart must flip ready
1291/// quickly because callers see `module_warming` until it does. Absence means
1292/// plain restart, the safe reading. The daemon trusts nothing about it; the
1293/// candidate is proven by its launch nonce at HELLO.
1294pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
1295/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
1296pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
1297/// How long a swap waits for its candidate to register and declare itself
1298/// ready when the operator does not say. A module warming as a swap candidate
1299/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
1300/// daemon allows that plus time to start the process and send HELLO.
1301pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
1302
1303/// Bounded restart policy for crash exits.
1304///
1305/// `max_restarts` is the number of replacement processes allowed after the
1306/// initial spawn WITHIN `window`. After that many crash restarts inside one
1307/// window the module enters [`ModuleState::Failed`] and the supervisor stops
1308/// the crash loop.
1309///
1310/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
1311/// and that only survived because crashes were rare: a module that crashed
1312/// three times across a week was disabled forever by crashes that had nothing
1313/// to do with each other. That stopped being survivable once modules began
1314/// exiting non-zero whenever the daemon's connection to them drops, because
1315/// then every daemon-side connection drop spends a unit of the same budget and
1316/// one flappy hour permanently stops a healthy module. Restarts older than
1317/// `window` release their slot, so a module that crashed twice yesterday has a
1318/// full budget today, while a genuine crash loop -- which is fast by
1319/// definition -- still reaches the cap and stops.
1320#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1321pub struct RestartPolicy {
1322    pub max_restarts: u32,
1323    /// Base delay before a crash replacement. The actual delay escalates with
1324    /// the number of recent crash replacements and is capped by `max_backoff`.
1325    pub backoff: Duration,
1326    /// Maximum delay before a crash replacement.
1327    pub max_backoff: Duration,
1328    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
1329    /// budget effectively infinite (nothing is ever in-window), which is why
1330    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
1331    pub window: Duration,
1332}
1333
1334impl RestartPolicy {
1335    /// A policy with the default crash window. Callers that care about the
1336    /// window say so with [`Self::with_window`]; the ones that do not are
1337    /// asking for the standard rate limit, not for no limit.
1338    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
1339        Self {
1340            max_restarts,
1341            backoff,
1342            max_backoff: DEFAULT_MAX_BACKOFF,
1343            window: DEFAULT_RESTART_WINDOW,
1344        }
1345    }
1346
1347    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
1348        self.max_backoff = max_backoff;
1349        self
1350    }
1351
1352    pub fn with_window(mut self, window: Duration) -> Self {
1353        self.window = window;
1354        self
1355    }
1356
1357    /// Calculate the capped exponential delay for the next crash replacement.
1358    /// `restart_in_window` is zero for the first replacement after an operator
1359    /// action (restart, reload, re-enable) cleared the crash ring, or after all
1360    /// older crash replacements have aged out of the window.
1361    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
1362        if self.backoff.is_zero() || self.max_backoff.is_zero() {
1363            return Duration::ZERO;
1364        }
1365
1366        let mut delay = self.backoff;
1367        for _ in 0..restart_in_window {
1368            if delay >= self.max_backoff {
1369                return self.max_backoff;
1370            }
1371            delay = delay
1372                .checked_mul(10)
1373                .unwrap_or(self.max_backoff)
1374                .min(self.max_backoff);
1375        }
1376        delay.min(self.max_backoff)
1377    }
1378
1379    /// The one sentence that explains a budget-exhausted stop, used for both the
1380    /// log line and the terminal record so the two cannot drift. It names the
1381    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
1382    /// exactly what this budget is not.
1383    fn budget_exhausted_detail(&self) -> String {
1384        format!(
1385            "crash budget exhausted: max_restarts={} within window_secs={}",
1386            self.max_restarts,
1387            self.window.as_secs()
1388        )
1389    }
1390}
1391
1392impl Default for RestartPolicy {
1393    fn default() -> Self {
1394        Self {
1395            max_restarts: DEFAULT_MAX_RESTARTS,
1396            backoff: DEFAULT_BACKOFF,
1397            max_backoff: DEFAULT_MAX_BACKOFF,
1398            window: DEFAULT_RESTART_WINDOW,
1399        }
1400    }
1401}
1402
1403#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1404struct CrashRestartSchedule {
1405    restart_in_window: u32,
1406    delay: Duration,
1407}
1408
1409/// Whether the daemon itself will bring this module back after the exit being
1410/// handled: it is enabled AND its in-window crash restarts are below the cap.
1411///
1412/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
1413/// the window are dropped here rather than by a timer, so the count is right
1414/// the moment somebody asks and no bookkeeping runs for idle modules.
1415fn daemon_will_restart(
1416    state: &mut SupervisorSnapshot,
1417    policy: &RestartPolicy,
1418    now: Instant,
1419) -> bool {
1420    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1421}
1422
1423const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1424const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1425const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1426const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1427
1428#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1429pub enum HealthAction {
1430    Report,
1431    Restart,
1432    Alert,
1433}
1434
1435impl fmt::Display for HealthAction {
1436    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1437        f.write_str(match self {
1438            Self::Report => "report",
1439            Self::Restart => "restart",
1440            Self::Alert => "alert",
1441        })
1442    }
1443}
1444
1445#[derive(Debug, Clone, PartialEq, Eq)]
1446pub struct HealthConfig {
1447    /// Optional loopback HTTP endpoint for a managed non-wire process.
1448    /// Changing it applies live on rescan; the process protocol changes only
1449    /// at its next spawn.
1450    pub http: Option<String>,
1451    pub cadence: Duration,
1452    pub deadline: Duration,
1453    pub failure_threshold: u32,
1454    pub on_degraded: HealthAction,
1455    pub on_failing: HealthAction,
1456    pub critical: bool,
1457}
1458
1459impl Default for HealthConfig {
1460    fn default() -> Self {
1461        Self {
1462            http: None,
1463            cadence: DEFAULT_HEALTH_CADENCE,
1464            deadline: DEFAULT_HEALTH_DEADLINE,
1465            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1466            on_degraded: HealthAction::Report,
1467            on_failing: HealthAction::Report,
1468            critical: false,
1469        }
1470    }
1471}
1472
1473/// The supervisor's view of one module's health, relayed to clients over
1474/// channel-0 and rendered by `ck health`.
1475///
1476/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1477/// stated here rather than only at the wire type a consumer reads. A reader can
1478/// look up what `None` means; only a writer can silently change it, and the
1479/// writer has no reason to go looking at a downstream contract before editing.
1480///
1481/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1482/// back to `None` on re-registration precisely so a respawned module does not
1483/// carry its predecessor's timestamp — so an old value and an absent one call for
1484/// opposite readings, and anything that defaulted this to a number would make a
1485/// never-probed module indistinguishable from one probed at the epoch.
1486///
1487/// `detail` and `metrics` are `None` when the module published none on this
1488/// probe, which does not mean it reported nothing wrong — it is also the shape
1489/// when the probe never reached it. `last_probe_ms` is what separates those.
1490#[derive(Debug, Clone, PartialEq)]
1491pub struct ModuleHealthStatus {
1492    pub status: SupervisorHealthStatus,
1493    pub last_probe_ms: Option<u64>,
1494    pub detail: Option<String>,
1495    pub metrics: Option<Value>,
1496    pub consecutive_failures: u32,
1497    /// Number of replies received after a recurring health probe's deadline.
1498    /// Unlike a timeout, every increment proves the module was alive.
1499    pub late_answer_count: u64,
1500    /// End-to-end latency of the newest late reply, measured from probe start.
1501    pub last_late_answer_latency_ms: Option<u64>,
1502    pub last_action: Option<String>,
1503    /// Set together with `last_action`; the pair moves as one, and both being
1504    /// absent means no escalation has ever been taken rather than that the last
1505    /// one succeeded.
1506    pub last_action_ms: Option<u64>,
1507}
1508
1509impl Default for ModuleHealthStatus {
1510    fn default() -> Self {
1511        Self {
1512            status: SupervisorHealthStatus::Unknown,
1513            last_probe_ms: None,
1514            detail: None,
1515            metrics: None,
1516            consecutive_failures: 0,
1517            late_answer_count: 0,
1518            last_late_answer_latency_ms: None,
1519            last_action: None,
1520            last_action_ms: None,
1521        }
1522    }
1523}
1524
1525/// Typed lifecycle state for a supervised module.
1526#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1527pub enum ModuleState {
1528    Starting,
1529    Running,
1530    Unresponsive,
1531    Restarting,
1532    Draining,
1533    Stopped,
1534    Failed,
1535    Disabled,
1536}
1537
1538impl fmt::Display for ModuleState {
1539    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1540        f.write_str(match self {
1541            Self::Starting => "starting",
1542            Self::Running => "running",
1543            Self::Unresponsive => "unresponsive",
1544            Self::Restarting => "restarting",
1545            Self::Draining => "draining",
1546            Self::Stopped => "stopped",
1547            Self::Failed => "failed",
1548            Self::Disabled => "disabled",
1549        })
1550    }
1551}
1552
1553/// Supervisor classification of a child-process exit.
1554#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1555pub enum ExitKind {
1556    Clean,
1557    Crash,
1558    DeliberateSeverance,
1559}
1560
1561impl From<ExitKind> for TerminalExitKind {
1562    fn from(kind: ExitKind) -> Self {
1563        match kind {
1564            ExitKind::Clean => Self::Clean,
1565            ExitKind::Crash => Self::Crash,
1566            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1567        }
1568    }
1569}
1570
1571/// Exact process identity retained when a supervised module registers its
1572/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1573#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1574pub(crate) struct ProcessIdentity {
1575    pub(crate) pid: u32,
1576    pub(crate) start_time: u64,
1577}
1578
1579/// Last observed child exit, if any.
1580#[derive(Debug, Clone, PartialEq, Eq)]
1581pub struct ExitReport {
1582    pub kind: ExitKind,
1583    pub code: Option<i32>,
1584    pub signal: Option<i32>,
1585    pub at_ms: u64,
1586}
1587
1588/// Point-in-time module status answerable by subc without forwarding to the
1589/// module process.
1590#[derive(Debug, Clone, PartialEq)]
1591pub struct ModuleStatus {
1592    pub(crate) admission_refused: Option<String>,
1593    pub(crate) legacy_registration_spawn: bool,
1594    pub(crate) registration_declaration: Option<String>,
1595    pub module_id: String,
1596    /// Retained configuration-exit infrastructure failure. Cleared only after a
1597    /// successful launch; an ordinary stop has no infrastructure stop reason.
1598    pub stop_reason: Option<ModuleStopReason>,
1599    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1600    pub waiting_for_previous_pid: Option<u32>,
1601    /// Why the previous instance still prevents a new start.
1602    pub waiting_for_previous_detail: Option<String>,
1603    /// The daemon's Windows per-spawn record: `pipe` after the process it
1604    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1605    /// List/status reports use this record, not the source the module sends
1606    /// in its registration message's manifest.provenance.launch_nonce_source.
1607    pub launch_nonce_source: Option<String>,
1608    pub state: ModuleState,
1609    pub enabled: bool,
1610    pub process_alive: bool,
1611    pub registration_active: bool,
1612    /// The module's declared wire protocol, carried beside `live` because it is
1613    /// what makes `live` readable: the two fields answer one question together.
1614    /// While a process is alive this is its launch declaration, not a later
1615    /// pending-reload edit. When down it is the configured next launch protocol.
1616    pub protocol: ModuleProtocol,
1617    /// Whether the module is serving, under the strongest definition the daemon
1618    /// can assert for its protocol.
1619    ///
1620    /// A subc module must also be REGISTERED: its process being alive says
1621    /// nothing about whether it can take a request. A `protocol: "none"` module
1622    /// never registers, so that term is dropped and this falls back to "enabled,
1623    /// running, and the process the daemon launched is alive" -- which is all
1624    /// the daemon observes about a process that speaks no subc wire. It stays a
1625    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1626    /// rather than printing it bare.
1627    pub live: bool,
1628    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1629    /// restarts have already released their slot, so this count can go down
1630    /// without anybody touching the module.
1631    pub restart_count: u32,
1632    /// Replacement processes spawned over this module's entire supervisor lifetime;
1633    /// unlike `restart_count`, this value is never reset by an operator action
1634    /// and never falls out of a window.
1635    pub lifetime_restarts: u32,
1636    pub spawn_generation: u64,
1637    /// Retained daemon-local outcome; the legacy control response omits it.
1638    pub swap_outcome: Option<SwapOutcome>,
1639    /// The budget `restart_count` is spent against. Carried alongside the count
1640    /// because the count alone does not say how close the module is to being
1641    /// disabled, and reporting one without the other is what makes an
1642    /// about-to-be-retired module look ordinary.
1643    pub max_restarts: u32,
1644    /// The span `restart_count` is counted over. Carried with the pair above for
1645    /// the same reason they are carried together: "2 of 3" means one thing for a
1646    /// ten-minute window and something else entirely for a lifetime.
1647    pub restart_window: Duration,
1648    /// Effective drain and restart timing policy used by this running module.
1649    /// These values are carried together with the restart budget so status
1650    /// readers can compare configured intent with what the supervisor applied.
1651    pub drain_timeout: Duration,
1652    pub restart_backoff: Duration,
1653    pub restart_max_backoff: Duration,
1654    /// The module's process. On macOS this stays absent while the `ck-subc`
1655    /// launch trampoline is still running in that pid, and appears once the
1656    /// exec acknowledgement confirms the module image has replaced it. Launch
1657    /// time and the supervisor's own hold on the process are unaffected.
1658    pub pid: Option<u32>,
1659    pub spawned_at_ms: Option<u64>,
1660    pub spawned_from: Option<PathBuf>,
1661    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1662    pub process_start_time: Option<u64>,
1663    pub last_exit: Option<ExitReport>,
1664    pub health: ModuleHealthStatus,
1665}
1666
1667#[cfg(test)]
1668#[derive(Debug, Clone, Copy, PartialEq)]
1669struct ActorSelectCheckpoint {
1670    generation: u64,
1671    turn: u64,
1672    registered_connection: Option<ConnectionId>,
1673    next_probe_at: Option<Instant>,
1674    wake_after: Option<Duration>,
1675}
1676
1677#[derive(Debug, Clone, PartialEq)]
1678struct SupervisorSnapshot {
1679    admission_refused: Option<String>,
1680    /// Distinguishes an admission-parked actor from an unrelated terminal exit.
1681    config_hold_parked: bool,
1682    /// Preserve daemon attribution for a paused start so a failing release can
1683    /// schedule the next slow retry instead of treating the start as operator work.
1684    config_hold_daemon_initiated: bool,
1685    /// The approval that owns the parked attempt, not whichever approval is latest.
1686    config_hold_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1687    /// Retain an approved launch failure across competing reconciliation passes.
1688    approved_start_failure: Option<(crate::config_gate::runtime::ApprovalIdentity, String)>,
1689    legacy_registration_spawn: bool,
1690    registration_declaration: Option<String>,
1691    stop_reason: Option<ModuleStopReason>,
1692    config_exit_failed_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1693    subtree_target: crate::subtree_resources::Target,
1694    failed_retry: FailedRetry,
1695    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1696    #[cfg(windows)]
1697    pipe_consumed: Option<Arc<OnceLock<()>>>,
1698    /// Counts running-child loop turns, not executor polls of a parked wait.
1699    #[cfg(test)]
1700    actor_turns: u64,
1701    /// Running select after exec confirmation and with no unconsumed registry
1702    /// event. Recording a spawn or starting a loop turn is not this barrier.
1703    #[cfg(test)]
1704    actor_select: Option<ActorSelectCheckpoint>,
1705    state: ModuleState,
1706    enabled: bool,
1707    process_alive: bool,
1708    spawned_protocol: Option<ModuleProtocol>,
1709    spawn_failure: Option<String>,
1710    waiting_for_previous_pid: Option<u32>,
1711    waiting_for_previous_detail: Option<String>,
1712    /// When each crash restart was spent, oldest first. This IS the crash
1713    /// budget: its in-window length is the count an operator sees and the count
1714    /// the restart decision is made against, so there is no second counter that
1715    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1716    /// operator actions that used to zero the old lifetime counter.
1717    crash_restarts: VecDeque<Instant>,
1718    lifetime_restarts: u32,
1719    /// Successful child spawns in this daemon incarnation.
1720    ///
1721    /// `lifetime_restarts` was considered and rejected: it starts at zero
1722    /// (line 640), successful initial/operator spawns in `set_running` do not
1723    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1724    /// increments before a successful replacement exists (lines 604, 3846,
1725    /// and 3921), so a failed spawn can consume it. This counter moves only
1726    /// when a live PID is accepted below.
1727    spawn_generation: u64,
1728    swap_outcome: Option<SwapOutcome>,
1729    pid: Option<u32>,
1730    #[cfg(target_os = "macos")]
1731    report_ready: Option<Arc<OnceLock<()>>>,
1732    /// Last reaped child, retained after current process facts are cleared.
1733    reaped_pid: Option<u32>,
1734    /// Whether a replacement has already been scheduled for the current process.
1735    respawn_pending: bool,
1736    /// A second restart is waiting for the replacement already scheduled.
1737    coalesced_restart_pending: bool,
1738    spawned_at_ms: Option<u64>,
1739    spawned_from: Option<PathBuf>,
1740    launch_spec_digest: Option<LaunchSpecDigest>,
1741    spawned_file_identity: Option<SpawnedFileIdentity>,
1742    process_start_time: Option<u64>,
1743    deliberate_severance: Option<ProcessIdentity>,
1744    last_exit: Option<ExitReport>,
1745    /// Diagnostic attached to the next drain's terminal record, if any.
1746    drain_disposition_detail: Option<String>,
1747    health: ModuleHealthStatus,
1748    last_probe_evidence: Option<HealthProbeEvidence>,
1749    /// Whether the current process was started as a swap candidate and so
1750    /// lives in the module's alternate cgroup. The next swap's candidate takes
1751    /// the other one, so the two processes of a swap never share a cgroup. A
1752    /// plain spawn always uses the primary cgroup.
1753    in_alternate_slot: bool,
1754    /// Whether the current `Draining` state ends in a replacement process
1755    /// (restart, reload, health restart) rather than a stop. Only meaningful
1756    /// while `state` is `Draining`; every entry into that state rewrites it.
1757    /// It is what lets route.open answer the retryable `module_reloading` to a
1758    /// consumer that reaches a still-registered process mid-restart, instead of
1759    /// the `supervisor_not_live` a stop or disable deserves.
1760    draining_to_replace: bool,
1761    /// Whether a configuration update has been applied since the current
1762    /// process was spawned, so that process runs an older spec than the one
1763    /// the supervisor now holds. A queued restart is only coalesced into a
1764    /// fresher process when this is false: a restart requested to pick up a
1765    /// new configuration must not be satisfied by a process that predates it.
1766    configuration_updated_since_spawn: bool,
1767}
1768
1769impl SupervisorSnapshot {
1770    /// The pid that status, provenance and resource readings may report. While
1771    /// the launch trampoline still runs in the pid, reading its executable or
1772    /// resource use would describe `ck-subc`, not the module, so none is
1773    /// reported until the exec acknowledgement confirms the module image.
1774    fn reported_pid(&self) -> Option<u32> {
1775        #[cfg(target_os = "macos")]
1776        if self
1777            .report_ready
1778            .as_ref()
1779            .is_some_and(|ready| ready.get().is_none())
1780        {
1781            return None;
1782        }
1783        self.pid
1784    }
1785
1786    fn starting() -> Self {
1787        Self::new(ModuleState::Starting, true)
1788    }
1789
1790    fn disabled() -> Self {
1791        Self::new(ModuleState::Disabled, false)
1792    }
1793
1794    /// Crash restarts still inside `window`, having dropped the ones that are
1795    /// not. Pruning on read is what makes the budget a rate: an instant older
1796    /// than the window stops holding a slot the moment anybody counts.
1797    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1798        while let Some(oldest) = self.crash_restarts.front() {
1799            if now.duration_since(*oldest) > window {
1800                self.crash_restarts.pop_front();
1801            } else {
1802                break;
1803            }
1804        }
1805        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1806    }
1807
1808    /// Spend one unit of the crash budget and record the restart in the ledger.
1809    ///
1810    /// The ring is bounded by the cap because more than `max_restarts` in-window
1811    /// instants can never be reached (the caller refuses the restart first), so
1812    /// anything beyond that is an unbounded queue waiting to happen.
1813    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1814        self.crash_restarts.push_back(now);
1815        while self.crash_restarts.len() > policy.max_restarts as usize {
1816            self.crash_restarts.pop_front();
1817        }
1818        self.lifetime_restarts += 1;
1819    }
1820
1821    /// Reserve one crash-restart slot and calculate the delay before respawning.
1822    /// The count is captured before recording this restart, so the first retry
1823    /// uses the base delay and each later in-window retry escalates once.
1824    fn next_crash_restart(
1825        &mut self,
1826        policy: &RestartPolicy,
1827        now: Instant,
1828    ) -> Option<CrashRestartSchedule> {
1829        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1830        if restart_in_window >= policy.max_restarts {
1831            return None;
1832        }
1833        self.record_crash_restart(policy, now);
1834        Some(CrashRestartSchedule {
1835            restart_in_window,
1836            delay: policy.delay_for_restart(restart_in_window),
1837        })
1838    }
1839
1840    /// Give the module its full budget back, as an operator restart, reload, or
1841    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1842    /// ledger of what actually happened, and an operator action does not unmake
1843    /// the crashes.
1844    fn clear_crash_restarts(&mut self) {
1845        self.crash_restarts.clear();
1846    }
1847
1848    fn new(state: ModuleState, enabled: bool) -> Self {
1849        Self {
1850            admission_refused: None,
1851            config_hold_parked: false,
1852            config_hold_daemon_initiated: false,
1853            config_hold_approval: None,
1854            approved_start_failure: None,
1855            legacy_registration_spawn: false,
1856            registration_declaration: None,
1857            stop_reason: None,
1858            config_exit_failed_approval: None,
1859            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1860            failed_retry: FailedRetry::default(),
1861            exec_outcome: None,
1862            #[cfg(test)]
1863            actor_turns: 0,
1864            #[cfg(test)]
1865            actor_select: None,
1866            state,
1867            enabled,
1868            process_alive: false,
1869            #[cfg(windows)]
1870            pipe_consumed: None,
1871            spawned_protocol: None,
1872            spawn_failure: None,
1873            waiting_for_previous_pid: None,
1874            waiting_for_previous_detail: None,
1875            crash_restarts: VecDeque::new(),
1876            lifetime_restarts: 0,
1877            spawn_generation: 0,
1878            swap_outcome: None,
1879            pid: None,
1880            #[cfg(target_os = "macos")]
1881            report_ready: None,
1882            reaped_pid: None,
1883            respawn_pending: false,
1884            coalesced_restart_pending: false,
1885            spawned_at_ms: None,
1886            spawned_from: None,
1887            launch_spec_digest: None,
1888            spawned_file_identity: None,
1889            process_start_time: None,
1890            deliberate_severance: None,
1891            last_exit: None,
1892            drain_disposition_detail: None,
1893            health: ModuleHealthStatus::default(),
1894            last_probe_evidence: None,
1895            in_alternate_slot: false,
1896            draining_to_replace: false,
1897            configuration_updated_since_spawn: false,
1898        }
1899    }
1900}
1901
1902type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1903
1904type SpawnSubscriberKey = (ConnectionId, u64);
1905
1906#[derive(Debug)]
1907struct SpawnSubscriber {
1908    version: u8,
1909    frames: mpsc::Sender<Frame>,
1910    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1911    /// from which event. The full frame channel cannot carry that news, so it
1912    /// travels beside it; see `SpawnEventFeed::subscribe`.
1913    lagged: Option<oneshot::Sender<SpawnCursor>>,
1914}
1915
1916#[derive(Debug)]
1917struct SpawnEventState {
1918    daemon_incarnation: String,
1919    seq: u64,
1920    capacity: usize,
1921    live: HashMap<String, LiveSpawn>,
1922    generations: HashMap<String, u64>,
1923    events: VecDeque<SpawnEvent>,
1924    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1925    process_changes: HashMap<String, tokio::sync::watch::Sender<u64>>,
1926}
1927
1928impl Default for SpawnEventState {
1929    fn default() -> Self {
1930        Self {
1931            daemon_incarnation: "unconfigured".to_string(),
1932            seq: 0,
1933            capacity: SPAWN_EVENT_RING_CAPACITY,
1934            live: HashMap::new(),
1935            generations: HashMap::new(),
1936            events: VecDeque::new(),
1937            subscribers: HashMap::new(),
1938            process_changes: HashMap::new(),
1939        }
1940    }
1941}
1942
1943#[derive(Debug, Clone, Default)]
1944struct SpawnEventFeed(
1945    Arc<Mutex<SpawnEventState>>,
1946    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1947);
1948
1949#[derive(Debug, Clone, PartialEq, Eq)]
1950pub(crate) enum SpawnSubscribeRefusal {
1951    ForeignIncarnation { current: String },
1952    TooOld { oldest: SpawnCursor },
1953    Frame(String),
1954}
1955
1956impl SpawnEventFeed {
1957    fn configure_incarnation(&self, daemon_incarnation: String) {
1958        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1959        state.daemon_incarnation = daemon_incarnation;
1960        state.seq = 0;
1961        state.live.clear();
1962        state.generations.clear();
1963        state.events.clear();
1964        state.subscribers.clear();
1965        for changed in state.process_changes.values() {
1966            changed.send_replace(0);
1967        }
1968    }
1969
1970    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1971        SpawnCursor {
1972            daemon_incarnation: state.daemon_incarnation.clone(),
1973            seq: state.seq,
1974        }
1975    }
1976
1977    fn snapshot(&self) -> SpawnSnapshot {
1978        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1979        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1980        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1981        SpawnSnapshot {
1982            cursor: Self::cursor(&state),
1983            ring_bound: state.capacity as u64,
1984            live,
1985        }
1986    }
1987
1988    fn allocate_generation_locked(state: &mut SpawnEventState, module_id: &str) -> u64 {
1989        let generation = state
1990            .generations
1991            .get(module_id)
1992            .copied()
1993            .unwrap_or(0)
1994            .checked_add(1)
1995            .expect("spawn generation exhausted");
1996        state.generations.insert(module_id.to_string(), generation);
1997        generation
1998    }
1999
2000    /// Allocate the spawn generation of a launch that is about to start, with
2001    /// the daemon incarnation it belongs to. The number names the launch from
2002    /// this moment, before its process runs; a launch that never runs keeps
2003    /// its number, so the generations a subscriber sees may skip.
2004    fn allocate_generation(&self, module_id: &str) -> (String, u64) {
2005        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2006        let generation = Self::allocate_generation_locked(&mut state, module_id);
2007        (state.daemon_incarnation.clone(), generation)
2008    }
2009
2010    /// Publish a running process. `allocated` is the generation its launch
2011    /// was given before it started; a launch without one (a process with no
2012    /// registration channel) is numbered here.
2013    fn emit_spawned(
2014        &self,
2015        module_id: &str,
2016        allocated: Option<u64>,
2017        pid: u32,
2018        spawned_at_ms: u64,
2019    ) -> u64 {
2020        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2021        let generation = match allocated {
2022            Some(generation) => generation,
2023            None => Self::allocate_generation_locked(&mut state, module_id),
2024        };
2025        let live = LiveSpawn {
2026            module_id: module_id.to_string(),
2027            spawn_generation: generation,
2028            pid,
2029            spawned_at_ms,
2030        };
2031        state.live.insert(module_id.to_string(), live);
2032        Self::emit_locked(
2033            &mut state,
2034            SpawnEventKind::Spawned,
2035            module_id.to_string(),
2036            generation,
2037            pid,
2038            None,
2039            None,
2040        );
2041        generation
2042    }
2043
2044    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
2045        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2046        let Some(live) = state.live.remove(module_id) else {
2047            warn!(
2048                module_id,
2049                "terminal record had no live spawn event identity"
2050            );
2051            return;
2052        };
2053        Self::emit_locked(
2054            &mut state,
2055            SpawnEventKind::Exited,
2056            module_id.to_string(),
2057            live.spawn_generation,
2058            live.pid,
2059            exit_code,
2060            exit_signal,
2061        );
2062    }
2063
2064    /// Report the exit of a process that a swap has already replaced.
2065    ///
2066    /// `emit_exited` removes the module's live entry, which after a swap's
2067    /// cutover describes the promoted candidate, not the old process now
2068    /// exiting. This emits the old generation's exit and leaves the live entry
2069    /// alone unless it still names that generation.
2070    fn emit_superseded_exited(
2071        &self,
2072        module_id: &str,
2073        spawn_generation: u64,
2074        pid: u32,
2075        exit_code: Option<i32>,
2076        exit_signal: Option<i32>,
2077    ) {
2078        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2079        if state
2080            .live
2081            .get(module_id)
2082            .is_some_and(|live| live.spawn_generation == spawn_generation)
2083        {
2084            state.live.remove(module_id);
2085        }
2086        Self::emit_locked(
2087            &mut state,
2088            SpawnEventKind::Exited,
2089            module_id.to_string(),
2090            spawn_generation,
2091            pid,
2092            exit_code,
2093            exit_signal,
2094        );
2095    }
2096
2097    #[allow(clippy::too_many_arguments)]
2098    fn emit_locked(
2099        state: &mut SpawnEventState,
2100        kind: SpawnEventKind,
2101        module_id: String,
2102        spawn_generation: u64,
2103        pid: u32,
2104        exit_code: Option<i32>,
2105        exit_signal: Option<i32>,
2106    ) {
2107        state.seq = state
2108            .seq
2109            .checked_add(1)
2110            .expect("spawn event sequence exhausted");
2111        let event = SpawnEvent {
2112            cursor: Self::cursor(state),
2113            kind,
2114            module_id,
2115            spawn_generation,
2116            pid,
2117            exit_code,
2118            exit_signal,
2119        };
2120        state
2121            .process_changes
2122            .retain(|_, changed| changed.receiver_count() > 0);
2123        if let Some(changed) = state.process_changes.get(&event.module_id) {
2124            changed.send_replace(state.seq);
2125        }
2126        state.events.push_back(event.clone());
2127        while state.events.len() > state.capacity {
2128            state.events.pop_front();
2129        }
2130        let body = match serde_json::to_vec(&event) {
2131            Ok(body) => body,
2132            Err(error) => {
2133                error!(%error, "failed to serialize supervisor spawn event");
2134                return;
2135            }
2136        };
2137        state.subscribers.retain(|(connection_id, corr), subscriber| {
2138            let frame = Frame::build_with_version(
2139                subscriber.version,
2140                FrameType::StreamData,
2141                control_flags(),
2142                0,
2143                0,
2144                *corr,
2145                body.clone(),
2146            );
2147            match frame {
2148                Ok(frame) => {
2149                    if subscriber.frames.try_send(frame).is_ok() {
2150                        true
2151                    } else {
2152                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
2153                        if let Some(lagged) = subscriber.lagged.take() {
2154                            let _ = lagged.send(event.cursor.clone());
2155                        }
2156                        false
2157                    }
2158                }
2159                Err(error) => {
2160                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
2161                    false
2162                }
2163            }
2164        });
2165    }
2166
2167    fn subscribe(
2168        &self,
2169        connection_id: ConnectionId,
2170        corr: u64,
2171        version: u8,
2172        since: Option<SpawnCursor>,
2173        sink: FrameSink,
2174    ) -> Result<(), SpawnSubscribeRefusal> {
2175        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
2176        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
2177        {
2178            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2179            let replay = if let Some(since) = since {
2180                if since.daemon_incarnation != state.daemon_incarnation {
2181                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
2182                        current: state.daemon_incarnation.clone(),
2183                    });
2184                }
2185                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
2186                    if since.seq < oldest.seq.saturating_sub(1) {
2187                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
2188                    }
2189                }
2190                state
2191                    .events
2192                    .iter()
2193                    .filter(|event| event.cursor.seq > since.seq)
2194                    .cloned()
2195                    .collect::<Vec<_>>()
2196            } else {
2197                Vec::new()
2198            };
2199            for event in replay {
2200                let body = serde_json::to_vec(&event)
2201                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2202                let frame = Frame::build_with_version(
2203                    version,
2204                    FrameType::StreamData,
2205                    control_flags(),
2206                    0,
2207                    0,
2208                    corr,
2209                    body,
2210                )
2211                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2212                frames
2213                    .try_send(frame)
2214                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2215            }
2216            state.subscribers.insert(
2217                (connection_id, corr),
2218                SpawnSubscriber {
2219                    version,
2220                    frames,
2221                    lagged: Some(lagged),
2222                },
2223            );
2224        }
2225        // The lagged terminal is sent here, by the forwarder, rather than by
2226        // the emitter: at the moment of the drop the subscriber's own channel
2227        // is full, and writing to the connection sink directly from the emitter
2228        // would put the Error AHEAD of the events still queued in that channel
2229        // (and the emitter holds the feed lock, so it cannot await the sink).
2230        // Dropping the subscriber drops the only sender, so `recv` drains every
2231        // queued event and then returns `None`; only then is the Error sent, so
2232        // the client sees each event it can keep, then the reason it was cut.
2233        // Cancel and connection removal drop the oneshot unsent, so they end
2234        // the stream with no Error.
2235        tokio::spawn(async move {
2236            while let Some(frame) = receiver.recv().await {
2237                if sink.send(frame).await.is_err() {
2238                    return;
2239                }
2240            }
2241            let Ok(first_undelivered) = lagged_rx.try_recv() else {
2242                return;
2243            };
2244            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
2245                Ok(frame) => {
2246                    let _ = sink.send(frame).await;
2247                }
2248                Err(error) => {
2249                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
2250                }
2251            }
2252        });
2253        Ok(())
2254    }
2255
2256    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
2257        let Some(subscriber) = self
2258            .0
2259            .lock()
2260            .unwrap_or_else(|p| p.into_inner())
2261            .subscribers
2262            .remove(&(connection_id, corr))
2263        else {
2264            return false;
2265        };
2266        if let Ok(frame) = Frame::build_with_version(
2267            subscriber.version,
2268            FrameType::StreamEnd,
2269            control_flags(),
2270            0,
2271            0,
2272            corr,
2273            Vec::new(),
2274        ) {
2275            tokio::spawn(async move {
2276                let _ = subscriber.frames.send(frame).await;
2277            });
2278        }
2279        true
2280    }
2281
2282    fn remove_connection(&self, connection_id: ConnectionId) {
2283        self.0
2284            .lock()
2285            .unwrap_or_else(|p| p.into_inner())
2286            .subscribers
2287            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
2288    }
2289
2290    #[cfg(any(test, feature = "test-support"))]
2291    fn set_capacity(&self, capacity: usize) {
2292        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
2293    }
2294
2295    #[cfg(any(test, feature = "test-support"))]
2296    fn subscriber_count(&self) -> usize {
2297        self.0
2298            .lock()
2299            .unwrap_or_else(|p| p.into_inner())
2300            .subscribers
2301            .len()
2302    }
2303}
2304
2305/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
2306/// The terminal Error a lagged spawn subscriber receives after its queued events.
2307fn spawn_subscriber_lagged_frame(
2308    version: u8,
2309    corr: u64,
2310    first_undelivered: SpawnCursor,
2311) -> Result<Frame, String> {
2312    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
2313    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
2314        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
2315        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
2316            .to_string(),
2317        detail: Some(serde_json::json!({
2318            "first_undelivered_cursor": first_undelivered
2319        })),
2320    })
2321    .map_err(|error| error.to_string())?;
2322    Frame::build_with_version(
2323        version,
2324        FrameType::Error,
2325        crate::router::daemon_error_flags(),
2326        0,
2327        0,
2328        corr,
2329        body,
2330    )
2331    .map_err(|error| error.to_string())
2332}
2333
2334pub trait ModuleProcessLiveness: Send + Sync {
2335    fn process_live(&self, module_id: &str) -> Option<bool>;
2336
2337    /// Whether the supervisor is replacing this module's process right now: an
2338    /// operator restart or reload, a health restart, or a crash respawn whose
2339    /// backoff is running. A module in that state is not live, but a consumer
2340    /// refused now should retry shortly rather than treat the target as gone.
2341    /// Stopped, failed, and disabled modules are not replacing.
2342    fn process_replacing(&self, _module_id: &str) -> bool {
2343        false
2344    }
2345}
2346
2347/// Shared process-liveness registry keyed by supervised `module_id`.
2348#[derive(Debug, Clone, Default)]
2349pub struct SupervisorProcessLiveness {
2350    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
2351}
2352
2353impl SupervisorProcessLiveness {
2354    pub fn new() -> Self {
2355        Self::default()
2356    }
2357
2358    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
2359        let mut snapshots = self
2360            .snapshots
2361            .lock()
2362            .unwrap_or_else(|poisoned| poisoned.into_inner());
2363        snapshots.insert(module_id, snapshot);
2364    }
2365
2366    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
2367        let mut snapshots = self
2368            .snapshots
2369            .lock()
2370            .unwrap_or_else(|poisoned| poisoned.into_inner());
2371        let is_current = snapshots
2372            .get(module_id)
2373            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
2374            .unwrap_or(false);
2375        if is_current {
2376            snapshots.remove(module_id);
2377        }
2378    }
2379}
2380
2381impl ModuleProcessLiveness for SupervisorProcessLiveness {
2382    fn process_live(&self, module_id: &str) -> Option<bool> {
2383        let snapshot = {
2384            let snapshots = self
2385                .snapshots
2386                .lock()
2387                .unwrap_or_else(|poisoned| poisoned.into_inner());
2388            snapshots.get(module_id).cloned()
2389        }?;
2390        let snapshot = snapshot
2391            .lock()
2392            .unwrap_or_else(|poisoned| poisoned.into_inner());
2393        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
2394    }
2395
2396    fn process_replacing(&self, module_id: &str) -> bool {
2397        let Some(snapshot) = self
2398            .snapshots
2399            .lock()
2400            .unwrap_or_else(|poisoned| poisoned.into_inner())
2401            .get(module_id)
2402            .cloned()
2403        else {
2404            return false;
2405        };
2406        let snapshot = snapshot
2407            .lock()
2408            .unwrap_or_else(|poisoned| poisoned.into_inner());
2409        snapshot.enabled
2410            && match snapshot.state {
2411                ModuleState::Restarting => true,
2412                ModuleState::Draining => snapshot.draining_to_replace,
2413                ModuleState::Starting
2414                | ModuleState::Running
2415                | ModuleState::Unresponsive
2416                | ModuleState::Stopped
2417                | ModuleState::Failed
2418                | ModuleState::Disabled => false,
2419            }
2420    }
2421}
2422
2423#[cfg(test)]
2424#[derive(Debug, Default)]
2425struct ReloadExitRecordGate {
2426    reached: tokio::sync::Notify,
2427    resume: tokio::sync::Notify,
2428}
2429
2430#[derive(Debug, Clone, Copy)]
2431enum RespawnKind {
2432    Spawn,
2433    Reload,
2434}
2435
2436#[derive(Debug, Clone)]
2437struct PendingRespawn {
2438    deadline: Instant,
2439    kind: RespawnKind,
2440    daemon_initiated: bool,
2441    approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
2442}
2443
2444impl PendingRespawn {
2445    fn failure_context(&self, runtime: &SupervisorRuntimeConfig) -> FailureContext {
2446        FailureContext {
2447            approval: self.approval.clone(),
2448            ..runtime.failure_context(self.daemon_initiated)
2449        }
2450    }
2451}
2452
2453type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
2454
2455#[derive(Debug, Clone)]
2456struct SupervisorRuntimeConfig {
2457    registry: Arc<Registry>,
2458    config_delivery: Option<config_delivery::ConfigDelivery>,
2459    /// Retained across an operation's deferred backoff, until its child starts.
2460    recovery_is_daemon: bool,
2461    start_requester: Option<Principal>,
2462    pending_activity_restart: bool,
2463    provenance_probe: ExecutableIdentityProbe,
2464    /// Restart operations hand their backoff to the loop, which keeps serving commands.
2465    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2466    /// A reload acknowledges completion only after its replacement registers.
2467    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2468    restart_policy: RestartPolicy,
2469    register_timeout: Duration,
2470    /// This module's RESOLVED drain budget: per-module config when present,
2471    /// else `default_drain_timeout`.
2472    drain_timeout: Duration,
2473    /// Shared with the status handle so the attested value changes atomically
2474    /// when a rescan updates the running drain policy.
2475    effective_drain_timeout: Arc<Mutex<Duration>>,
2476    /// The supervisor-wide fallback, kept so a configuration update that
2477    /// REMOVES the per-module override can re-resolve to it.
2478    default_drain_timeout: Duration,
2479    health: HealthConfig,
2480    host_health: HostHealth,
2481    connection_file_path: Option<PathBuf>,
2482    capture_logs_dir: Option<PathBuf>,
2483    forwarding: Option<Arc<ForwardingTable>>,
2484    /// The shared handle, so every spawn path (initial, restart, reload) records the
2485    /// reserved-module launch nonce the HELLO verifier checks against.
2486    supervisor_handle: Option<SupervisorHandle>,
2487    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
2488    /// status queries.
2489    ///
2490    /// One ring per module, held across every respawn. The lines explaining an exit
2491    /// are written BEFORE that exit, so a ring recreated per process would be empty
2492    /// exactly when it is asked for.
2493    stderr_ring: Arc<Mutex<StderrRing>>,
2494    terminal_ring: Arc<Mutex<TerminalRing>>,
2495    spawn_events: SpawnEventFeed,
2496    child_roster: ChildRoster,
2497    #[cfg(target_os = "linux")]
2498    cgroup_placement: Option<subc_cgroup::Placement>,
2499    #[cfg(test)]
2500    test_seed_stale_facts_before_enable_spawn: bool,
2501    #[cfg(test)]
2502    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2503}
2504
2505impl SupervisorRuntimeConfig {
2506    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2507        FailureContext {
2508            daemon_initiated,
2509            approval: None,
2510            shutdown: self.child_roster.shutdown_flag(),
2511        }
2512    }
2513}
2514
2515#[derive(Debug, Clone, PartialEq, Eq)]
2516struct SupervisedConfiguration {
2517    spec: ModuleSpec,
2518    enabled: bool,
2519    health: HealthConfig,
2520    register_timeout: Duration,
2521    restart_policy: RestartPolicy,
2522}
2523
2524/// Shared daemon lookup table for supervised module handles.
2525///
2526/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2527/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2528/// launch nonces recorded at spawn are checked by the same daemon instance.
2529#[derive(Debug, Clone, Default)]
2530pub struct SupervisorHandle {
2531    #[cfg(test)]
2532    resource_commit_pause: Arc<Mutex<Option<resource_launch::ResourceCommitPause>>>,
2533    #[cfg(test)]
2534    registration_test_channels: CapturedRegistrationChannels,
2535    registration_service: Arc<Mutex<Option<registration::RegistrationService>>>,
2536    registration_connections: Arc<Mutex<HashMap<ConnectionId, Arc<RegistrationSpawn>>>>,
2537    registration_launches: Arc<Mutex<Vec<std::sync::Weak<RegistrationSpawn>>>>,
2538    gate_runtime: Arc<Mutex<Option<std::sync::Weak<crate::config_gate::runtime::RuntimeOwner>>>>,
2539    program_bindings: program_binding::ProgramBindings,
2540    #[cfg(test)]
2541    identity_sample_pause: Arc<Mutex<Option<IdentitySamplePause>>>,
2542    #[cfg(all(test, unix))]
2543    plain_identity_policy_pause: Arc<Mutex<Option<(String, IdentitySamplePause)>>>,
2544    host_health: HostHealth,
2545    restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2546    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2547    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2548    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2549    /// Module ids the supervisor has taken on. An id is added BEFORE the
2550    /// module's first process is spawned and removed only when the module
2551    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2552    /// the keys of `modules`.
2553    ///
2554    /// `modules` cannot answer "is this module configured?" on its own: a
2555    /// [`SupervisedModule`] only exists once its process has been spawned, and
2556    /// a fast child can connect, register, sync its scopes and ask about them
2557    /// before the supervisor has inserted it. Answering "not configured" in that
2558    /// gap makes scope admission refuse with the terminal "will never sync"
2559    /// instead of the retryable "has not synced yet".
2560    configured_ids: Arc<Mutex<HashSet<String>>>,
2561    spawn_events: SpawnEventFeed,
2562    /// The current expected launch nonce for each reserved module_id. Set when the
2563    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2564    /// non-reserved module never has an entry here and is never nonce-checked.
2565    /// Reserved module ids and the nonce that authorizes their next HELLO.
2566    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2567    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2568    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2569    /// had NO entry and admitted anyone: the reservation protected the nonce
2570    /// holder, not the NAME (found live by CKCRED's canary probe registering
2571    /// against a reserved scratch id).
2572    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2573    /// Reservations installed only to prevent HELLO for a protocol:none process.
2574    /// Replacement by a subc process releases them; explicitly reserved ids are
2575    /// never placed in this set and still require an operator release.
2576    implicit_protocol_none: Arc<Mutex<HashSet<String>>>,
2577    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2578    ///
2579    /// This is deliberately in-memory only: subc is state-free across daemon
2580    /// restarts, and the tombstone only explains the hours-after-removal window
2581    /// while this executing daemon is still alive. Do not persist it in a store.
2582    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2583    /// The current launch nonce for every supervised spawn. This is separate from
2584    /// reserved_nonces because consumer route.open attestation applies to all spawned
2585    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2586    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2587    /// The launch each live launch nonce was minted for, keyed by nonce, with
2588    /// the launch's module id, so a nonce-attested consumer or registration
2589    /// resolves to that launch's authority record. An entry whose launch has
2590    /// ended stays while its nonce still authorizes, so a nonce whose record
2591    /// is gone is told apart from one that never had a record.
2592    nonce_launches: Arc<Mutex<NonceLaunches>>,
2593    /// Reserved namespace prefixes mapped to the supervised owner module whose
2594    /// current spawn nonce authorizes HELLO claims below the prefix.
2595    ///
2596    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2597    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2598    /// accidental collisions and lower-trust processes from squatting protected
2599    /// namespaces.
2600    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2601    /// Blue/green swaps in progress, by module id. An entry exists from just
2602    /// before the candidate process is spawned until the swap has failed, or
2603    /// has cut over and the old process is gone. While it exists, HELLO for the
2604    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2605    /// consumer attestation accepts both processes' nonces.
2606    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2607    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2608    promotion_observer: PromotionObserverSlot,
2609    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2610    /// this daemon-wide ordering, a rescan could retire or update a module while a
2611    /// concurrent reload still held its old handle and launch specification.
2612    operation_lock: Arc<AsyncMutex<()>>,
2613}
2614
2615#[cfg(test)]
2616#[derive(Debug)]
2617struct IdentitySamplePause {
2618    reached: std::sync::mpsc::Sender<()>,
2619    release: std::sync::mpsc::Receiver<()>,
2620}
2621
2622/// Told when a swap has promoted its candidate to be the module's active
2623/// registration.
2624///
2625/// An ordinary HELLO runs the control plane's registration side effects (the
2626/// capability cache, the deny census, the requirement recompute) as it
2627/// registers. A swap candidate's HELLO does not, because it is not routable;
2628/// promotion is when those must run instead, and promotion happens in the
2629/// supervisor, which has no other way into the control handler.
2630pub(crate) trait SwapPromotionObserver: Send + Sync {
2631    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2632}
2633
2634/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2635/// control handler) owns this handle, so a strong reference back would be a
2636/// cycle that keeps both alive.
2637#[derive(Clone, Default)]
2638struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2639
2640impl fmt::Debug for PromotionObserverSlot {
2641    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2642        f.write_str("PromotionObserverSlot")
2643    }
2644}
2645
2646/// The nonces of one open swap.
2647#[derive(Debug, Clone)]
2648struct OpenSwap {
2649    /// Legacy candidate token. A gated candidate has an empty lifecycle
2650    /// placeholder instead; only its own channel admits its HELLO.
2651    candidate_nonce: String,
2652    /// Incumbent token accepted before cutover, then cleared by promotion even
2653    /// while the incumbent's process is still draining.
2654    incumbent_nonce: Option<String>,
2655    /// Set once a HELLO has been admitted with the swap token, so the token
2656    /// admits one registration and cannot be replayed after cutover empties
2657    /// the candidate slot.
2658    candidate_admitted: bool,
2659}
2660
2661/// What the swap gate says about a HELLO. See
2662/// [`SupervisorHandle::swap_hello_admission`].
2663#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2664pub(crate) enum SwapHelloAdmission {
2665    /// No swap is open for the id (or the HELLO carries the incumbent's own
2666    /// nonce); the ordinary gates decide.
2667    NotSwapping,
2668    /// The HELLO carries the swap token: register it into the candidate slot.
2669    Candidate,
2670    /// A swap is open and the HELLO carries a nonce the supervisor did not
2671    /// mint for this id, no nonce, or a token already used.
2672    Refused,
2673    /// The id belongs to a supervised `protocol: none` module. Such a module
2674    /// never registers, so no HELLO may claim its id, with or without a nonce.
2675    ProtocolNone,
2676    /// The id is gated. Its supervised child registers only over the
2677    /// registration channel it inherited at launch, never by HELLO alone.
2678    ChannelRequired,
2679}
2680
2681#[derive(Debug, Clone, PartialEq, Eq)]
2682pub(crate) enum ReservedHelloRejection {
2683    Exact {
2684        module_id: String,
2685    },
2686    Prefix {
2687        prefix: String,
2688        owner_module_id: String,
2689    },
2690}
2691
2692/// Identity facts established by the reserved-module policy for one HELLO.
2693#[derive(Debug, Default)]
2694pub(crate) struct ReservedHelloVerification {
2695    pub(crate) nonce_verified: bool,
2696    pub(crate) prefix_owner: Option<String>,
2697}
2698
2699#[derive(Debug)]
2700struct DaemonRestartIntent {
2701    expires: Option<Duration>,
2702    window: Duration,
2703}
2704
2705impl Default for DaemonRestartIntent {
2706    fn default() -> Self {
2707        Self {
2708            expires: None,
2709            window: Duration::from_secs(120),
2710        }
2711    }
2712}
2713
2714impl SupervisorHandle {
2715    pub(crate) fn module_is_gated(&self, module: &str) -> bool {
2716        let runtime = {
2717            let Ok(runtime) = self.gate_runtime.lock() else {
2718                return true;
2719            };
2720            runtime.clone()
2721        };
2722        runtime.is_some_and(|runtime| {
2723            runtime
2724                .upgrade()
2725                .is_none_or(|owner| owner.module_is_gated(module))
2726        })
2727    }
2728    pub(crate) fn announce_daemon_restart(&self) -> u64 {
2729        let mut intent = self
2730            .restart_intent
2731            .lock()
2732            .unwrap_or_else(|p| p.into_inner());
2733        intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2734        unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2735    }
2736
2737    #[cfg(feature = "test-support")]
2738    pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2739        self.restart_intent
2740            .lock()
2741            .unwrap_or_else(|p| p.into_inner())
2742            .window = window;
2743    }
2744
2745    #[cfg(unix)]
2746    fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2747        let expiry = self
2748            .restart_intent
2749            .lock()
2750            .unwrap_or_else(|p| p.into_inner())
2751            .expires
2752            .take();
2753        if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2754            RouteCloseReason::Restart
2755        } else {
2756            RouteCloseReason::Disable
2757        }
2758    }
2759
2760    pub fn new() -> Self {
2761        Self::default()
2762    }
2763
2764    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2765        self.spawn_events.snapshot()
2766    }
2767
2768    /// Subscribe before reading the bound process. A coalesced wake is enough:
2769    /// readers compare its exact generation and PID, not the last event kind.
2770    pub(crate) fn subscribe_process_changes(
2771        &self,
2772        module: &str,
2773    ) -> tokio::sync::watch::Receiver<u64> {
2774        let mut state = self
2775            .spawn_events
2776            .0
2777            .lock()
2778            .unwrap_or_else(|p| p.into_inner());
2779        state
2780            .process_changes
2781            .retain(|_, changed| changed.receiver_count() > 0);
2782        let seq = state.seq;
2783        state
2784            .process_changes
2785            .entry(module.into())
2786            .or_insert_with(|| tokio::sync::watch::channel(seq).0)
2787            .subscribe()
2788    }
2789
2790    pub(crate) fn current_spawn(&self, module: &str) -> Option<LiveSpawn> {
2791        self.spawn_events
2792            .0
2793            .lock()
2794            .unwrap_or_else(|p| p.into_inner())
2795            .live
2796            .get(module)
2797            .cloned()
2798    }
2799
2800    pub(crate) fn subscribe_spawns(
2801        &self,
2802        connection_id: ConnectionId,
2803        corr: u64,
2804        version: u8,
2805        since: Option<SpawnCursor>,
2806        sink: FrameSink,
2807    ) -> Result<(), SpawnSubscribeRefusal> {
2808        self.spawn_events
2809            .subscribe(connection_id, corr, version, since, sink)
2810    }
2811
2812    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2813        self.spawn_events.cancel(connection_id, corr)
2814    }
2815
2816    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2817        self.spawn_events.remove_connection(connection_id);
2818    }
2819
2820    #[cfg(any(test, feature = "test-support"))]
2821    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2822        assert!(capacity > 0, "spawn event capacity must be non-zero");
2823        self.spawn_events.set_capacity(capacity);
2824    }
2825
2826    #[cfg(any(test, feature = "test-support"))]
2827    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2828        self.spawn_events.subscriber_count()
2829    }
2830
2831    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2832    /// a respawn invalidates stale consumer identities.
2833    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2834        self.spawn_nonces
2835            .lock()
2836            .unwrap_or_else(|poisoned| poisoned.into_inner())
2837            .insert(module_id.to_string(), nonce);
2838        self.refresh_program_binding_nonces(module_id);
2839    }
2840
2841    /// Record the launch nonce expected from the next HELLO for a reserved module,
2842    /// replacing any prior nonce (a respawn invalidates the previous one).
2843    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2844        {
2845            let mut reserved = self
2846                .reserved_nonces
2847                .lock()
2848                .unwrap_or_else(|p| p.into_inner());
2849            self.implicit_protocol_none
2850                .lock()
2851                .unwrap_or_else(|p| p.into_inner())
2852                .remove(module_id);
2853            reserved.insert(module_id.to_string(), Some(nonce));
2854        }
2855        self.refresh_program_binding_nonces(module_id);
2856    }
2857
2858    /// Record namespace prefixes owned by a supervised module.
2859    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2860        let mut owners = self
2861            .reserved_prefix_owners
2862            .lock()
2863            .unwrap_or_else(|poisoned| poisoned.into_inner());
2864        owners.retain(|_, owner| owner != owner_module_id);
2865        for prefix in prefixes {
2866            owners.insert(prefix.clone(), owner_module_id.to_string());
2867        }
2868    }
2869
2870    /// The launch nonce most recently minted for a module's spawn, if any.
2871    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2872        if self.module_is_gated(module_id) {
2873            return None;
2874        }
2875        self.spawn_nonces
2876            .lock()
2877            .unwrap_or_else(|poisoned| poisoned.into_inner())
2878            .get(module_id)
2879            .cloned()
2880    }
2881
2882    #[cfg(all(test, unix))]
2883    pub(crate) fn legacy_nonce_present_for_test(&self, module: &str) -> bool {
2884        self.spawn_nonces.lock().unwrap().contains_key(module)
2885    }
2886
2887    pub(crate) fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2888        // Prefix ownership follows staged configuration, not the running
2889        // child's launch settings. A respawn may still carry an older spec.
2890        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2891        let protocol = self
2892            .get(&spec.module_id)
2893            .and_then(|module| {
2894                let snapshot = module.inner.snapshot.lock().ok()?;
2895                snapshot
2896                    .process_alive
2897                    .then_some(snapshot.spawned_protocol.unwrap_or(spec.protocol))
2898            })
2899            .unwrap_or(spec.protocol);
2900        self.apply_identity_policy(spec, protocol);
2901    }
2902
2903    #[cfg(all(test, unix))]
2904    pub(crate) fn pause_plain_identity_policy_for_test(
2905        &self,
2906        module: &str,
2907    ) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
2908        let (reached, waiting) = std::sync::mpsc::channel();
2909        let (release, paused) = std::sync::mpsc::channel();
2910        *self.plain_identity_policy_pause.lock().unwrap() = Some((
2911            module.to_owned(),
2912            IdentitySamplePause {
2913                reached,
2914                release: paused,
2915            },
2916        ));
2917        (waiting, release)
2918    }
2919
2920    fn apply_identity_policy(&self, spec: &ModuleSpec, protocol: ModuleProtocol) {
2921        let channel_only = self.module_is_gated(&spec.module_id);
2922        let mut current_nonces = self
2923            .spawn_nonces
2924            .lock()
2925            .unwrap_or_else(|poisoned| poisoned.into_inner());
2926        if protocol == ModuleProtocol::None || channel_only {
2927            current_nonces.remove(&spec.module_id);
2928        }
2929        let spawn_nonce = current_nonces.get(&spec.module_id).cloned();
2930        #[cfg(test)]
2931        if let Some(pause) = self.identity_sample_pause.lock().unwrap().take() {
2932            pause.reached.send(()).unwrap();
2933            pause.release.recv().unwrap();
2934        }
2935        let mut reserved_nonces = self
2936            .reserved_nonces
2937            .lock()
2938            .unwrap_or_else(|poisoned| poisoned.into_inner());
2939        let mut implicit = self
2940            .implicit_protocol_none
2941            .lock()
2942            .unwrap_or_else(|p| p.into_inner());
2943        if spec.reserved {
2944            implicit.remove(&spec.module_id);
2945        } else if protocol == ModuleProtocol::None {
2946            if !reserved_nonces.contains_key(&spec.module_id) {
2947                implicit.insert(spec.module_id.clone());
2948            }
2949        } else if implicit.remove(&spec.module_id) {
2950            reserved_nonces.remove(&spec.module_id);
2951        }
2952        if spec.reserved || protocol == ModuleProtocol::None {
2953            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2954            // reserved name whose module has never spawned has no legitimate
2955            // holder, and the entry's absence is what used to leave the name
2956            // open to the first claimant.
2957            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2958        }
2959        drop(implicit);
2960        drop(reserved_nonces);
2961        drop(current_nonces);
2962        self.refresh_program_binding_nonces(&spec.module_id);
2963        // A later unreserved declaration must not silently unreserve an id that
2964        // was retained after its reserved configuration was removed. The explicit
2965        // release ceremony is the only operation that retires that gate.
2966        self.removal_tombstones
2967            .lock()
2968            .unwrap_or_else(|poisoned| poisoned.into_inner())
2969            .remove(&spec.module_id);
2970    }
2971
2972    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2973    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2974    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2975    /// with no matching prefix are always authorized.
2976    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2977        self.reserved_hello_verification(module_id, presented)
2978            .is_ok()
2979    }
2980
2981    /// Whether HELLO passed a nonce check that was actually required by reserved
2982    /// module policy, or the refusal explaining why a required check failed.
2983    pub(crate) fn reserved_hello_verification(
2984        &self,
2985        module_id: &str,
2986        presented: Option<&str>,
2987    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2988        if self.module_is_gated(module_id) {
2989            return Err(ReservedHelloRejection::Exact {
2990                module_id: module_id.into(),
2991            });
2992        }
2993        self.hello_verification(module_id, presented, false)
2994    }
2995
2996    pub(crate) fn channel_hello_verification(
2997        &self,
2998        module_id: &str,
2999    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
3000        if self
3001            .get(module_id)
3002            .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3003        {
3004            return Err(ReservedHelloRejection::Exact {
3005                module_id: module_id.into(),
3006            });
3007        }
3008        let mut verification = self.hello_verification(module_id, None, true)?;
3009        verification.nonce_verified = true;
3010        Ok(verification)
3011    }
3012
3013    pub(crate) fn prefix_claim_authorized(&self, owner: &str, module_id: &str) -> bool {
3014        self.prefix_claim_publication(owner, module_id).is_some()
3015    }
3016
3017    #[cfg(all(test, unix))]
3018    pub(crate) fn prefix_publication_locked_for_test(&self) -> bool {
3019        matches!(
3020            self.reserved_prefix_owners.try_lock(),
3021            Err(std::sync::TryLockError::WouldBlock)
3022        )
3023    }
3024
3025    // Keep this guard through registry/forwarding publication. Rescans replace
3026    // prefix owners without stopping their launches, so the spawn guard alone
3027    // cannot fence a reservation transfer.
3028    pub(crate) fn prefix_claim_publication(
3029        &self,
3030        owner: &str,
3031        module_id: &str,
3032    ) -> Option<std::sync::MutexGuard<'_, HashMap<String, String>>> {
3033        if subc_protocol::manifest::validate_module_id(module_id).is_err()
3034            || self.module_is_gated(module_id)
3035            || self
3036                .reserved_nonces
3037                .lock()
3038                .unwrap_or_else(|p| p.into_inner())
3039                .contains_key(module_id)
3040        {
3041            return None;
3042        }
3043        let owners = self
3044            .reserved_prefix_owners
3045            .lock()
3046            .unwrap_or_else(|p| p.into_inner());
3047        let authorized = owners
3048            .iter()
3049            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3050            .max_by_key(|(prefix, _)| prefix.len())
3051            .is_some_and(|(prefix, holder)| holder == owner && module_id.len() > prefix.len());
3052        authorized.then_some(owners)
3053    }
3054
3055    fn hello_verification(
3056        &self,
3057        module_id: &str,
3058        presented: Option<&str>,
3059        channel: bool,
3060    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
3061        let nonces = self
3062            .reserved_nonces
3063            .lock()
3064            .unwrap_or_else(|poisoned| poisoned.into_inner());
3065        if let Some(expected) = nonces.get(module_id) {
3066            // `None` = reserved with no legitimate holder: refuse every
3067            // presentation, because no process can hold a nonce that was never
3068            // minted. Only a real minted nonce admits, in constant time.
3069            let authorized = channel
3070                || match expected {
3071                    Some(expected) => presented.is_some_and(|p| {
3072                        !p.is_empty() && constant_time_eq(expected.as_bytes(), p.as_bytes())
3073                    }),
3074                    None => false,
3075                };
3076            if authorized {
3077                return Ok(ReservedHelloVerification {
3078                    nonce_verified: true,
3079                    prefix_owner: None,
3080                });
3081            }
3082            return Err(ReservedHelloRejection::Exact {
3083                module_id: module_id.to_string(),
3084            });
3085        }
3086        drop(nonces);
3087
3088        let matched_prefix = self
3089            .reserved_prefix_owners
3090            .lock()
3091            .unwrap_or_else(|poisoned| poisoned.into_inner())
3092            .iter()
3093            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3094            .max_by_key(|(prefix, _)| prefix.len())
3095            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
3096        let Some((prefix, owner_module_id)) = matched_prefix else {
3097            return Ok(ReservedHelloVerification::default());
3098        };
3099
3100        if module_id.len() == prefix.len()
3101            || (!channel
3102                && (self.module_is_gated(&owner_module_id) || self.protocol_none(&owner_module_id)))
3103        {
3104            return Err(ReservedHelloRejection::Prefix {
3105                prefix,
3106                owner_module_id,
3107            });
3108        }
3109
3110        let authorized = if channel {
3111            owner_module_id == module_id
3112        } else {
3113            presented.is_some_and(|presented| {
3114                self.spawn_nonces
3115                .lock()
3116                .unwrap_or_else(|poisoned| poisoned.into_inner())
3117                .get(&owner_module_id)
3118                .is_some_and(|expected| !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes()))
3119                // While the owner is being swapped, children started by
3120                // either of its two processes hold that process's nonce.
3121                || self.swap_nonce_matches(&owner_module_id, presented)
3122            })
3123        };
3124        if authorized {
3125            Ok(ReservedHelloVerification {
3126                nonce_verified: true,
3127                prefix_owner: (owner_module_id != module_id).then_some(owner_module_id),
3128            })
3129        } else {
3130            Err(ReservedHelloRejection::Prefix {
3131                prefix,
3132                owner_module_id,
3133            })
3134        }
3135    }
3136
3137    /// Whether a consumer connection proved it came from a daemon-spawned module.
3138    ///
3139    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
3140    /// accepted only for module ids the supervisor has spawned.
3141    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
3142        if self.module_is_gated(module_id) || self.protocol_none(module_id) {
3143            return false;
3144        }
3145        let nonces = self
3146            .spawn_nonces
3147            .lock()
3148            .unwrap_or_else(|poisoned| poisoned.into_inner());
3149        let current = nonces.get(module_id).is_some_and(|expected| {
3150            !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes())
3151        });
3152        drop(nonces);
3153        // Before cutover both launches may attest. Promotion removes the
3154        // incumbent's token even if its process still needs time to drain.
3155        current || self.swap_nonce_matches(module_id, presented)
3156    }
3157
3158    /// Whether `presented` is either nonce of an open swap for `module_id`.
3159    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
3160        if self.module_is_gated(module_id) || self.protocol_none(module_id) || presented.is_empty()
3161        {
3162            return false;
3163        }
3164        let swaps = self
3165            .swaps
3166            .lock()
3167            .unwrap_or_else(|poisoned| poisoned.into_inner());
3168        swaps.get(module_id).is_some_and(|swap| {
3169            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
3170                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
3171                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
3172                })
3173        })
3174    }
3175
3176    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
3177    /// Called before the candidate process exists.
3178    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
3179        let incumbent_nonce = self
3180            .spawn_nonces
3181            .lock()
3182            .unwrap_or_else(|poisoned| poisoned.into_inner())
3183            .get(module_id)
3184            .cloned();
3185        self.swaps
3186            .lock()
3187            .unwrap_or_else(|poisoned| poisoned.into_inner())
3188            .insert(
3189                module_id.to_string(),
3190                OpenSwap {
3191                    candidate_nonce,
3192                    incumbent_nonce,
3193                    candidate_admitted: false,
3194                },
3195            );
3196        self.refresh_program_binding_nonces(module_id);
3197    }
3198
3199    /// Close the swap for `module_id`, releasing whichever nonce is no longer
3200    /// the module's recorded one.
3201    pub(crate) fn close_swap(&self, module_id: &str) {
3202        self.swaps
3203            .lock()
3204            .unwrap_or_else(|poisoned| poisoned.into_inner())
3205            .remove(module_id);
3206        self.refresh_program_binding_nonces(module_id);
3207    }
3208
3209    /// Install the observer told about swap promotions, replacing any earlier
3210    /// one.
3211    pub(crate) fn set_swap_promotion_observer(
3212        &self,
3213        observer: std::sync::Weak<dyn SwapPromotionObserver>,
3214    ) {
3215        *self
3216            .promotion_observer
3217            .0
3218            .lock()
3219            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
3220    }
3221
3222    /// Tell the installed observer, if it is still alive, that a swap promoted
3223    /// `registration`.
3224    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
3225        let observer = self
3226            .promotion_observer
3227            .0
3228            .lock()
3229            .unwrap_or_else(|poisoned| poisoned.into_inner())
3230            .as_ref()
3231            .and_then(std::sync::Weak::upgrade);
3232        if let Some(observer) = observer {
3233            observer.swap_promoted(registration);
3234        }
3235    }
3236
3237    /// Whether a swap is open for `module_id`.
3238    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
3239        self.swaps
3240            .lock()
3241            .unwrap_or_else(|poisoned| poisoned.into_inner())
3242            .contains_key(module_id)
3243    }
3244
3245    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
3246    /// respawn would, once cutover has made the candidate the module's process.
3247    /// Promotion revokes the incumbent's tokens; the swap record stays open
3248    /// only for lifecycle cleanup while the old process drains.
3249    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
3250        let candidate_nonce = self
3251            .swaps
3252            .lock()
3253            .unwrap_or_else(|poisoned| poisoned.into_inner())
3254            .get_mut(module_id)
3255            .map(|swap| {
3256                swap.incumbent_nonce = None;
3257                swap.candidate_nonce.clone()
3258            });
3259        self.spawn_nonces
3260            .lock()
3261            .unwrap_or_else(|p| p.into_inner())
3262            .remove(module_id);
3263        {
3264            let mut nonces = self
3265                .reserved_nonces
3266                .lock()
3267                .unwrap_or_else(|p| p.into_inner());
3268            if reserved || nonces.contains_key(module_id) {
3269                nonces.insert(module_id.into(), None);
3270            }
3271        }
3272        if self.module_is_gated(module_id) {
3273            return;
3274        }
3275        let Some(nonce) = candidate_nonce.filter(|nonce| !nonce.is_empty()) else {
3276            return;
3277        };
3278        self.set_spawn_nonce(module_id, nonce.clone());
3279        if reserved {
3280            self.set_reserved_nonce(module_id, nonce);
3281        }
3282    }
3283
3284    /// The swap gate for a HELLO claiming `module_id`.
3285    ///
3286    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
3287    /// presents the candidate nonce, which the reserved gate (holding the
3288    /// incumbent's nonce) would refuse as `reserved_module` before swap
3289    /// admission was ever reached. And it applies to unreserved ids too: for an
3290    /// unreserved id the only thing that ever stopped a second process claiming
3291    /// a live id was the `duplicate_module_id` refusal, which is exactly the
3292    /// refusal a swap lifts for its candidate.
3293    ///
3294    /// The incumbent's own nonce falls through to the ordinary gates, which
3295    /// treat it as they always have (a live incumbent is refused as a
3296    /// duplicate). Anything else while a swap is open is refused, including an
3297    /// absent nonce.
3298    pub(crate) fn swap_hello_admission(
3299        &self,
3300        module_id: &str,
3301        presented: Option<&str>,
3302    ) -> SwapHelloAdmission {
3303        if self.protocol_none(module_id) {
3304            return SwapHelloAdmission::ProtocolNone;
3305        }
3306        if self.module_is_gated(module_id) {
3307            return SwapHelloAdmission::ChannelRequired;
3308        }
3309        let swaps = self
3310            .swaps
3311            .lock()
3312            .unwrap_or_else(|poisoned| poisoned.into_inner());
3313        let Some(swap) = swaps.get(module_id) else {
3314            return SwapHelloAdmission::NotSwapping;
3315        };
3316        let Some(presented) = presented.filter(|nonce| !nonce.is_empty()) else {
3317            return SwapHelloAdmission::Refused;
3318        };
3319        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
3320            return if swap.candidate_admitted {
3321                SwapHelloAdmission::Refused
3322            } else {
3323                SwapHelloAdmission::Candidate
3324            };
3325        }
3326        if swap
3327            .incumbent_nonce
3328            .as_deref()
3329            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
3330        {
3331            return SwapHelloAdmission::NotSwapping;
3332        }
3333        SwapHelloAdmission::Refused
3334    }
3335
3336    fn protocol_none(&self, module: &str) -> bool {
3337        self.get(module)
3338            .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3339    }
3340
3341    /// Exclude swap closure through the short, synchronous candidate publication.
3342    pub(crate) fn with_swap_candidate_admission<T>(
3343        &self,
3344        module_id: &str,
3345        publish: impl FnOnce() -> (T, bool),
3346    ) -> Option<T> {
3347        let mut swaps = self.swaps.lock().unwrap_or_else(|p| p.into_inner());
3348        let swap = swaps
3349            .get_mut(module_id)
3350            .filter(|swap| !swap.candidate_admitted)?;
3351        let (result, admitted) = publish();
3352        swap.candidate_admitted = admitted;
3353        Some(result)
3354    }
3355
3356    /// Test/support lookup for the current launch nonce of a supervised spawn.
3357    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3358        self.spawn_nonce(module_id)
3359    }
3360
3361    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
3362    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3363        if self.module_is_gated(module_id) {
3364            return None;
3365        }
3366        self.reserved_nonces
3367            .lock()
3368            .unwrap_or_else(|poisoned| poisoned.into_inner())
3369            .get(module_id)
3370            .cloned()
3371            .flatten()
3372    }
3373
3374    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
3375        // Normally already marked before the process was spawned; marking here
3376        // too keeps `configured_ids` a superset of the roster for any caller
3377        // that inserts a module directly.
3378        self.mark_configured(module.module_id());
3379        let mut modules = self
3380            .modules
3381            .lock()
3382            .unwrap_or_else(|poisoned| poisoned.into_inner());
3383        modules.insert(module.module_id().to_string(), module)
3384    }
3385
3386    /// Record that the supervisor has taken on `module_id`. Called before the
3387    /// module's first process is spawned, so that by the time that process can
3388    /// register, [`Self::is_configured`] already answers true.
3389    fn mark_configured(&self, module_id: &str) {
3390        self.update_configured_ids(|ids| {
3391            ids.insert(module_id.to_string());
3392        });
3393    }
3394
3395    fn publish_configured_ids(&self, ids: Option<&HashSet<String>>) {
3396        let owner = self
3397            .gate_runtime
3398            .lock()
3399            .unwrap_or_else(|p| p.into_inner())
3400            .clone()
3401            .and_then(|owner| owner.upgrade());
3402        if let Some(owner) = owner {
3403            owner.observe_supervisor_configured_ids(ids, Arc::downgrade(&self.configured_ids));
3404        }
3405    }
3406
3407    fn update_configured_ids(&self, update: impl FnOnce(&mut HashSet<String>)) {
3408        match self.configured_ids.lock() {
3409            Ok(mut ids) => {
3410                update(&mut ids);
3411                // Publish while this roster mutation is serialized, so an
3412                // older snapshot can never overwrite a later configured id.
3413                self.publish_configured_ids(Some(&ids));
3414            }
3415            Err(poisoned) => {
3416                update(&mut poisoned.into_inner());
3417                self.publish_configured_ids(None);
3418            }
3419        }
3420    }
3421
3422    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
3423    /// before it was ever put on the roster. A module already on the roster
3424    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
3425    fn unmark_configured_unless_rostered(&self, module_id: &str) {
3426        let modules = self
3427            .modules
3428            .lock()
3429            .unwrap_or_else(|poisoned| poisoned.into_inner());
3430        if !modules.contains_key(module_id) {
3431            self.update_configured_ids(|ids| {
3432                ids.remove(module_id);
3433            });
3434        }
3435    }
3436
3437    /// Whether `module_id` is a module this daemon supervises: on the roster,
3438    /// or about to be (its process is being spawned right now).
3439    ///
3440    /// This, not `get(..).is_some()`, is what "the owner is configured" means
3441    /// for scopes: a supervised module's process can register and sync before
3442    /// [`Self::get`] can return it, and in that window it is still a module
3443    /// that will sync, not one that never will.
3444    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
3445        self.configured_ids
3446            .lock()
3447            .unwrap_or_else(|poisoned| poisoned.into_inner())
3448            .contains(module_id)
3449    }
3450
3451    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
3452        let modules = self
3453            .modules
3454            .lock()
3455            .unwrap_or_else(|poisoned| poisoned.into_inner());
3456        modules.get(module_id).cloned()
3457    }
3458
3459    pub(crate) fn record_late_health_answer(
3460        &self,
3461        module_id: &str,
3462        latency_ms: u64,
3463    ) -> Result<bool, SuperviseError> {
3464        let Some(module) = self.get(module_id) else {
3465            return Ok(false);
3466        };
3467        self.host_health.record_late_module(module_id);
3468        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
3469            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
3470            state.health.last_late_answer_latency_ms = Some(latency_ms);
3471            // A late answer is an answer: the module served the probe, just past
3472            // the deadline. Leaving the miss streak in place while logging
3473            // "proves the module is alive" is how a CPU-starved module that
3474            // answers every probe a few seconds late still marches to the
3475            // threshold and gets killed — the exact kill class `NoAnswer` is
3476            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
3477            // is degradation, and degradation reports; it does not restart.
3478            state.health.consecutive_failures = 0;
3479        })?;
3480        Ok(true)
3481    }
3482
3483    pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
3484        self.host_health.spawn_monitor()
3485    }
3486
3487    /// Arm the one-shot marker for the module process that this caller
3488    /// deliberately initiated severance against. Generic connection teardown
3489    /// must not call this:
3490    /// a surviving process would otherwise retain an exemption for a later
3491    /// genuine crash.
3492    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
3493        let Some(module) = self.get(module_id) else {
3494            return Ok(false);
3495        };
3496        let snapshot = lock_snapshot(&module.inner.snapshot)?;
3497        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
3498            return Ok(false);
3499        };
3500        drop(snapshot);
3501        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
3502    }
3503
3504    pub fn list(&self) -> Vec<SupervisedModule> {
3505        let modules = self
3506            .modules
3507            .lock()
3508            .unwrap_or_else(|poisoned| poisoned.into_inner());
3509        let mut modules = modules.values().cloned().collect::<Vec<_>>();
3510        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
3511        modules
3512    }
3513
3514    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
3515        self.activity_images
3516            .lock()
3517            .unwrap_or_else(|p| p.into_inner())
3518            .remove(module_id);
3519        self.spawn_nonces
3520            .lock()
3521            .unwrap_or_else(|poisoned| poisoned.into_inner())
3522            .remove(module_id);
3523        self.close_swap(module_id);
3524        let mut reserved_nonces = self
3525            .reserved_nonces
3526            .lock()
3527            .unwrap_or_else(|poisoned| poisoned.into_inner());
3528        if reserved_nonces.contains_key(module_id) {
3529            // The old nonce must die with the removed process, but the exact-id
3530            // gate remains until an operator explicitly releases it.
3531            reserved_nonces.insert(module_id.to_string(), None);
3532        }
3533        drop(reserved_nonces);
3534        self.reserved_prefix_owners
3535            .lock()
3536            .unwrap_or_else(|poisoned| poisoned.into_inner())
3537            .retain(|_, owner| owner != module_id);
3538        let removed = self
3539            .modules
3540            .lock()
3541            .unwrap_or_else(|poisoned| poisoned.into_inner())
3542            .remove(module_id);
3543        self.update_configured_ids(|ids| {
3544            ids.remove(module_id);
3545        });
3546        self.remove_program_binding_records(module_id);
3547        removed
3548    }
3549
3550    /// Remember a module removed by a non-preview rescan so route.open can
3551    /// distinguish that intentional removal from an unknown id.
3552    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
3553        self.removal_tombstones
3554            .lock()
3555            .unwrap_or_else(|poisoned| poisoned.into_inner())
3556            .insert(module_id.to_string(), unix_ms_now());
3557    }
3558
3559    /// Return how long ago a rescan removed this module in milliseconds.
3560    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
3561        self.removal_tombstones
3562            .lock()
3563            .unwrap_or_else(|poisoned| poisoned.into_inner())
3564            .get(module_id)
3565            .copied()
3566            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
3567    }
3568
3569    /// Retire a reserved-id gate only after its module has left supervision.
3570    ///
3571    /// A retained gate has no live nonce (`None`), so releasing any other entry
3572    /// would weaken a currently configured or otherwise active reservation.
3573    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
3574        if self.get(module_id).is_some() {
3575            return false;
3576        }
3577        let mut reserved_nonces = self
3578            .reserved_nonces
3579            .lock()
3580            .unwrap_or_else(|poisoned| poisoned.into_inner());
3581        if !matches!(reserved_nonces.get(module_id), Some(None)) {
3582            return false;
3583        }
3584        reserved_nonces.remove(module_id);
3585        self.implicit_protocol_none
3586            .lock()
3587            .unwrap_or_else(|p| p.into_inner())
3588            .remove(module_id);
3589        true
3590    }
3591
3592    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
3593        Arc::clone(&self.operation_lock)
3594    }
3595}
3596
3597/// Process supervisor for subc-owned singleton modules.
3598#[derive(Debug, Clone)]
3599pub struct Supervisor {
3600    registry: Arc<Registry>,
3601    config_delivery: Option<config_delivery::ConfigDelivery>,
3602    restart_policy: RestartPolicy,
3603    drain_timeout: Duration,
3604    connection_file_path: Option<PathBuf>,
3605    capture_logs_dir: Option<PathBuf>,
3606    forwarding: Option<Arc<ForwardingTable>>,
3607    process_liveness: Arc<SupervisorProcessLiveness>,
3608    supervisor_handle: Option<SupervisorHandle>,
3609    health: HealthConfig,
3610    host_health: HostHealth,
3611    daemon_start_clock: crate::clock::StartClock,
3612    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
3613    spawn_events: SpawnEventFeed,
3614    provenance_probe: ExecutableIdentityProbe,
3615    /// Every process spawned through this supervisor (and its clones) and not
3616    /// yet reaped, so daemon shutdown can end them.
3617    child_roster: ChildRoster,
3618    #[cfg(target_os = "linux")]
3619    cgroup_placement: Option<subc_cgroup::Placement>,
3620    #[cfg(test)]
3621    test_after_first_spawn: AfterFirstSpawnHook,
3622}
3623
3624/// A roster entry whose monitor and first process have not started yet.
3625/// Keeping the command receiver here lets discovery serve status immediately
3626/// while start/stop requests wait until bootstrap has published the connection file.
3627pub(crate) struct RegisteredConfiguredModule {
3628    spec: ModuleSpec,
3629    runtime: SupervisorRuntimeConfig,
3630    module: SupervisedModule,
3631    commands: mpsc::Receiver<SupervisorCommand>,
3632}
3633
3634/// Test-only hook run on the path that takes on a new module, right after its
3635/// first `spawn_child` returns (the process exists and could already be
3636/// registering) and before that process is handed to the module's supervise
3637/// loop and put on the roster. Lets a test observe what a fast child would see
3638/// in that window without racing a real one.
3639#[cfg(test)]
3640#[derive(Clone, Default)]
3641struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
3642
3643#[cfg(test)]
3644type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
3645
3646#[cfg(test)]
3647impl fmt::Debug for AfterFirstSpawnHook {
3648    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3649        f.write_str("AfterFirstSpawnHook")
3650    }
3651}
3652
3653#[cfg(test)]
3654impl AfterFirstSpawnHook {
3655    fn run(&self, module_id: &str) {
3656        if let Some(hook) = &self.0 {
3657            hook(module_id);
3658        }
3659    }
3660}
3661
3662impl Supervisor {
3663    #[cfg(all(test, target_os = "linux"))]
3664    pub(crate) fn set_admission_hook_for_test(
3665        &self,
3666        hook: impl Fn(&str, &'static str, Option<u32>) -> bool + Send + Sync + 'static,
3667    ) {
3668        self.child_roster.set_admission_hook_for_test(hook);
3669    }
3670
3671    #[cfg(test)]
3672    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
3673        let supervisor = Self::new(registry, policy);
3674        #[cfg(target_os = "macos")]
3675        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
3676        supervisor
3677    }
3678    /// Verify the trampoline once when configured. Missing private OS support
3679    /// refuses every macOS launch by name but does not stop the daemon's control
3680    /// server. Embedders must explicitly provide a binary with the subc-os hidden
3681    /// entry point; the library must not exec an arbitrary hosting program.
3682    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
3683        let path = path.into();
3684        #[cfg(target_os = "macos")]
3685        {
3686            let result = probe_privacy_trampoline(&path).map(|()| path);
3687            if let Err(cause) = &result {
3688                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
3689            }
3690            self.child_roster.set_privacy_trampoline(result);
3691        }
3692        #[cfg(not(target_os = "macos"))]
3693        let _ = path;
3694        self
3695    }
3696    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
3697    /// capability-requirement log to read.
3698    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
3699        self.child_roster.shutdown_flag()
3700    }
3701
3702    /// The first step of an announced daemon shutdown, before the notice and
3703    /// before any connection is closed.
3704    ///
3705    /// Sets the daemon-shutdown flag first: from here on no module is
3706    /// respawned (crash restart, operator restart, or swap), and every child
3707    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
3708    /// the module exits on the EOF this shutdown gives it or is signalled by a
3709    /// service manager that kills the whole cgroup. Then writes the journal's
3710    /// shutdown marker, which records the instant and closes this daemon
3711    /// incarnation's stretch of the journal.
3712    #[cfg(unix)]
3713    pub(crate) fn begin_daemon_shutdown(&self) {
3714        self.child_roster.close();
3715        if let Some(journal) = &self.terminal_journal {
3716            journal.stamp_shutdown();
3717        }
3718    }
3719
3720    /// Announce a cut while established connections can still carry replies.
3721    /// These budgets bound the attempt to deliver notices and the wait for
3722    /// routes to drain; neither guarantees delivery or child completion, and
3723    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
3724    #[cfg(unix)]
3725    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
3726        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
3727        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
3728        let Some(forwarding) = &self.forwarding else {
3729            return Ok(());
3730        };
3731        let notice_started = Instant::now();
3732        let modules = forwarding
3733            .begin_daemon_drain()
3734            .map_err(SuperviseError::Forwarding)?;
3735        let deadline_ms =
3736            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
3737        let shutdown_reason = self
3738            .supervisor_handle
3739            .as_ref()
3740            .map(SupervisorHandle::take_daemon_shutdown_reason)
3741            .unwrap_or(RouteCloseReason::Disable);
3742        let mut modules_told = 0;
3743        let mut notices = tokio::task::JoinSet::new();
3744        let mut drains = Vec::new();
3745        for connection in forwarding
3746            .module_connections()
3747            .map_err(SuperviseError::Forwarding)?
3748        {
3749            let target = ModuleDrainTarget {
3750                endpoint: connection.endpoint,
3751                sink: connection.sink,
3752                negotiated_ver: connection.negotiated_ver,
3753                abandoned_bindings: Vec::new(),
3754                excluded_subscriptions: 0,
3755            };
3756            notices.spawn(async move {
3757                (
3758                    true,
3759                    send_module_draining(
3760                        &connection.module_id,
3761                        shutdown_reason,
3762                        deadline_ms,
3763                        &target,
3764                        true,
3765                    )
3766                    .await,
3767                )
3768            });
3769        }
3770        for (module_id, endpoint) in modules {
3771            let Some(target) = forwarding
3772                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
3773                .map_err(SuperviseError::Forwarding)?
3774            else {
3775                continue;
3776            };
3777            let routes = forwarding
3778                .endpoint_routes(target.endpoint)
3779                .map_err(SuperviseError::Forwarding)?;
3780            // Only an announced restart promises that modules are coming back.
3781            // Client route notices retain their existing reconnect semantics.
3782            let mut recipients = Vec::new();
3783            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3784            for route in routes {
3785                let client = route.goodbye_target;
3786                if let Some((_, channels)) = clients
3787                    .iter_mut()
3788                    .find(|(existing, _)| existing.connection_id == client.connection_id)
3789                {
3790                    channels.push(client.channel);
3791                } else {
3792                    let channel = client.channel;
3793                    clients.push((client, vec![channel]));
3794                }
3795            }
3796            for (client, mut channels) in clients {
3797                channels.sort_unstable();
3798                channels.dedup();
3799                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3800                    module_id: module_id.clone(),
3801                    channels,
3802                    reason: RouteCloseReason::Restart,
3803                })
3804                .expect("route closing serializes");
3805                recipients.push((client.sink, client.negotiated_ver, closing));
3806            }
3807            for (sink, version, body) in recipients {
3808                notices.spawn(async move {
3809                    let frame = Frame::build_with_version(
3810                        version,
3811                        FrameType::Push,
3812                        control_flags(),
3813                        0,
3814                        0,
3815                        0,
3816                        body,
3817                    )
3818                    .expect("bounded lifecycle notice frame builds");
3819                    (false, sink.send_flushed(frame).await.is_ok())
3820                });
3821            }
3822            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3823            drains.push((module_id, target.endpoint, gauges));
3824        }
3825        // A quiet forwarding table is not proof that queued notices reached the
3826        // socket. Wait for writer flush acknowledgements before testing quiescence.
3827        let notice_deadline = Instant::now() + NOTICE_BUDGET;
3828        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3829            match result {
3830                Ok((true, true)) => modules_told += 1,
3831                Ok((false, true)) => {}
3832                result => warn!(?result, "daemon shutdown notice delivery failed"),
3833            }
3834        }
3835        notices.abort_all();
3836        info!(
3837            ?shutdown_reason,
3838            modules_told, "daemon shutdown module notices delivered"
3839        );
3840        let notice_elapsed = notice_started.elapsed();
3841        let drain_started = Instant::now();
3842        let deadline = drain_started + DRAIN_BUDGET;
3843        let mut waits = tokio::task::JoinSet::new();
3844        for (module_id, endpoint, gauges) in drains {
3845            let forwarding = Arc::clone(forwarding);
3846            let mut runtime = self.runtime_config();
3847            runtime.health.cadence = Duration::from_millis(100);
3848            waits.spawn(async move {
3849                let wait_result = wait_for_forwarding_quiescence(
3850                    &forwarding,
3851                    &module_id,
3852                    &runtime,
3853                    endpoint,
3854                    deadline,
3855                    &gauges,
3856                    DrainScope::Endpoint(endpoint),
3857                    &mut DrainGaugeTrace::new(&gauges),
3858                )
3859                .await;
3860                // Store the measurement before the task returns: quiescence
3861                // may be reached just as the deadline stops the join loop from
3862                // collecting results. Unfinished or failed waits report false.
3863                forwarding
3864                    .record_daemon_drain_result(
3865                        endpoint,
3866                        drained_after_quiescence_wait(&wait_result),
3867                    )
3868                    .map_err(SuperviseError::Forwarding)?;
3869                wait_result
3870            });
3871        }
3872        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3873            if !matches!(result, Ok(Ok(true))) {
3874                warn!(?result, "daemon shutdown drain did not reach quiescence");
3875            }
3876        }
3877        // Measure only the forwarding drain, not child teardown or the caller's
3878        // signal delivery and process-reaping delays.
3879        info!(
3880            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3881            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3882            "daemon shutdown forwarding drain complete"
3883        );
3884        Ok(())
3885    }
3886
3887    /// The last step of a graceful daemon shutdown, after the notice and the
3888    /// drain: send every registered module a module GOODBYE, the same planned
3889    /// stop signal `ck module stop` gives, then close every connection so each
3890    /// subc module sees EOF and starts its own teardown, then end every
3891    /// supervised child that has not exited
3892    /// by its own deadline (its drain budget, capped). Modules lead their own
3893    /// process groups, so a
3894    /// service manager's group kill no longer reaches them; without this a
3895    /// child that does not stop on EOF (every `protocol: "none"` child, which
3896    /// has no connection) would outlive the daemon. Every wait is bounded (see
3897    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
3898    #[cfg(unix)]
3899    pub(crate) async fn end_children_for_daemon_shutdown(
3900        &self,
3901        already_escalated: bool,
3902        escalate: impl std::future::Future<Output = ()>,
3903    ) {
3904        tokio::pin!(escalate);
3905        let mut escalated = already_escalated;
3906        if let Some(forwarding) = &self.forwarding {
3907            let reason = CloseReason::new(
3908                "daemon_shutdown",
3909                "the daemon is exiting after its shutdown notice and drain",
3910            );
3911            if escalated {
3912                // The operator asked to stop waiting: queue the GOODBYEs but
3913                // do not wait for them to be written.
3914                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3915            } else {
3916                tokio::select! {
3917                    biased;
3918                    _ = escalate.as_mut() => {
3919                        info!("second SIGTERM: abandoning module GOODBYE delivery");
3920                        escalated = true;
3921                    }
3922                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3923                }
3924            }
3925            let closed = forwarding.close_all_connections(&reason);
3926            debug!(closed, "closed established connections for daemon shutdown");
3927        }
3928        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3929        // already completed and must not be polled again; the child shutdown
3930        // wait is told it is escalated and gets a future that never fires.
3931        let escalated_here = escalated && !already_escalated;
3932        let remaining_escalate = async move {
3933            if escalated_here {
3934                std::future::pending::<()>().await;
3935            } else {
3936                escalate.await;
3937            }
3938        };
3939        crate::child_roster::end_children_for_daemon_shutdown(
3940            &self.child_roster,
3941            escalated,
3942            remaining_escalate,
3943        )
3944        .await;
3945    }
3946
3947    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3948        Self {
3949            registry,
3950            config_delivery: None,
3951            restart_policy,
3952            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3953            connection_file_path: None,
3954            capture_logs_dir: None,
3955            forwarding: None,
3956            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3957            supervisor_handle: None,
3958            health: HealthConfig::default(),
3959            host_health: HostHealth::default(),
3960            daemon_start_clock: crate::clock::StartClock::capture(),
3961            terminal_journal: None,
3962            spawn_events: SpawnEventFeed::default(),
3963            provenance_probe: ExecutableIdentityProbe::default(),
3964            child_roster: ChildRoster::default(),
3965            #[cfg(target_os = "linux")]
3966            cgroup_placement: None,
3967            #[cfg(test)]
3968            test_after_first_spawn: AfterFirstSpawnHook::default(),
3969        }
3970    }
3971
3972    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3973        self.drain_timeout = drain_timeout;
3974        self
3975    }
3976
3977    /// The drain budget a module gets when its config entry sets none.
3978    pub(crate) fn default_drain_timeout(&self) -> Duration {
3979        self.drain_timeout
3980    }
3981
3982    pub fn with_process_liveness(
3983        mut self,
3984        process_liveness: Arc<SupervisorProcessLiveness>,
3985    ) -> Self {
3986        self.process_liveness = process_liveness;
3987        self
3988    }
3989
3990    /// Connect configuration delivery to observed process lifecycle events.
3991    pub fn with_config_runtime(
3992        mut self,
3993        owner: Arc<crate::config_gate::runtime::RuntimeOwner>,
3994    ) -> Self {
3995        if let Some(handle) = &self.supervisor_handle {
3996            handle.install_binding_retirement(&owner);
3997        }
3998        self.config_delivery = Some(config_delivery::ConfigDelivery(owner));
3999        self
4000    }
4001
4002    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
4003        self.connection_file_path = Some(connection_file_path.into());
4004        self
4005    }
4006
4007    /// Enables daemon-owned capture files for supervised stdout and stderr.
4008    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
4009        self.capture_logs_dir = Some(logs_dir.into());
4010        self
4011    }
4012
4013    /// Names this daemon lifetime in spawn events, independently of whether a
4014    /// terminal journal is configured.
4015    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
4016        // A millisecond start stamp can repeat after clock rollback or a rapid
4017        // restart. Use the connection file's random daemon_id instead: it already
4018        // identifies this daemon lifetime independently of the wall clock.
4019        self.spawn_events.configure_incarnation(daemon_incarnation);
4020        self
4021    }
4022
4023    /// Enables best-effort history shared by every supervised module. Without
4024    /// it, terminal history is kept only in each module's in-memory ring.
4025    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
4026        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
4027        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
4028            path,
4029            daemon_incarnation,
4030        )));
4031        this
4032    }
4033
4034    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
4035        self.child_roster.counters = forwarding.counters();
4036        self.forwarding = Some(forwarding);
4037        self
4038    }
4039
4040    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
4041        if let Some(delivery) = &self.config_delivery {
4042            supervisor_handle.install_binding_retirement(&delivery.0);
4043        }
4044        self.host_health = supervisor_handle.host_health.clone();
4045        self.spawn_events = supervisor_handle.spawn_events.clone();
4046        self.supervisor_handle = Some(supervisor_handle);
4047        self
4048    }
4049
4050    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
4051        self.health = health;
4052        self
4053    }
4054
4055    /// Keeps a record of every live child at `path`, rewritten on each spawn and
4056    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
4057    /// record is kept.
4058    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
4059        self.child_roster.record_to(path.into());
4060        self
4061    }
4062
4063    #[cfg(target_os = "linux")]
4064    pub fn with_cgroup_placement(
4065        mut self,
4066        cgroup_placement: Option<subc_cgroup::Placement>,
4067    ) -> Self {
4068        self.child_roster
4069            .set_cgroup_placement(cgroup_placement.clone());
4070        self.cgroup_placement = cgroup_placement;
4071        self
4072    }
4073
4074    /// Spawn `spec.program` and start monitoring it.
4075    ///
4076    /// The child is expected to parse `--subc <connection-file-path>`, read the
4077    /// TCP+key connection file, authenticate to the already-running listener, and
4078    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
4079    ///
4080    /// This synchronous embedder API blocks on launch admission and store I/O.
4081    /// Do not call it on an async runtime thread; use [`Self::spawn_async`].
4082    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4083        validate_spec(&spec)?;
4084        self.establish_identity(&spec);
4085
4086        let runtime = self.runtime_config();
4087        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4088        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4089        let registration_watch = watch_registration(&self.registry, &spec);
4090        let spawned = spawn_child(
4091            &spec,
4092            runtime.connection_file_path.as_deref(),
4093            self.supervisor_handle.as_ref(),
4094            &runtime.stderr_ring,
4095            runtime.capture_logs_dir.as_deref(),
4096            &runtime.child_roster,
4097            runtime.spawn_delivery(),
4098            #[cfg(target_os = "linux")]
4099            runtime.cgroup_placement.as_ref(),
4100            runtime.register_timeout,
4101        );
4102        #[cfg(test)]
4103        self.test_after_first_spawn.run(&spec.module_id);
4104        let mut child = match spawned {
4105            Ok(child) => child,
4106            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4107                defer_config_spawn(
4108                    &spec,
4109                    &runtime,
4110                    &snapshot,
4111                    RespawnKind::Spawn,
4112                    &runtime.failure_context(runtime.recovery_is_daemon),
4113                )?;
4114                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4115            }
4116            Err(SuperviseError::ConfigHeld { .. }) => {
4117                hold_config_spawn(
4118                    &spec,
4119                    &runtime,
4120                    &snapshot,
4121                    &runtime.failure_context(runtime.recovery_is_daemon),
4122                )?;
4123                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4124            }
4125            Err(err) => {
4126                // Unlike the configured paths, a failed `spawn` leaves nothing
4127                // on the roster, so the module must not stay marked configured.
4128                self.abandon_unrostered(&spec.module_id);
4129                return Err(err);
4130            }
4131        };
4132        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4133        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4134        if let Some(spawn) = activity_spawn {
4135            spawn.complete(&snapshot);
4136        }
4137
4138        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4139    }
4140
4141    /// Spawn without running configuration integrity or keychain checks on the
4142    /// runtime thread. Cancellation during preparation cannot create a child.
4143    pub async fn spawn_async(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4144        validate_spec(&spec)?;
4145        self.establish_identity(&spec);
4146        let mut identity = AsyncInitialIdentity {
4147            supervisor: self,
4148            module: spec.module_id.clone(),
4149            committed: false,
4150        };
4151        let runtime = self.runtime_config();
4152        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4153        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4154        let registration_watch = watch_registration(&self.registry, &spec);
4155        let spawned = spawn_child_async(&spec, &runtime).await;
4156        #[cfg(test)]
4157        self.test_after_first_spawn.run(&spec.module_id);
4158        let mut child = match spawned {
4159            Ok(child) => child,
4160            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4161                defer_config_spawn(
4162                    &spec,
4163                    &runtime,
4164                    &snapshot,
4165                    RespawnKind::Spawn,
4166                    &runtime.failure_context(runtime.recovery_is_daemon),
4167                )?;
4168                identity.committed = true;
4169                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4170            }
4171            Err(SuperviseError::ConfigHeld { .. }) => {
4172                hold_config_spawn(
4173                    &spec,
4174                    &runtime,
4175                    &snapshot,
4176                    &runtime.failure_context(runtime.recovery_is_daemon),
4177                )?;
4178                identity.committed = true;
4179                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4180            }
4181            Err(err) => {
4182                self.abandon_unrostered(&spec.module_id);
4183                return Err(err);
4184            }
4185        };
4186        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4187        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4188        if let Some(spawn) = activity_spawn {
4189            spawn.complete(&snapshot);
4190        }
4191        identity.committed = true;
4192        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4193    }
4194
4195    /// Make `spec`'s module count as configured, with its identity gates
4196    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
4197    /// exists.
4198    ///
4199    /// Every path that takes on a new module calls this before `spawn_child`.
4200    /// The order is the point: the child can connect, register, sync its
4201    /// scopes and ask about them as soon as it is spawned, and the module is
4202    /// only put on the roster after `spawn_child` returns. Were the mark set
4203    /// with the roster entry, a fast child would see its own owner reported
4204    /// as not configured, and a scoped `route.open` in that window would be
4205    /// refused as terminal `scope_not_live` ("will never sync") instead of
4206    /// retryable `scope_not_synced`.
4207    fn establish_identity(&self, spec: &ModuleSpec) {
4208        if let Some(supervisor_handle) = &self.supervisor_handle {
4209            supervisor_handle.apply_identity_configuration(spec);
4210            supervisor_handle.mark_configured(&spec.module_id);
4211        }
4212    }
4213
4214    /// Take back [`Self::establish_identity`]'s configured mark when the
4215    /// module will not be put on the roster after all.
4216    fn abandon_unrostered(&self, module_id: &str) {
4217        if let Some(supervisor_handle) = &self.supervisor_handle {
4218            supervisor_handle.unmark_configured_unless_rostered(module_id);
4219        }
4220    }
4221
4222    /// Record a freshly spawned first process as running. On failure the
4223    /// module never reaches the roster, so its configured mark is taken back.
4224    fn mark_first_process_running(
4225        &self,
4226        spec: &ModuleSpec,
4227        runtime: &SupervisorRuntimeConfig,
4228        snapshot: &SharedSnapshot,
4229        child: &mut SupervisedChild,
4230    ) -> Result<(), SuperviseError> {
4231        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
4232            containment::terminate_blocking(&runtime.child_roster.containment);
4233            self.abandon_unrostered(&spec.module_id);
4234            return Err(err);
4235        }
4236        self.process_liveness
4237            .track(spec.module_id.clone(), Arc::clone(snapshot));
4238        Ok(())
4239    }
4240
4241    /// Start supervising a module declared in daemon configuration.
4242    ///
4243    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
4244    /// failures in the supervisor handle so operator-facing `supervisor.list`
4245    /// reflects every configured module while daemon startup continues.
4246    pub fn supervise_configured(
4247        &self,
4248        spec: ModuleSpec,
4249        enabled: bool,
4250    ) -> Result<SupervisedModule, SuperviseError> {
4251        self.supervise_configured_requested(
4252            spec,
4253            enabled,
4254            self.health.clone(),
4255            None,
4256            self.restart_policy,
4257            DEFAULT_REGISTER_TIMEOUT,
4258            None,
4259        )
4260    }
4261
4262    /// Supervise a configured module with its own health, drain, and crash
4263    /// budget. The restart policy is per-module because the config file is:
4264    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
4265    /// module that is expensive to restart should not be forced onto the same
4266    /// budget as one that is cheap.
4267    pub fn supervise_configured_with_health(
4268        &self,
4269        spec: ModuleSpec,
4270        enabled: bool,
4271        health: HealthConfig,
4272        drain_timeout_ms: Option<u64>,
4273        restart_policy: RestartPolicy,
4274    ) -> Result<SupervisedModule, SuperviseError> {
4275        self.supervise_configured_requested(
4276            spec,
4277            enabled,
4278            health,
4279            drain_timeout_ms,
4280            restart_policy,
4281            DEFAULT_REGISTER_TIMEOUT,
4282            None,
4283        )
4284    }
4285
4286    #[allow(clippy::too_many_arguments)]
4287    pub(crate) fn supervise_configured_requested(
4288        &self,
4289        spec: ModuleSpec,
4290        enabled: bool,
4291        health: HealthConfig,
4292        drain_timeout_ms: Option<u64>,
4293        restart_policy: RestartPolicy,
4294        register_timeout: Duration,
4295        requester: Option<Principal>,
4296    ) -> Result<SupervisedModule, SuperviseError> {
4297        let registered = self.register_configured_with_health(
4298            spec,
4299            enabled,
4300            health,
4301            drain_timeout_ms,
4302            restart_policy,
4303            register_timeout,
4304        )?;
4305        let daemon_initiated = requester.is_none();
4306        self.start_registered_configured(registered, requester, daemon_initiated)
4307    }
4308
4309    #[allow(clippy::too_many_arguments)]
4310    pub(crate) async fn supervise_configured_requested_async(
4311        &self,
4312        spec: ModuleSpec,
4313        enabled: bool,
4314        health: HealthConfig,
4315        drain_timeout_ms: Option<u64>,
4316        restart_policy: RestartPolicy,
4317        register_timeout: Duration,
4318        requester: Option<Principal>,
4319    ) -> Result<SupervisedModule, SuperviseError> {
4320        let registered = self.register_configured_with_health(
4321            spec,
4322            enabled,
4323            health,
4324            drain_timeout_ms,
4325            restart_policy,
4326            register_timeout,
4327        )?;
4328        let daemon_initiated = requester.is_none();
4329        self.start_registered_configured_async(registered, requester, daemon_initiated)
4330            .await
4331    }
4332
4333    /// Record a configured module as starting or disabled, without launching a
4334    /// process or running its monitor. Bootstrap registers the entire roster
4335    /// before publishing discovery, then activates it once children can read
4336    /// the connection file passed in their `--subc` argument.
4337    pub(crate) fn register_configured_with_health(
4338        &self,
4339        spec: ModuleSpec,
4340        enabled: bool,
4341        health: HealthConfig,
4342        drain_timeout_ms: Option<u64>,
4343        restart_policy: RestartPolicy,
4344        register_timeout: Duration,
4345    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
4346        validate_spec(&spec)?;
4347        self.establish_identity(&spec);
4348
4349        let mut runtime = self.runtime_config();
4350        runtime.health = health;
4351        runtime.restart_policy = restart_policy;
4352        runtime.register_timeout = register_timeout;
4353        if let Some(ms) = drain_timeout_ms {
4354            runtime.drain_timeout = Duration::from_millis(ms);
4355            *runtime
4356                .effective_drain_timeout
4357                .lock()
4358                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
4359        }
4360        let snapshot = if enabled {
4361            SupervisorSnapshot::starting()
4362        } else {
4363            SupervisorSnapshot::disabled()
4364        };
4365        let snapshot = {
4366            let mut snapshot = snapshot;
4367            if enabled {
4368                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
4369                {
4370                    snapshot.waiting_for_previous_pid = Some(pid);
4371                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
4372                }
4373            }
4374            snapshot
4375        };
4376        let (module, commands) =
4377            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
4378        Ok(RegisteredConfiguredModule {
4379            spec,
4380            runtime,
4381            module,
4382            commands,
4383        })
4384    }
4385
4386    /// Activate a previously registered module. Queued control requests are
4387    /// served by the monitor after the initial configured spawn has settled.
4388    pub(crate) fn start_registered_configured(
4389        &self,
4390        registered: RegisteredConfiguredModule,
4391        requester: Option<Principal>,
4392        daemon_initiated: bool,
4393    ) -> Result<SupervisedModule, SuperviseError> {
4394        let RegisteredConfiguredModule {
4395            spec,
4396            mut runtime,
4397            module,
4398            commands,
4399        } = registered;
4400        runtime.start_requester = requester;
4401        runtime.recovery_is_daemon = daemon_initiated;
4402        let snapshot = &module.inner.snapshot;
4403        let should_spawn = {
4404            let state = lock_snapshot(snapshot)?;
4405            state.enabled && state.waiting_for_previous_pid.is_none()
4406        };
4407        let child = if should_spawn {
4408            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4409            let registration_watch = watch_registration(&self.registry, &spec);
4410            let spawned = spawn_child(
4411                &spec,
4412                runtime.connection_file_path.as_deref(),
4413                self.supervisor_handle.as_ref(),
4414                &runtime.stderr_ring,
4415                runtime.capture_logs_dir.as_deref(),
4416                &runtime.child_roster,
4417                runtime.spawn_delivery(),
4418                #[cfg(target_os = "linux")]
4419                runtime.cgroup_placement.as_ref(),
4420                runtime.register_timeout,
4421            );
4422            #[cfg(test)]
4423            self.test_after_first_spawn.run(&spec.module_id);
4424            match spawned {
4425                Ok(mut child) => {
4426                    arm_registration_deadline(
4427                        &mut child,
4428                        runtime.register_timeout,
4429                        registration_watch,
4430                    );
4431                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4432                    if let Some(spawn) = activity_spawn {
4433                        spawn.complete(snapshot);
4434                    }
4435                    Some(child)
4436                }
4437                Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4438                    defer_config_spawn(
4439                        &spec,
4440                        &runtime,
4441                        snapshot,
4442                        RespawnKind::Spawn,
4443                        &runtime.failure_context(daemon_initiated),
4444                    )?;
4445                    None
4446                }
4447                Err(mut err) => {
4448                    if matches!(err, SuperviseError::ConfigHeld { .. }) {
4449                        match hold_config_spawn(
4450                            &spec,
4451                            &runtime,
4452                            snapshot,
4453                            &runtime.failure_context(daemon_initiated),
4454                        ) {
4455                            Ok(()) => {
4456                                self.start_monitor(spec, runtime, &module, None, commands);
4457                                return Ok(module);
4458                            }
4459                            Err(parking_error) => err = parking_error,
4460                        }
4461                    }
4462                    if runtime.health.critical {
4463                        error!(
4464                            module_id = %spec.module_id,
4465                            program = %spec.program.display(),
4466                            error = %err,
4467                            "critical configured module failed to spawn; marking failed and alerting"
4468                        );
4469                    } else {
4470                        error!(
4471                            module_id = %spec.module_id,
4472                            program = %spec.program.display(),
4473                            error = %err,
4474                            "configured module failed to spawn; marking failed and continuing"
4475                        );
4476                    }
4477                    fail_snapshot(
4478                        snapshot,
4479                        Some(&spec.module_id),
4480                        None,
4481                        &runtime.failure_context(daemon_initiated),
4482                    );
4483                    None
4484                }
4485            }
4486        } else {
4487            None
4488        };
4489        self.start_monitor(spec, runtime, &module, child, commands);
4490        Ok(module)
4491    }
4492
4493    pub(crate) async fn start_registered_configured_async(
4494        &self,
4495        registered: RegisteredConfiguredModule,
4496        requester: Option<Principal>,
4497        daemon_initiated: bool,
4498    ) -> Result<SupervisedModule, SuperviseError> {
4499        self.start_registered_with_failure_policy(registered, requester, daemon_initiated, None)
4500            .await
4501    }
4502
4503    pub(crate) async fn start_registered_approved_async(
4504        &self,
4505        registered: RegisteredConfiguredModule,
4506        requester: Principal,
4507        generation: crate::config_gate::runtime::ApprovalIdentity,
4508    ) -> Result<SupervisedModule, SuperviseError> {
4509        self.start_registered_with_failure_policy(
4510            registered,
4511            Some(requester),
4512            false,
4513            Some(generation),
4514        )
4515        .await
4516    }
4517
4518    async fn start_registered_with_failure_policy(
4519        &self,
4520        registered: RegisteredConfiguredModule,
4521        requester: Option<Principal>,
4522        daemon_initiated: bool,
4523        approved_generation: Option<crate::config_gate::runtime::ApprovalIdentity>,
4524    ) -> Result<SupervisedModule, SuperviseError> {
4525        let RegisteredConfiguredModule {
4526            spec,
4527            mut runtime,
4528            module,
4529            commands,
4530        } = registered;
4531        runtime.start_requester = requester;
4532        runtime.recovery_is_daemon = daemon_initiated;
4533        let snapshot = &module.inner.snapshot;
4534        let mut startup_error = None;
4535        let failure = FailureContext {
4536            approval: approved_generation.clone(),
4537            ..runtime.failure_context(daemon_initiated)
4538        };
4539        let should_spawn = {
4540            let state = lock_snapshot(snapshot)?;
4541            state.enabled && state.waiting_for_previous_pid.is_none()
4542        };
4543        let child = if should_spawn {
4544            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4545            let registration_watch = watch_registration(&self.registry, &spec);
4546            let spawned = spawn_child_async(&spec, &runtime).await;
4547            #[cfg(test)]
4548            self.test_after_first_spawn.run(&spec.module_id);
4549            match spawned {
4550                Ok(mut child) => {
4551                    arm_registration_deadline(
4552                        &mut child,
4553                        runtime.register_timeout,
4554                        registration_watch,
4555                    );
4556                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4557                    if let Some(spawn) = activity_spawn {
4558                        spawn.complete(snapshot);
4559                    }
4560                    Some(child)
4561                }
4562                Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4563                    defer_config_spawn(&spec, &runtime, snapshot, RespawnKind::Spawn, &failure)?;
4564                    None
4565                }
4566                Err(mut err) => {
4567                    if matches!(err, SuperviseError::ConfigHeld { .. }) {
4568                        match hold_config_spawn(&spec, &runtime, snapshot, &failure) {
4569                            Ok(()) => {
4570                                self.start_monitor(spec, runtime, &module, None, commands);
4571                                return Ok(module);
4572                            }
4573                            Err(parking_error) => err = parking_error,
4574                        }
4575                    }
4576                    if runtime.health.critical {
4577                        error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4578                            "critical configured module failed to spawn; marking failed and alerting");
4579                    } else {
4580                        error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4581                            "configured module failed to spawn; marking failed and continuing");
4582                    }
4583                    fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
4584                    if let Some(generation) = approved_generation {
4585                        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4586                            state.approved_start_failure = Some((generation, err.to_string()));
4587                        });
4588                        startup_error = Some(err);
4589                    }
4590                    None
4591                }
4592            }
4593        } else {
4594            None
4595        };
4596        self.start_monitor(spec, runtime, &module, child, commands);
4597        // Applying an operator-approved configuration must report a spawn error
4598        // without removing the failed module or its monitor. Bootstrap instead
4599        // tolerates failed starts so other configured modules can still run.
4600        startup_error.map_or(Ok(module), Err)
4601    }
4602
4603    fn runtime_config(&self) -> SupervisorRuntimeConfig {
4604        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
4605        SupervisorRuntimeConfig {
4606            registry: self.registry.clone(),
4607            config_delivery: self.config_delivery.clone(),
4608            recovery_is_daemon: true,
4609            start_requester: None,
4610            pending_activity_restart: false,
4611            provenance_probe: self.provenance_probe.clone(),
4612            scheduled_respawn: Arc::default(),
4613            deferred_reload_reply: Arc::default(),
4614            restart_policy: self.restart_policy,
4615            register_timeout: DEFAULT_REGISTER_TIMEOUT,
4616            drain_timeout: self.drain_timeout,
4617            // Shared with this module's roster copy: daemon shutdown waits on
4618            // each child for the module's own drain budget, as resolved now.
4619            child_roster: self
4620                .child_roster
4621                .for_module(Arc::clone(&effective_drain_timeout)),
4622            effective_drain_timeout,
4623            default_drain_timeout: self.drain_timeout,
4624            health: self.health.clone(),
4625            host_health: self.host_health.clone(),
4626            connection_file_path: self.connection_file_path.clone(),
4627            capture_logs_dir: self.capture_logs_dir.clone(),
4628            forwarding: self.forwarding.clone(),
4629            supervisor_handle: self.supervisor_handle.clone(),
4630            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
4631            terminal_ring: Arc::new(Mutex::new(
4632                TerminalRing::new(
4633                    TerminalRingConfig::default(),
4634                    self.daemon_start_clock.started_at_ms(),
4635                )
4636                .with_start_clock(self.daemon_start_clock)
4637                .with_journal(self.terminal_journal.clone())
4638                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
4639            )),
4640            spawn_events: self.spawn_events.clone(),
4641            #[cfg(target_os = "linux")]
4642            cgroup_placement: self.cgroup_placement.clone(),
4643            #[cfg(test)]
4644            test_seed_stale_facts_before_enable_spawn: false,
4645            #[cfg(test)]
4646            test_reload_exit_record_gate: None,
4647        }
4648    }
4649
4650    fn supervised_module(
4651        &self,
4652        spec: ModuleSpec,
4653        runtime: SupervisorRuntimeConfig,
4654        snapshot: SharedSnapshot,
4655        child: Option<SupervisedChild>,
4656    ) -> SupervisedModule {
4657        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
4658        self.start_monitor(spec, runtime, &module, child, commands);
4659        module
4660    }
4661
4662    fn register_module(
4663        &self,
4664        spec: &ModuleSpec,
4665        runtime: &SupervisorRuntimeConfig,
4666        snapshot: SharedSnapshot,
4667    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
4668        runtime.host_health.track(spec, &snapshot);
4669        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
4670            spec: spec.clone(),
4671            enabled: snapshot.lock().unwrap_or_else(|p| p.into_inner()).enabled,
4672            health: runtime.health.clone(),
4673            register_timeout: runtime.register_timeout,
4674            restart_policy: runtime.restart_policy,
4675        }));
4676        let stderr_ring = Arc::clone(&runtime.stderr_ring);
4677        let terminal_ring = Arc::clone(&runtime.terminal_ring);
4678        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
4679        let (tx, rx) = mpsc::channel(4);
4680        let module_id = spec.module_id.clone();
4681        let module = SupervisedModule {
4682            inner: Arc::new(SupervisedModuleInner {
4683                module_id: module_id.clone(),
4684                registry: Arc::clone(&self.registry),
4685                snapshot,
4686                configuration,
4687                stderr_ring,
4688                terminal_ring,
4689                commands: tx,
4690                monitor: Mutex::new(None),
4691                effective_drain_timeout,
4692                provenance_probe: self.provenance_probe.clone(),
4693                retained: runtime.child_roster.retained_view(),
4694            }),
4695        };
4696        // Identity gates and the configured mark must precede the roster entry,
4697        // so a client never sees a module without its admission policy.
4698        if let Some(supervisor_handle) = &self.supervisor_handle {
4699            supervisor_handle.insert(module.clone());
4700        }
4701        (module, rx)
4702    }
4703
4704    fn start_monitor(
4705        &self,
4706        spec: ModuleSpec,
4707        runtime: SupervisorRuntimeConfig,
4708        module: &SupervisedModule,
4709        child: Option<SupervisedChild>,
4710        commands: mpsc::Receiver<SupervisorCommand>,
4711    ) {
4712        let monitor = tokio::spawn(supervise_loop(
4713            spec,
4714            runtime,
4715            Arc::clone(&self.registry),
4716            Arc::clone(&self.process_liveness),
4717            Arc::clone(&module.inner.snapshot),
4718            child,
4719            commands,
4720        ));
4721        *module
4722            .inner
4723            .monitor
4724            .lock()
4725            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
4726    }
4727}
4728
4729impl Default for Supervisor {
4730    fn default() -> Self {
4731        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
4732    }
4733}
4734
4735/// Handle to one supervised child process.
4736#[derive(Clone)]
4737pub struct SupervisedModule {
4738    inner: Arc<SupervisedModuleInner>,
4739}
4740
4741struct SupervisedModuleInner {
4742    retained: crate::child_roster::RetainedView,
4743    module_id: String,
4744    registry: Arc<Registry>,
4745    snapshot: SharedSnapshot,
4746    configuration: Arc<Mutex<SupervisedConfiguration>>,
4747    stderr_ring: Arc<Mutex<StderrRing>>,
4748    terminal_ring: Arc<Mutex<TerminalRing>>,
4749    commands: mpsc::Sender<SupervisorCommand>,
4750    monitor: Mutex<Option<JoinHandle<()>>>,
4751    effective_drain_timeout: Arc<Mutex<Duration>>,
4752    provenance_probe: ExecutableIdentityProbe,
4753}
4754
4755impl fmt::Debug for SupervisedModule {
4756    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4757        f.debug_struct("SupervisedModule")
4758            .field("module_id", &self.inner.module_id)
4759            .field("status", &self.status())
4760            .finish_non_exhaustive()
4761    }
4762}
4763
4764impl SupervisedModule {
4765    pub(crate) fn check_approved_start_failure(
4766        &self,
4767        generation: &crate::config_gate::runtime::ApprovalIdentity,
4768    ) -> Result<bool, SuperviseError> {
4769        let state = lock_snapshot(&self.inner.snapshot)?;
4770        if let Some((failed_generation, detail)) = &state.approved_start_failure {
4771            if failed_generation == generation {
4772                return Err(SuperviseError::ReloadFailed {
4773                    module_id: self.inner.module_id.clone(),
4774                    reason: detail.clone(),
4775                });
4776            }
4777        }
4778        Ok(state.approved_start_failure.is_some())
4779    }
4780
4781    pub(crate) fn has_approved_start_failure(
4782        &self,
4783        identity: &crate::config_gate::runtime::ApprovalIdentity,
4784    ) -> bool {
4785        lock_snapshot(&self.inner.snapshot).is_ok_and(|state| {
4786            state
4787                .approved_start_failure
4788                .as_ref()
4789                .is_some_and(|(failed, _)| failed == identity)
4790        })
4791    }
4792
4793    pub(crate) fn needs_approved_gate_restart(
4794        &self,
4795        approval: &crate::config_gate::runtime::ApprovalIdentity,
4796    ) -> bool {
4797        lock_snapshot(&self.inner.snapshot)
4798            .is_ok_and(|state| state.needs_approved_gate_restart(approval))
4799    }
4800
4801    #[cfg(all(test, unix))]
4802    pub(crate) fn config_hold_origin_for_test(
4803        &self,
4804    ) -> (bool, Option<crate::config_gate::runtime::ApprovalIdentity>) {
4805        let state = lock_snapshot(&self.inner.snapshot).unwrap();
4806        (
4807            state.config_hold_daemon_initiated,
4808            state.config_hold_approval.clone(),
4809        )
4810    }
4811
4812    pub(crate) async fn config_policy_probe(
4813        &self,
4814        connection: crate::ConnectionId,
4815        generation: u64,
4816        attempt: u64,
4817    ) -> Result<(), SuperviseError> {
4818        self.inner
4819            .commands
4820            .send(SupervisorCommand::ConfigPolicyProbe {
4821                connection,
4822                generation,
4823                attempt,
4824            })
4825            .await
4826            .map_err(|_| SuperviseError::CommandClosed {
4827                module_id: self.inner.module_id.clone(),
4828            })
4829    }
4830    pub(crate) async fn config_exit_hold(&self) -> Result<(), SuperviseError> {
4831        let (reply, result) = oneshot::channel();
4832        self.inner
4833            .commands
4834            .send(SupervisorCommand::ConfigExitHold { reply })
4835            .await
4836            .map_err(|_| SuperviseError::CommandClosed {
4837                module_id: self.module_id().to_string(),
4838            })?;
4839        result.await.map_err(|_| SuperviseError::CommandClosed {
4840            module_id: self.module_id().to_string(),
4841        })?
4842    }
4843    pub(crate) async fn config_delivery_failed(
4844        &self,
4845        connection: crate::ConnectionId,
4846        hold: bool,
4847    ) -> Result<(), SuperviseError> {
4848        self.inner
4849            .commands
4850            .send(SupervisorCommand::ConfigDeliveryFailed { connection, hold })
4851            .await
4852            .map_err(|_| SuperviseError::CommandClosed {
4853                module_id: self.module_id().to_string(),
4854            })
4855    }
4856
4857    pub(crate) async fn config_apply_restart(&self, generation: u64) -> Result<(), SuperviseError> {
4858        self.inner
4859            .commands
4860            .send(SupervisorCommand::ConfigApplyRestart {
4861                generation,
4862                supersedes_swap: false,
4863            })
4864            .await
4865            .map_err(|_| SuperviseError::CommandClosed {
4866                module_id: self.inner.module_id.clone(),
4867            })
4868    }
4869    pub fn module_id(&self) -> &str {
4870        &self.inner.module_id
4871    }
4872
4873    /// Test-only: put one probe miss on the streak, the way
4874    /// `handle_health_probe_failure` does, so tests can assert what a later
4875    /// event does to the streak without driving the whole probe loop.
4876    #[cfg(test)]
4877    pub(crate) fn record_health_probe_failure_for_test(
4878        &self,
4879        detail: &str,
4880    ) -> Result<(), SuperviseError> {
4881        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
4882            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
4883            state.health.detail = Some(detail.to_string());
4884        })
4885    }
4886
4887    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
4888        Ok(lock_snapshot(&self.inner.snapshot)?.state)
4889    }
4890
4891    pub(crate) fn registration_end_reason(
4892        &self,
4893    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
4894        let snapshot = lock_snapshot(&self.inner.snapshot)?;
4895        Ok(match snapshot.state {
4896            ModuleState::Draining if snapshot.draining_to_replace => {
4897                Some(RegistrationEndReason::SupervisorRestart)
4898            }
4899            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
4900            ModuleState::Restarting if snapshot.draining_to_replace => {
4901                Some(RegistrationEndReason::SupervisorRestart)
4902            }
4903            _ => None,
4904        })
4905    }
4906
4907    /// The module's retained stderr, newest lines last.
4908    ///
4909    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
4910    /// module, `supervisor.list` renders every module, and putting it in the
4911    /// shared snapshot would make each status read carry a payload almost nobody
4912    /// asked for. Callers that want the text ask for it.
4913    pub fn stderr_tail(
4914        &self,
4915        max_lines: Option<usize>,
4916        max_bytes: Option<usize>,
4917    ) -> StderrTailSnapshot {
4918        self.inner
4919            .stderr_ring
4920            .lock()
4921            .unwrap_or_else(|poisoned| poisoned.into_inner())
4922            .snapshot(max_lines, max_bytes)
4923    }
4924
4925    /// The module's bounded terminal history, oldest retained exit first.
4926    ///
4927    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
4928    /// daemon whose in-memory history was necessarily reset.
4929    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
4930        self.inner
4931            .terminal_ring
4932            .lock()
4933            .unwrap_or_else(|poisoned| poisoned.into_inner())
4934            .snapshot()
4935    }
4936
4937    /// Retained observations from the current ring and all journal generations.
4938    ///
4939    /// Blocking: this reads the journal files. Async callers use
4940    /// `Self::read_durable_terminal_history`.
4941    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
4942        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
4943    }
4944
4945    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
4946    /// read (up to every retained generation) never occupies a runtime worker.
4947    /// Fails only if the blocking task could not finish (runtime shutdown or a
4948    /// panic in the read).
4949    pub(crate) async fn read_durable_terminal_history(
4950        &self,
4951    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
4952        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
4953        let module_id = self.inner.module_id.clone();
4954        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
4955            .await
4956    }
4957
4958    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
4959        self.status_with_snapshot_lock(&self.inner.snapshot, None)
4960            .map(|(status, _)| status)
4961    }
4962
4963    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
4964        let state = lock_snapshot(&self.inner.snapshot)?;
4965        Ok(state
4966            .failed_retry
4967            .pending
4968            .filter(|_| state.state == ModuleState::Failed && state.enabled)
4969            .map(|retry| {
4970                unix_ms_now().saturating_add(
4971                    u64::try_from(
4972                        retry
4973                            .deadline
4974                            .saturating_duration_since(Instant::now())
4975                            .as_millis(),
4976                    )
4977                    .unwrap_or(u64::MAX),
4978                )
4979            }))
4980    }
4981
4982    #[cfg(all(test, unix))]
4983    pub(crate) fn retry_deadline_for_test(&self) -> Option<Instant> {
4984        lock_snapshot(&self.inner.snapshot)
4985            .unwrap()
4986            .failed_retry
4987            .pending
4988            .map(|retry| retry.deadline)
4989    }
4990
4991    pub(crate) fn record_deliberate_severance(
4992        &self,
4993        identity: ProcessIdentity,
4994    ) -> Result<bool, SuperviseError> {
4995        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4996        if snapshot.pid != Some(identity.pid)
4997            || snapshot.process_start_time != Some(identity.start_time)
4998        {
4999            return Ok(false);
5000        }
5001        snapshot.deliberate_severance = Some(identity);
5002        Ok(true)
5003    }
5004
5005    /// Read status for a channel-0 renderer and report a contended snapshot lock.
5006    ///
5007    /// Internal supervision callers use [`Self::status`] so writer-side machinery
5008    /// does not produce reader-observability logs.
5009    pub(crate) fn status_for_control(
5010        &self,
5011        caller: &'static str,
5012    ) -> Result<ModuleStatus, SuperviseError> {
5013        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
5014            .map(|(status, _)| status)
5015    }
5016
5017    fn status_with_snapshot_lock(
5018        &self,
5019        snapshot: &SharedSnapshot,
5020        caller: Option<&'static str>,
5021    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
5022        let restart_policy = self
5023            .inner
5024            .configuration
5025            .lock()
5026            .map_err(|_| SuperviseError::StatePoisoned {
5027                module_id: Some(self.inner.module_id.clone()),
5028            })?
5029            .restart_policy;
5030        let mut guard = match caller {
5031            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
5032            None => lock_snapshot(snapshot)?,
5033        };
5034        // Read the budget through the pruning path so a reader sees the same
5035        // in-window count the restart decision would use, not a stale total.
5036        let restart_count = guard.crash_restarts_in_window(restart_policy.window, Instant::now());
5037        let snapshot = guard.clone();
5038        drop(guard);
5039        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
5040            SuperviseError::StatePoisoned {
5041                module_id: Some(self.inner.module_id.clone()),
5042            }
5043        })?;
5044        let registration_active = self
5045            .inner
5046            .registry
5047            .get_module(&self.inner.module_id)
5048            .map_err(SuperviseError::Registry)?
5049            .is_some();
5050        let protocol = snapshot
5051            .spawned_protocol
5052            .unwrap_or(self.declared_protocol()?);
5053        let running_process =
5054            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
5055        // Registration is the difference between the two protocols and the only
5056        // one: a subc module that has not registered cannot serve a request even
5057        // though its process is up, and a `none` module never registers at all,
5058        // so requiring it there would pin `live` to false for the whole life of
5059        // a perfectly healthy process.
5060        let live = match protocol {
5061            ModuleProtocol::Subc => running_process && registration_active,
5062            ModuleProtocol::None => running_process,
5063        };
5064        // Read only the retained cache: a status read must not resend signals,
5065        // rewrite the record, or keep the containment resource alive.
5066        let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
5067        let carry_configured = self
5068            .inner
5069            .configuration
5070            .lock()
5071            .map_err(|_| SuperviseError::StatePoisoned {
5072                module_id: Some(self.inner.module_id.clone()),
5073            })?
5074            .spec
5075            .carry_containment;
5076        let recovery_block = carries_containment(
5077            carry_configured,
5078            self.inner
5079                .retained
5080                .has_carried_container(&self.inner.module_id),
5081        )
5082        .then(|| self.inner.retained.recovery_block())
5083        .flatten();
5084        let launch_nonce_source = observed_launch_nonce_source(&snapshot);
5085        let pid = snapshot.reported_pid();
5086        let mut health = snapshot.health;
5087        if let Some(detail) = recovery_block
5088            .as_ref()
5089            .or_else(|| retained.as_ref().map(|(_, detail)| detail))
5090        {
5091            health.detail = Some(match health.detail {
5092                Some(previous) => format!("{previous}; {detail}"),
5093                None => detail.clone(),
5094            });
5095        }
5096
5097        Ok((
5098            ModuleStatus {
5099                admission_refused: snapshot.admission_refused,
5100                legacy_registration_spawn: snapshot.legacy_registration_spawn,
5101                registration_declaration: snapshot.registration_declaration.clone(),
5102                stop_reason: snapshot.stop_reason,
5103                waiting_for_previous_pid: retained
5104                    .as_ref()
5105                    .map(|(pid, _)| *pid)
5106                    .filter(|pid| *pid > 0)
5107                    .or(snapshot.waiting_for_previous_pid),
5108                waiting_for_previous_detail: recovery_block
5109                    .or_else(|| retained.map(|(_, detail)| detail))
5110                    .or(snapshot.waiting_for_previous_detail),
5111                module_id: self.inner.module_id.clone(),
5112                launch_nonce_source,
5113                state: snapshot.state,
5114                enabled: snapshot.enabled,
5115                process_alive: snapshot.process_alive,
5116                registration_active,
5117                protocol,
5118                live,
5119                restart_count,
5120                lifetime_restarts: snapshot.lifetime_restarts,
5121                spawn_generation: snapshot.spawn_generation,
5122                swap_outcome: snapshot.swap_outcome,
5123                max_restarts: restart_policy.max_restarts,
5124                restart_window: restart_policy.window,
5125                drain_timeout,
5126                restart_backoff: restart_policy.backoff,
5127                restart_max_backoff: restart_policy.max_backoff,
5128                pid,
5129                spawned_at_ms: snapshot.spawned_at_ms,
5130                spawned_from: snapshot.spawned_from,
5131                launch_spec_digest: snapshot.launch_spec_digest,
5132                process_start_time: snapshot.process_start_time,
5133                last_exit: snapshot.last_exit,
5134                health,
5135            },
5136            snapshot.spawned_file_identity.clone(),
5137        ))
5138    }
5139
5140    #[cfg(test)]
5141    pub(crate) fn hold_snapshot_for_test(
5142        &self,
5143        acquired: std::sync::mpsc::Sender<()>,
5144        hold: Duration,
5145    ) -> std::thread::JoinHandle<()> {
5146        let snapshot = Arc::clone(&self.inner.snapshot);
5147        std::thread::spawn(move || {
5148            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
5149            acquired
5150                .send(())
5151                .expect("test receiver waits for snapshot lock");
5152            std::thread::sleep(hold);
5153        })
5154    }
5155
5156    /// The status and the running-image check for `supervisor.provenance`,
5157    /// taken from one status read. The exec acknowledgement can land between
5158    /// two separate reads, and the reply would then pair "no pid yet" with an
5159    /// image observed after the module started, which describes no single
5160    /// moment.
5161    pub(crate) async fn status_and_running_image_agreement(
5162        &self,
5163    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
5164        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
5165        let image = self
5166            .inner
5167            .provenance_probe
5168            .observe(
5169                status.pid,
5170                status.spawned_from.as_deref(),
5171                identity,
5172                status.process_start_time,
5173            )
5174            .await;
5175        Ok((status, image))
5176    }
5177
5178    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
5179        let snapshot = match lock_snapshot(&self.inner.snapshot) {
5180            Ok(snapshot) => snapshot.clone(),
5181            Err(_) => {
5182                return subc_control::RunningImageAgreement::Unavailable {
5183                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
5184                };
5185            }
5186        };
5187        self.inner
5188            .provenance_probe
5189            .observe(
5190                snapshot.reported_pid(),
5191                snapshot.spawned_from.as_deref(),
5192                snapshot.spawned_file_identity.clone(),
5193                snapshot.process_start_time,
5194            )
5195            .await
5196    }
5197
5198    /// Memory and CPU time of the module's current process, read now. Only the
5199    /// process the supervisor spawned is read, not processes it has started.
5200    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
5201        #[cfg(windows)]
5202        {
5203            let image = lock_snapshot(&self.inner.snapshot)
5204                .ok()
5205                .and_then(|snapshot| {
5206                    snapshot
5207                        .spawned_file_identity
5208                        .as_ref()
5209                        .and_then(|identity| identity.windows_image.clone())
5210                });
5211            match image {
5212                Some(image) => {
5213                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
5214                }
5215                None => subc_control::ChildResourceUsage::Unavailable {
5216                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
5217                },
5218            }
5219        }
5220        #[cfg(not(windows))]
5221        {
5222            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
5223                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
5224                Err(_) => {
5225                    return subc_control::ChildResourceUsage::Unavailable {
5226                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
5227                    }
5228                }
5229            };
5230            crate::child_resources::read(pid, start_time)
5231        }
5232    }
5233
5234    #[cfg(all(test, any(target_os = "linux", windows)))]
5235    pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
5236        self.inner
5237            .snapshot
5238            .try_lock()
5239            .map(|snapshot| snapshot.subtree_target.clone())
5240            .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
5241    }
5242
5243    #[cfg(test)]
5244    pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
5245        self.inner.snapshot.lock().unwrap().subtree_target = target;
5246    }
5247
5248    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
5249        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
5250        Ok(match snapshot.state {
5251            ModuleState::Restarting => true,
5252            ModuleState::Failed | ModuleState::Disabled => false,
5253            _ if snapshot.enabled && snapshot.respawn_pending => true,
5254            _ => daemon_will_restart(
5255                &mut snapshot,
5256                &self
5257                    .inner
5258                    .configuration
5259                    .lock()
5260                    .map_err(|_| SuperviseError::StatePoisoned {
5261                        module_id: Some(self.inner.module_id.clone()),
5262                    })?
5263                    .restart_policy,
5264                Instant::now(),
5265            ),
5266        })
5267    }
5268
5269    #[cfg(test)]
5270    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
5271        self.is_warming_with_snapshot_lock(None)
5272    }
5273
5274    pub(crate) fn is_warming_for_control(
5275        &self,
5276        caller: &'static str,
5277    ) -> Result<bool, SuperviseError> {
5278        self.is_warming_with_snapshot_lock(Some(caller))
5279    }
5280
5281    fn is_warming_with_snapshot_lock(
5282        &self,
5283        caller: Option<&'static str>,
5284    ) -> Result<bool, SuperviseError> {
5285        let snapshot = match caller {
5286            Some(caller) => {
5287                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
5288            }
5289            None => lock_snapshot(&self.inner.snapshot)?,
5290        }
5291        .clone();
5292        Ok(matches!(
5293            snapshot.state,
5294            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
5295        ))
5296    }
5297
5298    /// Drain the module and stop monitoring it.
5299    pub async fn drain(&self) -> Result<(), SuperviseError> {
5300        self.stop().await
5301    }
5302
5303    pub(crate) async fn retire_requested(
5304        &self,
5305        requester: Option<Principal>,
5306    ) -> Result<(), SuperviseError> {
5307        match self.state()? {
5308            ModuleState::Stopped => return Ok(()),
5309            ModuleState::Failed
5310                if lock_snapshot(&self.inner.snapshot)?
5311                    .failed_retry
5312                    .pending
5313                    .is_none() =>
5314            {
5315                return Ok(())
5316            }
5317            ModuleState::Failed
5318            | ModuleState::Starting
5319            | ModuleState::Running
5320            | ModuleState::Unresponsive
5321            | ModuleState::Restarting
5322            | ModuleState::Draining
5323            | ModuleState::Disabled => {}
5324        }
5325
5326        let (reply_tx, reply_rx) = oneshot::channel();
5327        self.inner
5328            .commands
5329            .send(SupervisorCommand::Retire {
5330                requester,
5331                reply: reply_tx,
5332            })
5333            .await
5334            .map_err(|_| SuperviseError::CommandClosed {
5335                module_id: self.inner.module_id.clone(),
5336            })?;
5337        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5338            module_id: self.inner.module_id.clone(),
5339        })?
5340    }
5341
5342    pub async fn stop(&self) -> Result<(), SuperviseError> {
5343        self.stop_requested(None).await
5344    }
5345
5346    pub(crate) async fn stop_requested(
5347        &self,
5348        requester: Option<Principal>,
5349    ) -> Result<(), SuperviseError> {
5350        match self.state()? {
5351            ModuleState::Stopped => return Ok(()),
5352            ModuleState::Failed
5353                if lock_snapshot(&self.inner.snapshot)?
5354                    .failed_retry
5355                    .pending
5356                    .is_none() =>
5357            {
5358                return Ok(())
5359            }
5360            ModuleState::Failed
5361            | ModuleState::Starting
5362            | ModuleState::Running
5363            | ModuleState::Unresponsive
5364            | ModuleState::Restarting
5365            | ModuleState::Draining
5366            | ModuleState::Disabled => {}
5367        }
5368
5369        let (reply_tx, reply_rx) = oneshot::channel();
5370        self.inner
5371            .commands
5372            .send(SupervisorCommand::Drain {
5373                requester,
5374                reply: reply_tx,
5375            })
5376            .await
5377            .map_err(|_| SuperviseError::CommandClosed {
5378                module_id: self.inner.module_id.clone(),
5379            })?;
5380        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5381            module_id: self.inner.module_id.clone(),
5382        })?
5383    }
5384
5385    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
5386        self.restart_requested(drain_timeout_ms, None).await
5387    }
5388
5389    pub(crate) async fn restart_requested(
5390        &self,
5391        drain_timeout_ms: Option<u64>,
5392        requester: Option<Principal>,
5393    ) -> Result<(), SuperviseError> {
5394        let queued_at = Instant::now();
5395        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
5396        let (reply_tx, reply_rx) = oneshot::channel();
5397        self.inner
5398            .commands
5399            .send(SupervisorCommand::Restart {
5400                requester,
5401                drain_timeout_ms,
5402                received_at_generation,
5403                queued_at,
5404                reply: reply_tx,
5405            })
5406            .await
5407            .map_err(|_| SuperviseError::CommandClosed {
5408                module_id: self.inner.module_id.clone(),
5409            })?;
5410        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5411            module_id: self.inner.module_id.clone(),
5412        })?
5413    }
5414
5415    /// Blue/green restart: see `SupervisorCommand::Swap` and the
5416    /// `supervisor_swap` module. Returns once the swap has cut over (the old
5417    /// process then drains in the background of the supervise loop), has been
5418    /// superseded by an approved configuration restart, or has failed. Both
5419    /// supersession and failure leave the old process serving until replacement.
5420    pub async fn swap(
5421        &self,
5422        ready_timeout: Option<Duration>,
5423    ) -> Result<SwapOutcome, SuperviseError> {
5424        self.swap_requested(ready_timeout, None).await
5425    }
5426
5427    pub(crate) async fn swap_requested(
5428        &self,
5429        ready_timeout: Option<Duration>,
5430        requester: Option<Principal>,
5431    ) -> Result<SwapOutcome, SuperviseError> {
5432        let (reply_tx, reply_rx) = oneshot::channel();
5433        self.inner
5434            .commands
5435            .send(SupervisorCommand::Swap {
5436                requester,
5437                ready_timeout,
5438                reply: reply_tx,
5439            })
5440            .await
5441            .map_err(|_| SuperviseError::CommandClosed {
5442                module_id: self.inner.module_id.clone(),
5443            })?;
5444        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5445            module_id: self.inner.module_id.clone(),
5446        })?
5447    }
5448
5449    pub async fn reload(&self) -> Result<(), SuperviseError> {
5450        self.reload_requested(None).await
5451    }
5452
5453    pub(crate) async fn reload_requested(
5454        &self,
5455        requester: Option<Principal>,
5456    ) -> Result<(), SuperviseError> {
5457        let (reply_tx, reply_rx) = oneshot::channel();
5458        self.inner
5459            .commands
5460            .send(SupervisorCommand::Reload {
5461                requester,
5462                reply: reply_tx,
5463            })
5464            .await
5465            .map_err(|_| SuperviseError::CommandClosed {
5466                module_id: self.inner.module_id.clone(),
5467            })?;
5468        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5469            module_id: self.inner.module_id.clone(),
5470        })?
5471    }
5472
5473    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
5474        self.set_enabled_requested(enabled, None).await
5475    }
5476
5477    pub(crate) async fn set_enabled_requested(
5478        &self,
5479        enabled: bool,
5480        requester: Option<Principal>,
5481    ) -> Result<bool, SuperviseError> {
5482        self.set_enabled_with_budget_policy(enabled, requester, true)
5483            .await
5484    }
5485
5486    pub(crate) async fn release_config_hold(&self) -> Result<bool, SuperviseError> {
5487        self.set_enabled_with_budget_policy(true, None, false).await
5488    }
5489
5490    #[cfg(all(test, unix))]
5491    pub(crate) fn seed_crash_budget_for_test(&self, count: u32) {
5492        let policy = self.inner.configuration.lock().unwrap().restart_policy;
5493        let mut state = self.inner.snapshot.lock().unwrap();
5494        for _ in 0..count {
5495            state.record_crash_restart(&policy, Instant::now());
5496        }
5497    }
5498
5499    async fn set_enabled_with_budget_policy(
5500        &self,
5501        enabled: bool,
5502        requester: Option<Principal>,
5503        reset_budget: bool,
5504    ) -> Result<bool, SuperviseError> {
5505        let (reply_tx, reply_rx) = oneshot::channel();
5506        self.inner
5507            .commands
5508            .send(SupervisorCommand::SetEnabled {
5509                requester,
5510                enabled,
5511                reset_budget,
5512                reply: reply_tx,
5513            })
5514            .await
5515            .map_err(|_| SuperviseError::CommandClosed {
5516                module_id: self.inner.module_id.clone(),
5517            })?;
5518        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5519            module_id: self.inner.module_id.clone(),
5520        })?
5521    }
5522
5523    /// The current process's protocol, or the configured protocol when down.
5524    /// A rescan stores the next launch spec without changing how an existing
5525    /// process registers, serves routes, is probed, or exits.
5526    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
5527        let configured = self
5528            .inner
5529            .configuration
5530            .lock()
5531            .map_err(|_| SuperviseError::StatePoisoned {
5532                module_id: Some(self.inner.module_id.clone()),
5533            })?
5534            .spec
5535            .protocol;
5536        let state = lock_snapshot(&self.inner.snapshot)?;
5537        Ok(state.spawned_protocol.unwrap_or(configured))
5538    }
5539
5540    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
5541        let configuration =
5542            self.inner
5543                .configuration
5544                .lock()
5545                .map_err(|_| SuperviseError::StatePoisoned {
5546                    module_id: Some(self.inner.module_id.clone()),
5547                })?;
5548        Ok((configuration.spec.clone(), configuration.health.clone()))
5549    }
5550
5551    pub(crate) fn configured_register_timeout(&self) -> Result<Duration, SuperviseError> {
5552        self.inner
5553            .configuration
5554            .lock()
5555            .map(|configuration| configuration.register_timeout)
5556            .map_err(|_| SuperviseError::StatePoisoned {
5557                module_id: Some(self.inner.module_id.clone()),
5558            })
5559    }
5560
5561    pub(crate) fn configured_enabled(&self) -> Result<bool, SuperviseError> {
5562        self.inner
5563            .configuration
5564            .lock()
5565            .map(|configuration| configuration.enabled)
5566            .map_err(|_| SuperviseError::StatePoisoned {
5567                module_id: Some(self.inner.module_id.clone()),
5568            })
5569    }
5570
5571    /// Replace this module's launch spec, keeping its health and drain policy,
5572    /// the way a rescan does for a changed config entry. The running process is
5573    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
5574    #[cfg(any(test, feature = "test-support"))]
5575    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
5576        let (_, health) = self.configuration()?;
5577        let drain_timeout_ms = u64::try_from(
5578            self.inner
5579                .effective_drain_timeout
5580                .lock()
5581                .unwrap_or_else(|poisoned| poisoned.into_inner())
5582                .as_millis(),
5583        )
5584        .ok();
5585        self.update_configuration(spec, health, drain_timeout_ms)
5586            .await
5587    }
5588
5589    pub(crate) async fn reconcile_approved_configuration(
5590        &self,
5591        approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5592    ) -> Result<(), SuperviseError> {
5593        let entry = approved
5594            .module_entry(&self.inner.module_id)
5595            .map_err(|error| SuperviseError::InvalidSpec {
5596                reason: error.to_string(),
5597            })?;
5598        validate_spec(&entry.module_spec())?;
5599        let (reply, received) = oneshot::channel();
5600        self.inner
5601            .commands
5602            .send(SupervisorCommand::ApprovedConfiguration { approved, reply })
5603            .await
5604            .map_err(|_| SuperviseError::CommandClosed {
5605                module_id: self.inner.module_id.clone(),
5606            })?;
5607        received.await.map_err(|_| SuperviseError::CommandClosed {
5608            module_id: self.inner.module_id.clone(),
5609        })??;
5610        let mut configuration =
5611            self.inner
5612                .configuration
5613                .lock()
5614                .map_err(|_| SuperviseError::StatePoisoned {
5615                    module_id: Some(self.inner.module_id.clone()),
5616                })?;
5617        configuration.spec = entry.module_spec();
5618        configuration.enabled = entry.enabled;
5619        configuration.health = entry.health;
5620        configuration.register_timeout = entry.register_timeout;
5621        configuration.restart_policy = entry.restart;
5622        Ok(())
5623    }
5624
5625    pub(crate) async fn update_configuration(
5626        &self,
5627        spec: ModuleSpec,
5628        health: HealthConfig,
5629        drain_timeout_ms: Option<u64>,
5630    ) -> Result<(), SuperviseError> {
5631        if spec.module_id != self.inner.module_id {
5632            return Err(SuperviseError::InvalidSpec {
5633                reason: "a supervised module's module_id cannot be changed".to_string(),
5634            });
5635        }
5636        validate_spec(&spec)?;
5637        let (reply_tx, reply_rx) = oneshot::channel();
5638        self.inner
5639            .commands
5640            .send(SupervisorCommand::UpdateConfiguration {
5641                spec: spec.clone(),
5642                health: health.clone(),
5643                drain_timeout_ms,
5644                reply: reply_tx,
5645            })
5646            .await
5647            .map_err(|_| SuperviseError::CommandClosed {
5648                module_id: self.inner.module_id.clone(),
5649            })?;
5650        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5651            module_id: self.inner.module_id.clone(),
5652        })??;
5653        let mut configuration =
5654            self.inner
5655                .configuration
5656                .lock()
5657                .map_err(|_| SuperviseError::StatePoisoned {
5658                    module_id: Some(self.inner.module_id.clone()),
5659                })?;
5660        configuration.spec = spec;
5661        configuration.health = health;
5662        Ok(())
5663    }
5664}
5665
5666impl Drop for SupervisedModuleInner {
5667    fn drop(&mut self) {
5668        let Ok(mut monitor) = self.monitor.lock() else {
5669            return;
5670        };
5671        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
5672            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
5673                state.state = ModuleState::Stopped;
5674                clear_current_process_facts(state);
5675            });
5676            monitor.abort();
5677        }
5678        let _ = monitor.take();
5679    }
5680}
5681
5682#[derive(Debug)]
5683enum SupervisorCommand {
5684    ApprovedConfiguration {
5685        approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5686        reply: oneshot::Sender<Result<(), SuperviseError>>,
5687    },
5688    ConfigPolicyProbe {
5689        connection: crate::ConnectionId,
5690        generation: u64,
5691        attempt: u64,
5692    },
5693    ConfigApplyRestart {
5694        generation: u64,
5695        supersedes_swap: bool,
5696    },
5697    ConfigExitHold {
5698        reply: oneshot::Sender<Result<(), SuperviseError>>,
5699    },
5700    ConfigDeliveryFailed {
5701        connection: crate::ConnectionId,
5702        hold: bool,
5703    },
5704    Drain {
5705        requester: Option<Principal>,
5706        reply: oneshot::Sender<Result<(), SuperviseError>>,
5707    },
5708    Retire {
5709        requester: Option<Principal>,
5710        reply: oneshot::Sender<Result<(), SuperviseError>>,
5711    },
5712    Restart {
5713        requester: Option<Principal>,
5714        /// Operator override for this one restart's drain budget, in ms. `None`
5715        /// uses the module's configured/default budget; `Some(0)` cuts
5716        /// immediately (wedge bounce: a stuck request never settles, so
5717        /// waiting only delays recovery).
5718        drain_timeout_ms: Option<u64>,
5719        /// The module's `spawn_generation` when the request was received, before
5720        /// it waited in the command queue. A queued restart whose module has
5721        /// since spawned a newer process is already satisfied (see the handler).
5722        received_at_generation: u64,
5723        /// When the request entered the command queue, so the handler can log
5724        /// how long it waited behind the loop's other work.
5725        queued_at: Instant,
5726        reply: oneshot::Sender<Result<(), SuperviseError>>,
5727    },
5728    Reload {
5729        requester: Option<Principal>,
5730        reply: oneshot::Sender<Result<(), SuperviseError>>,
5731    },
5732    SetEnabled {
5733        reset_budget: bool,
5734        requester: Option<Principal>,
5735        enabled: bool,
5736        reply: oneshot::Sender<Result<bool, SuperviseError>>,
5737    },
5738    UpdateConfiguration {
5739        spec: ModuleSpec,
5740        health: HealthConfig,
5741        /// Per-module drain override from the new config; `None` re-resolves to
5742        /// the supervisor-wide default.
5743        drain_timeout_ms: Option<u64>,
5744        reply: oneshot::Sender<Result<(), SuperviseError>>,
5745    },
5746    Swap {
5747        requester: Option<Principal>,
5748        /// How long the candidate may take to register and declare itself
5749        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
5750        ready_timeout: Option<Duration>,
5751        /// Answered at cutover or failure; the incumbent's drain follows.
5752        reply: oneshot::Sender<Result<SwapOutcome, SuperviseError>>,
5753    },
5754}
5755
5756/// Successful daemon-local swap disposition. Supersession leaves the incumbent
5757/// serving until an approved replacement starts with the latest configured
5758/// command; the legacy control acknowledgment does not carry this disposition.
5759#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5760pub enum SwapOutcome {
5761    Cutover,
5762    SupersededByConfigRestart,
5763}
5764
5765impl SwapOutcome {
5766    pub fn as_str(self) -> &'static str {
5767        match self {
5768            Self::Cutover => "cutover",
5769            Self::SupersededByConfigRestart => "superseded_by_config_restart",
5770        }
5771    }
5772}
5773
5774#[derive(Debug)]
5775pub enum SuperviseError {
5776    ConfigHeld {
5777        module_id: String,
5778        code: String,
5779        detail: String,
5780    },
5781    ConfigDeliveryDeferred {
5782        module_id: String,
5783    },
5784    InvalidSpec {
5785        reason: String,
5786    },
5787    Spawn {
5788        program: PathBuf,
5789        source: io::Error,
5790        cgroup_path: Option<PathBuf>,
5791    },
5792    Cgroup {
5793        module_id: String,
5794        source: io::Error,
5795    },
5796    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
5797    /// than spawn a reserved module without its identity binding.
5798    LaunchNonce {
5799        reason: String,
5800    },
5801    Wait {
5802        module_id: String,
5803        source: io::Error,
5804    },
5805    Kill {
5806        module_id: String,
5807        source: io::Error,
5808    },
5809    Forwarding(ForwardingError),
5810    Registry(RegistryError),
5811    ReloadUnavailable {
5812        module_id: String,
5813        reason: String,
5814    },
5815    /// An operator restart/reload was requested for a module that is currently
5816    /// disabled. Restart/reload cycle a *running* module; a disabled module must
5817    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
5818    /// by a restart, so these commands are rejected instead of re-enabling it.
5819    Disabled {
5820        module_id: String,
5821    },
5822    ReloadFailed {
5823        module_id: String,
5824        reason: String,
5825    },
5826    RegistrationStillActive {
5827        module_id: String,
5828        waited: Duration,
5829    },
5830    StatePoisoned {
5831        module_id: Option<String>,
5832    },
5833    CommandClosed {
5834        module_id: String,
5835    },
5836    /// A restart or reload arrived while a swap's candidate was warming. The
5837    /// swap owns the module until it cuts over or fails; a stop or disable
5838    /// would have aborted it instead.
5839    SwapInProgress {
5840        module_id: String,
5841    },
5842    /// A swap was refused before anything was spawned.
5843    SwapRefused {
5844        module_id: String,
5845        reason: SwapRefusal,
5846    },
5847    /// A swap spawned a candidate and gave up on it. The candidate has been
5848    /// killed and its slot freed; the incumbent was left serving and was never
5849    /// drained, except in the one `CutoverLost` case described on that arm.
5850    SwapFailed {
5851        module_id: String,
5852        arm: SwapFailureArm,
5853        detail: String,
5854        /// How the candidate exited, when it exited on its own before the
5855        /// supervisor gave up on it.
5856        candidate_exit: Option<ExitReport>,
5857    },
5858}
5859
5860/// Why a swap was refused before a candidate was spawned.
5861#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5862pub enum SwapRefusal {
5863    /// The module's config does not declare `overlap: "safe"`.
5864    OverlapExclusive,
5865    /// The module is not registered, so there is no incumbent to keep serving
5866    /// and nothing a swap would improve on; a plain restart is the tool.
5867    NotRegistered,
5868    /// The module does not speak the subc wire, so a candidate could never
5869    /// register or declare itself ready.
5870    ProtocolNone,
5871    /// The supervisor lacks the forwarding table (to cut routes over) or the
5872    /// shared handle (to admit the candidate's HELLO) that a swap needs.
5873    NotConfigured,
5874    /// A swap is already open for this module.
5875    AlreadySwapping,
5876}
5877
5878impl SwapRefusal {
5879    pub fn as_str(self) -> &'static str {
5880        match self {
5881            Self::OverlapExclusive => "overlap_exclusive",
5882            Self::NotRegistered => "not_registered",
5883            Self::ProtocolNone => "protocol_none",
5884            Self::NotConfigured => "not_configured",
5885            Self::AlreadySwapping => "already_swapping",
5886        }
5887    }
5888}
5889
5890/// Which failure arm ended a swap. Every arm but one leaves the incumbent
5891/// serving and undrained; see `CutoverLost`.
5892#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5893pub enum SwapFailureArm {
5894    /// The candidate process could not be started.
5895    SpawnFailed,
5896    /// The candidate did not register within the readiness budget.
5897    NeverRegistered,
5898    /// The candidate registered but did not declare itself ready in time.
5899    NeverReady,
5900    /// The candidate exited before cutover.
5901    CandidateExited,
5902    /// The candidate declared itself ready but failed its health probe.
5903    CandidateUnhealthy,
5904    /// An operator stop, disable or retire arrived while the candidate warmed.
5905    /// The candidate was killed and the operator's command then carried out on
5906    /// the incumbent.
5907    Interrupted,
5908    /// The candidate's connection closed at the moment of cutover. If it
5909    /// closed before forwarding moved, the incumbent is untouched. If it closed
5910    /// between the forwarding and registry halves of cutover, forwarding can no
5911    /// longer route to the incumbent, so the module is restarted plainly.
5912    CutoverLost,
5913}
5914
5915impl SwapFailureArm {
5916    pub fn as_str(self) -> &'static str {
5917        match self {
5918            Self::SpawnFailed => "spawn_failed",
5919            Self::NeverRegistered => "never_registered",
5920            Self::NeverReady => "never_ready",
5921            Self::CandidateExited => "candidate_exited",
5922            Self::CandidateUnhealthy => "candidate_unhealthy",
5923            Self::Interrupted => "interrupted",
5924            Self::CutoverLost => "cutover_lost",
5925        }
5926    }
5927}
5928
5929impl SuperviseError {
5930    pub(crate) fn is_config_gate_error(&self) -> bool {
5931        match self {
5932            Self::ConfigHeld { .. } => true,
5933            Self::Spawn { source, .. } => source
5934                .get_ref()
5935                .is_some_and(|cause| cause.is::<crate::config_gate::GateError>()),
5936            _ => false,
5937        }
5938    }
5939}
5940
5941impl fmt::Display for SuperviseError {
5942    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5943        match self {
5944            Self::ConfigHeld { module_id, code, detail } => write!(f, "{code}: module '{module_id}' held: {detail}"),
5945            Self::ConfigDeliveryDeferred { module_id } => write!(f, "config_gated: module '{module_id}' is waiting for its exited delivery target's failure record"),
5946            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
5947            Self::Spawn {
5948                program,
5949                source,
5950                cgroup_path: Some(cgroup_path),
5951            } => write!(
5952                f,
5953                "failed to place module in cgroup '{}' while spawning '{}': {source}",
5954                cgroup_path.display(),
5955                program.display()
5956            ),
5957            Self::Spawn {
5958                program,
5959                source,
5960                cgroup_path: None,
5961            } => write!(
5962                f,
5963                "failed to spawn module '{}': {source}",
5964                program.display()
5965            ),
5966            Self::Cgroup { module_id, source } => {
5967                write!(
5968                    f,
5969                    "failed to prepare cgroup for module '{module_id}': {source}"
5970                )
5971            }
5972            Self::LaunchNonce { reason } => {
5973                write!(
5974                    f,
5975                    "failed to generate reserved-module launch nonce: {reason}"
5976                )
5977            }
5978            Self::Wait { module_id, source } => {
5979                write!(f, "failed to wait for module '{module_id}': {source}")
5980            }
5981            Self::Kill { module_id, source } => {
5982                write!(f, "failed to kill module '{module_id}': {source}")
5983            }
5984            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
5985            Self::Registry(err) => write!(f, "registry error: {err}"),
5986            Self::ReloadUnavailable { module_id, reason } => {
5987                write!(f, "reload unavailable for module '{module_id}': {reason}")
5988            }
5989            Self::Disabled { module_id } => {
5990                write!(
5991                    f,
5992                    "module '{module_id}' is disabled; enable it before restart or reload"
5993                )
5994            }
5995            Self::ReloadFailed { module_id, reason } => {
5996                write!(f, "reload failed for module '{module_id}': {reason}")
5997            }
5998            Self::RegistrationStillActive { module_id, waited } => write!(
5999                f,
6000                "module '{module_id}' registration remained active after waiting {waited:?}"
6001            ),
6002            Self::StatePoisoned { module_id } => match module_id {
6003                Some(module_id) => {
6004                    write!(f, "supervisor state for module '{module_id}' was poisoned")
6005                }
6006                None => write!(f, "supervisor state was poisoned"),
6007            },
6008            Self::CommandClosed { module_id } => {
6009                write!(
6010                    f,
6011                    "supervisor command channel for module '{module_id}' is closed"
6012                )
6013            }
6014            Self::SwapInProgress { module_id } => write!(
6015                f,
6016                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
6017            ),
6018            Self::SwapRefused { module_id, reason } => match reason {
6019                SwapRefusal::OverlapExclusive => write!(
6020                    f,
6021                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
6022                ),
6023                SwapRefusal::NotRegistered => write!(
6024                    f,
6025                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
6026                ),
6027                SwapRefusal::ProtocolNone => write!(
6028                    f,
6029                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
6030                ),
6031                SwapRefusal::NotConfigured => write!(
6032                    f,
6033                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
6034                ),
6035                SwapRefusal::AlreadySwapping => {
6036                    write!(f, "module '{module_id}' is already being swapped")
6037                }
6038            },
6039            Self::SwapFailed {
6040                module_id,
6041                arm,
6042                detail,
6043                ..
6044            } => write!(
6045                f,
6046                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
6047                arm.as_str()
6048            ),
6049        }
6050    }
6051}
6052
6053impl Error for SuperviseError {
6054    fn source(&self) -> Option<&(dyn Error + 'static)> {
6055        match self {
6056            Self::Spawn { source, .. }
6057            | Self::Cgroup { source, .. }
6058            | Self::Wait { source, .. }
6059            | Self::Kill { source, .. } => Some(source),
6060            Self::Forwarding(err) => Some(err),
6061            Self::Registry(err) => Some(err),
6062            Self::LaunchNonce { .. }
6063            | Self::ConfigDeliveryDeferred { .. }
6064            | Self::ConfigHeld { .. }
6065            | Self::InvalidSpec { .. }
6066            | Self::ReloadUnavailable { .. }
6067            | Self::Disabled { .. }
6068            | Self::ReloadFailed { .. }
6069            | Self::RegistrationStillActive { .. }
6070            | Self::StatePoisoned { .. }
6071            | Self::CommandClosed { .. }
6072            | Self::SwapInProgress { .. }
6073            | Self::SwapRefused { .. }
6074            | Self::SwapFailed { .. } => None,
6075        }
6076    }
6077}
6078
6079pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
6080    if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
6081        return Err(SuperviseError::InvalidSpec {
6082            reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
6083        });
6084    }
6085    if spec.module_id.trim().is_empty() {
6086        return Err(SuperviseError::InvalidSpec {
6087            reason: "module_id must not be empty".to_string(),
6088        });
6089    }
6090
6091    Ok(())
6092}
6093
6094#[derive(Debug, Default)]
6095struct HealthProbeRuntime {
6096    spawn_generation: Option<u64>,
6097    configured_health: Option<HealthConfig>,
6098    registered_connection: Option<crate::ConnectionId>,
6099    advertised: bool,
6100    next_probe_at: Option<Instant>,
6101    probe_index: u64,
6102}
6103
6104fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
6105    lock_snapshot(snapshot)
6106        .ok()
6107        .and_then(|state| state.spawned_protocol)
6108        .unwrap_or(spec.protocol)
6109}
6110
6111impl HealthProbeRuntime {
6112    fn refresh_registration(
6113        &mut self,
6114        spec: &ModuleSpec,
6115        runtime: &SupervisorRuntimeConfig,
6116        registry: &Registry,
6117        snapshot: &SharedSnapshot,
6118    ) {
6119        let generation = lock_snapshot(snapshot)
6120            .ok()
6121            .map(|state| state.spawn_generation);
6122        if self.spawn_generation != generation {
6123            // A replacement cannot reuse a deadline that is already due for its
6124            // predecessor. HTTP children get a full cadence to start; wire
6125            // children still wait for their own health-advertising HELLO.
6126            *self = Self {
6127                spawn_generation: generation,
6128                ..Self::default()
6129            };
6130        }
6131        if self.configured_health.as_ref() != Some(&runtime.health) {
6132            self.configured_health = Some(runtime.health.clone());
6133            self.next_probe_at = None;
6134            self.registered_connection = None;
6135            self.probe_index = 0;
6136        }
6137        // A non-wire process never registers. Only an explicitly configured
6138        // HTTP endpoint can arm its health probe; an absent HELLO is not a
6139        // health failure for that kind of process.
6140        if running_protocol(spec, snapshot) == ModuleProtocol::None {
6141            self.registered_connection = None;
6142            self.advertised = runtime.health.http.is_some();
6143            if !self.advertised {
6144                self.next_probe_at = None;
6145                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6146                    let unknown = ModuleHealthStatus::default();
6147                    if state.health != unknown {
6148                        state.health = unknown;
6149                        state.last_probe_evidence = None;
6150                    }
6151                });
6152            } else if self.next_probe_at.is_none() {
6153                self.next_probe_at = Some(
6154                    Instant::now()
6155                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6156                );
6157            }
6158            return;
6159        }
6160
6161        let registration = match registry.get_module(&spec.module_id) {
6162            Ok(registration) => registration,
6163            Err(err) => {
6164                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
6165                self.advertised = false;
6166                self.next_probe_at = None;
6167                return;
6168            }
6169        };
6170
6171        let Some(registration) = registration else {
6172            self.registered_connection = None;
6173            self.advertised = false;
6174            self.next_probe_at = None;
6175            return;
6176        };
6177
6178        let advertised = registration
6179            .control_ops
6180            .iter()
6181            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
6182        if !advertised {
6183            self.registered_connection = Some(registration.connection_id);
6184            self.advertised = false;
6185            self.next_probe_at = None;
6186            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6187                if state.health.status != SupervisorHealthStatus::Unknown
6188                    || state.health.consecutive_failures != 0
6189                    || state.health.last_probe_ms.is_some()
6190                    || state.health.detail.is_some()
6191                    || state.health.metrics.is_some()
6192                {
6193                    state.health.status = SupervisorHealthStatus::Unknown;
6194                    state.last_probe_evidence = None;
6195                    state.health.consecutive_failures = 0;
6196                    state.health.last_probe_ms = None;
6197                    state.health.detail = None;
6198                    state.health.metrics = None;
6199                }
6200            });
6201            return;
6202        }
6203
6204        let reregistered = self.registered_connection != Some(registration.connection_id);
6205        self.registered_connection = Some(registration.connection_id);
6206        self.advertised = true;
6207        if reregistered || self.next_probe_at.is_none() {
6208            self.probe_index = 0;
6209            self.next_probe_at = Some(
6210                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6211            );
6212            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6213                state.health.status = SupervisorHealthStatus::Unknown;
6214                state.last_probe_evidence = None;
6215                state.health.consecutive_failures = 0;
6216                state.health.detail = None;
6217                state.health.metrics = None;
6218            });
6219        }
6220    }
6221
6222    fn wake_at(&self) -> Option<Instant> {
6223        if !self.advertised {
6224            return None;
6225        }
6226        self.next_probe_at
6227    }
6228
6229    fn due(&self) -> bool {
6230        self.advertised
6231            && self
6232                .next_probe_at
6233                .is_some_and(|next| Instant::now() >= next)
6234    }
6235
6236    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
6237        self.probe_index = self.probe_index.wrapping_add(1);
6238        self.next_probe_at = Some(
6239            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
6240        );
6241    }
6242}
6243
6244/// Evidence is separate from diagnostic prose so host starvation can hold only
6245/// silence-based restarts. A missing lane is still actionable during overload;
6246/// a timeout can instead mean the host has not scheduled a healthy module.
6247#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6248enum HealthProbeEvidence {
6249    /// The module's control lane is gone. Proof of death.
6250    LaneDead,
6251    /// No reply within the deadline. Proves nothing about the module's state.
6252    NoAnswer,
6253    /// The module replied, but not with a usable health report. Proves it is alive.
6254    BadAnswer,
6255    /// The daemon could not ask. Says nothing about the module.
6256    Misconfigured,
6257}
6258
6259#[derive(Debug)]
6260struct HealthProbeError {
6261    evidence: HealthProbeEvidence,
6262    message: String,
6263}
6264
6265impl HealthProbeError {
6266    fn lane_dead(message: impl Into<String>) -> Self {
6267        Self::with(HealthProbeEvidence::LaneDead, message)
6268    }
6269
6270    fn no_answer(message: impl Into<String>) -> Self {
6271        Self::with(HealthProbeEvidence::NoAnswer, message)
6272    }
6273
6274    fn bad_answer(message: impl Into<String>) -> Self {
6275        Self::with(HealthProbeEvidence::BadAnswer, message)
6276    }
6277
6278    fn misconfigured(message: impl Into<String>) -> Self {
6279        Self::with(HealthProbeEvidence::Misconfigured, message)
6280    }
6281
6282    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
6283        Self {
6284            evidence,
6285            message: message.into(),
6286        }
6287    }
6288
6289    /// Whether this observation is proof the module cannot serve.
6290    ///
6291    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
6292    /// variant that fires under CPU starvation, and treating it as proof is the
6293    /// defect this enum exists to make impossible to reintroduce silently.
6294    ///
6295    /// Outside a host-overload hold, silence still reaches the normal failure
6296    /// threshold so an isolated, genuinely wedged module can recover.
6297    #[allow(dead_code)]
6298    fn is_proof_of_death(&self) -> bool {
6299        matches!(self.evidence, HealthProbeEvidence::LaneDead)
6300    }
6301
6302    /// Short stable label for logs and the health snapshot.
6303    ///
6304    /// An operator reading `ck health` currently cannot tell "the module is gone"
6305    /// from "the module did not answer in five seconds", because both render as
6306    /// prose in the same field. These labels are what make the two
6307    /// distinguishable at a glance, and they are what a later restart-policy
6308    /// change will be argued from.
6309    fn label(&self) -> &'static str {
6310        match self.evidence {
6311            HealthProbeEvidence::LaneDead => "lane-dead",
6312            HealthProbeEvidence::NoAnswer => "no-answer",
6313            HealthProbeEvidence::BadAnswer => "bad-answer",
6314            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
6315        }
6316    }
6317}
6318
6319impl fmt::Display for HealthProbeError {
6320    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
6321        f.write_str(&self.message)
6322    }
6323}
6324
6325async fn run_health_probe_cycle(
6326    spec: &ModuleSpec,
6327    runtime: &SupervisorRuntimeConfig,
6328    registry: &Registry,
6329    process_liveness: &SupervisorProcessLiveness,
6330    snapshot: &SharedSnapshot,
6331    child: &mut Option<SupervisedChild>,
6332) {
6333    let now_ms = unix_ms_now();
6334    let delivery_probe = match &runtime.config_delivery {
6335        Some(delivery) => {
6336            delivery
6337                .health_probe(&spec.module_id, runtime.forwarding.as_deref(), registry)
6338                .await
6339        }
6340        None => None,
6341    };
6342    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
6343        .then_some(runtime.health.http.as_deref())
6344        .flatten();
6345    let result = match http {
6346        Some(url) => probe_http_health(url, runtime.health.deadline)
6347            .await
6348            .map(|report| (report, None)),
6349        None => probe_module_health_from(&spec.module_id, runtime, None)
6350            .await
6351            .map(|(report, probed)| (report, Some(probed))),
6352    };
6353    match result {
6354        Ok((report, probed)) => {
6355            handle_health_report(
6356                spec,
6357                runtime,
6358                registry,
6359                process_liveness,
6360                snapshot,
6361                child,
6362                report,
6363                now_ms,
6364                delivery_probe,
6365                probed,
6366            )
6367            .await;
6368        }
6369        Err(err) => {
6370            if http.is_some() {
6371                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6372                    state.health.status = SupervisorHealthStatus::Failing;
6373                });
6374            }
6375            handle_health_probe_failure(
6376                spec,
6377                runtime,
6378                registry,
6379                process_liveness,
6380                snapshot,
6381                child,
6382                err,
6383                now_ms,
6384                delivery_probe,
6385            )
6386            .await;
6387        }
6388    }
6389}
6390
6391pub(crate) struct HttpProbeTarget<'a> {
6392    address: std::net::SocketAddr,
6393    localhost: bool,
6394    authority: &'a str,
6395    path: String,
6396}
6397
6398/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
6399/// or TLS. A URL cannot turn a local health check into an outbound connection.
6400pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
6401    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
6402        return Err("must not contain whitespace, controls, or a fragment".into());
6403    }
6404    let rest = url
6405        .strip_prefix("http://")
6406        .ok_or("must use plain http://")?;
6407    let split = rest.find(['/', '?']).unwrap_or(rest.len());
6408    let (authority, suffix) = rest.split_at(split);
6409    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
6410        ("::1", rest)
6411    } else {
6412        let split = authority.find(':').unwrap_or(authority.len());
6413        authority.split_at(split)
6414    };
6415    let ip: std::net::IpAddr = match host {
6416        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
6417        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
6418        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
6419    };
6420    let port = if port.is_empty() {
6421        80
6422    } else {
6423        port.strip_prefix(':')
6424            .and_then(|p| p.parse::<u16>().ok())
6425            .filter(|p| *p > 0)
6426            .ok_or("must have a valid nonzero TCP port")?
6427    };
6428    let path = if suffix.is_empty() {
6429        "/".into()
6430    } else if suffix.starts_with('?') {
6431        format!("/{suffix}")
6432    } else {
6433        suffix.into()
6434    };
6435    Ok(HttpProbeTarget {
6436        address: std::net::SocketAddr::new(ip, port),
6437        localhost: host == "localhost",
6438        authority,
6439        path,
6440    })
6441}
6442
6443async fn probe_http_health(
6444    url: &str,
6445    deadline: Duration,
6446) -> Result<HealthReport, HealthProbeError> {
6447    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
6448    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
6449    // Keep partial diagnostics outside the timed future so cancellation does
6450    // not discard a status line or body bytes already received.
6451    let mut response_status = String::new();
6452    let mut body = Vec::new();
6453    let probe = async {
6454        // Resolve localhost ourselves so a hosts-file override cannot turn
6455        // this into an outbound request, while IPv6-only local servers work.
6456        let connection = match tokio::net::TcpStream::connect(target.address).await {
6457            Err(_) if target.localhost => {
6458                tokio::net::TcpStream::connect((
6459                    std::net::Ipv6Addr::LOCALHOST,
6460                    target.address.port(),
6461                ))
6462                .await
6463            }
6464            result => result,
6465        };
6466        let mut stream = connection.map_err(|error| {
6467            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
6468        })?;
6469        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
6470            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
6471        let mut reader = BufReader::new(stream);
6472        let mut budget = 16 * 1024;
6473        let status = http_line(&mut reader, &mut budget).await?;
6474        let mut words = status.split_ascii_whitespace();
6475        let version = words.next();
6476        let code = words
6477            .next()
6478            .filter(|word| word.len() == 3)
6479            .and_then(|word| word.parse::<u16>().ok());
6480        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
6481            || !code.is_some_and(|code| (100..600).contains(&code))
6482        {
6483            return Err(HealthProbeError::bad_answer(format!(
6484                "invalid HTTP status: {status}"
6485            )));
6486        }
6487        let code = code.expect("validated status code");
6488        response_status = status.clone();
6489        let mut length = None;
6490        let mut chunked = false;
6491        loop {
6492            let line = http_line(&mut reader, &mut budget).await?;
6493            if line.is_empty() {
6494                break;
6495            }
6496            if let Some((name, value)) = line.split_once(':') {
6497                if name.eq_ignore_ascii_case("content-length") {
6498                    length = Some(value.trim().parse::<u64>().map_err(|_| {
6499                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
6500                    })?);
6501                } else if name.eq_ignore_ascii_case("transfer-encoding") {
6502                    chunked = value.trim().eq_ignore_ascii_case("chunked");
6503                }
6504            }
6505        }
6506        if chunked {
6507            while body.len() < 200 {
6508                let line = http_line(&mut reader, &mut budget).await?;
6509                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
6510                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
6511                if size == 0 {
6512                    break;
6513                }
6514                let count = size.min((200 - body.len()) as u64) as usize;
6515                let start = body.len();
6516                (&mut reader)
6517                    .take(count as u64)
6518                    .read_to_end(&mut body)
6519                    .await
6520                    .map_err(|error| {
6521                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6522                    })?;
6523                if body.len() - start != count {
6524                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
6525                }
6526                if size > count as u64 || body.len() == 200 {
6527                    break;
6528                }
6529                if !http_line(&mut reader, &mut budget).await?.is_empty() {
6530                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
6531                }
6532            }
6533        } else {
6534            reader
6535                .take(length.unwrap_or(200).min(200))
6536                .read_to_end(&mut body)
6537                .await
6538                .map_err(|error| {
6539                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6540                })?;
6541        }
6542        if (200..300).contains(&code) {
6543            Ok(HealthReport::ok())
6544        } else {
6545            Err(HealthProbeError::bad_answer(
6546                "HTTP health endpoint returned non-2xx",
6547            ))
6548        }
6549    };
6550    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
6551        Err(HealthProbeError::no_answer(format!(
6552            "HTTP probe timed out after {deadline:?}"
6553        )))
6554    });
6555    if let Err(error) = &mut result {
6556        if !response_status.is_empty() {
6557            error.message = format!(
6558                "{}; {response_status}: {}",
6559                error.message,
6560                String::from_utf8_lossy(&body)
6561            );
6562        }
6563    }
6564    result
6565}
6566
6567async fn http_line(
6568    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
6569    remaining: &mut usize,
6570) -> Result<String, HealthProbeError> {
6571    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
6572    let mut line = Vec::new();
6573    (&mut *reader)
6574        .take(*remaining as u64)
6575        .read_until(b'\n', &mut line)
6576        .await
6577        .map_err(|error| {
6578            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
6579        })?;
6580    *remaining -= line.len();
6581    if !line.ends_with(b"\r\n") {
6582        return Err(HealthProbeError::bad_answer(
6583            "HTTP headers are incomplete or exceed 16 KiB",
6584        ));
6585    }
6586    line.truncate(line.len() - 2);
6587    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
6588}
6589
6590async fn probe_module_health(
6591    module_id: &str,
6592    runtime: &SupervisorRuntimeConfig,
6593    drain_deadline: Option<Instant>,
6594) -> Result<HealthReport, HealthProbeError> {
6595    probe_module_health_from(module_id, runtime, drain_deadline)
6596        .await
6597        .map(|(report, _)| report)
6598}
6599
6600/// [`probe_module_health`], also returning the connection whose registration
6601/// answered, so its answer is credited to that registration and no other.
6602async fn probe_module_health_from(
6603    module_id: &str,
6604    runtime: &SupervisorRuntimeConfig,
6605    drain_deadline: Option<Instant>,
6606) -> Result<(HealthReport, ConnectionId), HealthProbeError> {
6607    let Some(forwarding) = runtime.forwarding.as_ref() else {
6608        return Err(HealthProbeError::misconfigured(
6609            "supervisor was not configured with a forwarding table",
6610        ));
6611    };
6612    let probe_started_at = Instant::now();
6613    let mut deadline = probe_started_at + runtime.health.deadline;
6614    if let Some(drain_deadline) = drain_deadline {
6615        deadline = deadline.min(drain_deadline);
6616    }
6617    let pending = if drain_deadline.is_some() {
6618        forwarding.begin_drain_health_probe_rpc_for(
6619            module_id,
6620            MODULE_CONTROL_OP_HEALTH_CHECK,
6621            probe_started_at,
6622            deadline,
6623        )
6624    } else {
6625        forwarding.begin_health_probe_rpc_for(
6626            module_id,
6627            MODULE_CONTROL_OP_HEALTH_CHECK,
6628            probe_started_at,
6629            deadline,
6630        )
6631    }
6632    .map_err(|err| {
6633        // The endpoint is not registered, so there is no live control lane to
6634        // ask. That is the module being absent, not slow.
6635        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6636    })?;
6637    let probed = pending.endpoint.connection_id;
6638    await_health_probe(forwarding, pending, deadline, runtime.health.deadline)
6639        .await
6640        .map(|report| (report, probed))
6641}
6642
6643/// [`probe_module_health`] for one endpoint rather than the id's active one.
6644///
6645/// A swap probes two processes that no by-id lookup reaches: its candidate
6646/// before cutover, and its superseded incumbent (for busy gauges) while the
6647/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
6648/// bounds the by-id drain probe.
6649async fn probe_endpoint_health(
6650    endpoint: crate::ModuleEndpointId,
6651    runtime: &SupervisorRuntimeConfig,
6652    deadline_cap: Option<Instant>,
6653) -> Result<HealthReport, HealthProbeError> {
6654    let Some(forwarding) = runtime.forwarding.as_ref() else {
6655        return Err(HealthProbeError::misconfigured(
6656            "supervisor was not configured with a forwarding table",
6657        ));
6658    };
6659    let probe_started_at = Instant::now();
6660    let mut deadline = probe_started_at + runtime.health.deadline;
6661    if let Some(cap) = deadline_cap {
6662        deadline = deadline.min(cap);
6663    }
6664    let pending = forwarding
6665        .begin_endpoint_health_probe_rpc_for(
6666            endpoint,
6667            MODULE_CONTROL_OP_HEALTH_CHECK,
6668            probe_started_at,
6669            deadline,
6670        )
6671        .map_err(|err| {
6672            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6673        })?;
6674    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
6675}
6676
6677/// Send a begun health probe and classify its answer.
6678async fn await_health_probe(
6679    forwarding: &ForwardingTable,
6680    pending: PendingModuleControlRpc,
6681    deadline: Instant,
6682    probe_budget: Duration,
6683) -> Result<HealthReport, HealthProbeError> {
6684    let PendingModuleControlRpc {
6685        endpoint,
6686        module_sink,
6687        negotiated_ver,
6688        corr,
6689        receiver,
6690    } = pending;
6691    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6692        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
6693    })?;
6694    let frame = Frame::build_with_version(
6695        negotiated_ver,
6696        FrameType::Request,
6697        control_flags(),
6698        0,
6699        0,
6700        corr,
6701        body,
6702    )
6703    .map_err(|err| {
6704        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
6705    })?;
6706
6707    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
6708    // blocks waiting for capacity when the module's egress queue is full, and an
6709    // unbounded await here freezes the whole supervision actor (it stops polling
6710    // Child::wait and supervisor commands), making the module unrecoverable
6711    // in-band. On timeout the probe fails like any transport failure.
6712    match timeout_at(deadline, module_sink.send(frame)).await {
6713        Ok(Ok(())) => {}
6714        Ok(Err(err)) => {
6715            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6716            // A closed sink means the module's egress channel is gone -- the
6717            // receiving half is dropped when its connection tears down. Proof.
6718            return Err(HealthProbeError::lane_dead(format!(
6719                "failed to send health.check: {err}"
6720            )));
6721        }
6722        Err(_elapsed) => {
6723            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6724            // A full egress queue means the module is not draining its socket, which
6725            // is consistent with a wedged module AND with one whose reader is merely
6726            // starved. Silence, not proof.
6727            return Err(HealthProbeError::no_answer(
6728                "health.check send timed out before enqueue (module egress full)",
6729            ));
6730        }
6731    }
6732
6733    match timeout_at(deadline, receiver).await {
6734        Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => Err(HealthProbeError::bad_answer(
6735            "health.check returned an unexpected config response",
6736        )),
6737        // Each arm records WHAT WAS OBSERVED. Four of them are the module
6738        // demonstrably answering -- rejected, non-health, malformed, wrong op --
6739        // and those prove it is alive even though the probe failed.
6740        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6741            response.health_report().ok_or_else(|| {
6742                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
6743            })
6744        }
6745        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
6746            format!("health.check rejected: {}", body.message),
6747        )),
6748        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6749            Err(HealthProbeError::lane_dead(message))
6750        }
6751        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6752            Err(HealthProbeError::bad_answer(message))
6753        }
6754        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6755            Err(HealthProbeError::bad_answer(format!(
6756                "expected module-control op '{expected}', got '{actual}'"
6757            )))
6758        }
6759        // A reply that crosses the deadline before this waiter observes it is
6760        // still proof of life. The forwarding path records its end-to-end latency
6761        // before delivering this classification.
6762        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
6763            "module answered health.check after its daemon deadline",
6764        )),
6765        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
6766            "health.check waiter was canceled before the module responded",
6767        )),
6768        Err(_) => {
6769            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
6770            Err(HealthProbeError::no_answer(format!(
6771                "module did not answer health.check within {probe_budget:?}"
6772            )))
6773        }
6774    }
6775}
6776
6777/// Act on a health probe's answer. `probed` is the connection whose
6778/// registration answered (`None` for an HTTP probe, which reaches no
6779/// registration); only that registration's spawn is credited with readiness.
6780#[allow(clippy::too_many_arguments)]
6781async fn handle_health_report(
6782    spec: &ModuleSpec,
6783    runtime: &SupervisorRuntimeConfig,
6784    registry: &Registry,
6785    process_liveness: &SupervisorProcessLiveness,
6786    snapshot: &SharedSnapshot,
6787    child: &mut Option<SupervisedChild>,
6788    report: HealthReport,
6789    now_ms: u64,
6790    delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6791    probed: Option<ConnectionId>,
6792) {
6793    let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6794        (Some(delivery), Some(probe)) => {
6795            delivery
6796                .health_result(
6797                    &spec.module_id,
6798                    runtime.forwarding.as_deref(),
6799                    registry,
6800                    probe,
6801                    report.status == HealthStatus::Ok,
6802                )
6803                .await
6804        }
6805        _ => false,
6806    };
6807    if let (Some(delivery), Some(spawn)) = (
6808        &runtime.config_delivery,
6809        child
6810            .as_ref()
6811            .and_then(|child| child.config_delivery.as_ref()),
6812    ) {
6813        delivery
6814            .mark_readiness(
6815                &spec.module_id,
6816                runtime.forwarding.as_deref(),
6817                probed,
6818                spawn,
6819                report.status == HealthStatus::Ok,
6820            )
6821            .await;
6822    }
6823    let status = supervisor_health_status(report.status);
6824    let detail = report.detail.clone();
6825    let metrics = truncate_health_metrics(report.metrics);
6826    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6827        state.health.status = status;
6828        state.last_probe_evidence = None;
6829        state.health.last_probe_ms = Some(now_ms);
6830        state.health.detail = detail.clone();
6831        state.health.metrics = metrics.clone();
6832        state.health.consecutive_failures = 0;
6833        state.failed_retry.observe_health(
6834            report.status == HealthStatus::Ok,
6835            runtime.restart_policy.window,
6836        );
6837    });
6838
6839    if delivery_failed {
6840        return;
6841    }
6842    let action = match report.status {
6843        HealthStatus::Ok => return,
6844        HealthStatus::Degraded => runtime.health.on_degraded,
6845        HealthStatus::Failing => runtime.health.on_failing,
6846    };
6847    apply_l3_health_action(
6848        spec,
6849        runtime,
6850        registry,
6851        process_liveness,
6852        snapshot,
6853        child,
6854        status,
6855        detail.as_deref(),
6856        action,
6857        now_ms,
6858    )
6859    .await;
6860}
6861
6862#[allow(clippy::too_many_arguments)]
6863async fn handle_health_probe_failure(
6864    spec: &ModuleSpec,
6865    runtime: &SupervisorRuntimeConfig,
6866    registry: &Registry,
6867    process_liveness: &SupervisorProcessLiveness,
6868    snapshot: &SharedSnapshot,
6869    child: &mut Option<SupervisedChild>,
6870    err: HealthProbeError,
6871    now_ms: u64,
6872    delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6873) {
6874    let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6875        (Some(delivery), Some(probe)) => {
6876            delivery
6877                .health_result(
6878                    &spec.module_id,
6879                    runtime.forwarding.as_deref(),
6880                    registry,
6881                    probe,
6882                    false,
6883                )
6884                .await
6885        }
6886        _ => false,
6887    };
6888    let threshold = runtime.health.failure_threshold.max(1);
6889    let held = if err.evidence == HealthProbeEvidence::NoAnswer {
6890        runtime.host_health.record_late_module(&spec.module_id)
6891    } else {
6892        false
6893    };
6894    let mut failures = 0;
6895    // Carry the evidence class into the operator-visible detail. Without it,
6896    // "module did not answer within 5s" and "the control lane is gone" are two
6897    // prose strings in the same field, and the reader has to know the codebase to
6898    // tell which one is proof of anything.
6899    let detail = if held {
6900        format!("[{}] {err}; host overloaded", err.label())
6901    } else {
6902        format!("[{}] {err}", err.label())
6903    };
6904    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6905        state.last_probe_evidence = Some(err.evidence);
6906        state
6907            .failed_retry
6908            .observe_health(false, runtime.restart_policy.window);
6909        // A failed wire probe invalidates the last report, even before the
6910        // restart threshold. HTTP probes already mark failures as Failing.
6911        if held {
6912            state.health.status = SupervisorHealthStatus::Degraded;
6913        } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
6914            state.health.status = SupervisorHealthStatus::Unknown;
6915        }
6916        state.health.last_probe_ms = Some(now_ms);
6917        // Held silence breaks the streak: after the host recovers, only new
6918        // failures count. Replaying pre-overload misses would restart immediately.
6919        state.health.consecutive_failures = if held {
6920            0
6921        } else {
6922            state.health.consecutive_failures.saturating_add(1)
6923        };
6924        state.health.detail = Some(detail.clone());
6925        state.health.metrics = None;
6926        failures = state.health.consecutive_failures;
6927    });
6928
6929    if delivery_failed {
6930        return;
6931    }
6932    if held || failures < threshold {
6933        warn!(
6934            module_id = %spec.module_id,
6935            consecutive_failures = failures,
6936            threshold,
6937            evidence = err.label(),
6938            detail = %detail,
6939            "health.check probe failed"
6940        );
6941        return;
6942    }
6943
6944    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6945        state.state = ModuleState::Unresponsive;
6946        state.health.status = SupervisorHealthStatus::Unresponsive;
6947    });
6948    // The evidence class is logged at the kill site because this is the line an
6949    // operator reads after an unexplained restart. A streak of `no-answer` under
6950    // machine load is the known false-positive shape; a `lane-dead` is not.
6951    if runtime.health.critical {
6952        error!(
6953            module_id = %spec.module_id,
6954            status = "unresponsive",
6955            evidence = err.label(),
6956            detail = %detail,
6957            "critical module health alert"
6958        );
6959    } else {
6960        warn!(
6961            module_id = %spec.module_id,
6962            status = "unresponsive",
6963            evidence = err.label(),
6964            detail = %detail,
6965            "module health threshold breached"
6966        );
6967    }
6968    if let Err(err) = health_restart_child(
6969        spec,
6970        runtime,
6971        registry,
6972        process_liveness,
6973        snapshot,
6974        child,
6975        SupervisorHealthStatus::Unresponsive,
6976        Some(&detail),
6977        now_ms,
6978    )
6979    .await
6980    {
6981        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6982    }
6983}
6984
6985#[allow(clippy::too_many_arguments)]
6986async fn apply_l3_health_action(
6987    spec: &ModuleSpec,
6988    runtime: &SupervisorRuntimeConfig,
6989    registry: &Registry,
6990    process_liveness: &SupervisorProcessLiveness,
6991    snapshot: &SharedSnapshot,
6992    child: &mut Option<SupervisedChild>,
6993    status: SupervisorHealthStatus,
6994    detail: Option<&str>,
6995    action: HealthAction,
6996    now_ms: u64,
6997) {
6998    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
6999    match action {
7000        HealthAction::Report => {
7001            info!(
7002                module_id = %spec.module_id,
7003                status = ?status,
7004                detail,
7005                "module reported non-ok health"
7006            );
7007        }
7008        HealthAction::Alert => {
7009            error!(
7010                module_id = %spec.module_id,
7011                status = ?status,
7012                detail,
7013                "module health alert"
7014            );
7015        }
7016        HealthAction::Restart => {
7017            if let Err(err) = health_restart_child(
7018                spec,
7019                runtime,
7020                registry,
7021                process_liveness,
7022                snapshot,
7023                child,
7024                status,
7025                detail,
7026                now_ms,
7027            )
7028            .await
7029            {
7030                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
7031            }
7032        }
7033    }
7034}
7035
7036#[allow(clippy::too_many_arguments)]
7037async fn health_restart_child(
7038    spec: &ModuleSpec,
7039    runtime: &SupervisorRuntimeConfig,
7040    registry: &Registry,
7041    process_liveness: &SupervisorProcessLiveness,
7042    snapshot: &SharedSnapshot,
7043    child: &mut Option<SupervisedChild>,
7044    status: SupervisorHealthStatus,
7045    detail: Option<&str>,
7046    now_ms: u64,
7047) -> Result<(), SuperviseError> {
7048    budgeted_restart_child(
7049        spec,
7050        runtime,
7051        registry,
7052        process_liveness,
7053        snapshot,
7054        child,
7055        status,
7056        detail,
7057        now_ms,
7058        true,
7059    )
7060    .await
7061}
7062
7063#[allow(clippy::too_many_arguments)]
7064async fn budgeted_restart_child(
7065    spec: &ModuleSpec,
7066    runtime: &SupervisorRuntimeConfig,
7067    registry: &Registry,
7068    process_liveness: &SupervisorProcessLiveness,
7069    snapshot: &SharedSnapshot,
7070    child: &mut Option<SupervisedChild>,
7071    status: SupervisorHealthStatus,
7072    detail: Option<&str>,
7073    now_ms: u64,
7074    health_failure: bool,
7075) -> Result<(), SuperviseError> {
7076    if let Some(error) = runtime
7077        .config_delivery
7078        .as_ref()
7079        .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
7080        .filter(|error| error.code != "config_delivery_deferred")
7081    {
7082        return Err(config_delivery::gate_hold_error(
7083            spec,
7084            error,
7085            runtime.config_delivery.as_ref(),
7086        ));
7087    }
7088    let recovery_reason = if health_failure {
7089        "health_check"
7090    } else {
7091        "configuration_delivery"
7092    };
7093    let (enabled, schedule) = {
7094        let mut state = lock_snapshot(snapshot)?;
7095        let enabled = state.enabled;
7096        let schedule = if enabled {
7097            state.next_crash_restart(&runtime.restart_policy, Instant::now())
7098        } else {
7099            None
7100        };
7101        (enabled, schedule)
7102    };
7103
7104    if !enabled {
7105        return Err(SuperviseError::Disabled {
7106            module_id: spec.module_id.clone(),
7107        });
7108    }
7109
7110    if schedule.is_none() {
7111        if health_failure {
7112            record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
7113        }
7114        error!(
7115            module_id = %spec.module_id,
7116            status = ?status,
7117            detail,
7118            max_restarts = runtime.restart_policy.max_restarts,
7119            window_secs = runtime.restart_policy.window.as_secs(),
7120            reason = %runtime.restart_policy.budget_exhausted_detail(),
7121            recovery_reason,
7122            "automatic restart budget exhausted; marking module failed"
7123        );
7124        let stop_notice = begin_forwarding_drain_if_configured(
7125            spec,
7126            runtime,
7127            registry,
7128            snapshot,
7129            Some(true),
7130            RouteCloseReason::Disable,
7131        )
7132        .await?;
7133        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7134            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
7135        })?;
7136        let result = drain_optional_child(
7137            &spec.module_id,
7138            spec.protocol,
7139            stop_notice,
7140            registry,
7141            runtime.forwarding.as_deref(),
7142            snapshot,
7143            &runtime.terminal_ring,
7144            &runtime.spawn_events,
7145            child,
7146            runtime.drain_timeout,
7147            ModuleState::Failed,
7148            Some(true),
7149            &runtime.failure_context(true),
7150        )
7151        .await;
7152        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7153            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
7154                state.failed_retry.schedule(&spec.module_id);
7155            }
7156        })?;
7157        result?;
7158        process_liveness.untrack_if_current(&spec.module_id, snapshot);
7159        return Ok(());
7160    }
7161
7162    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
7163    let mut restart_count = 0;
7164    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7165        restart_count = state.crash_restarts.len();
7166        state.state = ModuleState::Unresponsive;
7167        if health_failure {
7168            state.health.status = status;
7169            state.health.last_action = Some(HealthAction::Restart.to_string());
7170            state.health.last_action_ms = Some(now_ms);
7171        }
7172    })?;
7173    warn!(
7174        module_id = %spec.module_id,
7175        status = ?status,
7176        detail,
7177        restart_count,
7178        restart_in_window = schedule.restart_in_window,
7179        delay_ms = schedule.delay.as_millis() as u64,
7180        recovery_reason,
7181        "automatic module restart"
7182    );
7183
7184    let stop_notice = begin_forwarding_drain_if_configured(
7185        spec,
7186        runtime,
7187        registry,
7188        snapshot,
7189        Some(true),
7190        RouteCloseReason::Restart,
7191    )
7192    .await?;
7193    drain_optional_child(
7194        &spec.module_id,
7195        spec.protocol,
7196        stop_notice,
7197        registry,
7198        runtime.forwarding.as_deref(),
7199        snapshot,
7200        &runtime.terminal_ring,
7201        &runtime.spawn_events,
7202        child,
7203        runtime.drain_timeout,
7204        ModuleState::Restarting,
7205        Some(true),
7206        &runtime.failure_context(true),
7207    )
7208    .await?;
7209    schedule_respawn(
7210        runtime,
7211        snapshot,
7212        &spec.module_id,
7213        schedule.delay,
7214        RespawnKind::Spawn,
7215        true,
7216    )
7217}
7218
7219fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
7220    if let Some(reply) = runtime
7221        .deferred_reload_reply
7222        .lock()
7223        .unwrap_or_else(|p| p.into_inner())
7224        .take()
7225    {
7226        let _ = reply.send(Err(SuperviseError::ReloadFailed {
7227            module_id: module_id.to_string(),
7228            reason: reason.to_string(),
7229        }));
7230    }
7231}
7232
7233fn park_config_spawn(
7234    spec: &ModuleSpec,
7235    runtime: &SupervisorRuntimeConfig,
7236    snapshot: &SharedSnapshot,
7237    failure: &FailureContext,
7238    admission_refused: Option<String>,
7239) -> Result<(), SuperviseError> {
7240    *runtime
7241        .scheduled_respawn
7242        .lock()
7243        .unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
7244    if let Some(reply) = runtime
7245        .deferred_reload_reply
7246        .lock()
7247        .unwrap_or_else(|poisoned| poisoned.into_inner())
7248        .take()
7249    {
7250        let _ = reply.send(Err(SuperviseError::ConfigHeld {
7251            module_id: spec.module_id.clone(),
7252            code: "config_gated".into(),
7253            detail: "module is held by configuration admission".into(),
7254        }));
7255    }
7256    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7257        state.admission_refused = admission_refused;
7258        state.config_hold_daemon_initiated = failure.daemon_initiated;
7259        state.config_hold_approval = failure.approval.clone();
7260        state.config_hold_parked = true;
7261        state.state = ModuleState::Stopped;
7262        state.respawn_pending = false;
7263        state.failed_retry.pending = None;
7264        // Parking ends this attempt and discards its failure timestamp. If a
7265        // later release fails, its retry delay starts at that new failure.
7266        state.failed_retry.failed_at = None;
7267        clear_current_process_facts(state);
7268    })
7269}
7270
7271fn try_park_config_attempt(
7272    spec: &ModuleSpec,
7273    runtime: &SupervisorRuntimeConfig,
7274    snapshot: &SharedSnapshot,
7275    failure: &FailureContext,
7276    program_base: Option<&subc_config::Digest>,
7277) -> Result<bool, SuperviseError> {
7278    let result = (|| {
7279        if !lock_snapshot(snapshot)?.enabled {
7280            return Ok(false);
7281        }
7282        #[cfg(test)]
7283        if let Some(delivery) = &runtime.config_delivery {
7284            delivery.0.pause_before_config_park_for_test();
7285        }
7286        let Some(delivery) = &runtime.config_delivery else {
7287            return Ok(false);
7288        };
7289        let parked = delivery
7290            .0
7291            .with_recorded_admission_hold(&spec.module_id, program_base, |refusal| {
7292                park_config_spawn(spec, runtime, snapshot, failure, refusal)
7293            })
7294            .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(delivery)))?;
7295        if let Some(result) = parked {
7296            result?;
7297            Ok(true)
7298        } else {
7299            Ok(false)
7300        }
7301    })();
7302    if let Err(error) = &result {
7303        // A failed park still belongs to the attempt that tried to park. Mark it
7304        // Failed here, before the monitor can use a daemon/operator origin saved
7305        // for a different attempt to decide whether to schedule a slow retry.
7306        fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7307        record_approved_attempt_failure(spec, snapshot, failure, error);
7308    }
7309    result
7310}
7311
7312fn record_approved_attempt_failure(
7313    spec: &ModuleSpec,
7314    snapshot: &SharedSnapshot,
7315    failure: &FailureContext,
7316    error: &SuperviseError,
7317) {
7318    let Some(approval) = &failure.approval else {
7319        return;
7320    };
7321    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7322        if state.enabled
7323            && state.state == ModuleState::Failed
7324            && state
7325                .approved_start_failure
7326                .as_ref()
7327                .is_none_or(|(identity, _)| identity != approval)
7328        {
7329            state.approved_start_failure = Some((approval.clone(), error.to_string()));
7330        }
7331    });
7332}
7333
7334fn hold_config_spawn(
7335    spec: &ModuleSpec,
7336    runtime: &SupervisorRuntimeConfig,
7337    snapshot: &SharedSnapshot,
7338    failure: &FailureContext,
7339) -> Result<(), SuperviseError> {
7340    if !lock_snapshot(snapshot)?.enabled {
7341        return Ok(());
7342    }
7343    if !try_park_config_attempt(spec, runtime, snapshot, failure, None)? {
7344        // This caller consumed an actual start attempt. If its hold ended before
7345        // parking, retain that attempt instead of waiting for a lost release.
7346        defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
7347    }
7348    Ok(())
7349}
7350
7351fn schedule_respawn(
7352    runtime: &SupervisorRuntimeConfig,
7353    snapshot: &SharedSnapshot,
7354    module_id: &str,
7355    delay: Duration,
7356    kind: RespawnKind,
7357    daemon_initiated: bool,
7358) -> Result<(), SuperviseError> {
7359    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
7360    update_snapshot(snapshot, Some(module_id), |state| {
7361        state.respawn_pending = true
7362    })?;
7363    *runtime
7364        .scheduled_respawn
7365        .lock()
7366        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7367        deadline: Instant::now() + delay,
7368        kind,
7369        daemon_initiated,
7370        approval: None,
7371    });
7372    Ok(())
7373}
7374
7375const CONFIG_ADMISSION_RETRY_DELAY: Duration = Duration::from_millis(100);
7376
7377fn defer_config_spawn(
7378    spec: &ModuleSpec,
7379    runtime: &SupervisorRuntimeConfig,
7380    snapshot: &SharedSnapshot,
7381    kind: RespawnKind,
7382    failure: &FailureContext,
7383) -> Result<(), SuperviseError> {
7384    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7385        state.state = ModuleState::Restarting;
7386        state.respawn_pending = true;
7387        clear_current_process_facts(state);
7388    })?;
7389    *runtime
7390        .scheduled_respawn
7391        .lock()
7392        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7393        deadline: Instant::now() + CONFIG_ADMISSION_RETRY_DELAY,
7394        kind,
7395        daemon_initiated: failure.daemon_initiated,
7396        approval: failure.approval.clone(),
7397    });
7398    Ok(())
7399}
7400
7401fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
7402    let _ = update_snapshot(snapshot, Some(module_id), |state| {
7403        state.health.last_action = Some(action);
7404        state.health.last_action_ms = Some(now_ms);
7405    });
7406}
7407
7408fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
7409    match status {
7410        HealthStatus::Ok => SupervisorHealthStatus::Ok,
7411        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
7412        HealthStatus::Failing => SupervisorHealthStatus::Failing,
7413    }
7414}
7415
7416/// Caps the metrics blob stored in the cached supervisor snapshot, which is
7417/// returned to every `supervisor.list` and `supervisor.health` caller.
7418///
7419/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
7420/// path: that request exists to return a module's complete metrics object, and
7421/// `ck health <module-id>` documents it as the way to see what the cached view
7422/// truncates. The asymmetry is the feature.
7423///
7424/// So a new caller must decide which side it is on rather than assume the cap is
7425/// universal. Reaching for it on a fresh-probe path would silently reintroduce
7426/// the truncation that path exists to avoid.
7427fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
7428    let metrics = metrics?;
7429    match serde_json::to_vec(&metrics) {
7430        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
7431            "truncated": true,
7432            "original_bytes": encoded.len(),
7433        })),
7434        Ok(_) | Err(_) => Some(metrics),
7435    }
7436}
7437
7438/// Spread health probes so a fleet-wide restart does not converge them.
7439///
7440/// The delay is derived from the module id and probe index rather than a random
7441/// source, so it is deterministic per module: a module keeps its own offset
7442/// across daemon restarts instead of re-rolling into a collision.
7443fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
7444    if cadence.is_zero() {
7445        return Duration::ZERO;
7446    }
7447    let cadence_ms = cadence.as_millis() as u64;
7448    // This early return is REDUNDANT, deliberately, and a mutation run will show
7449    // it surviving removal. Recording why here so the next person to notice does
7450    // not have to re-derive it:
7451    //
7452    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
7453    //   a zero cadence and builds the Duration from whole milliseconds, so a
7454    //   sub-millisecond cadence cannot come from config.
7455    // - Even if reached it changes no answer. The `.max(1)` below makes the span
7456    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
7457    //   -- exactly what this returns.
7458    //
7459    // Kept as a guard against a future widening of the config parser (accepting
7460    // microseconds, say), which would make the sub-millisecond case reachable.
7461    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
7462    // divides by zero. Remove this and nothing changes.
7463    if cadence_ms == 0 {
7464        return cadence;
7465    }
7466    // Note that this never returns less than one cadence, including for the FIRST
7467    // probe. So a freshly registered module reports health `unknown` for a full
7468    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
7469    // ready to answer.
7470    //
7471    // That is a property of the supervisor's schedule, not of any module: an
7472    // operator watching a restart sees `unknown` and cannot tell it from a module
7473    // that is slow to warm. Measured on two unrelated modules, both flipping to
7474    // `ok` between 22s and 32s after restart.
7475    //
7476    // Left as-is because spreading the first probe is what keeps a fleet-wide
7477    // restart from firing fourteen simultaneous probes into a cold machine. The
7478    // alternative -- probe at t+0 and jitter only from the second onward -- trades
7479    // that thundering herd for a faster first reading.
7480    let jitter_span = (cadence_ms / 10).max(1);
7481    let hash = module_id.as_bytes().iter().fold(
7482        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
7483        |acc, byte| {
7484            acc.wrapping_mul(1099511628211)
7485                .wrapping_add(u64::from(*byte))
7486        },
7487    );
7488    cadence + Duration::from_millis(hash % jitter_span)
7489}
7490
7491#[cfg(test)]
7492mod tests {
7493    use super::*;
7494
7495    #[test]
7496    fn registration_retirement_self_prefixed_owner_is_not_a_prefix_submodule() {
7497        let handle = SupervisorHandle::new();
7498        handle.set_spawn_nonce("prefix:owner", "owner-nonce".into());
7499        handle.set_reserved_prefixes("prefix:owner", &["prefix:".into()]);
7500        let owner = handle
7501            .reserved_hello_verification("prefix:owner", Some("owner-nonce"))
7502            .unwrap();
7503        assert!(owner.nonce_verified);
7504        assert!(
7505            owner.prefix_owner.is_none(),
7506            "an unreserved programmatic owner is not its own prefix child"
7507        );
7508        let child = handle
7509            .reserved_hello_verification("prefix:child", Some("owner-nonce"))
7510            .unwrap();
7511        assert!(child.nonce_verified);
7512        assert_eq!(child.prefix_owner.as_deref(), Some("prefix:owner"));
7513    }
7514
7515    #[test]
7516    fn readding_a_module_clears_its_rescan_removal_tombstone() {
7517        let handle = SupervisorHandle::new();
7518        let module_id = "readded-tombstone";
7519        handle.record_rescan_removal(module_id);
7520        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
7521
7522        handle.apply_identity_configuration(&ModuleSpec {
7523            script: None,
7524            module_id: module_id.to_string(),
7525            program: PathBuf::from("/test/module"),
7526            args: Vec::new(),
7527            env: Vec::new(),
7528            reserved: false,
7529            reserved_prefixes: Vec::new(),
7530            protocol: ModuleProtocol::Subc,
7531            overlap: Default::default(),
7532            carry_containment: false,
7533        });
7534
7535        assert!(
7536            handle.removal_tombstone_age_ms(module_id).is_none(),
7537            "a re-added module must not retain a stale removal tombstone"
7538        );
7539    }
7540
7541    /// What one module's owner looked like from the control plane at the
7542    /// instant after its first process was spawned.
7543    #[derive(Debug, PartialEq, Eq)]
7544    struct OwnerInSpawnWindow {
7545        module_id: String,
7546        configured: bool,
7547        on_roster: bool,
7548        admission_refusal: Option<&'static str>,
7549    }
7550
7551    /// A supervised module's process can connect, register, sync its scopes
7552    /// and describe them as soon as it is spawned. A plain `spawn` does not put
7553    /// the module on the roster until afterward; configured supervision does
7554    /// so before spawning. In both cases the owner must
7555    /// already read as configured, so a scoped `route.open` against it is
7556    /// refused as retryable `scope_not_synced` and not as terminal
7557    /// `scope_not_live` ("will never sync").
7558    ///
7559    /// The hook runs just after each initial spawn attempt, before returning to
7560    /// the caller, so no race with a real child's registration is needed.
7561    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7562    async fn a_new_module_is_configured_before_its_first_process_can_register() {
7563        use crate::scopes::ScopeTable;
7564        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
7565
7566        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
7567        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
7568            script: None,
7569            module_id: module_id.to_string(),
7570            program,
7571            args: Vec::new(),
7572            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7573                .into_iter()
7574                .map(|key| (key.to_string(), dir.path().display().to_string()))
7575                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7576                .collect(),
7577            reserved: false,
7578            reserved_prefixes: Vec::new(),
7579            protocol: ModuleProtocol::Subc,
7580            overlap: Default::default(),
7581            carry_containment: false,
7582        };
7583        let live = super::terminal_history_tests::fake_aft_stub_path();
7584        let missing = dir.path().join("definitely-missing-module");
7585
7586        let handle = SupervisorHandle::new();
7587        let mut supervisor =
7588            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7589                .with_handle(handle.clone());
7590        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
7591        let hook_handle = handle.clone();
7592        let hook_observed = Arc::clone(&observed);
7593        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7594            // Exactly what the control plane computes for a scoped route.open
7595            // naming this module as the owner of a scope it has not synced.
7596            let configured = hook_handle.is_configured(module_id);
7597            let selector = ScopeSelector::new(
7598                Principal::Reserved {
7599                    module_id: module_id.to_string(),
7600                },
7601                "s".to_string(),
7602                Some(1),
7603            );
7604            let carrier = Principal::Reserved {
7605                module_id: "carrier".to_string(),
7606            };
7607            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
7608                .admit(&carrier, module_id, &selector, configured)
7609            {
7610                Ok(_) => None,
7611                Err(refusal) => Some(refusal.code),
7612            };
7613            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
7614                module_id: module_id.to_string(),
7615                configured,
7616                on_roster: hook_handle.get(module_id).is_some(),
7617                admission_refusal,
7618            });
7619        })));
7620
7621        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
7622        let configured = supervisor
7623            .supervise_configured(stub("configured", live.clone()), true)
7624            .unwrap();
7625        let with_health = supervisor
7626            .supervise_configured_with_health(
7627                stub("with-health", live.clone()),
7628                true,
7629                HealthConfig::default(),
7630                None,
7631                RestartPolicy::default(),
7632            )
7633            .unwrap();
7634        // The failed-spawn path still puts the module on the roster (as
7635        // failed), so it is configured throughout.
7636        let failed = supervisor
7637            .supervise_configured_with_health(
7638                stub("failed-spawn", missing.clone()),
7639                true,
7640                HealthConfig::default(),
7641                None,
7642                RestartPolicy::default(),
7643            )
7644            .unwrap();
7645        // A failed plain `spawn` puts nothing on the roster, so its mark is
7646        // taken back once the spawn has failed.
7647        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
7648
7649        let expected = [
7650            "plain",
7651            "configured",
7652            "with-health",
7653            "failed-spawn",
7654            "spawn-error",
7655        ]
7656        .into_iter()
7657        .map(|module_id| OwnerInSpawnWindow {
7658            module_id: module_id.to_string(),
7659            configured: true,
7660            on_roster: !matches!(module_id, "plain" | "spawn-error"),
7661            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
7662        })
7663        .collect::<Vec<_>>();
7664        assert_eq!(*observed.lock().unwrap(), expected);
7665
7666        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
7667            assert!(
7668                handle.get(module_id).is_some(),
7669                "{module_id} is on the roster"
7670            );
7671            assert!(
7672                handle.is_configured(module_id),
7673                "{module_id} stays configured"
7674            );
7675        }
7676        assert!(handle.get("spawn-error").is_none());
7677        assert!(
7678            !handle.is_configured("spawn-error"),
7679            "a plain spawn that failed must not leave its module marked configured"
7680        );
7681
7682        // Leaving the roster clears the mark with it.
7683        handle.retire("failed-spawn");
7684        assert!(!handle.is_configured("failed-spawn"));
7685
7686        for module in [plain, configured, with_health] {
7687            module.stop().await.unwrap();
7688        }
7689        drop(failed);
7690    }
7691
7692    /// A configured module reads as starting until the supervisor has admitted
7693    /// its first process (exec confirmed, identity durably recorded), even
7694    /// though that process is already running by then and can register over
7695    /// HELLO, which is what lists it in catalog.list. So a client that needs
7696    /// the supervisor state must wait for that state; seeing the module in
7697    /// the catalog does not mean supervisor.list already says running.
7698    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7699    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
7700        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
7701        let spec = ModuleSpec {
7702            script: None,
7703            module_id: "spawned-not-admitted".to_string(),
7704            program: super::terminal_history_tests::fake_aft_stub_path(),
7705            args: Vec::new(),
7706            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7707                .into_iter()
7708                .map(|key| (key.to_string(), dir.path().display().to_string()))
7709                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7710                .collect(),
7711            reserved: false,
7712            reserved_prefixes: Vec::new(),
7713            protocol: ModuleProtocol::Subc,
7714            overlap: Default::default(),
7715            carry_containment: false,
7716        };
7717        let handle = SupervisorHandle::new();
7718        let mut supervisor =
7719            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7720                .with_handle(handle.clone());
7721        let observed = Arc::new(Mutex::new(None));
7722        let hook_handle = handle.clone();
7723        let hook_observed = Arc::clone(&observed);
7724        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7725            let state = hook_handle
7726                .get(module_id)
7727                .expect("a configured module is on the roster before it is spawned")
7728                .state()
7729                .unwrap();
7730            *hook_observed.lock().unwrap() = Some(state);
7731        })));
7732
7733        let module = supervisor.supervise_configured(spec, true).unwrap();
7734        assert_eq!(
7735            *observed.lock().unwrap(),
7736            Some(ModuleState::Starting),
7737            "between the spawn and its admission the supervisor still reports starting"
7738        );
7739        assert_eq!(module.state().unwrap(), ModuleState::Running);
7740        module.stop().await.unwrap();
7741    }
7742
7743    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
7744        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
7745        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
7746            snapshot.process_alive = true;
7747            snapshot.pid = Some(41);
7748            snapshot.spawned_at_ms = Some(42);
7749            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
7750            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
7751            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
7752                device: 43,
7753                inode: 44,
7754                #[cfg(windows)]
7755                windows_image: None,
7756            });
7757        })
7758        .unwrap();
7759        snapshot
7760    }
7761
7762    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
7763        let snapshot = lock_snapshot(snapshot).unwrap();
7764        assert!(!snapshot.process_alive);
7765        assert_eq!(snapshot.pid, None);
7766        assert_eq!(snapshot.spawned_at_ms, None);
7767        assert_eq!(snapshot.spawned_from, None);
7768        assert_eq!(snapshot.launch_spec_digest, None);
7769        assert_eq!(snapshot.spawned_file_identity, None);
7770    }
7771
7772    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7773    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
7774        let supervisor =
7775            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7776        let mut runtime = supervisor.runtime_config();
7777        runtime.test_seed_stale_facts_before_enable_spawn = true;
7778        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
7779        let mut child = None;
7780        let spec = ModuleSpec {
7781            script: None,
7782            module_id: "failed-enable-clears-facts".to_string(),
7783            program: PathBuf::from("/definitely/missing/failed-enable-module"),
7784            args: Vec::new(),
7785            env: Vec::new(),
7786            reserved: false,
7787            reserved_prefixes: Vec::new(),
7788            protocol: ModuleProtocol::Subc,
7789            overlap: Default::default(),
7790            carry_containment: false,
7791        };
7792
7793        let result = set_child_enabled(
7794            &spec,
7795            &runtime,
7796            &supervisor.registry,
7797            &supervisor.process_liveness,
7798            &snapshot,
7799            &mut child,
7800            true,
7801            &runtime.failure_context(false),
7802        )
7803        .await;
7804
7805        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
7806        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7807        assert_snapshot_process_facts_cleared(&snapshot);
7808    }
7809
7810    #[tokio::test]
7811    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
7812        let supervisor =
7813            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7814        let runtime = supervisor.runtime_config();
7815        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
7816            ModuleState::Restarting,
7817            true,
7818        )));
7819        let spec = ModuleSpec {
7820            script: None,
7821            module_id: "start-stranded-restarting".to_string(),
7822            program: super::terminal_history_tests::fake_aft_stub_path(),
7823            args: Vec::new(),
7824            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
7825            reserved: false,
7826            reserved_prefixes: Vec::new(),
7827            protocol: ModuleProtocol::None,
7828            overlap: Default::default(),
7829            carry_containment: false,
7830        };
7831        let mut child = None;
7832        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
7833        assert!(!super::set_child_enabled(
7834            &spec,
7835            &runtime,
7836            &Registry::default(),
7837            &supervisor.process_liveness,
7838            &snapshot,
7839            &mut child,
7840            true,
7841            &runtime.failure_context(false),
7842        )
7843        .await
7844        .unwrap());
7845        assert!(child.is_none());
7846        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
7847        assert!(super::set_child_enabled(
7848            &spec,
7849            &runtime,
7850            &Registry::default(),
7851            &supervisor.process_liveness,
7852            &snapshot,
7853            &mut child,
7854            true,
7855            &runtime.failure_context(false),
7856        )
7857        .await
7858        .unwrap());
7859        assert_eq!(
7860            lock_snapshot(&snapshot).unwrap().state,
7861            ModuleState::Running
7862        );
7863        let mut child = child.unwrap();
7864        child.start_kill().unwrap();
7865        child.wait().await.unwrap();
7866    }
7867
7868    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7869    async fn failed_reload_spawn_clears_current_process_facts() {
7870        let supervisor =
7871            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7872        let mut runtime = supervisor.runtime_config();
7873        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
7874        let snapshot = stale_process_snapshot(ModuleState::Running, true);
7875        let mut child = None;
7876        let spec = ModuleSpec {
7877            script: None,
7878            module_id: "failed-reload-clears-facts".to_string(),
7879            program: PathBuf::from("/unused/failed-reload-module"),
7880            args: Vec::new(),
7881            env: Vec::new(),
7882            reserved: false,
7883            reserved_prefixes: Vec::new(),
7884            protocol: ModuleProtocol::Subc,
7885            overlap: Default::default(),
7886            carry_containment: false,
7887        };
7888
7889        let result = handle_reload_spawn_failure(
7890            &spec,
7891            &runtime,
7892            &supervisor.process_liveness,
7893            &snapshot,
7894            &mut child,
7895            "forced reload spawn failure".to_string(),
7896        )
7897        .await;
7898
7899        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
7900        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7901        assert_snapshot_process_facts_cleared(&snapshot);
7902    }
7903
7904    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7905    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
7906        let supervisor =
7907            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7908        let snapshot = stale_process_snapshot(ModuleState::Running, true);
7909        let module = supervisor.supervised_module(
7910            ModuleSpec {
7911                script: None,
7912                module_id: "drop-clears-facts".to_string(),
7913                program: PathBuf::from("/unused/drop-module"),
7914                args: Vec::new(),
7915                env: Vec::new(),
7916                reserved: false,
7917                reserved_prefixes: Vec::new(),
7918                protocol: ModuleProtocol::Subc,
7919                overlap: Default::default(),
7920                carry_containment: false,
7921            },
7922            supervisor.runtime_config(),
7923            Arc::clone(&snapshot),
7924            None,
7925        );
7926        assert!(!module
7927            .inner
7928            .monitor
7929            .lock()
7930            .unwrap()
7931            .as_ref()
7932            .unwrap()
7933            .is_finished());
7934
7935        drop(module);
7936
7937        assert_eq!(
7938            lock_snapshot(&snapshot).unwrap().state,
7939            ModuleState::Stopped
7940        );
7941        assert_snapshot_process_facts_cleared(&snapshot);
7942    }
7943
7944    #[cfg(unix)]
7945    #[tokio::test]
7946    async fn rescan_preserves_running_protocol_until_respawn() {
7947        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
7948        let initial = ModuleSpec {
7949            script: None,
7950            module_id: "rescan-protocol".into(),
7951            program: PathBuf::from("/bin/sleep"),
7952            args: vec!["60".into()],
7953            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7954                .into_iter()
7955                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
7956                .collect(),
7957            reserved: false,
7958            reserved_prefixes: vec![],
7959            protocol: ModuleProtocol::None,
7960            overlap: Default::default(),
7961            carry_containment: false,
7962        };
7963        let supervisor =
7964            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7965        let module = supervisor.spawn(initial.clone()).unwrap();
7966        assert!(module.status().unwrap().live);
7967        let mut next = initial;
7968        next.protocol = ModuleProtocol::Subc;
7969        module
7970            .update_configuration(next.clone(), HealthConfig::default(), None)
7971            .await
7972            .unwrap();
7973        assert!(
7974            module.status().unwrap().live,
7975            "rescan must not require HELLO from the old non-wire process"
7976        );
7977        let runtime = supervisor.runtime_config();
7978        let action = on_child_exit(
7979            &next,
7980            RestartPolicy::default(),
7981            &supervisor.registry,
7982            &module.inner.snapshot,
7983            &runtime.terminal_ring,
7984            &runtime.spawn_events,
7985            &runtime.child_roster,
7986            ExitReport {
7987                kind: ExitKind::Clean,
7988                code: Some(0),
7989                signal: None,
7990                at_ms: unix_ms_now(),
7991            },
7992            true,
7993        )
7994        .await;
7995        assert!(
7996            matches!(action, NextAction::Restart { .. }),
7997            "the old non-wire process's clean exit must restart"
7998        );
7999        module.drain().await.unwrap();
8000    }
8001
8002    #[cfg(unix)]
8003    #[tokio::test]
8004    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
8005        use std::os::unix::fs::PermissionsExt;
8006        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
8007        let script = dir.join("module.sh");
8008        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
8009        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
8010        let record_path = dir.join("live-children.json");
8011        let supervisor =
8012            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8013                .with_live_children_record(&record_path);
8014        for (program, args) in [
8015            (PathBuf::from("sleep"), vec!["60".into()]),
8016            (script, vec![]),
8017        ] {
8018            let spec = ModuleSpec {
8019                script: None,
8020                module_id: "image-identity".into(),
8021                program: program.clone(),
8022                args,
8023                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
8024                    .into_iter()
8025                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
8026                    .collect(),
8027                reserved: false,
8028                reserved_prefixes: vec![],
8029                protocol: ModuleProtocol::None,
8030                overlap: Default::default(),
8031                carry_containment: false,
8032            };
8033            let module = supervisor.spawn(spec).unwrap();
8034            #[cfg(target_os = "macos")]
8035            {
8036                // SETEXEC confirmation is asynchronous; the orphan record must
8037                // identify the final image, never the intermediate trampoline.
8038                let deadline = Instant::now() + Duration::from_secs(5);
8039                while crate::live_children::read_record(&record_path)
8040                    .unwrap()
8041                    .iter()
8042                    .all(|entry| entry.executable.is_none())
8043                {
8044                    assert!(Instant::now() < deadline, "module image was not confirmed");
8045                    tokio::time::sleep(Duration::from_millis(5)).await;
8046                }
8047            }
8048            let entry = crate::live_children::read_record(&record_path)
8049                .unwrap()
8050                .pop()
8051                .unwrap();
8052            let observed = subc_os::Process::open(entry.pid)
8053                .unwrap()
8054                .unwrap()
8055                .observe()
8056                .unwrap();
8057            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
8058            module.drain().await.unwrap();
8059            assert_eq!(
8060                verdict,
8061                crate::live_children::IdentityVerdict::Matches,
8062                "program {program:?}: recorded {entry:?}, observed {observed:?}"
8063            );
8064        }
8065    }
8066
8067    #[cfg(unix)]
8068    fn http_fixture(
8069        dir: &std::path::Path,
8070        url: &str,
8071        threshold: u32,
8072    ) -> crate::daemon_config::ConfiguredModule {
8073        let path = dir.join("subc.jsonc");
8074        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
8075            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
8076            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
8077            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
8078            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
8079        }}}).to_string()).unwrap();
8080        crate::daemon_config::load(&path)
8081            .unwrap()
8082            .unwrap()
8083            .modules
8084            .pop()
8085            .unwrap()
8086    }
8087
8088    #[cfg(unix)]
8089    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
8090        timeout(Duration::from_secs(5), async {
8091            loop {
8092                if module.status().unwrap().health.status == status {
8093                    break;
8094                }
8095                sleep(Duration::from_millis(5)).await;
8096            }
8097        })
8098        .await
8099        .unwrap_or_else(|_| {
8100            panic!(
8101                "expected {status:?}, got {:?}",
8102                module.status().unwrap().health
8103            )
8104        });
8105    }
8106
8107    #[cfg(unix)]
8108    #[tokio::test]
8109    async fn http_health_status_flips_ok_failing_ok() {
8110        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8111        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
8112        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8113        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
8114        let serving_status = status.clone();
8115        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8116        let server = tokio::spawn(async move {
8117            loop {
8118                let (mut stream, _) = listener.accept().await.unwrap();
8119                let mut request = [0u8; 2048];
8120                let count = stream.read(&mut request).await.unwrap();
8121                assert!(count > 0, "a probe must send an HTTP request");
8122                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
8123                let body = if code == 200 {
8124                    "ready"
8125                } else {
8126                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
8127                };
8128                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
8129                let _ = stream.write_all(response.as_bytes()).await;
8130            }
8131        });
8132        let configured = http_fixture(&dir, &url, 1000);
8133        let module =
8134            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8135                .supervise_configured_with_health(
8136                    configured.module_spec(),
8137                    true,
8138                    configured.health,
8139                    configured.drain_timeout_ms,
8140                    configured.restart,
8141                )
8142                .unwrap();
8143        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8144        status.store(503, std::sync::atomic::Ordering::SeqCst);
8145        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
8146        assert!(module
8147            .status()
8148            .unwrap()
8149            .health
8150            .detail
8151            .unwrap()
8152            .contains("scratch failure"));
8153        status.store(200, std::sync::atomic::Ordering::SeqCst);
8154        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8155        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
8156        let before = module.status().unwrap();
8157        let (spec, mut health) = module.configuration().unwrap();
8158        health.http = None;
8159        module
8160            .update_configuration(spec.clone(), health.clone(), Some(10))
8161            .await
8162            .unwrap();
8163        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
8164        health.http = Some(url);
8165        module
8166            .update_configuration(spec, health, Some(10))
8167            .await
8168            .unwrap();
8169        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8170        assert_eq!(
8171            module.status().unwrap().pid,
8172            before.pid,
8173            "changing a probe must apply live, not restart its process"
8174        );
8175        let (spec, mut health) = module.configuration().unwrap();
8176        health.failure_threshold = 2;
8177        module
8178            .update_configuration(spec, health, Some(10))
8179            .await
8180            .unwrap();
8181        status.store(503, std::sync::atomic::Ordering::SeqCst);
8182        timeout(Duration::from_secs(5), async {
8183            while module.status().unwrap().spawn_generation == before.spawn_generation {
8184                sleep(Duration::from_millis(5)).await;
8185            }
8186        })
8187        .await
8188        .expect("sustained HTTP 503 responses must trigger the health restart policy");
8189        module.drain().await.unwrap();
8190        server.abort();
8191    }
8192
8193    #[cfg(unix)]
8194    #[tokio::test]
8195    async fn http_health_replacement_recovers_after_one_restart() {
8196        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8197        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
8198        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8199        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8200        let mut configured = http_fixture(&dir, &url, 2);
8201        configured.restart.max_restarts = 2;
8202        let module =
8203            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8204                .supervise_configured_with_health(
8205                    configured.module_spec(),
8206                    true,
8207                    configured.health,
8208                    configured.drain_timeout_ms,
8209                    configured.restart,
8210                )
8211                .unwrap();
8212        let first_generation = module.status().unwrap().spawn_generation;
8213        let observed = module.clone();
8214        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
8215        let server = tokio::spawn(async move {
8216            let mut generation = first_generation;
8217            let mut probes = 0;
8218            loop {
8219                let (mut stream, _) = listener.accept().await.unwrap();
8220                let mut request = [0; 2048];
8221                assert!(stream.read(&mut request).await.unwrap() > 0);
8222                let current = observed.status().unwrap().spawn_generation;
8223                if current != generation {
8224                    generation = current;
8225                    probes = 0;
8226                }
8227                probes += 1;
8228                // The original process never recovers. Each replacement has one
8229                // transient miss, then answers successfully for its remaining life.
8230                let healthy = generation != first_generation && probes > 1;
8231                let code = if healthy { 200 } else { 503 };
8232                let response = format!(
8233                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
8234                );
8235                let _ = stream.write_all(response.as_bytes()).await;
8236                if healthy {
8237                    healthy_tx.send(()).await.unwrap();
8238                }
8239            }
8240        });
8241        timeout(Duration::from_secs(5), async {
8242            let mut healthy_probes = 0;
8243            loop {
8244                if module.state().unwrap() == ModuleState::Failed {
8245                    break;
8246                }
8247                while healthy_rx.try_recv().is_ok() {
8248                    healthy_probes += 1;
8249                }
8250                if healthy_probes >= 3
8251                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
8252                {
8253                    break;
8254                }
8255                sleep(Duration::from_millis(5)).await;
8256            }
8257        })
8258        .await
8259        .expect("the replacement must settle healthy without exhausting its restart budget");
8260        let status = module.status().unwrap();
8261        module.drain().await.unwrap();
8262        server.abort();
8263        assert_eq!(status.state, ModuleState::Running);
8264        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
8265        assert_eq!(status.health.consecutive_failures, 0);
8266        assert_eq!(status.lifetime_restarts, 1);
8267        assert_eq!(status.restart_count, 1);
8268    }
8269
8270    #[cfg(unix)]
8271    #[tokio::test]
8272    async fn http_health_sustained_failure_exhausts_restart_budget() {
8273        use std::collections::BTreeMap;
8274        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8275        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
8276        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8277        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8278        let mut configured = http_fixture(&dir, &url, 2);
8279        configured.restart.max_restarts = 2;
8280        let module =
8281            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8282                .supervise_configured_with_health(
8283                    configured.module_spec(),
8284                    true,
8285                    configured.health,
8286                    configured.drain_timeout_ms,
8287                    configured.restart,
8288                )
8289                .unwrap();
8290        let observed = module.clone();
8291        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
8292        let served = probes.clone();
8293        let server = tokio::spawn(async move {
8294            loop {
8295                let (mut stream, _) = listener.accept().await.unwrap();
8296                let mut request = [0; 2048];
8297                assert!(stream.read(&mut request).await.unwrap() > 0);
8298                *served
8299                    .lock()
8300                    .unwrap()
8301                    .entry(observed.status().unwrap().spawn_generation)
8302                    .or_default() += 1;
8303                let _ = stream
8304                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
8305                    .await;
8306            }
8307        });
8308        timeout(Duration::from_secs(5), async {
8309            while module.state().unwrap() != ModuleState::Failed {
8310                sleep(Duration::from_millis(5)).await;
8311            }
8312        })
8313        .await
8314        .expect("sustained failures must still exhaust the health restart budget");
8315        let status = module.status().unwrap();
8316        server.abort();
8317        assert_eq!(status.restart_count, 2);
8318        assert_eq!(status.lifetime_restarts, 2);
8319        assert!(!status.process_alive);
8320        assert_eq!(
8321            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
8322            vec![2, 2, 2]
8323        );
8324    }
8325
8326    #[cfg(unix)]
8327    #[tokio::test]
8328    async fn spawn_clears_the_previous_process_health() {
8329        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
8330        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
8331        configured.health.cadence = Duration::from_secs(3600);
8332        let module =
8333            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8334                .supervise_configured_with_health(
8335                    configured.module_spec(),
8336                    false,
8337                    configured.health,
8338                    configured.drain_timeout_ms,
8339                    configured.restart,
8340                )
8341                .unwrap();
8342        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
8343            state.health.status = SupervisorHealthStatus::Unresponsive;
8344            state.health.consecutive_failures = 3;
8345            state.health.detail = Some("old process refused connections".into());
8346            state.health.metrics = Some(serde_json::json!({"old": true}));
8347            state.health.last_probe_ms = Some(123);
8348            state.health.last_action = Some("restart".into());
8349            state.health.last_action_ms = Some(124);
8350        })
8351        .unwrap();
8352        module.set_enabled(true).await.unwrap();
8353        let health = module.status().unwrap().health;
8354        module.drain().await.unwrap();
8355        assert_eq!(health.consecutive_failures, 0);
8356        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
8357        assert_eq!(health.detail, None);
8358        assert_eq!(health.metrics, None);
8359        assert_eq!(health.last_probe_ms, None);
8360        assert_eq!(health.last_action.as_deref(), Some("restart"));
8361        assert_eq!(health.last_action_ms, Some(124));
8362    }
8363
8364    #[cfg(unix)]
8365    #[tokio::test]
8366    async fn http_health_no_listener_restarts_after_consecutive_failures() {
8367        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
8368        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8369        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
8370        drop(unused);
8371        let configured = http_fixture(&dir, &url, 2);
8372        let module =
8373            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8374                .supervise_configured_with_health(
8375                    configured.module_spec(),
8376                    true,
8377                    configured.health,
8378                    configured.drain_timeout_ms,
8379                    configured.restart,
8380                )
8381                .unwrap();
8382        let before = module.status().unwrap().spawn_generation;
8383        timeout(Duration::from_secs(5), async {
8384            loop {
8385                let status = module.status().unwrap();
8386                if status.spawn_generation > before {
8387                    assert!(status.lifetime_restarts > 0);
8388                    break;
8389                }
8390                sleep(Duration::from_millis(5)).await;
8391            }
8392        })
8393        .await
8394        .expect("sustained HTTP refusal must trigger the health restart policy");
8395        module.drain().await.unwrap();
8396    }
8397
8398    #[cfg(unix)]
8399    #[tokio::test]
8400    async fn http_health_timeout_honours_deadline() {
8401        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8402        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8403        let server = tokio::spawn(async move {
8404            let _held = listener.accept().await.unwrap();
8405            std::future::pending::<()>().await;
8406        });
8407        let error = timeout(
8408            Duration::from_secs(1),
8409            probe_http_health(&url, Duration::from_millis(10)),
8410        )
8411        .await
8412        .expect("the probe must enforce its own deadline")
8413        .unwrap_err();
8414        server.abort();
8415        assert!(error.to_string().contains("timed out"));
8416    }
8417
8418    #[cfg(unix)]
8419    #[tokio::test]
8420    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
8421        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8422        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
8423        let url = format!(
8424            "http://localhost:{}/healthz",
8425            listener.local_addr().unwrap().port()
8426        );
8427        let server = tokio::spawn(async move {
8428            let (mut stream, _) = listener.accept().await.unwrap();
8429            let mut request = [0u8; 2048];
8430            assert!(stream.read(&mut request).await.unwrap() > 0);
8431            stream
8432                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
8433                .await
8434                .unwrap();
8435        });
8436        // The deadline only bounds a hang. A probe that never tried the IPv6
8437        // address would be refused on 127.0.0.1 and fail at once, so a longer
8438        // deadline does not weaken the assertion; one second timed out under a
8439        // loaded parallel test run.
8440        assert_eq!(
8441            probe_http_health(&url, Duration::from_secs(10))
8442                .await
8443                .unwrap()
8444                .status,
8445            HealthStatus::Ok
8446        );
8447        server.await.unwrap();
8448    }
8449
8450    #[cfg(unix)]
8451    #[tokio::test]
8452    async fn http_health_timeout_keeps_partial_status_and_body() {
8453        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8454        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8455        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8456        let server = tokio::spawn(async move {
8457            let (mut stream, _) = listener.accept().await.unwrap();
8458            let mut request = [0u8; 2048];
8459            assert!(stream.read(&mut request).await.unwrap() > 0);
8460            stream
8461                .write_all(
8462                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
8463                )
8464                .await
8465                .unwrap();
8466            std::future::pending::<()>().await;
8467        });
8468        let error = probe_http_health(&url, Duration::from_secs(1))
8469            .await
8470            .unwrap_err()
8471            .to_string();
8472        server.abort();
8473        assert!(
8474            error.contains("timed out")
8475                && error.contains("503 Unavailable")
8476                && error.contains("partial diagnostic"),
8477            "{error}"
8478        );
8479    }
8480
8481    #[cfg(unix)]
8482    #[tokio::test]
8483    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
8484        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8485        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8486        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8487        let server = tokio::spawn(async move {
8488            let (mut stream, _) = listener.accept().await.unwrap();
8489            let mut request = [0u8; 2048];
8490            assert!(stream.read(&mut request).await.unwrap() > 0);
8491            let body = format!("{}not-in-diagnostic", "x".repeat(400));
8492            let response = format!(
8493                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
8494                body.len()
8495            );
8496            stream.write_all(response.as_bytes()).await.unwrap();
8497        });
8498        let error = probe_http_health(&url, Duration::from_secs(1))
8499            .await
8500            .unwrap_err()
8501            .to_string();
8502        server.await.unwrap();
8503        assert!(error.ends_with(&"x".repeat(200)), "{error}");
8504        assert_eq!(error.split(": ").last().unwrap().len(), 200);
8505        assert!(!error.contains("not-in-diagnostic"));
8506    }
8507
8508    #[cfg(unix)]
8509    #[tokio::test]
8510    async fn http_health_real_nats_server_monitoring() {
8511        if std::process::Command::new("nats-server")
8512            .arg("--version")
8513            .env("XDG_DATA_HOME", std::env::temp_dir())
8514            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
8515            .env("XDG_CONFIG_HOME", std::env::temp_dir())
8516            .output()
8517            .is_err()
8518        {
8519            eprintln!(
8520                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
8521            );
8522            return;
8523        }
8524        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
8525        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8526        let port = monitor.local_addr().unwrap().port();
8527        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8528        let client_port = client.local_addr().unwrap().port();
8529        let config = dir.join("server.conf");
8530        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
8531        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
8532        configured.program = PathBuf::from("nats-server");
8533        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
8534        drop(monitor);
8535        drop(client);
8536        let module =
8537            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8538                .supervise_configured_with_health(
8539                    configured.module_spec(),
8540                    true,
8541                    configured.health,
8542                    configured.drain_timeout_ms,
8543                    configured.restart,
8544                )
8545                .unwrap();
8546        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8547        module.drain().await.unwrap();
8548    }
8549
8550    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8551    async fn configuration_update_does_not_replace_captured_running_process_facts() {
8552        let supervisor =
8553            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8554        let snapshot = stale_process_snapshot(ModuleState::Running, true);
8555        let initial = ModuleSpec {
8556            script: None,
8557            module_id: "rescan-preserves-spawn-facts".to_string(),
8558            program: PathBuf::from("/spawned/module"),
8559            args: Vec::new(),
8560            env: Vec::new(),
8561            reserved: false,
8562            reserved_prefixes: Vec::new(),
8563            protocol: ModuleProtocol::Subc,
8564            overlap: Default::default(),
8565            carry_containment: false,
8566        };
8567        let module = supervisor.supervised_module(
8568            initial.clone(),
8569            supervisor.runtime_config(),
8570            snapshot,
8571            None,
8572        );
8573        let before = module.status().unwrap();
8574        let mut replacement = initial;
8575        replacement.program = PathBuf::from("/rescanned/replacement-module");
8576
8577        module
8578            .update_configuration(replacement, HealthConfig::default(), None)
8579            .await
8580            .unwrap();
8581
8582        let after = module.status().unwrap();
8583        assert_eq!(after.pid, before.pid);
8584        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
8585        assert_eq!(after.spawned_from, before.spawned_from);
8586        drop(module);
8587    }
8588}
8589
8590fn unix_ms_now() -> u64 {
8591    SystemTime::now()
8592        .duration_since(UNIX_EPOCH)
8593        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
8594        .unwrap_or(0)
8595}
8596
8597async fn supervise_loop(
8598    mut spec: ModuleSpec,
8599    mut runtime: SupervisorRuntimeConfig,
8600    registry: Arc<Registry>,
8601    process_liveness: Arc<SupervisorProcessLiveness>,
8602    snapshot: SharedSnapshot,
8603    mut child: Option<SupervisedChild>,
8604    mut commands: mpsc::Receiver<SupervisorCommand>,
8605) {
8606    let mut health_probe = HealthProbeRuntime::default();
8607    // Registry writes (including embedded callers) notify this module only.
8608    // Subscribe before the first refresh; watch retains changes that arrive
8609    // while commands or probes are running. No polling fallback is needed.
8610    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
8611        Ok(changes) => Some(changes),
8612        Err(err) => {
8613            // A poisoned registry cannot accept further writes, so it cannot
8614            // recover via a timer. Keep exit and command handling alive.
8615            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
8616            None
8617        }
8618    };
8619    // All restart backoffs run here, including health and operator requests.
8620    // While one is pending the loop serves commands, so disable or drain can
8621    // cancel the replacement without spawning a process just to stop it.
8622    let mut pending_respawn: Option<PendingRespawn> = None;
8623    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
8624    // before anything else so a stop that interrupted a swap runs at once.
8625    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
8626    let mut previous_instance_recheck = tokio::time::interval_at(
8627        Instant::now() + Duration::from_secs(5),
8628        Duration::from_secs(5),
8629    );
8630    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
8631    'supervision: loop {
8632        #[cfg(test)]
8633        {
8634            lock_snapshot(&snapshot).unwrap().actor_select = None;
8635        }
8636        #[cfg(target_os = "macos")]
8637        if let Some(active) = child.as_mut() {
8638            active.confirm_privacy_exec().await;
8639        }
8640        if let Some(scheduled) = runtime
8641            .scheduled_respawn
8642            .lock()
8643            .unwrap_or_else(|p| p.into_inner())
8644            .take()
8645        {
8646            runtime.recovery_is_daemon = scheduled.daemon_initiated;
8647            pending_respawn = Some(scheduled);
8648        }
8649        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
8650            pending_respawn = None;
8651            cancel_deferred_reload(
8652                &runtime,
8653                &spec.module_id,
8654                "respawn cancelled by a supervisor command",
8655            );
8656        }
8657        if child.is_none()
8658            && pending_respawn.is_none()
8659            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
8660        {
8661            cancel_deferred_reload(
8662                &runtime,
8663                &spec.module_id,
8664                "respawn cancelled before a replacement was spawned",
8665            );
8666            let mut stranded = false;
8667            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8668                state.respawn_pending = false;
8669                state.coalesced_restart_pending = false;
8670                if matches!(
8671                    state.state,
8672                    ModuleState::Restarting
8673                        | ModuleState::Starting
8674                        | ModuleState::Draining
8675                        | ModuleState::Unresponsive
8676                ) {
8677                    error!(module_id = %spec.module_id, state = ?state.state,
8678                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
8679                    stranded = true;
8680                }
8681            });
8682            if stranded {
8683                fail_snapshot(
8684                    &snapshot,
8685                    Some(&spec.module_id),
8686                    None,
8687                    &runtime.failure_context(runtime.recovery_is_daemon),
8688                );
8689            }
8690        }
8691        if child.is_none()
8692            && (runtime.child_roster.is_closed()
8693                || pending_respawn.is_none()
8694                || !lock_snapshot(&snapshot)
8695                    .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
8696        {
8697            containment::terminate(&runtime.child_roster.containment).await;
8698        }
8699        if let Some(command) = requeued.pop_front() {
8700            if !handle_supervisor_command(
8701                command,
8702                &mut spec,
8703                &mut runtime,
8704                &registry,
8705                &process_liveness,
8706                &snapshot,
8707                &mut child,
8708                &mut commands,
8709                &mut requeued,
8710            )
8711            .await
8712            {
8713                containment::terminate(&runtime.child_roster.containment).await;
8714                return;
8715            }
8716            if child.is_some() || !respawn_still_pending(&snapshot) {
8717                pending_respawn = None;
8718                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8719                    state.respawn_pending = false
8720                });
8721            }
8722            continue;
8723        }
8724        if child.is_some() {
8725            runtime.recovery_is_daemon = true;
8726            #[cfg(test)]
8727            {
8728                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
8729            }
8730            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
8731            let active = child.as_mut().expect("child checked above");
8732            if active.registered_since_spawn() {
8733                if let Some(delivery) = &runtime.config_delivery {
8734                    delivery.queue_registered(
8735                        &spec.module_id,
8736                        runtime.forwarding.as_ref(),
8737                        &registry,
8738                        active,
8739                    );
8740                }
8741            }
8742            if runtime.child_roster.is_closed() || !active.awaiting_registration() {
8743                active.registration_deadline = None;
8744            }
8745            let registration_deadline = active.registration_deadline;
8746            let no_checks = runtime.health.http.is_none()
8747                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
8748                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
8749            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
8750                state
8751                    .failed_retry
8752                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
8753            });
8754            let wake_at = match (health_probe.wake_at(), uptime_reset) {
8755                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
8756                (probe, None) => probe,
8757            };
8758            let wake_at = match (wake_at, registration_deadline) {
8759                (Some(wake), Some(register)) => Some(wake.min(register)),
8760                (wake, register) => wake.or(register),
8761            };
8762            #[cfg(test)]
8763            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
8764            let probe_sleep = supervisor_sleep(wake_at);
8765            tokio::pin!(probe_sleep);
8766            // A `protocol: "none"` child (a plain process that never registers
8767            // over the subc wire, such as nats-server) with no HTTP health
8768            // check has nothing to probe, so it waits only for its exit or a
8769            // supervisor command. A rescan that adds an HTTP check is a command.
8770            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
8771            #[cfg(test)]
8772            {
8773                let registration_pending = wire_child
8774                    && registration_changes
8775                        .as_ref()
8776                        .is_some_and(|changes| changes.has_changed().unwrap());
8777                let mut state = lock_snapshot(&snapshot).unwrap();
8778                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
8779                    generation: state.spawn_generation,
8780                    turn: state.actor_turns,
8781                    registered_connection: health_probe.registered_connection,
8782                    next_probe_at: health_probe.next_probe_at,
8783                    wake_after,
8784                });
8785            }
8786            let registration = active
8787                .awaiting_registration()
8788                .then(|| active.registration_spawn.clone())
8789                .flatten();
8790            #[cfg(all(test, target_os = "linux"))]
8791            let registration_before_exit = !runtime.child_roster.admission_event_for_test(
8792                &spec.module_id,
8793                "observe_registration_before_exit",
8794                Some(active.pid),
8795            ) && registration.is_some();
8796            #[cfg(not(all(test, target_os = "linux")))]
8797            let registration_before_exit = false;
8798            #[cfg(all(test, target_os = "linux"))]
8799            let registration_child_is_running = |child: &mut SupervisedChild| {
8800                // Model the interval after SIGKILL is accepted but before the
8801                // child becomes waitable, without delaying the actual kill.
8802                registration_before_exit || registration_child_is_running(child)
8803            };
8804            let registration_failed = async {
8805                match &registration {
8806                    Some(spawn) => spawn.wait_registration_failure().await,
8807                    None => std::future::pending().await,
8808                }
8809            };
8810            let active_child = child.as_mut().expect("child checked above");
8811            tokio::select! {
8812                _ = registration_failed => {
8813                    if !active_child.registered_since_spawn() && registration_child_is_running(active_child) {
8814                        active_child.registration_deadline = None;
8815                        active_child.spawn_failure = Some("registration_channel_closed".into());
8816                        if let Err(error) = stop_unregistered_child(active_child).await {
8817                            fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8818                                &runtime.spawn_events, format!("registration_channel_closed: {error}"),
8819                                &runtime.failure_context(true));
8820                            child = None;
8821                        }
8822                    }
8823                    continue;
8824                }
8825                wait_result = active_child.wait(), if !registration_before_exit => {
8826                    if let Ok(mut state) = lock_snapshot(&snapshot) {
8827                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
8828                    }
8829                    #[cfg(test)]
8830                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8831                    // Every arm below that gives up on the CHILD must keep the
8832                    // supervision task itself alive (child = None, loop
8833                    // continues into command-serving mode). Returning here
8834                    // closes the command channel, which makes the module
8835                    // permanently unrestartable in-band: a clean child exit
8836                    // of an enabled module once wedged the fleet this way
8837                    // ('supervisor command channel is closed') and required a
8838                    // full daemon restart to recover.
8839                    let exit_report = match wait_result {
8840                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
8841                        Err(err) => {
8842                            active_child.drain_stderr(&spec.module_id).await;
8843                            fail_after_wait_error(
8844                                &spec.module_id,
8845                                &snapshot,
8846                                &runtime.terminal_ring,
8847                                &runtime.spawn_events,
8848                                format!("failed to wait for supervised module: {err}"),
8849                            &runtime.failure_context(true),
8850                            );
8851                            untrack_if_registration_released(
8852                                &process_liveness,
8853                                &registry,
8854                                &spec.module_id,
8855                                &snapshot,
8856                            );
8857                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
8858                            child = None;
8859                            continue;
8860                        }
8861                    };
8862                    active_child.drain_stderr(&spec.module_id).await;
8863
8864                    let delivery=runtime.config_delivery.clone();
8865                    let exited_spawn=active_child.spawned_at;
8866                    let spawn=active_child.config_delivery.clone();
8867                    let refusal=active_child.spawn_failure.as_deref().filter(|detail|detail.starts_with("program_refused:") && !active_child.live_program_binding.as_ref().is_some_and(|binding|binding.is_refused())).zip(active_child.admission_base.clone()).map(|(detail,base)|(base,subc_config::raw_digest(detail.as_bytes())));
8868                    let module_id=spec.module_id.clone();
8869                    // Publication may own the store mutex for local durable I/O.
8870                    // Keep commands selectable until the child's exit is recorded;
8871                    // no respawn disposition is chosen before that observation.
8872                    let observed=async move {
8873                        if let Some(delivery)=delivery {
8874                            delivery.0.on_blocking(move |owner| {
8875                                if let Some(spawn)=spawn { owner.delivery_child_exited(&spawn)?; }
8876                                if let Some((base,identity))=refusal {owner.record_program_refusal(&module_id,&base,identity)?;}
8877                                Ok(())
8878                            }).await
8879                        } else {Ok(())}
8880                    };
8881                    tokio::pin!(observed);
8882                    loop {
8883                        tokio::select! {
8884                            result=&mut observed=>{
8885                                if let Err(error)=result {warn!(module_id=%spec.module_id,%error,"configuration exit observation failed");}
8886                                break;
8887                            }
8888                            command=commands.recv()=>{
8889                                let Some(command)=command else {containment::terminate(&runtime.child_roster.containment).await;return;};
8890                                let deferred=||SuperviseError::ConfigDeliveryDeferred {module_id:spec.module_id.clone()};
8891                                let command=match command {
8892                                    SupervisorCommand::Restart {reply,..}|SupervisorCommand::Reload {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8893                                    SupervisorCommand::Swap {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8894                                    command @ (SupervisorCommand::ConfigApplyRestart {..}|SupervisorCommand::ConfigPolicyProbe {..}|SupervisorCommand::ConfigDeliveryFailed {..})=>{requeued.push_back(command);continue;}
8895                                    command=>command,
8896                                };
8897                                if !handle_supervisor_command(command,&mut spec,&mut runtime,&registry,&process_liveness,&snapshot,&mut child,&mut commands,&mut requeued).await {
8898                                    containment::terminate(&runtime.child_roster.containment).await;return;
8899                                }
8900                                if child.as_ref().is_none_or(|current|current.spawned_at!=exited_spawn) {continue 'supervision;}
8901                            }
8902                        }
8903                    }
8904                    let active_child=child.as_mut().expect("exit observation retained its child");
8905                    // An unavailable image can race the program's own exit.
8906                    // Only the admission kill is budget-free; another exit must
8907                    // retain its ordinary crash/clean-exit accounting.
8908                    let admission_kill = admission_exit::is_admission_kill(active_child.admission_kill_sent, &exit_report);
8909                    let deferred_spawn=admission_kill && active_child.spawn_failure.as_deref().is_some_and(|detail|detail.starts_with("config_delivery_deferred:"));
8910                    let refused_program = active_child.live_program_binding.as_ref().is_some_and(|binding| binding.is_refused());
8911                    let exit_held = !deferred_spawn && (refused_program || runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.exit_is_held(&spec.module_id)));
8912                    let parked = exit_held && (!refused_program || active_child.admission_base.is_some())
8913                        && try_park_config_attempt(&spec, &runtime, &snapshot, &runtime.failure_context(true),
8914                            if refused_program { active_child.admission_base.as_ref() } else { None })
8915                            .unwrap_or_else(|error| { warn!(module_id=%spec.module_id, %error, "exit hold could not be recorded; using ordinary exit accounting"); false });
8916                    let next = if deferred_spawn {
8917                        record_terminal_with_detail(&spec.module_id,&runtime.terminal_ring,&runtime.spawn_events,&exit_report,TerminalDisposition::Restarting,active_child.spawn_failure.clone());
8918                        let _=update_snapshot(&snapshot,Some(&spec.module_id),|state| {
8919                            state.state=ModuleState::Restarting;
8920                            clear_current_process_facts(state);
8921                            state.last_exit=Some(exit_report.clone());
8922                        });
8923                        NextAction::Restart {schedule:None}
8924                    } else if parked {
8925                        record_terminal_with_detail(&spec.module_id, &runtime.terminal_ring, &runtime.spawn_events, &exit_report, TerminalDisposition::Stopped, active_child.spawn_failure.clone().or_else(|| Some("config_gated".into())));
8926                        NextAction::Stop { registration_released: registry.get_module(&spec.module_id).ok().flatten().is_none() }
8927                    } else { on_child_exit(
8928                        &spec,
8929                        runtime.restart_policy,
8930                        &registry,
8931                        &snapshot,
8932                        &runtime.terminal_ring,
8933                        &runtime.spawn_events,
8934                        &runtime.child_roster,
8935                        exit_report,
8936                        true,
8937                    ).await };
8938                    runtime.start_requester = None;
8939                    runtime.pending_activity_restart = false;
8940                    // The exit is recorded, so a daemon shutdown may stop
8941                    // waiting for this child (see `SupervisedChild::wait`).
8942                    let carry = matches!(next, NextAction::Restart { .. })
8943                        && active_child.spawn_failure.is_none()
8944                        && !runtime.child_roster.is_closed();
8945                    if let Err(error) = active_child.finish_exit(carry).await {
8946                        containment::terminate(&runtime.child_roster.containment).await;
8947                        fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8948                            &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
8949                            &runtime.failure_context(true));
8950                        child = None;
8951                        continue;
8952                    }
8953                    match next {
8954                        NextAction::Stop { registration_released } => {
8955                            if registration_released {
8956                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
8957                            }
8958                            child = None;
8959                        }
8960                        NextAction::Restart { schedule } => {
8961                            let delay = if deferred_spawn {CONFIG_ADMISSION_RETRY_DELAY} else {schedule.map_or(
8962                                runtime.restart_policy.delay_for_restart(0),
8963                                |schedule| schedule.delay,
8964                            )};
8965                            if let Some(schedule) = schedule {
8966                                log_crash_respawn(&spec.module_id, schedule);
8967                            }
8968                            // The exited child is fully recorded at this point,
8969                            // so release it and count the backoff down in the
8970                            // command-serving branch below rather than sleeping
8971                            // here: commands cannot be received from inside this
8972                            // select arm, and an operator disable or drain that
8973                            // arrives during the backoff must cancel the pending
8974                            // respawn instead of waiting for it to spawn first.
8975                            child = None;
8976                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true, approval: None });
8977                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
8978                        }
8979                    }
8980                }
8981                command = commands.recv() => {
8982                    #[cfg(test)]
8983                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8984                    let Some(command) = command else {
8985                        containment::terminate(&runtime.child_roster.containment).await;
8986                        return;
8987                    };
8988                    if !handle_supervisor_command(
8989                        command,
8990                        &mut spec,
8991                        &mut runtime,
8992                        &registry,
8993                        &process_liveness,
8994                        &snapshot,
8995                        &mut child,
8996                        &mut commands,
8997                        &mut requeued,
8998                    ).await {
8999                        containment::terminate(&runtime.child_roster.containment).await;
9000                        return;
9001                    }
9002                }
9003                _ = async {
9004                    match registration_changes.as_mut() {
9005                        Some(changes) => { let _ = changes.changed().await; }
9006                        None => std::future::pending().await,
9007                    }
9008                }, if wire_child => {
9009                    #[cfg(test)]
9010                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
9011                }
9012                _ = &mut probe_sleep => {
9013                    #[cfg(test)]
9014                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
9015                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
9016                        // Recheck after the wake: HELLO, exit or shutdown may
9017                        // have become ready in the same executor tick.
9018                        active_child.registration_deadline = None;
9019                        if !runtime.child_roster.is_closed()
9020                            && !active_child.registered_since_spawn()
9021                            && registration_child_is_running(active_child)
9022                        {
9023                            warn!(module_id = %spec.module_id, pid = active_child.pid,
9024                                wait_secs = runtime.register_timeout.as_secs(),
9025                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
9026                            active_child.spawn_failure = Some("registration_timeout".into());
9027                            if let Err(error) = stop_unregistered_child(active_child).await {
9028                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
9029                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
9030                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
9031                                    &runtime.failure_context(true));
9032                                child = None;
9033                            }
9034                            // The ordinary exit handler records the child's exit,
9035                            // spends crash budget, and schedules recovery.
9036                            continue;
9037                        }
9038                    }
9039                    if health_probe.due() {
9040                        run_health_probe_cycle(
9041                            &spec,
9042                            &runtime,
9043                            &registry,
9044                            &process_liveness,
9045                            &snapshot,
9046                            &mut child,
9047                        ).await;
9048                        if child.is_none() {
9049                            runtime.start_requester = None;
9050                            runtime.pending_activity_restart = false;
9051                        }
9052                        if child.is_some() {
9053                            health_probe.schedule_next(&spec, runtime.health.cadence);
9054                        }
9055                    }
9056                }
9057            }
9058        } else if let Some(pending) = pending_respawn.clone() {
9059            tokio::select! {
9060                _ = runtime.child_roster.closed() => {
9061                    pending_respawn = None;
9062                    let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9063                        state.state = ModuleState::Stopped;
9064                        state.respawn_pending = false;
9065                    });
9066                    containment::terminate(&runtime.child_roster.containment).await;
9067                }
9068                _ = async {
9069                    sleep_until(pending.deadline).await;
9070                    if let Some(delivery) = &runtime.config_delivery {
9071                        delivery.unfenced(&spec.module_id).await;
9072                    }
9073                } => {
9074                    pending_respawn = None;
9075                    let failure = pending.failure_context(&runtime);
9076                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9077                    // A command handled below while the backoff elapsed may
9078                    // have stopped the module; never respawn past an operator's
9079                    // disable or drain.
9080                    if !respawn_still_pending(&snapshot) {
9081                        continue;
9082                    }
9083                    // The daemon began shutting down during the backoff: the
9084                    // spawn would be refused anyway, and refusing it here
9085                    // leaves the module stopped instead of reporting a
9086                    // failed restart.
9087                    if runtime.child_roster.is_closed() {
9088                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9089                            state.state = ModuleState::Stopped;
9090                        });
9091                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
9092                        continue;
9093                    }
9094                    if let Err(err) = release_dead_registration(
9095                        &registry,
9096                        runtime.forwarding.as_deref(),
9097                        &snapshot,
9098                        &spec.module_id,
9099                        &failure,
9100                    ).await {
9101                        record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9102                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
9103                        continue;
9104                    }
9105
9106                    if matches!(pending.kind, RespawnKind::Reload) {
9107                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
9108                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
9109                        if matches!(result, Err(SuperviseError::ConfigDeliveryDeferred { .. })) {
9110                            *runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()) = reply;
9111                            continue;
9112                        }
9113                        if result.is_ok() && runtime.pending_activity_restart {
9114                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9115                        }
9116                        runtime.pending_activity_restart = false;
9117                        if let Some(reply) = reply { let _ = reply.send(result); }
9118                        continue;
9119                    }
9120                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9121                    match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9122                        Ok(next_child) => {
9123                            child = Some(next_child);
9124                            if runtime.pending_activity_restart {
9125                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9126                                runtime.pending_activity_restart = false;
9127                            }
9128                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
9129                        }
9130                        Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9131                            pending_respawn = Some(PendingRespawn { deadline:Instant::now()+CONFIG_ADMISSION_RETRY_DELAY, ..pending });
9132                            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
9133                        }
9134                        Err(SuperviseError::ConfigHeld { .. }) => {
9135                            let _ = hold_config_spawn(&spec, &runtime, &snapshot, &failure);
9136                            runtime.pending_activity_restart = false;
9137                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9138                        }
9139                        Err(err) => {
9140                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &failure);
9141                            record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9142                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9143                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
9144                        }
9145                    }
9146                }
9147                command = commands.recv() => {
9148                    let Some(command) = command else {
9149                        containment::terminate(&runtime.child_roster.containment).await;
9150                        return;
9151                    };
9152                    if !handle_supervisor_command(
9153                        command,
9154                        &mut spec,
9155                        &mut runtime,
9156                        &registry,
9157                        &process_liveness,
9158                        &snapshot,
9159                        &mut child,
9160                        &mut commands,
9161                        &mut requeued,
9162                    ).await {
9163                        containment::terminate(&runtime.child_roster.containment).await;
9164                        return;
9165                    }
9166                    // Reconcile the pending respawn with what the command did:
9167                    // a start may already have spawned a fresh child,
9168                    // while a disable or drain moved the snapshot out of the
9169                    // state the respawn was counting down from.
9170                    if child.is_some() || !respawn_still_pending(&snapshot) {
9171                        pending_respawn = None;
9172                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9173                    }
9174                }
9175            }
9176        } else {
9177            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
9178                (state.enabled && state.state == ModuleState::Failed)
9179                    .then_some(state.failed_retry.pending)
9180                    .flatten()
9181            });
9182            let command = if lock_snapshot(&snapshot)
9183                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
9184            {
9185                tokio::select! {
9186                    command = commands.recv() => command,
9187                    _ = previous_instance_recheck.tick() => {
9188                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
9189                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
9190                            if enabled && !runtime.child_roster.is_closed() {
9191                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9192                                match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9193                                    Ok(next_child) => child = Some(next_child),
9194                                    Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9195                                        let _ = defer_config_spawn(&spec, &runtime, &snapshot, RespawnKind::Spawn, &runtime.failure_context(true));
9196                                    }
9197                                    Err(SuperviseError::ConfigHeld { .. }) => {
9198                                        let _ = hold_config_spawn(&spec, &runtime, &snapshot, &runtime.failure_context(true));
9199                                    }
9200                                    Err(error) => {
9201                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
9202                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
9203                                    }
9204                                }
9205                            } else {
9206                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9207                                    state.waiting_for_previous_pid = None;
9208                                    state.waiting_for_previous_detail = None;
9209                                });
9210                            }
9211                        }
9212                        continue;
9213                    }
9214                }
9215            } else {
9216                tokio::select! {
9217                    biased;
9218                    command = commands.recv() => command,
9219                    _ = async {
9220                        match slow_retry {
9221                            Some(retry) => sleep_until(retry.deadline).await,
9222                            None => std::future::pending().await,
9223                        }
9224                    } => {
9225                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
9226                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9227                            state.failed_retry.pending = None;
9228                        });
9229                        if runtime.child_roster.is_closed() {
9230                            continue;
9231                        }
9232                        info!(module_id = %spec.module_id, attempt = retry.attempt,
9233                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
9234                        if let Err(error) = set_child_enabled(
9235                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
9236                            &runtime.failure_context(true),
9237                        ).await {
9238                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
9239                        }
9240                        continue;
9241                    }
9242                }
9243            };
9244            let Some(command) = command else {
9245                containment::terminate(&runtime.child_roster.containment).await;
9246                return;
9247            };
9248            if !handle_supervisor_command(
9249                command,
9250                &mut spec,
9251                &mut runtime,
9252                &registry,
9253                &process_liveness,
9254                &snapshot,
9255                &mut child,
9256                &mut commands,
9257                &mut requeued,
9258            )
9259            .await
9260            {
9261                containment::terminate(&runtime.child_roster.containment).await;
9262                return;
9263            }
9264        }
9265    }
9266}
9267
9268// Preserve the deadline until the future is polled. Starting a relative sleep
9269// after the executor yields would restart part of the countdown.
9270async fn supervisor_sleep(wake_at: Option<Instant>) {
9271    match wake_at {
9272        Some(deadline) => sleep_until(deadline).await,
9273        None => std::future::pending().await,
9274    }
9275}
9276
9277fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
9278    info!(
9279        module_id,
9280        restart_in_window = schedule.restart_in_window,
9281        delay_ms = schedule.delay.as_millis() as u64,
9282        "respawning after crash"
9283    );
9284}
9285
9286/// Whether the respawn a backoff was counting down to is still wanted. A
9287/// disable or drain handled while the backoff elapsed moves the snapshot out
9288/// of `Restarting`, and the operator's stop must win over the pending respawn,
9289/// so every sleep-then-spawn path re-validates against the live snapshot
9290/// instead of assuming the state it left behind still holds.
9291fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
9292    matches!(
9293        lock_snapshot(snapshot),
9294        Ok(state) if state.enabled && state.state == ModuleState::Restarting
9295    )
9296}
9297
9298enum NextAction {
9299    Stop {
9300        registration_released: bool,
9301    },
9302    Restart {
9303        schedule: Option<CrashRestartSchedule>,
9304    },
9305}
9306
9307#[allow(clippy::too_many_arguments)]
9308async fn handle_supervisor_command(
9309    command: SupervisorCommand,
9310    spec: &mut ModuleSpec,
9311    runtime: &mut SupervisorRuntimeConfig,
9312    registry: &Arc<Registry>,
9313    process_liveness: &SupervisorProcessLiveness,
9314    snapshot: &SharedSnapshot,
9315    child: &mut Option<SupervisedChild>,
9316    commands: &mut mpsc::Receiver<SupervisorCommand>,
9317    requeued: &mut VecDeque<SupervisorCommand>,
9318) -> bool {
9319    if let SupervisorCommand::SetEnabled {
9320        enabled: true,
9321        reset_budget: false,
9322        reply,
9323        ..
9324    } = command
9325    {
9326        let parked = lock_snapshot(snapshot).ok().and_then(|state| {
9327            (state.config_hold_parked
9328                && state.enabled
9329                && !state.process_alive
9330                && child.is_none()
9331                && state.state == ModuleState::Stopped)
9332                .then(|| {
9333                    (
9334                        state.config_hold_daemon_initiated,
9335                        state.config_hold_approval.clone(),
9336                    )
9337                })
9338        });
9339        // Clearing a configuration-delivery block must not restart a child that
9340        // exited on its own or exhausted its crash budget. Resume only an actor
9341        // stopped by a configuration hold; otherwise leave its slow retry alone.
9342        let result = if let Some((daemon_initiated, approval)) = parked {
9343            runtime.recovery_is_daemon = daemon_initiated;
9344            let failure = FailureContext {
9345                approval,
9346                ..runtime.failure_context(daemon_initiated)
9347            };
9348            let result = set_child_enabled_with_budget_policy(
9349                spec,
9350                runtime,
9351                registry,
9352                process_liveness,
9353                snapshot,
9354                child,
9355                true,
9356                &failure,
9357                false,
9358            )
9359            .await;
9360            if let Err(error) = &result {
9361                record_approved_attempt_failure(spec, snapshot, &failure, error);
9362            }
9363            result
9364        } else {
9365            Ok(false)
9366        };
9367        let _ = reply.send(result);
9368        return true;
9369    }
9370    let blocked = runtime
9371        .config_delivery
9372        .as_ref()
9373        .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
9374        .filter(|error| error.code != "config_delivery_deferred")
9375        .map(|error| {
9376            config_delivery::gate_hold_error(spec, error, runtime.config_delivery.as_ref())
9377        })
9378        .or_else(|| recovery_block_error(spec, &runtime.child_roster))
9379        .or_else(|| {
9380            runtime
9381                .child_roster
9382                .previous_instance(&spec.module_id)
9383                .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
9384        });
9385    let command = if let Some(error) = blocked {
9386        match command {
9387            SupervisorCommand::Restart { reply, .. } | SupervisorCommand::Reload { reply, .. } => {
9388                let _ = reply.send(Err(error));
9389                return true;
9390            }
9391            SupervisorCommand::Swap { reply, .. } => {
9392                let _ = reply.send(Err(error));
9393                return true;
9394            }
9395            SupervisorCommand::ConfigApplyRestart { .. } => return true,
9396            SupervisorCommand::SetEnabled {
9397                enabled: true,
9398                requester,
9399                reply,
9400                ..
9401            } => {
9402                if matches!(error, SuperviseError::ConfigHeld { .. })
9403                    && lock_snapshot(snapshot).is_ok_and(|state| !state.enabled)
9404                    && runtime
9405                        .config_delivery
9406                        .as_ref()
9407                        .is_some_and(|delivery| delivery.0.can_stage_unstamp_enable())
9408                {
9409                    // Stage only the pending unstamp's desired state. A stamped
9410                    // unapproved actor stays refused; admission release starts
9411                    // an unstamped actor only after reconciliation completes.
9412                    let mut changed = false;
9413                    let result = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9414                        changed = !state.enabled;
9415                        state.enabled = true;
9416                        if child.is_none() && state.waiting_for_previous_pid.is_none() {
9417                            state.state = ModuleState::Starting;
9418                            state.failed_retry.cancel();
9419                        }
9420                    })
9421                    .and_then(|_| {
9422                        if child.is_none()
9423                            && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none()
9424                        {
9425                            hold_config_spawn(
9426                                spec,
9427                                runtime,
9428                                snapshot,
9429                                &runtime.failure_context(false),
9430                            )?;
9431                        }
9432                        Ok(changed)
9433                    });
9434                    if result.is_ok() && changed {
9435                        runtime.start_requester = requester;
9436                    }
9437                    let _ = reply.send(result);
9438                    return true;
9439                }
9440                let _ = reply.send(Err(error));
9441                return true;
9442            }
9443            other => other,
9444        }
9445    } else {
9446        command
9447    };
9448    // A refused start or restart leaves the Failed actor's existing recovery
9449    // deadline intact. Cancel only after the command passes admission.
9450    if matches!(
9451        command,
9452        SupervisorCommand::Drain { .. }
9453            | SupervisorCommand::Retire { .. }
9454            | SupervisorCommand::SetEnabled { .. }
9455            | SupervisorCommand::Restart { .. }
9456    ) {
9457        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9458            state.failed_retry.cancel();
9459        });
9460    }
9461    match command {
9462        SupervisorCommand::ApprovedConfiguration { approved, reply } => {
9463            let result = async {
9464                let next = approved.module_entry(&spec.module_id).map_err(|error| {
9465                    SuperviseError::InvalidSpec { reason: error.to_string() }
9466                })?;
9467                let retry_approved_failure = {
9468                    let mut state = lock_snapshot(snapshot)?;
9469                    if let Some((generation, detail)) = &state.approved_start_failure {
9470                        if *generation == approved.identity() && next.enabled {
9471                            return Err(SuperviseError::Spawn {
9472                                program: spec.program.clone(),
9473                                source: std::io::Error::other(detail.clone()),
9474                                cgroup_path: None,
9475                            });
9476                        }
9477                    }
9478                    let retry = state.approved_start_failure.is_some();
9479                    state.approved_start_failure = None;
9480                    retry
9481                };
9482                stage_supervisor_configuration(
9483                    spec, runtime, snapshot, next.module_spec(), next.health,
9484                    next.drain_timeout_ms,
9485                );
9486                runtime.restart_policy = next.restart;
9487                runtime.register_timeout = next.register_timeout;
9488                if let Some(active) = child.as_mut()
9489                    .filter(|child| child.registration_deadline.is_some())
9490                {
9491                    active.registration_deadline = Some(active.spawned_at + next.register_timeout);
9492                }
9493                let enabled = lock_snapshot(snapshot)?.enabled;
9494                // An approval retries a changed failed entry or its own earlier
9495                // failed start, not an unrelated crash-budget failure. For an
9496                // unchanged entry, reconciled means the approved configuration
9497                // is effective; it does not claim the process has recovered.
9498                // A terminal gate stop is different: a new approval owns its
9499                // recovery even when the daemon entry itself did not change.
9500                let retry_gate_stopped = lock_snapshot(snapshot)?
9501                    .needs_approved_gate_restart(&approved.identity());
9502                let retry_failed = enabled && child.is_none()
9503                    && ((lock_snapshot(snapshot)?.state == ModuleState::Failed
9504                        && (retry_approved_failure || approved.changes_entry(&spec.module_id)))
9505                        || retry_gate_stopped);
9506                if retry_failed {
9507                    // This approval owns a new start, so the failed attempt's
9508                    // daemon retry must not survive either result of that start.
9509                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
9510                        state.failed_retry.cancel();
9511                        if retry_gate_stopped {
9512                            state.stop_reason = None;
9513                            state.config_exit_failed_approval = None;
9514                        }
9515                    })?;
9516                }
9517                if enabled != next.enabled || retry_failed {
9518                    let failure = FailureContext {
9519                        approval: next.enabled.then(|| approved.identity()),
9520                        ..runtime.failure_context(false)
9521                    };
9522                    let held = next.enabled && runtime.config_delivery.as_ref()
9523                        .is_some_and(|delivery| delivery.0.has_recorded_admission_hold(&spec.module_id)
9524                            || delivery.0.check_module_hold(&spec.module_id).is_err_and(|error| error.code == "config_delivery_deferred"));
9525                    if held {
9526                        // Preserve module-delivery failure/recovery holds. The
9527                        // admission release starts the actor when its own hold ends.
9528                        reset_restart_count(snapshot, &spec.module_id)?;
9529                        update_snapshot(snapshot, Some(&spec.module_id), |state| {
9530                            state.enabled = true;
9531                            if child.is_none() && state.waiting_for_previous_pid.is_none() {
9532                                state.state = ModuleState::Starting;
9533                            }
9534                        })?;
9535                        if child.is_none() && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none() {
9536                            hold_config_spawn(spec, runtime, snapshot, &failure)?;
9537                        }
9538                    } else {
9539                        match set_child_enabled_with_budget_policy(
9540                            spec, runtime, registry, process_liveness, snapshot, child,
9541                            next.enabled, &failure, true,
9542                        ).await {
9543                            Ok(_) => {},
9544                            Err(SuperviseError::ConfigHeld { .. }) if next.enabled => {},
9545                            Err(error) => return Err(error),
9546                        }
9547                        if !next.enabled {
9548                            containment::terminate(&runtime.child_roster.containment).await;
9549                        }
9550                    }
9551                }
9552                if (child.as_ref().is_some_and(|child| child.registration_spawn.is_some() && child.admission_base.is_none())
9553                        || (child.is_none() && lock_snapshot(snapshot)?.state == ModuleState::Restarting
9554                            && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.stamp_channel_cutover_pending())))
9555                    && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.module_is_gated(&spec.module_id))
9556                {
9557                    let failure = FailureContext {
9558                        approval: Some(approved.identity()),
9559                        ..runtime.failure_context(false)
9560                    };
9561                    // A stamp cannot adopt an unverified legacy process, even
9562                    // when its approved replacement no longer speaks subc.
9563                    // Stopping that process spends no crash restart budget.
9564                    if child.is_some() {
9565                        restart_child_with_budget_policy(spec, runtime, registry,
9566                            process_liveness, snapshot, child, runtime.drain_timeout, false).await?;
9567                    }
9568                    // The stamp is already committed. A deferred or held launch
9569                    // belongs to the normal retry/hold loop, not a failed apply.
9570                    update_snapshot(snapshot, Some(&spec.module_id), |state| state.respawn_pending = false)?;
9571                    match spawn_and_mark_running_async(spec, runtime, snapshot).await {
9572                        Ok(replacement) => *child = Some(replacement),
9573                        Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9574                            defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, &failure)?;
9575                        }
9576                        Err(SuperviseError::ConfigHeld { .. }) => hold_config_spawn(spec, runtime, snapshot, &failure)?,
9577                        Err(error) => {
9578                            fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
9579                            return Err(error);
9580                        }
9581                    }
9582                }
9583                info!(module_id=%spec.module_id, config_generation=approved.generation(), "approved daemon entry reconciled");
9584                Ok(())
9585            }
9586            .await;
9587            if let Err(error) = &result {
9588                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9589                    if state.enabled
9590                        && state.state == ModuleState::Failed
9591                        && state
9592                            .approved_start_failure
9593                            .as_ref()
9594                            .is_none_or(|(generation, _)| *generation != approved.identity())
9595                    {
9596                        state.approved_start_failure =
9597                            Some((approved.identity(), error.to_string()));
9598                    }
9599                });
9600            }
9601            let _ = reply.send(result);
9602            true
9603        }
9604        SupervisorCommand::ConfigPolicyProbe {
9605            connection,
9606            generation,
9607            attempt,
9608        } => {
9609            let Some(delivery) = &runtime.config_delivery else {
9610                return true;
9611            };
9612            let Ok(gate) = delivery.0.module_gate(&spec.module_id) else {
9613                return true;
9614            };
9615            let guard = gate.lock().await;
9616            let pending = delivery
9617                .0
9618                .on_blocking(move |owner| {
9619                    owner.policy_probe_pending(connection, generation, attempt)
9620                })
9621                .await
9622                .unwrap_or(false)
9623                && child.is_some()
9624                && registry
9625                    .get_module(&spec.module_id)
9626                    .ok()
9627                    .flatten()
9628                    .is_some_and(|registration| registration.connection_id == connection);
9629            drop(guard);
9630            if pending {
9631                run_health_probe_cycle(spec, runtime, registry, process_liveness, snapshot, child)
9632                    .await;
9633            }
9634            true
9635        }
9636        SupervisorCommand::ConfigApplyRestart {
9637            generation,
9638            supersedes_swap,
9639        } => {
9640            let Some(owner) = runtime
9641                .config_delivery
9642                .as_ref()
9643                .map(|delivery| delivery.0.clone())
9644            else {
9645                return true;
9646            };
9647            let Ok(gate) = owner.module_gate(&spec.module_id) else {
9648                return true;
9649            };
9650            let _guard = gate.lock().await;
9651            let spawn = child
9652                .as_ref()
9653                .and_then(|child| child.config_delivery.clone());
9654            let module = spec.module_id.clone();
9655            let current = owner
9656                .on_blocking(move |owner| {
9657                    Ok((supersedes_swap
9658                        || owner
9659                            .pending_config_restarts()?
9660                            .contains(&(module.clone(), generation)))
9661                        && owner.requires_fresher_spawn(&module, spawn.as_ref())?
9662                            == Some(generation))
9663                })
9664                .await
9665                .unwrap_or(false);
9666            if !current
9667                || !lock_snapshot(snapshot).is_ok_and(|state| {
9668                    state.enabled && !state.respawn_pending && state.process_alive
9669                })
9670            {
9671                return true;
9672            }
9673            drop(_guard);
9674            info!(module_id=%spec.module_id, config_generation=generation, "approved configuration restart preserves the crash budget");
9675            if let Err(error) = restart_child_with_budget_policy(
9676                spec,
9677                runtime,
9678                registry,
9679                process_liveness,
9680                snapshot,
9681                child,
9682                runtime.drain_timeout,
9683                false,
9684            )
9685            .await
9686            {
9687                warn!(module_id=%spec.module_id, %error, "approved configuration restart could not be scheduled");
9688            }
9689            true
9690        }
9691        SupervisorCommand::ConfigExitHold { reply } => {
9692            // A notification owns no start, backoff, reload reply or previous-
9693            // instance wait. Only an idle, registered initial start has no work
9694            // to preserve; its initialized startup context can be parked here.
9695            let idle_start = child.is_none()
9696                && lock_snapshot(snapshot).is_ok_and(|state| {
9697                    state.enabled
9698                        && state.state == ModuleState::Starting
9699                        && !state.process_alive
9700                        && !state.respawn_pending
9701                        && state.waiting_for_previous_pid.is_none()
9702                        && state.waiting_for_previous_detail.is_none()
9703                        && state.failed_retry.pending.is_none()
9704                })
9705                && runtime
9706                    .scheduled_respawn
9707                    .lock()
9708                    .unwrap_or_else(|p| p.into_inner())
9709                    .is_none()
9710                && runtime
9711                    .deferred_reload_reply
9712                    .lock()
9713                    .unwrap_or_else(|p| p.into_inner())
9714                    .is_none();
9715            let result = if idle_start {
9716                try_park_config_attempt(
9717                    spec,
9718                    runtime,
9719                    snapshot,
9720                    &runtime.failure_context(runtime.recovery_is_daemon),
9721                    None,
9722                )
9723                .map(|_| ())
9724            } else {
9725                Ok(())
9726            };
9727            let _ = reply.send(result);
9728            true
9729        }
9730        SupervisorCommand::ConfigDeliveryFailed { connection, hold } => {
9731            if registry
9732                .get_module(&spec.module_id)
9733                .ok()
9734                .flatten()
9735                .is_none_or(|registration| registration.connection_id != connection)
9736            {
9737                return true;
9738            }
9739            runtime.recovery_is_daemon = true;
9740            // With a recoverable generation, delivery-failure recovery spends
9741            // one crash-budget entry: the module rejected its configuration.
9742            // Approved configuration restarts spend none.
9743            if hold {
9744                let notice = begin_forwarding_drain_if_configured(
9745                    spec,
9746                    runtime,
9747                    registry,
9748                    snapshot,
9749                    Some(true),
9750                    RouteCloseReason::Disable,
9751                )
9752                .await;
9753                if let Ok(notice) = notice {
9754                    let drained = drain_optional_child(
9755                        &spec.module_id,
9756                        spec.protocol,
9757                        notice,
9758                        registry,
9759                        runtime.forwarding.as_deref(),
9760                        snapshot,
9761                        &runtime.terminal_ring,
9762                        &runtime.spawn_events,
9763                        child,
9764                        runtime.drain_timeout,
9765                        ModuleState::Failed,
9766                        Some(true),
9767                        &runtime.failure_context(true),
9768                    )
9769                    .await;
9770                    if drained.is_ok() {
9771                        match try_park_config_attempt(
9772                            spec,
9773                            runtime,
9774                            snapshot,
9775                            &runtime.failure_context(true),
9776                            None,
9777                        ) {
9778                            Ok(true) => {}
9779                            Ok(false) => {
9780                                if let Err(error) = budgeted_restart_child(
9781                                    spec,
9782                                    runtime,
9783                                    registry,
9784                                    process_liveness,
9785                                    snapshot,
9786                                    child,
9787                                    SupervisorHealthStatus::Unknown,
9788                                    Some("configuration delivery failed after its hold ended"),
9789                                    unix_ms_now(),
9790                                    false,
9791                                )
9792                                .await
9793                                {
9794                                    warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9795                                }
9796                            }
9797                            Err(error) => {
9798                                warn!(module_id=%spec.module_id, %error, "configuration failure could not be parked")
9799                            }
9800                        }
9801                    }
9802                }
9803            } else if let Err(error) = budgeted_restart_child(
9804                spec,
9805                runtime,
9806                registry,
9807                process_liveness,
9808                snapshot,
9809                child,
9810                SupervisorHealthStatus::Unknown,
9811                Some("configuration delivery failed"),
9812                unix_ms_now(),
9813                false,
9814            )
9815            .await
9816            {
9817                warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9818            }
9819            true
9820        }
9821        SupervisorCommand::Drain { requester, reply } => {
9822            // A plain stop runs no forwarding drain, so nothing reaches the
9823            // module over its connection before the wait: ask by signal.
9824            let result = drain_optional_child(
9825                &spec.module_id,
9826                spec.protocol,
9827                StopNotice::NotSent,
9828                registry,
9829                runtime.forwarding.as_deref(),
9830                snapshot,
9831                &runtime.terminal_ring,
9832                &runtime.spawn_events,
9833                child,
9834                runtime.drain_timeout,
9835                ModuleState::Stopped,
9836                None,
9837                &FailureContext::operator(),
9838            )
9839            .await;
9840            containment::terminate(&runtime.child_roster.containment).await;
9841            let registration_released = result.is_ok();
9842            if registration_released {
9843                runtime.record_activity_transition(
9844                    "subc.module_stopped",
9845                    &spec.module_id,
9846                    requester,
9847                );
9848            }
9849            let _ = reply.send(result);
9850            if registration_released {
9851                process_liveness.untrack_if_current(&spec.module_id, snapshot);
9852            }
9853            false
9854        }
9855        SupervisorCommand::Retire { requester, reply } => {
9856            let result = async {
9857                let stop_notice = begin_forwarding_drain_if_configured(
9858                    spec,
9859                    runtime,
9860                    registry,
9861                    snapshot,
9862                    None,
9863                    RouteCloseReason::Disable,
9864                )
9865                .await?;
9866                drain_optional_child(
9867                    &spec.module_id,
9868                    spec.protocol,
9869                    stop_notice,
9870                    registry,
9871                    runtime.forwarding.as_deref(),
9872                    snapshot,
9873                    &runtime.terminal_ring,
9874                    &runtime.spawn_events,
9875                    child,
9876                    runtime.drain_timeout,
9877                    ModuleState::Stopped,
9878                    None,
9879                    &FailureContext::operator(),
9880                )
9881                .await
9882            }
9883            .await;
9884            containment::terminate(&runtime.child_roster.containment).await;
9885            let registration_released = result.is_ok();
9886            if registration_released {
9887                runtime.record_activity_transition(
9888                    "subc.module_stopped",
9889                    &spec.module_id,
9890                    requester,
9891                );
9892            }
9893            let _ = reply.send(result);
9894            if registration_released {
9895                process_liveness.untrack_if_current(&spec.module_id, snapshot);
9896            }
9897            false
9898        }
9899        SupervisorCommand::Restart {
9900            requester,
9901            drain_timeout_ms,
9902            received_at_generation,
9903            queued_at,
9904            reply,
9905        } => {
9906            // Without this line a restart that waited in the queue (behind a
9907            // health probe cycle or another command) was invisible: the log
9908            // showed only the drain timing out, minutes after the operator's call.
9909            info!(
9910                module_id = %spec.module_id,
9911                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
9912                "restart command dequeued"
9913            );
9914            // ACK AT INITIATION, not completion. The blocking form deadlocked any
9915            // caller whose own request lane rides the module being restarted: the
9916            // caller's in-flight request keeps the drain from quiescing, the drain
9917            // keeps the restart from completing, and the completion keeps the reply
9918            // from releasing the caller — so the drain always timed out and cut the
9919            // initiator with a GOODBYE, even on a healthy module. Replying once the
9920            // restart is validated lets a self-lane caller settle, which is exactly
9921            // what makes the drain succeed. Completion is observable via
9922            // supervisor.list / module status; a post-ack failure lands the module
9923            // in a visible terminal state below rather than in a reply nobody can
9924            // receive.
9925            let validation = match lock_snapshot(snapshot) {
9926                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
9927                    module_id: spec.module_id.clone(),
9928                }),
9929                Ok(_) => Ok(()),
9930                Err(err) => Err(err),
9931            };
9932            let initiated = validation.is_ok();
9933            let _ = reply.send(validation);
9934            // A restart asks for a fresh process. Commands run one at a time,
9935            // so a restart queued behind another restart (two operator calls
9936            // in quick succession) is dequeued the moment the first one has
9937            // spawned its replacement -- before that process has sent HELLO.
9938            // Running it would drain and kill the process the first restart
9939            // just produced, which is the opposite of what both callers asked
9940            // for. If a process spawned after this request was received is
9941            // still supervised, the request is already satisfied. Not when the
9942            // configuration changed since that spawn: then the newer process
9943            // predates the spec this restart may exist to apply.
9944            // Generation publication can lag a child that is already serving
9945            // while its identity is recorded. Only process creation after request
9946            // receipt satisfies a restart, not publication after receipt.
9947            let satisfied_by_generation = if initiated
9948                && child
9949                    .as_ref()
9950                    .is_some_and(|child| child.spawned_at > queued_at)
9951            {
9952                lock_snapshot(snapshot).ok().and_then(|state| {
9953                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
9954                })
9955            } else {
9956                None
9957            };
9958            let satisfied_by_pending = initiated
9959                && child.is_none()
9960                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
9961                    // No child exists yet, so this replacement will necessarily
9962                    // be created after the queued request, unlike a published
9963                    // generation whose process may already have been serving.
9964                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
9965                    if pending {
9966                        state.coalesced_restart_pending = true;
9967                    }
9968                    pending
9969                });
9970            if satisfied_by_pending {
9971                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
9972            } else if let Some(generation) = satisfied_by_generation {
9973                info!(
9974                    module_id = %spec.module_id,
9975                    received_at_generation,
9976                    "restart already satisfied by generation {generation}; not restarting again"
9977                );
9978            } else if initiated {
9979                // Precedence: this restart's operator override, else the module's
9980                // configured budget (already resolved into the runtime).
9981                let drain_timeout = drain_timeout_ms
9982                    .map(Duration::from_millis)
9983                    .unwrap_or(runtime.drain_timeout);
9984                if let Err(err) = restart_child(
9985                    spec,
9986                    runtime,
9987                    registry,
9988                    process_liveness,
9989                    snapshot,
9990                    child,
9991                    drain_timeout,
9992                )
9993                .await
9994                {
9995                    warn!(
9996                        module_id = %spec.module_id,
9997                        error = %err,
9998                        "operator restart failed after initiation ack; module state carries the outcome"
9999                    );
10000                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10001                        state.state = ModuleState::Failed;
10002                        clear_current_process_facts(state);
10003                    });
10004                } else {
10005                    runtime.start_requester = requester;
10006                    runtime.pending_activity_restart = true;
10007                }
10008            }
10009            true
10010        }
10011        SupervisorCommand::Reload { requester, reply } => {
10012            let result =
10013                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
10014            if result.is_ok() {
10015                runtime.start_requester = requester;
10016                runtime.pending_activity_restart = true;
10017            }
10018            if result.is_ok()
10019                && runtime
10020                    .scheduled_respawn
10021                    .lock()
10022                    .unwrap_or_else(|p| p.into_inner())
10023                    .as_ref()
10024                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
10025            {
10026                *runtime
10027                    .deferred_reload_reply
10028                    .lock()
10029                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
10030            } else {
10031                let _ = reply.send(result);
10032            }
10033            true
10034        }
10035        SupervisorCommand::SetEnabled {
10036            enabled,
10037            requester,
10038            reply,
10039            reset_budget,
10040        } => {
10041            let previous_requester = runtime.start_requester.clone();
10042            if enabled {
10043                runtime.start_requester = requester.clone();
10044            }
10045            let result = set_child_enabled_with_budget_policy(
10046                spec,
10047                runtime,
10048                registry,
10049                process_liveness,
10050                snapshot,
10051                child,
10052                enabled,
10053                &runtime.failure_context(false),
10054                reset_budget,
10055            )
10056            .await;
10057            if !enabled {
10058                containment::terminate(&runtime.child_roster.containment).await;
10059            }
10060            if matches!(result, Ok(true)) {
10061                runtime.pending_activity_restart = false;
10062                if enabled {
10063                    runtime.start_requester = requester;
10064                } else {
10065                    runtime.start_requester = None;
10066                    runtime.record_activity_transition(
10067                        "subc.module_disabled",
10068                        &spec.module_id,
10069                        requester,
10070                    );
10071                }
10072            } else {
10073                runtime.start_requester = previous_requester;
10074            }
10075            let _ = reply.send(result);
10076            true
10077        }
10078        SupervisorCommand::UpdateConfiguration {
10079            spec: next_spec,
10080            health,
10081            drain_timeout_ms,
10082            reply,
10083        } => {
10084            if runtime
10085                .config_delivery
10086                .as_ref()
10087                .is_some_and(|delivery| delivery.0.file_configuration_is_gated(&spec.module_id))
10088            {
10089                let _ = reply.send(Err(SuperviseError::ConfigHeld {
10090                    module_id: spec.module_id.clone(),
10091                    code: "config_gated".into(),
10092                    detail: "live file reconfiguration cannot replace stamped authority".into(),
10093                }));
10094                return true;
10095            }
10096            stage_supervisor_configuration(
10097                spec,
10098                runtime,
10099                snapshot,
10100                next_spec,
10101                health,
10102                drain_timeout_ms,
10103            );
10104            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10105                state.configuration_updated_since_spawn = true;
10106            });
10107            let _ = reply.send(Ok(()));
10108            true
10109        }
10110        SupervisorCommand::Swap {
10111            requester,
10112            ready_timeout,
10113            reply,
10114        } => {
10115            if spec.carry_containment
10116                || containment::current(&runtime.child_roster.containment).is_some()
10117            {
10118                let _ = reply.send(Err(SuperviseError::InvalidSpec {
10119                    reason: "a carried container cannot overlap a swap candidate".to_owned(),
10120                }));
10121                return true;
10122            }
10123            let mut swap_runtime = runtime.clone();
10124            swap_runtime.start_requester = requester;
10125            let before = lock_snapshot(snapshot)
10126                .ok()
10127                .map(|state| state.spawn_generation);
10128            let end = swap::run_swap(
10129                spec,
10130                &swap_runtime,
10131                registry,
10132                process_liveness,
10133                snapshot,
10134                child,
10135                commands,
10136                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
10137                reply,
10138            )
10139            .await;
10140            if before
10141                != lock_snapshot(snapshot)
10142                    .ok()
10143                    .map(|state| state.spawn_generation)
10144            {
10145                runtime.start_requester = swap_runtime.start_requester;
10146            }
10147            requeued.extend(end.requeue);
10148            true
10149        }
10150    }
10151}
10152
10153fn stage_supervisor_configuration(
10154    spec: &mut ModuleSpec,
10155    runtime: &mut SupervisorRuntimeConfig,
10156    snapshot: &SharedSnapshot,
10157    next_spec: ModuleSpec,
10158    health: HealthConfig,
10159    drain_timeout_ms: Option<u64>,
10160) {
10161    if let Some(handle) = &runtime.supervisor_handle {
10162        handle.apply_identity_configuration(&next_spec);
10163    }
10164    let launch_changed = *spec != next_spec;
10165    *spec = next_spec;
10166    if launch_changed {
10167        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10168            state.configuration_updated_since_spawn = true
10169        });
10170    }
10171    let health_changed = runtime.health != health;
10172    runtime.health = health;
10173    // Reset the cadence and old endpoint's failure streak on a live
10174    // health-policy change rather than waiting for its old deadline.
10175    if health_changed {
10176        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10177            state.health = ModuleHealthStatus::default();
10178            state.last_probe_evidence = None;
10179        });
10180    }
10181    runtime.drain_timeout = drain_timeout_ms
10182        .map(Duration::from_millis)
10183        .unwrap_or(runtime.default_drain_timeout);
10184    *runtime
10185        .effective_drain_timeout
10186        .lock()
10187        .unwrap_or_else(|p| p.into_inner()) = runtime.drain_timeout;
10188}
10189
10190async fn restart_child(
10191    spec: &ModuleSpec,
10192    runtime: &SupervisorRuntimeConfig,
10193    registry: &Registry,
10194    process_liveness: &SupervisorProcessLiveness,
10195    snapshot: &SharedSnapshot,
10196    child: &mut Option<SupervisedChild>,
10197    drain_timeout: Duration,
10198) -> Result<(), SuperviseError> {
10199    restart_child_with_budget_policy(
10200        spec,
10201        runtime,
10202        registry,
10203        process_liveness,
10204        snapshot,
10205        child,
10206        drain_timeout,
10207        true,
10208    )
10209    .await
10210}
10211
10212#[allow(clippy::too_many_arguments)]
10213async fn restart_child_with_budget_policy(
10214    spec: &ModuleSpec,
10215    runtime: &SupervisorRuntimeConfig,
10216    registry: &Registry,
10217    process_liveness: &SupervisorProcessLiveness,
10218    snapshot: &SharedSnapshot,
10219    child: &mut Option<SupervisedChild>,
10220    drain_timeout: Duration,
10221    reset_budget: bool,
10222) -> Result<(), SuperviseError> {
10223    // Restart cycles a running module; it must not silently start a disabled one.
10224    if !lock_snapshot(snapshot)?.enabled {
10225        return Err(SuperviseError::Disabled {
10226            module_id: spec.module_id.clone(),
10227        });
10228    }
10229    let stop_notice = begin_forwarding_drain_with_timeout(
10230        spec,
10231        runtime,
10232        registry,
10233        snapshot,
10234        None,
10235        RouteCloseReason::Restart,
10236        drain_timeout,
10237    )
10238    .await?;
10239
10240    if child.is_some() {
10241        drain_optional_child(
10242            &spec.module_id,
10243            spec.protocol,
10244            stop_notice,
10245            registry,
10246            runtime.forwarding.as_deref(),
10247            snapshot,
10248            &runtime.terminal_ring,
10249            &runtime.spawn_events,
10250            child,
10251            drain_timeout,
10252            ModuleState::Restarting,
10253            Some(true),
10254            &FailureContext::operator(),
10255        )
10256        .await?;
10257    } else {
10258        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10259            state.enabled = true;
10260            state.state = ModuleState::Restarting;
10261            clear_current_process_facts(state);
10262        })?;
10263        release_dead_registration(
10264            registry,
10265            runtime.forwarding.as_deref(),
10266            snapshot,
10267            &spec.module_id,
10268            &runtime.failure_context(false),
10269        )
10270        .await?;
10271    }
10272
10273    if reset_budget {
10274        reset_restart_count(snapshot, &spec.module_id)?;
10275    }
10276    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10277    schedule_respawn(
10278        runtime,
10279        snapshot,
10280        &spec.module_id,
10281        runtime.restart_policy.backoff,
10282        RespawnKind::Spawn,
10283        false,
10284    )
10285}
10286
10287async fn reload_child(
10288    spec: &ModuleSpec,
10289    runtime: &SupervisorRuntimeConfig,
10290    registry: &Registry,
10291    process_liveness: &SupervisorProcessLiveness,
10292    snapshot: &SharedSnapshot,
10293    child: &mut Option<SupervisedChild>,
10294) -> Result<(), SuperviseError> {
10295    // Reload cycles a running module; it must not silently start a disabled one.
10296    if !lock_snapshot(snapshot)?.enabled {
10297        return Err(SuperviseError::Disabled {
10298            module_id: spec.module_id.clone(),
10299        });
10300    }
10301    let stop_notice = begin_forwarding_drain(
10302        spec,
10303        runtime,
10304        registry,
10305        snapshot,
10306        Some(true),
10307        RouteCloseReason::Reload,
10308    )
10309    .await?;
10310
10311    if child.is_some() {
10312        drain_optional_child(
10313            &spec.module_id,
10314            spec.protocol,
10315            stop_notice,
10316            registry,
10317            runtime.forwarding.as_deref(),
10318            snapshot,
10319            &runtime.terminal_ring,
10320            &runtime.spawn_events,
10321            child,
10322            runtime.drain_timeout,
10323            ModuleState::Restarting,
10324            Some(true),
10325            &FailureContext::operator(),
10326        )
10327        .await?;
10328    } else {
10329        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10330            state.enabled = true;
10331            state.state = ModuleState::Restarting;
10332            clear_current_process_facts(state);
10333        })?;
10334        release_dead_registration(
10335            registry,
10336            runtime.forwarding.as_deref(),
10337            snapshot,
10338            &spec.module_id,
10339            &runtime.failure_context(false),
10340        )
10341        .await?;
10342    }
10343
10344    reset_restart_count(snapshot, &spec.module_id)?;
10345    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10346    schedule_respawn(
10347        runtime,
10348        snapshot,
10349        &spec.module_id,
10350        runtime.restart_policy.backoff,
10351        RespawnKind::Reload,
10352        false,
10353    )
10354}
10355
10356async fn finish_reload_child(
10357    spec: &ModuleSpec,
10358    runtime: &SupervisorRuntimeConfig,
10359    registry: &Registry,
10360    process_liveness: &SupervisorProcessLiveness,
10361    snapshot: &SharedSnapshot,
10362    child: &mut Option<SupervisedChild>,
10363) -> Result<(), SuperviseError> {
10364    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10365    let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10366        Ok(next_child) => next_child,
10367        Err(error @ SuperviseError::ConfigDeliveryDeferred { .. }) => {
10368            defer_config_spawn(
10369                spec,
10370                runtime,
10371                snapshot,
10372                RespawnKind::Reload,
10373                &runtime.failure_context(runtime.recovery_is_daemon),
10374            )?;
10375            return Err(error);
10376        }
10377        Err(error @ SuperviseError::ConfigHeld { .. }) => {
10378            let failure = runtime.failure_context(runtime.recovery_is_daemon);
10379            if try_park_config_attempt(spec, runtime, snapshot, &failure, None)? {
10380                return Err(error);
10381            }
10382            defer_config_spawn(spec, runtime, snapshot, RespawnKind::Reload, &failure)?;
10383            return Err(SuperviseError::ConfigDeliveryDeferred {
10384                module_id: spec.module_id.clone(),
10385            });
10386        }
10387        Err(err) => {
10388            return handle_reload_spawn_failure(
10389                spec,
10390                runtime,
10391                process_liveness,
10392                snapshot,
10393                child,
10394                format!("new child failed to spawn: {err}"),
10395            )
10396            .await;
10397        }
10398    };
10399    *child = Some(next_child);
10400
10401    let wait_outcome = {
10402        let active_child = child.as_mut().expect("new reload child was just stored");
10403        wait_for_registration_after_reload(
10404            registry,
10405            &spec.module_id,
10406            snapshot,
10407            active_child,
10408            REGISTRY_RELEASE_TIMEOUT,
10409        )
10410        .await?
10411    };
10412
10413    match wait_outcome {
10414        RegistrationWaitOutcome::Registered => {
10415            child
10416                .as_mut()
10417                .expect("registered reload child")
10418                .registration_deadline = None;
10419            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
10420            Ok(())
10421        }
10422        RegistrationWaitOutcome::Exited(exit_report) => {
10423            if let Some(active_child) = child.as_mut() {
10424                active_child.drain_stderr(&spec.module_id).await;
10425            }
10426            // Keep the exited child's roster guard until its terminal is written.
10427            // Shutdown waits on that guard, not on the child Option used for respawn.
10428            let mut exited_child = child.take().expect("exited reload child is still stored");
10429            #[cfg(test)]
10430            if let Some(gate) = &runtime.test_reload_exit_record_gate {
10431                gate.reached.notify_one();
10432                gate.resume.notified().await;
10433            }
10434            let result = handle_reload_child_registration_failure(
10435                spec,
10436                runtime,
10437                registry,
10438                process_liveness,
10439                snapshot,
10440                child,
10441                ReloadRegistrationFailure {
10442                    exit_report: registration_failure_exit_report(exit_report),
10443                    reason: exited_child
10444                        .spawn_failure
10445                        .clone()
10446                        .unwrap_or_else(|| "new child exited before registering".to_string()),
10447                },
10448            )
10449            .await;
10450            exited_child
10451                .finish_exit(false)
10452                .await
10453                .map_err(|source| SuperviseError::Wait {
10454                    module_id: spec.module_id.clone(),
10455                    source,
10456                })?;
10457            result
10458        }
10459        RegistrationWaitOutcome::TimedOut => {
10460            let mut timed_out_child = child
10461                .take()
10462                .expect("timed-out reload child is still running");
10463            timed_out_child
10464                .start_kill()
10465                .map_err(|source| SuperviseError::Kill {
10466                    module_id: spec.module_id.clone(),
10467                    source,
10468                })?;
10469            let status = timed_out_child
10470                .wait()
10471                .await
10472                .map_err(|source| SuperviseError::Wait {
10473                    module_id: spec.module_id.clone(),
10474                    source,
10475                })?;
10476            timed_out_child.drain_stderr(&spec.module_id).await;
10477            let result = handle_reload_child_registration_failure(
10478                spec,
10479                runtime,
10480                registry,
10481                process_liveness,
10482                snapshot,
10483                child,
10484                ReloadRegistrationFailure {
10485                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
10486                        snapshot,
10487                        &timed_out_child,
10488                        &status,
10489                    )),
10490                    reason: format!(
10491                        "new child did not register within {:?}",
10492                        REGISTRY_RELEASE_TIMEOUT
10493                    ),
10494                },
10495            )
10496            .await;
10497            timed_out_child
10498                .finish_exit(false)
10499                .await
10500                .map_err(|source| SuperviseError::Wait {
10501                    module_id: spec.module_id.clone(),
10502                    source,
10503                })?;
10504            result
10505        }
10506    }
10507}
10508
10509#[allow(clippy::too_many_arguments)]
10510async fn set_child_enabled(
10511    spec: &ModuleSpec,
10512    runtime: &SupervisorRuntimeConfig,
10513    registry: &Registry,
10514    process_liveness: &SupervisorProcessLiveness,
10515    snapshot: &SharedSnapshot,
10516    child: &mut Option<SupervisedChild>,
10517    enabled: bool,
10518    failure: &FailureContext,
10519) -> Result<bool, SuperviseError> {
10520    set_child_enabled_with_budget_policy(
10521        spec,
10522        runtime,
10523        registry,
10524        process_liveness,
10525        snapshot,
10526        child,
10527        enabled,
10528        failure,
10529        true,
10530    )
10531    .await
10532}
10533
10534#[allow(clippy::too_many_arguments)]
10535async fn set_child_enabled_with_budget_policy(
10536    spec: &ModuleSpec,
10537    runtime: &SupervisorRuntimeConfig,
10538    registry: &Registry,
10539    process_liveness: &SupervisorProcessLiveness,
10540    snapshot: &SharedSnapshot,
10541    child: &mut Option<SupervisedChild>,
10542    enabled: bool,
10543    failure: &FailureContext,
10544    reset_budget: bool,
10545) -> Result<bool, SuperviseError> {
10546    let (current_enabled, current_state, respawn_pending) = {
10547        let state = lock_snapshot(snapshot)?;
10548        (state.enabled, state.state, state.respawn_pending)
10549    };
10550    // `start` (enable on an already-enabled module) heals TERMINAL states instead
10551    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
10552    // clean (Stopped) has no live process and no other in-band recovery — the
10553    // operator's start is the explicit recovery act and resets the budget. Without
10554    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
10555    // which the 2026-07-14 aft outage proved is a trap when the failed module is
10556    // the one providing every agent's shell.
10557    let revive_terminal = enabled
10558        && current_enabled
10559        && child.is_none()
10560        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
10561            || (current_state == ModuleState::Restarting && !respawn_pending));
10562    if current_enabled == enabled && !revive_terminal {
10563        return Ok(false);
10564    }
10565
10566    if enabled {
10567        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10568            state.enabled = true;
10569            state.state = ModuleState::Starting;
10570            clear_current_process_facts(state);
10571        })?;
10572        #[cfg(test)]
10573        if runtime.test_seed_stale_facts_before_enable_spawn {
10574            update_snapshot(snapshot, Some(&spec.module_id), |state| {
10575                state.process_alive = true;
10576                state.pid = Some(41);
10577                state.spawned_at_ms = Some(42);
10578                state.spawned_from = Some(PathBuf::from("/spawned/module"));
10579                state.spawned_file_identity = Some(SpawnedFileIdentity {
10580                    device: 43,
10581                    inode: 44,
10582                    #[cfg(windows)]
10583                    windows_image: None,
10584                });
10585            })?;
10586        }
10587        release_dead_registration(
10588            registry,
10589            runtime.forwarding.as_deref(),
10590            snapshot,
10591            &spec.module_id,
10592            failure,
10593        )
10594        .await?;
10595        if reset_budget {
10596            reset_restart_count(snapshot, &spec.module_id)?;
10597        }
10598        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10599        let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10600            Ok(next_child) => next_child,
10601            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
10602                defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
10603                return Ok(true);
10604            }
10605            Err(error @ SuperviseError::ConfigHeld { .. }) => {
10606                if let Err(parking_error) = hold_config_spawn(spec, runtime, snapshot, failure) {
10607                    process_liveness.untrack_if_current(&spec.module_id, snapshot);
10608                    return Err(parking_error);
10609                }
10610                return Err(error);
10611            }
10612            Err(err) => {
10613                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
10614                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10615                return Err(err);
10616            }
10617        };
10618        *child = Some(next_child);
10619        debug!(module_id = %spec.module_id, "supervised module enabled");
10620        Ok(true)
10621    } else {
10622        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10623            state.config_hold_parked = false;
10624            state.config_hold_daemon_initiated = false;
10625            state.config_hold_approval = None;
10626        })?;
10627        let stop_notice = begin_forwarding_drain_if_configured(
10628            spec,
10629            runtime,
10630            registry,
10631            snapshot,
10632            Some(false),
10633            RouteCloseReason::Disable,
10634        )
10635        .await?;
10636        drain_optional_child(
10637            &spec.module_id,
10638            spec.protocol,
10639            stop_notice,
10640            registry,
10641            runtime.forwarding.as_deref(),
10642            snapshot,
10643            &runtime.terminal_ring,
10644            &runtime.spawn_events,
10645            child,
10646            runtime.drain_timeout,
10647            ModuleState::Disabled,
10648            Some(false),
10649            failure,
10650        )
10651        .await?;
10652        debug!(module_id = %spec.module_id, "supervised module disabled");
10653        Ok(true)
10654    }
10655}
10656
10657#[allow(clippy::too_many_arguments)]
10658async fn on_child_exit(
10659    spec: &ModuleSpec,
10660    policy: RestartPolicy,
10661    registry: &Registry,
10662    snapshot: &SharedSnapshot,
10663    terminal_ring: &Arc<Mutex<TerminalRing>>,
10664    spawn_events: &SpawnEventFeed,
10665    roster: &ChildRoster,
10666    exit_report: ExitReport,
10667    daemon_initiated: bool,
10668) -> NextAction {
10669    // Once the daemon has begun shutting down, no exit is a crash to recover
10670    // from: the module is exiting because the daemon is going away (EOF on its
10671    // connection, or a service manager signalling the whole cgroup). Record it
10672    // as such and never schedule a respawn, which would only start a process
10673    // for the shutdown to end again.
10674    if roster.is_closed() {
10675        return on_child_exit_during_daemon_shutdown(
10676            spec,
10677            registry,
10678            snapshot,
10679            terminal_ring,
10680            spawn_events,
10681            exit_report,
10682        )
10683        .await;
10684    }
10685    // Deliberate stops the supervisor asks for (operator stop, disable,
10686    // restart, reload, swap, a health restart, a drain that runs out of budget)
10687    // takes the child out of the supervise loop and reaps it in
10688    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
10689    // that reaches this point is either unrequested or a first-HELLO timeout,
10690    // which is deliberately classified as a crash even after a clean TERM exit.
10691    //
10692    // For a subc-wire module a clean exit is still a stop: those modules are
10693    // written to re-raise SIGTERM, so a stray outside signal already reads as a
10694    // crash, and exiting 0 is a deliberate choice the module made. A
10695    // `protocol: "none"` module is a stock program we cannot change, and many
10696    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
10697    // stop would leave the module down for good after any stray signal, so it
10698    // goes through the crash path instead: it spends restart budget, respawns
10699    // with the crash backoff, and ends `failed` when the budget runs out.
10700    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
10701        && running_protocol(spec, snapshot) == ModuleProtocol::None;
10702    match exit_report.kind {
10703        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
10704            info!(
10705                module_id = %spec.module_id,
10706                exit_code = ?exit_report.code,
10707                exit_signal = ?exit_report.signal,
10708                "supervised module exited cleanly"
10709            );
10710            if let Err(err) = publish_after_terminal_record(
10711                &spec.module_id,
10712                || {
10713                    record_terminal(
10714                        &spec.module_id,
10715                        terminal_ring,
10716                        spawn_events,
10717                        &exit_report,
10718                        TerminalDisposition::Stopped,
10719                    )
10720                },
10721                || {
10722                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10723                        state.state = ModuleState::Stopped;
10724                        clear_current_process_facts(state);
10725                        state.last_exit = Some(exit_report.clone());
10726                    })
10727                },
10728            ) {
10729                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
10730            }
10731            let registration_released = match wait_for_registration_release(
10732                registry,
10733                &spec.module_id,
10734                REGISTRY_RELEASE_TIMEOUT,
10735            )
10736            .await
10737            {
10738                Ok(()) => true,
10739                Err(err) => {
10740                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
10741                    false
10742                }
10743            };
10744            NextAction::Stop {
10745                registration_released,
10746            }
10747        }
10748        ExitKind::Clean | ExitKind::Crash => {
10749            if unrequested_clean_exit_of_protocol_none {
10750                warn!(
10751                    module_id = %spec.module_id,
10752                    exit_code = ?exit_report.code,
10753                    exit_signal = ?exit_report.signal,
10754                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
10755                );
10756            } else {
10757                warn!(
10758                    module_id = %spec.module_id,
10759                    exit_code = ?exit_report.code,
10760                    exit_signal = ?exit_report.signal,
10761                    "supervised module exited abnormally (crash)"
10762                );
10763            }
10764            let mut restart_schedule = None;
10765            let mut disposition = TerminalDisposition::Disabled;
10766            let mut final_state = ModuleState::Disabled;
10767            // Set only when the budget is what stopped the module, so the
10768            // terminal record says which limit was hit rather than leaving
10769            // `failed` to be read as "crashed once, badly".
10770            let mut disposition_detail = lock_snapshot(snapshot)
10771                .ok()
10772                .and_then(|mut state| state.spawn_failure.take());
10773            let now = Instant::now();
10774            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10775                if state.enabled {
10776                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
10777                        // Reserving the last slot exhausts the budget for the
10778                        // replacement, not for this exit. Publish its scheduled
10779                        // recovery atomically with the reservation so connection
10780                        // cleanup sees it while terminal recording still holds
10781                        // the visible state at Running.
10782                        state.respawn_pending = true;
10783                        final_state = ModuleState::Restarting;
10784                        restart_schedule = Some(schedule);
10785                        disposition = TerminalDisposition::Restarting;
10786                    } else {
10787                        final_state = ModuleState::Failed;
10788                        disposition = TerminalDisposition::Failed;
10789                        let budget = policy.budget_exhausted_detail();
10790                        disposition_detail =
10791                            Some(disposition_detail.take().map_or_else(
10792                                || budget.clone(),
10793                                |cause| format!("{cause}; {budget}"),
10794                            ));
10795                    }
10796                } else {
10797                    disposition = TerminalDisposition::Disabled;
10798                }
10799            }) {
10800                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
10801                return NextAction::Stop {
10802                    registration_released: false,
10803                };
10804            }
10805            if disposition == TerminalDisposition::Failed {
10806                // The window is in the message, not only in the fields: this line
10807                // is read in a scrollback where a bare `max_restarts=3` reads as a
10808                // lifetime cap and sends the operator looking for three crashes
10809                // that never happened together.
10810                error!(
10811                    module_id = %spec.module_id,
10812                    max_restarts = policy.max_restarts,
10813                    window_secs = policy.window.as_secs(),
10814                    "module stopped: {}",
10815                    policy.budget_exhausted_detail()
10816                );
10817            }
10818            if let Err(err) = publish_after_terminal_record(
10819                &spec.module_id,
10820                || {
10821                    record_terminal_with_detail(
10822                        &spec.module_id,
10823                        terminal_ring,
10824                        spawn_events,
10825                        &exit_report,
10826                        disposition,
10827                        disposition_detail,
10828                    )
10829                },
10830                || {
10831                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10832                        state.state = final_state;
10833                        clear_current_process_facts(state);
10834                        if final_state == ModuleState::Failed
10835                            && daemon_initiated
10836                            && !roster.is_closed()
10837                        {
10838                            state.failed_retry.schedule(&spec.module_id);
10839                        }
10840                        state.last_exit = Some(exit_report.clone());
10841                    })
10842                },
10843            ) {
10844                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
10845            }
10846
10847            if let Some(schedule) = restart_schedule {
10848                NextAction::Restart {
10849                    schedule: Some(schedule),
10850                }
10851            } else {
10852                let registration_released = match wait_for_registration_release(
10853                    registry,
10854                    &spec.module_id,
10855                    REGISTRY_RELEASE_TIMEOUT,
10856                )
10857                .await
10858                {
10859                    Ok(()) => true,
10860                    Err(err) => {
10861                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
10862                        false
10863                    }
10864                };
10865                NextAction::Stop {
10866                    registration_released,
10867                }
10868            }
10869        }
10870        ExitKind::DeliberateSeverance => {
10871            warn!(
10872                module_id = %spec.module_id,
10873                exit_code = ?exit_report.code,
10874                exit_signal = ?exit_report.signal,
10875                "supervised module exited after deliberate connection severance"
10876            );
10877            let should_restart = match lock_snapshot(snapshot) {
10878                Ok(state) => state.enabled,
10879                Err(err) => {
10880                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
10881                    return NextAction::Stop {
10882                        registration_released: false,
10883                    };
10884                }
10885            };
10886            let final_state = if should_restart {
10887                ModuleState::Restarting
10888            } else {
10889                ModuleState::Disabled
10890            };
10891            if let Err(err) = publish_after_terminal_record(
10892                &spec.module_id,
10893                || {
10894                    record_terminal(
10895                        &spec.module_id,
10896                        terminal_ring,
10897                        spawn_events,
10898                        &exit_report,
10899                        terminal_disposition(final_state),
10900                    )
10901                },
10902                || {
10903                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10904                        state.state = final_state;
10905                        clear_current_process_facts(state);
10906                        state.last_exit = Some(exit_report.clone());
10907                        state.lifetime_restarts += 1;
10908                    })
10909                },
10910            ) {
10911                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
10912                return NextAction::Stop {
10913                    registration_released: false,
10914                };
10915            }
10916
10917            if should_restart {
10918                NextAction::Restart { schedule: None }
10919            } else {
10920                let registration_released = match wait_for_registration_release(
10921                    registry,
10922                    &spec.module_id,
10923                    REGISTRY_RELEASE_TIMEOUT,
10924                )
10925                .await
10926                {
10927                    Ok(()) => true,
10928                    Err(err) => {
10929                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
10930                        false
10931                    }
10932                };
10933                NextAction::Stop {
10934                    registration_released,
10935                }
10936            }
10937        }
10938    }
10939}
10940
10941async fn on_child_exit_during_daemon_shutdown(
10942    spec: &ModuleSpec,
10943    registry: &Registry,
10944    snapshot: &SharedSnapshot,
10945    terminal_ring: &Arc<Mutex<TerminalRing>>,
10946    spawn_events: &SpawnEventFeed,
10947    exit_report: ExitReport,
10948) -> NextAction {
10949    info!(
10950        module_id = %spec.module_id,
10951        exit_code = ?exit_report.code,
10952        exit_signal = ?exit_report.signal,
10953        exit_kind = ?exit_report.kind,
10954        "supervised module exited during daemon shutdown; not restarting it"
10955    );
10956    if let Err(err) = publish_after_terminal_record(
10957        &spec.module_id,
10958        || {
10959            record_terminal(
10960                &spec.module_id,
10961                terminal_ring,
10962                spawn_events,
10963                &exit_report,
10964                TerminalDisposition::DaemonShutdown,
10965            )
10966        },
10967        || {
10968            update_snapshot(snapshot, Some(&spec.module_id), |state| {
10969                state.state = ModuleState::Stopped;
10970                clear_current_process_facts(state);
10971                state.last_exit = Some(exit_report.clone());
10972            })
10973        },
10974    ) {
10975        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
10976    }
10977    let registration_released =
10978        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
10979            .await
10980            .is_ok();
10981    NextAction::Stop {
10982        registration_released,
10983    }
10984}
10985
10986// Keep the status lock out of terminal I/O. Readers may observe the previous
10987// state while recording is in progress, but never a completed exit without its
10988// history. Journal errors are logged by the journal and retained in the ring;
10989// a recording panic still publishes the final state before resuming the unwind.
10990fn publish_after_terminal_record<T>(
10991    module_id: &str,
10992    record: impl FnOnce(),
10993    publish: impl FnOnce() -> T,
10994) -> T {
10995    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
10996    let published = publish();
10997    if let Err(panic) = recorded {
10998        error!(
10999            module_id,
11000            "terminal recording panicked; published final state before resuming unwind"
11001        );
11002        std::panic::resume_unwind(panic);
11003    }
11004    published
11005}
11006
11007fn fail_after_wait_error(
11008    module_id: &str,
11009    snapshot: &SharedSnapshot,
11010    terminal_ring: &Arc<Mutex<TerminalRing>>,
11011    spawn_events: &SpawnEventFeed,
11012    detail: String,
11013    failure: &FailureContext,
11014) {
11015    let report = wait_error_exit_report();
11016    publish_after_terminal_record(
11017        module_id,
11018        || {
11019            record_terminal_with_detail(
11020                module_id,
11021                terminal_ring,
11022                spawn_events,
11023                &report,
11024                TerminalDisposition::Failed,
11025                Some(detail),
11026            )
11027        },
11028        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
11029    );
11030}
11031
11032#[cfg(test)]
11033fn record_wait_error_terminal(
11034    module_id: &str,
11035    terminal_ring: &Arc<Mutex<TerminalRing>>,
11036    spawn_events: &SpawnEventFeed,
11037) {
11038    record_terminal(
11039        module_id,
11040        terminal_ring,
11041        spawn_events,
11042        &wait_error_exit_report(),
11043        TerminalDisposition::Failed,
11044    );
11045}
11046
11047fn record_terminal(
11048    module_id: &str,
11049    terminal_ring: &Arc<Mutex<TerminalRing>>,
11050    spawn_events: &SpawnEventFeed,
11051    exit_report: &ExitReport,
11052    disposition: TerminalDisposition,
11053) {
11054    record_terminal_with_detail(
11055        module_id,
11056        terminal_ring,
11057        spawn_events,
11058        exit_report,
11059        disposition,
11060        None,
11061    );
11062}
11063
11064/// The ring lock is held only to capture the read (see
11065/// `TerminalJournal::capture_read`), so this module's exits keep recording
11066/// while the journal files are read. Blocking: it reads files.
11067fn durable_terminal_history_of(
11068    terminal_ring: &Mutex<TerminalRing>,
11069    module_id: &str,
11070) -> subc_control::TerminalHistory {
11071    let read = terminal_ring
11072        .lock()
11073        .unwrap_or_else(|p| p.into_inner())
11074        .capture_durable_history();
11075    read.read(module_id)
11076}
11077
11078fn record_terminal_with_detail(
11079    module_id: &str,
11080    terminal_ring: &Arc<Mutex<TerminalRing>>,
11081    spawn_events: &SpawnEventFeed,
11082    exit_report: &ExitReport,
11083    disposition: TerminalDisposition,
11084    disposition_detail: Option<String>,
11085) {
11086    #[cfg(test)]
11087    terminal_history_tests::before_terminal_record(spawn_events);
11088    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
11089    let record = TerminalRecord {
11090        exit_code: exit_report.code,
11091        exit_signal: exit_report.signal,
11092        at_ms: exit_report.at_ms,
11093        disposition,
11094        exit_kind: exit_report.kind.into(),
11095        disposition_detail,
11096    };
11097    terminal_ring
11098        .lock()
11099        .unwrap_or_else(|poisoned| poisoned.into_inner())
11100        .record_exit(module_id, record);
11101}
11102
11103fn untrack_if_registration_released(
11104    process_liveness: &SupervisorProcessLiveness,
11105    registry: &Registry,
11106    module_id: &str,
11107    snapshot: &SharedSnapshot,
11108) {
11109    match registry.get_module(module_id) {
11110        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
11111        Ok(Some(_)) => {}
11112        Err(err) => {
11113            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
11114        }
11115    }
11116}
11117
11118/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
11119/// then apply the module's configured entries minus daemon-private capture keys.
11120///
11121/// Separated from `spawn_child` only so it can be asserted without spawning a
11122/// process — a duplicate of this logic in a test would pass while the real one
11123/// drifted, which is the defect class this function exists to avoid.
11124/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
11125/// nonce. A `protocol: "none"` module gets neither, because it cannot use
11126/// either and the argument would stop a stock binary from starting at all.
11127/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
11128///
11129/// The plain-spawn form, kept for the tests that assert its plan; spawns go
11130/// through [`apply_wire_spawn_args_for_admission`].
11131#[cfg(test)]
11132fn apply_wire_spawn_args(
11133    command: &mut Command,
11134    spec: &ModuleSpec,
11135    connection_file_path: Option<&std::path::Path>,
11136    handle: Option<&SupervisorHandle>,
11137) -> Result<Option<NonceHandoff>, SuperviseError> {
11138    apply_wire_spawn_args_for_role(
11139        command,
11140        spec,
11141        connection_file_path,
11142        handle,
11143        SpawnRole::Plain,
11144    )
11145}
11146
11147/// The read end of a spawn's launch-nonce pipe, prepared by
11148/// [`apply_wire_spawn_args_for_admission`] and installed as the child's descriptor 3
11149/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
11150/// server before spawn and authenticates its client after the child exists.
11151#[cfg(unix)]
11152type NonceHandoff = subc_os::LaunchNonceHandoff;
11153#[cfg(windows)]
11154type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
11155#[cfg(not(any(unix, windows)))]
11156type NonceHandoff = std::convert::Infallible;
11157
11158/// Create the launch nonce that a new module process must present when it
11159/// registers or opens a consumer connection, including a replacement process
11160/// started alongside a still-serving module.
11161///
11162/// A plain spawn replaces the module's recorded nonce. A swap candidate records
11163/// a separate candidate token so the still-serving incumbent and its consumers
11164/// keep their nonce. Both records are installed before the process exists, so
11165/// the child's initial HELLO registration cannot arrive ahead of its nonce.
11166///
11167/// On Unix the nonce is delivered only through a pipe. It is written into
11168/// a pipe whose read end the child gets as descriptor 3, named by
11169/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
11170/// process of the same user cannot read it with `ps eww`. That handoff is
11171/// returned rather than installed here. The combined descriptor placement runs
11172/// after the cgroup callback and placement barrier, so replacing slots 3-5
11173/// cannot close a descriptor either callback still needs.
11174/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
11175#[cfg(test)]
11176fn apply_wire_spawn_args_for_role(
11177    command: &mut Command,
11178    spec: &ModuleSpec,
11179    connection_file_path: Option<&std::path::Path>,
11180    handle: Option<&SupervisorHandle>,
11181    role: SpawnRole,
11182) -> Result<Option<NonceHandoff>, SuperviseError> {
11183    apply_wire_spawn_args_for_admission(command, spec, connection_file_path, handle, role, None)
11184}
11185
11186fn apply_wire_spawn_args_for_admission(
11187    command: &mut Command,
11188    spec: &ModuleSpec,
11189    connection_file_path: Option<&std::path::Path>,
11190    handle: Option<&SupervisorHandle>,
11191    role: SpawnRole,
11192    admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11193) -> Result<Option<NonceHandoff>, SuperviseError> {
11194    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
11195    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
11196    // included: a daemon started from a module's process tree inherits it,
11197    // and passing it on would point the child at a descriptor it does not
11198    // have.
11199    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
11200    #[cfg(windows)]
11201    {
11202        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
11203        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
11204    }
11205    // Remove inherited or configured copies too: withholding must mean absent.
11206    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
11207    command.env_remove(subc_os::registration_channel::REGISTRATION_FD_ENV);
11208    command.env_remove(subc_os::registration_channel::REGISTRATION_PIPE_ENV);
11209    command.env_remove(subc_os::registration_channel::DAEMON_PID_ENV);
11210    if role == SpawnRole::Plain {
11211        if let Some(handle) = handle {
11212            #[cfg(all(test, unix))]
11213            {
11214                let pause = {
11215                    let mut slot = handle.plain_identity_policy_pause.lock().unwrap();
11216                    if slot
11217                        .as_ref()
11218                        .is_some_and(|(module, _)| module == &spec.module_id)
11219                    {
11220                        slot.take().map(|(_, pause)| pause)
11221                    } else {
11222                        None
11223                    }
11224                };
11225                if let Some(pause) = pause {
11226                    pause.reached.send(()).unwrap();
11227                    pause.release.recv().unwrap();
11228                }
11229            }
11230            // A rescan preserves the running child's nonce and reservation.
11231            // Replacement spawn preparation applies the new protocol now,
11232            // before protocol:none can return without minting a nonce.
11233            handle.apply_identity_policy(spec, spec.protocol);
11234        }
11235    }
11236    if spec.protocol == ModuleProtocol::None {
11237        return Ok(None);
11238    }
11239    if let Some(connection_file_path) = connection_file_path {
11240        command.arg(SUBC_ARG).arg(connection_file_path);
11241    }
11242
11243    if admission.is_some_and(|admission| admission.launch.is_some()) {
11244        if let Some(handle) = handle {
11245            match role {
11246                SpawnRole::Plain => {
11247                    handle
11248                        .spawn_nonces
11249                        .lock()
11250                        .unwrap_or_else(|p| p.into_inner())
11251                        .remove(&spec.module_id);
11252                    if spec.reserved {
11253                        handle
11254                            .reserved_nonces
11255                            .lock()
11256                            .unwrap_or_else(|p| p.into_inner())
11257                            .insert(spec.module_id.clone(), None);
11258                    }
11259                }
11260                // Swaps still need their lifecycle record, but the empty value
11261                // is not a bearer token and is never accepted by nonce HELLO.
11262                SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, String::new()),
11263            }
11264        }
11265        return Ok(None);
11266    }
11267    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
11268    // route.open attestation. Reserved modules additionally use the same nonce
11269    // for HELLO id-squatting protection. A respawn rotates both records.
11270    let nonce = generate_launch_nonce()?;
11271    if let Some(handle) = handle {
11272        match role {
11273            SpawnRole::Plain => {
11274                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
11275                if spec.reserved {
11276                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
11277                }
11278            }
11279            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
11280        }
11281    }
11282    #[cfg(unix)]
11283    let handoff = {
11284        let handoff =
11285            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
11286                program: spec.program.clone(),
11287                source,
11288                cgroup_path: None,
11289            })?;
11290        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
11291        Some(handoff)
11292    };
11293    #[cfg(windows)]
11294    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
11295    #[cfg(not(any(unix, windows)))]
11296    let handoff = None;
11297    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
11298    // until the daemon's per-spawn delivery records show that every running
11299    // Windows module has read its secret from the named pipe.
11300    #[cfg(not(unix))]
11301    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
11302    Ok(handoff)
11303}
11304
11305pub(crate) fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
11306    command.env_remove(CK_LOG_ENV);
11307    // The spawn role is the supervisor's to set, and only on a swap candidate
11308    // (see `apply_spawn_role`). Removing it here, rather than just not setting
11309    // it, is what makes it absent on a plain spawn: the daemon's own
11310    // environment could carry it, and so could a spec built outside daemon
11311    // config (config refuses it as an `env` key). A module reading it on a
11312    // plain restart would pick the long swap budget and leave callers waiting.
11313    command.env_remove(SUBC_SPAWN_ROLE_ENV);
11314    for (key, value) in &spec.env {
11315        // cortexkit-log currently exposes retention only as a Rust struct, not
11316        // environment names. These values are daemon-private sink metadata and
11317        // must never become a public child-process contract by being inherited.
11318        if matches!(
11319            key.as_str(),
11320            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
11321        ) || key == SUBC_SPAWN_ROLE_ENV
11322        {
11323            continue;
11324        }
11325        command.env(key, value);
11326    }
11327}
11328
11329#[cfg(windows)]
11330fn prepare_windows_registration_pipe(
11331    command: &mut Command,
11332    spec: &ModuleSpec,
11333) -> Result<Option<SpawnRegistrationChannel>, SuperviseError> {
11334    if spec.protocol == ModuleProtocol::None {
11335        return Ok(None);
11336    }
11337    let pipe = subc_os::registration_channel::RegistrationPipe::new().map_err(|source| {
11338        SuperviseError::Spawn {
11339            program: spec.program.clone(),
11340            source,
11341            cgroup_path: None,
11342        }
11343    })?;
11344    command
11345        .env(
11346            subc_os::registration_channel::REGISTRATION_PIPE_ENV,
11347            pipe.name(),
11348        )
11349        .env(
11350            subc_os::registration_channel::DAEMON_PID_ENV,
11351            std::process::id().to_string(),
11352        );
11353    Ok(Some(pipe))
11354}
11355
11356#[cfg(windows)]
11357fn prepare_windows_nonce_pipe(
11358    command: &mut Command,
11359    spec: &ModuleSpec,
11360    nonce: &str,
11361) -> Option<NonceHandoff> {
11362    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
11363        Ok(handoff) => {
11364            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
11365            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
11366            // still register until every module reads the pipe. A reader without
11367            // this variable set to env returns the pipe error instead of using
11368            // an inherited or configured environment copy.
11369            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
11370            Some(handoff)
11371        }
11372        Err(error) => {
11373            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
11374            None
11375        }
11376    }
11377}
11378
11379#[cfg(windows)]
11380fn start_windows_nonce_pipe(
11381    handoff: Option<NonceHandoff>,
11382    pid: u32,
11383    spec: &ModuleSpec,
11384) -> Option<subc_os::LaunchNoncePipeDelivery> {
11385    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
11386    // same maximum wait used for registration during a reload. Older modules
11387    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
11388    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
11389        Ok(delivery) => Some(delivery),
11390        Err(error) => {
11391            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
11392            None
11393        }
11394    })
11395}
11396
11397fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
11398    #[cfg(windows)]
11399    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
11400        return Some(
11401            if snapshot
11402                .pipe_consumed
11403                .as_ref()
11404                .is_some_and(|consumed| consumed.get().is_some())
11405            {
11406                "pipe"
11407            } else {
11408                "env"
11409            }
11410            .to_owned(),
11411        );
11412    }
11413    let _ = snapshot;
11414    None
11415}
11416
11417/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
11418/// of a blue/green swap.
11419#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11420enum SpawnRole {
11421    Plain,
11422    SwapCandidate,
11423}
11424
11425/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
11426/// `apply_child_env` has already removed the variable for every spawn.
11427fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
11428    if role == SpawnRole::SwapCandidate {
11429        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
11430    }
11431}
11432
11433fn previous_instance_error(
11434    spec: &ModuleSpec,
11435    pid: u32,
11436    reason: crate::live_children::RetainedReason,
11437) -> SuperviseError {
11438    SuperviseError::Spawn {
11439        program: spec.program.clone(),
11440        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
11441        cgroup_path: None,
11442    }
11443}
11444
11445fn carries_containment(configured: bool, has_container: bool) -> bool {
11446    configured || has_container
11447}
11448
11449fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
11450    if !carries_containment(
11451        spec.carry_containment,
11452        containment::current(&roster.containment).is_some(),
11453    ) {
11454        return None;
11455    }
11456    Some(SuperviseError::Spawn {
11457        program: spec.program.clone(),
11458        source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
11459        cgroup_path: None,
11460    })
11461}
11462
11463#[allow(clippy::too_many_arguments)]
11464fn spawn_child(
11465    spec: &ModuleSpec,
11466    connection_file_path: Option<&std::path::Path>,
11467    handle: Option<&SupervisorHandle>,
11468    ring: &Arc<Mutex<StderrRing>>,
11469    capture_logs_dir: Option<&std::path::Path>,
11470    roster: &ChildRoster,
11471    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11472    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11473    register_timeout: Duration,
11474) -> Result<SupervisedChild, SuperviseError> {
11475    #[cfg(all(test, unix))]
11476    if let Some(source) = containment_tests::join_failure(spec, roster) {
11477        containment::terminate_blocking(&roster.containment);
11478        return Err(SuperviseError::Spawn {
11479            program: spec.program.clone(),
11480            source,
11481            cgroup_path: None,
11482        });
11483    }
11484    let result = spawn_child_in_slot(
11485        spec,
11486        connection_file_path,
11487        handle,
11488        ring,
11489        capture_logs_dir,
11490        roster,
11491        config_delivery,
11492        #[cfg(target_os = "linux")]
11493        cgroup_placement,
11494        SpawnRole::Plain,
11495        false,
11496        register_timeout,
11497    );
11498    if result.is_err() {
11499        if roster.admission_pending(&spec.module_id) {
11500            containment::terminate_deferred(&roster.containment);
11501        } else {
11502            containment::terminate_blocking(&roster.containment);
11503        }
11504    }
11505    result
11506}
11507
11508/// Spawn one process of `spec` into a slot.
11509///
11510/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
11511/// A swap candidate needs a different cgroup from the process it is replacing,
11512/// which is still alive: in the same cgroup the two would be one kill domain,
11513/// and killing a failed candidate could take the incumbent with it.
11514///
11515/// The stderr capture file is `<module_id>.stderr.log` for every process of
11516/// the module, whichever slot it is in, because that is the one file
11517/// `ck module logs` reads. During a swap's overlap both processes append to it;
11518/// the daemon writes whole lines, so the two interleave by line, which is also
11519/// the merged view an operator wants while a swap runs.
11520#[allow(clippy::too_many_arguments)]
11521fn spawn_child_in_slot(
11522    spec: &ModuleSpec,
11523    connection_file_path: Option<&std::path::Path>,
11524    handle: Option<&SupervisorHandle>,
11525    ring: &Arc<Mutex<StderrRing>>,
11526    capture_logs_dir: Option<&std::path::Path>,
11527    roster: &ChildRoster,
11528    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11529    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11530    role: SpawnRole,
11531    alternate_slot: bool,
11532    register_timeout: Duration,
11533) -> Result<SupervisedChild, SuperviseError> {
11534    let prepared = prepare_child_in_slot(
11535        spec,
11536        connection_file_path,
11537        handle,
11538        capture_logs_dir,
11539        roster,
11540        config_delivery,
11541        #[cfg(target_os = "linux")]
11542        cgroup_placement,
11543        role,
11544        alternate_slot,
11545        false,
11546    )?;
11547    let gate = config_delivery
11548        .map(|delivery| delivery.module_gate(spec))
11549        .transpose()?;
11550    let _gate = gate
11551        .as_ref()
11552        .map(|gate| {
11553            gate.try_lock()
11554                .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11555                    module_id: spec.module_id.clone(),
11556                })
11557        })
11558        .transpose()?;
11559    if let (Some(delivery), Some(admission)) = (config_delivery, &prepared.spawn_admission) {
11560        delivery.check(spec, admission)?;
11561    }
11562    let checked_base = prepared
11563        .spawn_admission
11564        .as_ref()
11565        .and_then(|admission| admission.approved_base())
11566        .cloned();
11567    let mut result = launch_child_in_slot(spec, ring, roster, prepared, register_timeout);
11568    drop(_gate);
11569    if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11570        (config_delivery, &result)
11571    {
11572        if let Some(base) = checked_base.as_ref().filter(|_| code == "program_refused") {
11573            if !delivery.record_refusal(spec, base, detail)? {
11574                result = Err(spawn_plan_stale(spec));
11575            }
11576        }
11577    }
11578    result
11579}
11580
11581struct AsyncInitialIdentity<'a> {
11582    supervisor: &'a Supervisor,
11583    module: String,
11584    committed: bool,
11585}
11586impl Drop for AsyncInitialIdentity<'_> {
11587    fn drop(&mut self) {
11588        if !self.committed {
11589            self.supervisor.abandon_unrostered(&self.module);
11590        }
11591    }
11592}
11593
11594/// Child-free preparation can be abandoned without orphaning a process. After
11595/// this value returns, the async caller validates its launch witness and creates
11596/// and publishes the child without another await.
11597struct PreparedChild {
11598    script_delivery: Option<script::Delivery>,
11599    registration_spawn: Option<Arc<RegistrationSpawn>>,
11600    registration_handle: Option<SupervisorHandle>,
11601    #[cfg(any(unix, windows))]
11602    registration_channel: Option<SpawnRegistrationChannel>,
11603    config_context: Option<config_delivery::ConfigDelivery>,
11604    binding_publication: Option<ProgramBindingPublication>,
11605    launch_spec: ModuleSpec,
11606    spawn_admission: Option<crate::config_gate::runtime::SpawnAdmission>,
11607    command: Command,
11608    carrying: bool,
11609    carried: Option<Arc<containment::CarriedContainment>>,
11610    #[cfg(unix)]
11611    launch_identity: Option<containment::LaunchIdentity>,
11612    #[cfg(unix)]
11613    group_target: containment::GroupTarget,
11614    #[cfg(target_os = "linux")]
11615    cgroup_placement: Option<subc_cgroup::Placement>,
11616    #[cfg(target_os = "linux")]
11617    cgroup_name: String,
11618    #[cfg(target_os = "linux")]
11619    cgroup_cleanup: PreparedCgroup,
11620    cgroup_path: Option<PathBuf>,
11621    #[cfg(unix)]
11622    admission_gate: Option<admission::Gate>,
11623    output_sink: ChildOutputSink,
11624    #[cfg(target_os = "macos")]
11625    privacy_exec: Option<PrivacyExec>,
11626    #[cfg(windows)]
11627    image_capture: subc_os::ExecutableCapture,
11628    #[cfg(windows)]
11629    nonce_handoff: Option<NonceHandoff>,
11630    spawned_at: Instant,
11631    config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
11632    delivery_registration: Option<config_delivery::RegistrationGuard>,
11633    program_binding: Option<SpawnProgramBinding>,
11634}
11635
11636#[cfg(target_os = "linux")]
11637struct PreparedCgroup(Option<(subc_cgroup::Placement, String)>);
11638#[cfg(target_os = "linux")]
11639impl Drop for PreparedCgroup {
11640    fn drop(&mut self) {
11641        if let Some((placement, name)) = &self.0 {
11642            remove_module_cgroup(placement, name);
11643        }
11644    }
11645}
11646
11647pub(crate) struct SpawnProgramBinding {
11648    pub(crate) path: PathBuf,
11649    // Keep the inode alive if the configured name is replaced while preparation
11650    // is suspended, so replacement cannot pass through inode reuse.
11651    _file: std::fs::File,
11652    pub(crate) identity: subc_os::RetainedFileIdentity,
11653}
11654
11655impl SpawnProgramBinding {
11656    pub(crate) fn capture(spec: &ModuleSpec) -> Result<Self, SuperviseError> {
11657        Self::capture_after_resolution(spec, |_| {})
11658    }
11659
11660    fn capture_after_resolution(
11661        spec: &ModuleSpec,
11662        before_open: impl FnOnce(&std::path::Path),
11663    ) -> Result<Self, SuperviseError> {
11664        require_program_search_path(spec)?;
11665        #[cfg(windows)]
11666        let windows_capture =
11667            windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
11668                program: spec.program.clone(),
11669                source,
11670                cgroup_path: None,
11671            })?;
11672        #[cfg(windows)]
11673        let path = windows_capture.path().to_path_buf();
11674        #[cfg(not(windows))]
11675        let path = if spec.program.is_absolute() || spec.program.components().count() > 1 {
11676            spec.program.clone()
11677        } else {
11678            let path = spec
11679                .env
11680                .iter()
11681                .rev()
11682                .find(|(key, _)| key == "PATH")
11683                .map(|(_, path)| std::ffi::OsString::from(path))
11684                .or_else(|| std::env::var_os("PATH"))
11685                .unwrap_or_else(|| {
11686                    if cfg!(target_os = "macos") {
11687                        // Match Darwin's spawn search when PATH is absent, not
11688                        // the current-directory search of an explicitly empty PATH.
11689                        "/usr/bin:/bin".into()
11690                    } else {
11691                        Default::default()
11692                    }
11693                });
11694            std::env::split_paths(&path)
11695                .map(|dir| dir.join(&spec.program))
11696                .find(|path| {
11697                    if !path.is_file() {
11698                        return false;
11699                    }
11700                    #[cfg(unix)]
11701                    {
11702                        rustix::fs::accessat(
11703                            rustix::fs::CWD,
11704                            path,
11705                            rustix::fs::Access::EXEC_OK,
11706                            rustix::fs::AtFlags::EACCESS,
11707                        )
11708                        .is_ok()
11709                    }
11710                    #[cfg(not(unix))]
11711                    {
11712                        true
11713                    }
11714                })
11715                .ok_or_else(|| SuperviseError::Spawn {
11716                    program: spec.program.clone(),
11717                    source: io::Error::new(io::ErrorKind::NotFound, "executable not found in PATH"),
11718                    cgroup_path: None,
11719                })?
11720        };
11721        let resolved = std::fs::canonicalize(&path).map_err(|source| SuperviseError::Spawn {
11722            program: spec.program.clone(),
11723            source,
11724            cgroup_path: None,
11725        })?;
11726        before_open(&resolved);
11727        // Resolving links does not open their target. The descriptor open must
11728        // remain nonblocking and no-follow if that resolved name is substituted.
11729        let file =
11730            subc_os::open_regular_nofollow(&resolved).map_err(|source| SuperviseError::Spawn {
11731                program: spec.program.clone(),
11732                source: io::Error::other(source),
11733                cgroup_path: None,
11734            })?;
11735        let identity =
11736            subc_os::retained_file_identity(&file).map_err(|source| SuperviseError::Spawn {
11737                program: spec.program.clone(),
11738                source,
11739                cgroup_path: None,
11740            })?;
11741        Ok(Self {
11742            path,
11743            _file: file,
11744            identity,
11745        })
11746    }
11747    fn current(&self) -> bool {
11748        #[cfg(unix)]
11749        {
11750            subc_os::file_identity(&self.path) == Some(self.identity)
11751        }
11752        #[cfg(windows)]
11753        {
11754            std::fs::canonicalize(&self.path)
11755                .ok()
11756                .and_then(|path| subc_os::open_regular_nofollow(&path).ok())
11757                .and_then(|file| subc_os::retained_file_identity(&file).ok())
11758                == Some(self.identity)
11759        }
11760    }
11761}
11762
11763fn spawn_plan_stale(spec: &ModuleSpec) -> SuperviseError {
11764    SuperviseError::ConfigDeliveryDeferred {
11765        module_id: spec.module_id.clone(),
11766    }
11767}
11768
11769fn require_program_search_path(spec: &ModuleSpec) -> Result<(), SuperviseError> {
11770    if !cfg!(target_os = "macos")
11771        && !spec.program.is_absolute()
11772        && spec.program.components().count() == 1
11773        && !spec.env.iter().any(|(key, _)| {
11774            if cfg!(windows) {
11775                key.eq_ignore_ascii_case("PATH")
11776            } else {
11777                key == "PATH"
11778            }
11779        })
11780        && std::env::var_os("PATH").is_none()
11781    {
11782        // The OS default search is not a current-directory search. Refusing an
11783        // unspecified search path prevents approving a decoy in the current directory.
11784        return Err(SuperviseError::Spawn {
11785            program: spec.program.clone(),
11786            source: io::Error::new(io::ErrorKind::InvalidInput, "bare program requires PATH"),
11787            cgroup_path: None,
11788        });
11789    }
11790    Ok(())
11791}
11792
11793async fn spawn_child_async(
11794    spec: &ModuleSpec,
11795    runtime: &SupervisorRuntimeConfig,
11796) -> Result<SupervisedChild, SuperviseError> {
11797    #[cfg(all(test, unix))]
11798    if let Some(source) = containment_tests::join_failure(spec, &runtime.child_roster) {
11799        containment::terminate_blocking(&runtime.child_roster.containment);
11800        return Err(SuperviseError::Spawn {
11801            program: spec.program.clone(),
11802            source,
11803            cgroup_path: None,
11804        });
11805    }
11806    let result = spawn_child_in_slot_async(spec, runtime, SpawnRole::Plain, false).await;
11807    if result.is_err() {
11808        if runtime.child_roster.admission_pending(&spec.module_id) {
11809            containment::terminate_deferred(&runtime.child_roster.containment);
11810        } else {
11811            containment::terminate_blocking(&runtime.child_roster.containment);
11812        }
11813    }
11814    result
11815}
11816
11817async fn spawn_child_in_slot_async(
11818    spec: &ModuleSpec,
11819    runtime: &SupervisorRuntimeConfig,
11820    role: SpawnRole,
11821    alternate_slot: bool,
11822) -> Result<SupervisedChild, SuperviseError> {
11823    let gate = runtime
11824        .config_delivery
11825        .as_ref()
11826        .map(|delivery| delivery.0.module_gate(&spec.module_id))
11827        .transpose()
11828        .map_err(|error| SuperviseError::InvalidSpec {
11829            reason: error.to_string(),
11830        })?;
11831    if gate.as_ref().is_some_and(|gate| gate.try_lock().is_err()) {
11832        return Err(SuperviseError::ConfigDeliveryDeferred {
11833            module_id: spec.module_id.clone(),
11834        });
11835    }
11836    let prepared_spec = spec.clone();
11837    let prepared_runtime = runtime.clone();
11838    let prepared = tokio::task::spawn_blocking(move || {
11839        let prepared = prepare_child_in_slot(
11840            &prepared_spec,
11841            prepared_runtime.connection_file_path.as_deref(),
11842            prepared_runtime.supervisor_handle.as_ref(),
11843            prepared_runtime.capture_logs_dir.as_deref(),
11844            &prepared_runtime.child_roster,
11845            prepared_runtime.spawn_delivery(),
11846            #[cfg(target_os = "linux")]
11847            prepared_runtime.cgroup_placement.as_ref(),
11848            role,
11849            alternate_slot,
11850            true,
11851        );
11852        prepared
11853    })
11854    .await
11855    .map_err(|_| SuperviseError::InvalidSpec {
11856        reason: "configuration launch preparation stopped".into(),
11857    })??;
11858    if let Some(registration) = &prepared.delivery_registration {
11859        if !registration.launch_current()? {
11860            return Err(spawn_plan_stale(spec));
11861        }
11862    }
11863    let _gate = match gate {
11864        Some(gate) => {
11865            Some(
11866                gate.try_lock_owned()
11867                    .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11868                        module_id: spec.module_id.clone(),
11869                    })?,
11870            )
11871        }
11872        None => None,
11873    };
11874    if let (Some(delivery), Some(admission)) = (runtime.spawn_delivery(), &prepared.spawn_admission)
11875    {
11876        delivery.check(spec, admission)?;
11877    }
11878    if prepared
11879        .spawn_admission
11880        .as_ref()
11881        .is_none_or(|admission| admission.launch.is_none())
11882        && prepared
11883            .program_binding
11884            .as_ref()
11885            .is_some_and(|binding| !binding.current())
11886    {
11887        return Err(spawn_plan_stale(spec));
11888    }
11889    // No await may separate this last identity check from ownership publication.
11890    let checked_base = prepared
11891        .spawn_admission
11892        .as_ref()
11893        .and_then(|admission| admission.approved_base())
11894        .cloned();
11895    let mut result = launch_child_in_slot(
11896        spec,
11897        &runtime.stderr_ring,
11898        &runtime.child_roster,
11899        prepared,
11900        runtime.register_timeout,
11901    );
11902    drop(_gate);
11903    if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11904        (runtime.config_delivery.as_ref(), &result)
11905    {
11906        if let Some(base) = checked_base.filter(|_| code == "program_refused") {
11907            let module = spec.module_id.clone();
11908            let identity = subc_config::raw_digest(detail.as_bytes());
11909            let recorded = delivery
11910                .0
11911                .on_blocking(move |owner| owner.record_program_refusal(&module, &base, identity))
11912                .await;
11913            match recorded {
11914                Ok(true) => {}
11915                Ok(false) => result = Err(spawn_plan_stale(spec)),
11916                Err(error) => {
11917                    result = Err(config_delivery::gate_hold_error(
11918                        spec,
11919                        error,
11920                        Some(delivery),
11921                    ))
11922                }
11923            }
11924        }
11925    }
11926    result
11927}
11928
11929fn validate_spawn_program(
11930    spec: &ModuleSpec,
11931    admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11932    delivery: Option<config_delivery::SpawnDelivery<'_>>,
11933) -> Result<(), SuperviseError> {
11934    if let Some(admission) = admission {
11935        if let Some(program) = &admission.launch {
11936            if let Err(error) = program.validate_for_spawn() {
11937                // A captured unsigned image is a definite refusal. Uncertain
11938                // reads are deferred by validate_for_spawn, not attributed to
11939                // the operator as a changed or unapprovable program.
11940                if let (Some(context), Some(base)) = (delivery, admission.approved_base()) {
11941                    if error.code == "program_refused"
11942                        && !context.record_refusal(spec, base, &error.detail)?
11943                    {
11944                        return Err(spawn_plan_stale(spec));
11945                    }
11946                }
11947                return Err(config_delivery::gate_hold_error(
11948                    spec,
11949                    error,
11950                    delivery.map(|context| context.runtime()).as_ref(),
11951                ));
11952            }
11953        }
11954    }
11955    Ok(())
11956}
11957
11958#[allow(clippy::too_many_arguments)]
11959fn prepare_child_in_slot(
11960    spec: &ModuleSpec,
11961    connection_file_path: Option<&std::path::Path>,
11962    handle: Option<&SupervisorHandle>,
11963    capture_logs_dir: Option<&std::path::Path>,
11964    roster: &ChildRoster,
11965    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11966    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11967    role: SpawnRole,
11968    alternate_slot: bool,
11969    asynchronous: bool,
11970) -> Result<PreparedChild, SuperviseError> {
11971    require_program_search_path(spec)?;
11972    let spawn_admission = config_delivery
11973        .map(|delivery| delivery.prepare(spec))
11974        .transpose()?;
11975    #[cfg(target_os = "macos")]
11976    let gated = spawn_admission
11977        .as_ref()
11978        .is_some_and(|admission| admission.approved_base().is_some());
11979    let mut launch_spec = spawn_admission
11980        .as_ref()
11981        .and_then(|admission| admission.launch.as_ref())
11982        .map(|launch| launch.spec.clone())
11983        .unwrap_or_else(|| spec.clone());
11984    if let Some(script) = &launch_spec.script {
11985        launch_spec.program =
11986            script::runner_path(script.runtime).map_err(|source| SuperviseError::Spawn {
11987                program: launch_spec.program.clone(),
11988                source,
11989                cgroup_path: None,
11990            })?;
11991    }
11992    let script_content = if let Some(entry) = &launch_spec.script {
11993        Some(
11994            spawn_admission
11995                .as_ref()
11996                .and_then(|admission| admission.launch.as_ref())
11997                .and_then(|launch| launch.script.clone())
11998                .map_or_else(|| script::ScriptBytes::read(&entry.path), Ok)
11999                .map_err(|error| config_delivery::gate_hold_error(&launch_spec, error, None))?,
12000        )
12001    } else {
12002        None
12003    };
12004    let spec = &launch_spec;
12005    if let Some(error) = recovery_block_error(spec, roster) {
12006        return Err(error);
12007    }
12008    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
12009        return Err(previous_instance_error(spec, pid, reason));
12010    }
12011    if roster.is_closed() {
12012        return Err(SuperviseError::Spawn {
12013            program: spec.program.clone(),
12014            source: io::Error::other("the daemon is shutting down; not starting a new process"),
12015            cgroup_path: None,
12016        });
12017    }
12018    let carried = containment::current(&roster.containment);
12019    let carrying = carries_containment(spec.carry_containment, carried.is_some());
12020    if role != SpawnRole::Plain && carrying {
12021        return Err(SuperviseError::InvalidSpec {
12022            reason: "a carried container cannot overlap a swap candidate".to_owned(),
12023        });
12024    }
12025    #[cfg(unix)]
12026    let launch_identity = if carrying && carried.is_none() {
12027        Some(
12028            containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
12029                program: spec.program.clone(),
12030                source,
12031                cgroup_path: None,
12032            })?,
12033        )
12034    } else {
12035        None
12036    };
12037    #[cfg(target_os = "linux")]
12038    let cgroup_placement = match &carried {
12039        Some(container) => container.cgroup().map(|(placement, _)| placement),
12040        None => cgroup_placement,
12041    };
12042    #[cfg(unix)]
12043    let group_target = if let Some(container) = carried
12044        .as_ref()
12045        .filter(|container| container.group().is_some())
12046    {
12047        containment::GroupTarget::Pinned(Arc::clone(container))
12048    } else {
12049        #[cfg(target_os = "linux")]
12050        let cgroup_container = carrying && cgroup_placement.is_some();
12051        #[cfg(not(target_os = "linux"))]
12052        let cgroup_container = false;
12053        if cgroup_container {
12054            containment::GroupTarget::NoGroup
12055        } else {
12056            containment::GroupTarget::OwnLeader
12057        }
12058    };
12059    #[cfg(target_os = "linux")]
12060    let cgroup_name = {
12061        // Slot names alone are not kill domains: a retired incumbent may still
12062        // be draining when a later enable/restart spawns into the same slot.
12063        // Decimal entropy keeps the suffix unambiguous; Placement performs
12064        // the module-id escaping and constructs the filesystem path.
12065        if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
12066            name.to_owned()
12067        } else if cgroup_placement.is_none() {
12068            swap::cgroup_name(&spec.module_id, alternate_slot)
12069        } else {
12070            let nonce = generate_launch_nonce()?;
12071            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
12072            // Leave room for byte escaping and the suffix under NAME_MAX. The
12073            // label is only for humans; the nonce identifies the kill domain.
12074            let mut end = spec.module_id.len().min(64);
12075            while !spec.module_id.is_char_boundary(end) {
12076                end -= 1;
12077            }
12078            format!(
12079                "{}_{suffix}",
12080                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
12081            )
12082        }
12083    };
12084    #[cfg(not(target_os = "linux"))]
12085    let _ = alternate_slot;
12086    #[cfg(windows)]
12087    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
12088        program: spec.program.clone(),
12089        source,
12090        cgroup_path: None,
12091    })?;
12092    #[cfg(windows)]
12093    let program_binding = if asynchronous {
12094        let mut resolved = spec.clone();
12095        resolved.program = image_capture.path().to_path_buf();
12096        Some(SpawnProgramBinding::capture(&resolved)?)
12097    } else {
12098        None
12099    };
12100    #[cfg(not(windows))]
12101    let program_binding = (asynchronous || cfg!(target_os = "macos"))
12102        .then(|| SpawnProgramBinding::capture(spec))
12103        .transpose()?;
12104    #[cfg(target_os = "macos")]
12105    validate_spawn_program(spec, spawn_admission.as_ref(), config_delivery)?;
12106    #[cfg(target_os = "macos")]
12107    let (mut command, privacy_exec, exec_ack) = privacy_command(
12108        spec,
12109        program_binding
12110            .as_ref()
12111            .expect("macOS captures every launch"),
12112        roster,
12113        // Ungated launches retain legacy unsuspended exec, including scripts. Suspended
12114        // verification admits approved programs only. prepare() above fails
12115        // closed if the gate cannot decide; an image sample is not authority.
12116        gated,
12117    )?;
12118    #[cfg(windows)]
12119    let mut command = Command::new(image_capture.path());
12120    #[cfg(not(any(target_os = "macos", windows)))]
12121    let mut command = Command::new(
12122        program_binding
12123            .as_ref()
12124            .map_or(spec.program.as_path(), |binding| binding.path.as_path()),
12125    );
12126    command.args(&spec.args);
12127    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
12128    // that is the whole of the intent, so remove that one key rather than the
12129    // environment.
12130    //
12131    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
12132    // and took the POSIX environment with it. Modules spawned that way had no
12133    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
12134    // logging:
12135    //
12136    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
12137    //     both unset it fell back to the temp dir alone and `ck` could not find
12138    //     a daemon running on the same machine from inside any module's process
12139    //     tree — reporting a path the file has never lived at, which reads as
12140    //     "the daemon did not write its file".
12141    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
12142    //     the RELATIVE `.local/share`, so a module deriving its own store path
12143    //     resolved it against its own CWD. That is the store-fragmentation
12144    //     defect the daemon already refuses in config (`parse_doc` rejects a
12145    //     relative `storage.data_home`) arriving by derivation instead.
12146    //   * anything a module spawns inherited it: git without ~/.gitconfig,
12147    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
12148    //     quietly rather than erroring.
12149    //
12150    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
12151    // offered one candidate under /tmp while the file sat in /run/user/1000.
12152    //
12153    // A configured module is unaffected either way: `module_spec()` puts the
12154    // resolved CK_LOG into `spec.env`, which is applied below and therefore
12155    // wins over anything ambient.
12156    apply_child_env(&mut command, spec);
12157    if spec.script.is_some() {
12158        script::remove_runtime_env(&mut command, spec);
12159    }
12160    let origin_witness = crate::program_origin::LaunchWitness::capture_command(spec, &command)
12161        .map_err(|source| SuperviseError::Spawn {
12162            program: spec.program.clone(),
12163            source,
12164            cgroup_path: None,
12165        })?;
12166    command.current_dir(&origin_witness.working_directory);
12167    apply_spawn_role(&mut command, role);
12168    let nonce_handoff = apply_wire_spawn_args_for_admission(
12169        &mut command,
12170        spec,
12171        connection_file_path,
12172        handle,
12173        role,
12174        spawn_admission.as_ref(),
12175    )?;
12176    #[cfg(unix)]
12177    let mut slots = subc_os::registration_channel::ChildDescriptorSlots::default();
12178    command.env_remove(subc_os::script_pipe::SCRIPT_FD_ENV);
12179    let script_delivery = script_content
12180        .map(|content| {
12181            script::Delivery::prepare(
12182                content,
12183                &mut command,
12184                #[cfg(unix)]
12185                &mut slots,
12186            )
12187        })
12188        .transpose()
12189        .map_err(|source| SuperviseError::Spawn {
12190            program: spec.program.clone(),
12191            source,
12192            cgroup_path: None,
12193        })?;
12194    #[cfg(unix)]
12195    let registration_channel = if spec.protocol == ModuleProtocol::Subc {
12196        let (parent, child, metadata) = subc_os::registration_channel::registration_socketpair()
12197            .map_err(|source| SuperviseError::Spawn {
12198                program: spec.program.clone(),
12199                source,
12200                cgroup_path: None,
12201            })?;
12202        slots
12203            .channel(child)
12204            .map_err(|source| SuperviseError::Spawn {
12205                program: spec.program.clone(),
12206                source,
12207                cgroup_path: None,
12208            })?;
12209        command
12210            .env(subc_os::registration_channel::REGISTRATION_FD_ENV, metadata)
12211            .env(
12212                subc_os::registration_channel::DAEMON_PID_ENV,
12213                std::process::id().to_string(),
12214            );
12215        Some(parent)
12216    } else {
12217        None
12218    };
12219    #[cfg(windows)]
12220    let registration_channel = prepare_windows_registration_pipe(&mut command, spec)?;
12221    let binding_publication = if spawn_admission
12222        .as_ref()
12223        .is_some_and(|admission| admission.launch.is_some())
12224    {
12225        handle.and_then(|handle| handle.begin_program_binding(&spec.module_id, role))
12226    } else {
12227        None
12228    };
12229    let registration_spawn = (spec.protocol == ModuleProtocol::Subc)
12230        .then(|| {
12231            handle.map(|handle| {
12232                let spawn = RegistrationSpawn::new_with_binding_requirement(
12233                    handle,
12234                    &spec.module_id,
12235                    role,
12236                    binding_publication.clone(),
12237                    spawn_admission
12238                        .as_ref()
12239                        .is_some_and(|admission| admission.launch.is_some()),
12240                );
12241                if let Some(admission) = &spawn_admission {
12242                    admission.initialize_origin(&spawn.authority, &origin_witness);
12243                }
12244                handle.record_nonce_launch(&spawn);
12245                spawn
12246            })
12247        })
12248        .flatten();
12249
12250    #[cfg(target_os = "linux")]
12251    let cgroup_path = cgroup_placement
12252        .map(|placement| {
12253            if let Some(container) = &carried {
12254                let path = container
12255                    .cgroup_path()
12256                    .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
12257                placement.existing_module_path(path)?.ok_or_else(|| {
12258                    io::Error::new(
12259                        io::ErrorKind::NotFound,
12260                        "carried cgroup disappeared before replacement",
12261                    )
12262                })
12263            } else {
12264                if carrying {
12265                    containment::prepare_cgroup_path(placement, &cgroup_name)
12266                } else {
12267                    placement.module_path(&cgroup_name)
12268                }
12269            }
12270        })
12271        .transpose()
12272        .map_err(|source| SuperviseError::Cgroup {
12273            module_id: spec.module_id.clone(),
12274            source,
12275        })?;
12276    #[cfg(not(target_os = "linux"))]
12277    let cgroup_path: Option<PathBuf> = None;
12278    // Own a newly created directory before any fallible admission or command
12279    // preparation. Existing carried containment must never be removed here.
12280    #[cfg(target_os = "linux")]
12281    let cgroup_cleanup = PreparedCgroup(
12282        cgroup_placement
12283            .filter(|_| carried.is_none())
12284            .map(|placement| (placement.clone(), cgroup_name.clone())),
12285    );
12286    #[cfg(unix)]
12287    let mut admission_gate = if carrying {
12288        Some(
12289            admission::Gate::install(&mut command, spec, roster).map_err(|source| {
12290                SuperviseError::Spawn {
12291                    program: spec.program.clone(),
12292                    source,
12293                    cgroup_path: cgroup_path.clone(),
12294                }
12295            })?,
12296        )
12297    } else {
12298        None
12299    };
12300    #[cfg(target_os = "linux")]
12301    if let Some(path) = &cgroup_path {
12302        apply_cgroup_placement(&mut command, spec, path)?;
12303    }
12304
12305    let output_sink = if let Some(logs_dir) = capture_logs_dir {
12306        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
12307        match ChildOutputSink::open(&path, capture_retention(spec)) {
12308            Ok(sink) => sink,
12309            Err(error) => {
12310                warn!(
12311                    module_id = %spec.module_id,
12312                    path = %path.display(),
12313                    error = %error,
12314                    "could not open child output capture file; forwarding to stderr"
12315                );
12316                ChildOutputSink::Stderr
12317            }
12318        }
12319    } else {
12320        ChildOutputSink::Stderr
12321    };
12322
12323    command.stdout(Stdio::piped());
12324    command.stderr(Stdio::piped());
12325    command.kill_on_drop(!carrying);
12326    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
12327    // before exec). In the daemon's group, a service manager that kills the
12328    // job's process group when the daemon exits (launchd's default) killed
12329    // every module at the same moment its control connection closed, so no
12330    // module ever ran its EOF teardown on a daemon stop. Outside that group a
12331    // module is reached only by the daemon: the EOF it sees when its
12332    // connection closes, and the bounded stop in `child_roster` for anything
12333    // still running after that. On Linux this composes with the cgroup
12334    // placement above: that is a pre_exec write to cgroup.procs, std performs
12335    // setpgid in the child before running pre_exec callbacks, and the two
12336    // change independent process attributes.
12337    //
12338    // stdin is /dev/null because a process outside the terminal's foreground
12339    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
12340    // by hand would otherwise hand down. Under a service manager stdin is
12341    // already /dev/null.
12342    #[cfg(unix)]
12343    command.process_group(
12344        carried
12345            .as_ref()
12346            .map(|container| container.verified_join_group())
12347            .transpose()
12348            .map_err(|source| SuperviseError::Spawn {
12349                program: spec.program.clone(),
12350                source,
12351                cgroup_path: cgroup_path.clone(),
12352            })?
12353            .flatten()
12354            .unwrap_or(0) as i32,
12355    );
12356    command.stdin(Stdio::null());
12357    #[cfg(unix)]
12358    if let Some(gate) = &mut admission_gate {
12359        gate.install_placement(&mut command);
12360    }
12361    // Fill all reserved slots in one allocation-free step, after placement has
12362    // stopped using descriptors that the fixed slots may replace.
12363    #[cfg(unix)]
12364    {
12365        if let Some(handoff) = nonce_handoff {
12366            slots.nonce(handoff);
12367        }
12368        #[cfg(target_os = "macos")]
12369        slots
12370            .acknowledgement(exec_ack.into_descriptor())
12371            .map_err(|source| SuperviseError::Spawn {
12372                program: spec.program.clone(),
12373                source,
12374                cgroup_path: cgroup_path.clone(),
12375            })?;
12376        slots.install(command.as_std_mut());
12377    }
12378    #[cfg(not(any(unix, windows)))]
12379    let _ = nonce_handoff;
12380
12381    // Containment, step 1 of 3 (issue #109): create the child suspended so it
12382    // cannot run a single instruction -- and therefore cannot spawn a
12383    // grandchild -- before it is in the job. See `contain_spawned_child` for the
12384    // other two steps and why the window matters.
12385    #[cfg(windows)]
12386    {
12387        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
12388            program: spec.program.clone(),
12389            source,
12390            cgroup_path: None,
12391        })?;
12392        subc_jobobject::suspend_on_create_async(&mut command);
12393    }
12394    // Timestamp before the spawn call: on Unix the child can serve before that
12395    // call even returns. This conservative bound never labels an older child
12396    // as a replacement for a request received during process creation.
12397    let spawned_at = Instant::now();
12398    let delivery_context = config_delivery;
12399    let config_delivery = config_delivery
12400        .map(|delivery| {
12401            delivery.spawned(
12402                spec,
12403                spawn_admission
12404                    .as_ref()
12405                    .expect("delivery prepares spawn admission"),
12406                spawned_at,
12407                role,
12408                registration_spawn
12409                    .as_ref()
12410                    .map(|spawn| spawn.authority.clone()),
12411            )
12412        })
12413        .transpose()?
12414        .flatten();
12415    let mut delivery_registration =
12416        delivery_context.map(|context| context.registration_guard(config_delivery.as_ref(), role));
12417    if let Some(registration) = &mut delivery_registration {
12418        registration.defer_cleanup();
12419    }
12420    if asynchronous {
12421        #[cfg(test)]
12422        if let Some(context) = delivery_context {
12423            context.pause_preflight_for_test();
12424        }
12425    }
12426    validate_spawn_program(spec, spawn_admission.as_ref(), delivery_context)?;
12427    Ok(PreparedChild {
12428        script_delivery,
12429        registration_spawn,
12430        registration_handle: handle.cloned(),
12431        #[cfg(any(unix, windows))]
12432        registration_channel,
12433        config_context: delivery_context.map(|delivery| delivery.runtime()),
12434        binding_publication,
12435        launch_spec,
12436        spawn_admission,
12437        command,
12438        carrying,
12439        carried: carried.clone(),
12440        #[cfg(unix)]
12441        launch_identity,
12442        #[cfg(unix)]
12443        group_target,
12444        #[cfg(target_os = "linux")]
12445        cgroup_placement: cgroup_placement.cloned(),
12446        #[cfg(target_os = "linux")]
12447        cgroup_cleanup,
12448        #[cfg(target_os = "linux")]
12449        cgroup_name,
12450        cgroup_path,
12451        #[cfg(unix)]
12452        admission_gate,
12453        output_sink,
12454        #[cfg(target_os = "macos")]
12455        privacy_exec,
12456        #[cfg(windows)]
12457        image_capture,
12458        #[cfg(windows)]
12459        nonce_handoff,
12460        spawned_at,
12461        config_delivery,
12462        delivery_registration,
12463        program_binding,
12464    })
12465}
12466
12467fn launch_child_in_slot(
12468    _spec: &ModuleSpec,
12469    ring: &Arc<Mutex<StderrRing>>,
12470    roster: &ChildRoster,
12471    prepared: PreparedChild,
12472    register_timeout: Duration,
12473) -> Result<SupervisedChild, SuperviseError> {
12474    let launch_spec = prepared.launch_spec.clone();
12475    let registration_declaration = prepared
12476        .spawn_admission
12477        .as_ref()
12478        .and_then(|admission| admission.registration_declaration(&launch_spec.module_id));
12479    let admission_base = prepared
12480        .spawn_admission
12481        .as_ref()
12482        .and_then(|admission| admission.approved_base())
12483        .cloned();
12484    let spec = &launch_spec;
12485    let expected_image = prepared
12486        .spawn_admission
12487        .as_ref()
12488        .and_then(|admission| admission.launch.as_ref())
12489        .map(|launch| launch.image);
12490    let program_launch = prepared
12491        .spawn_admission
12492        .and_then(|admission| admission.launch);
12493    let config_owner = prepared
12494        .config_context
12495        .as_ref()
12496        .map(|context| context.0.clone());
12497    let binding_publication = prepared.binding_publication;
12498    #[cfg(test)]
12499    if let Some(context) = &prepared.config_context {
12500        context.0.before_spawn_exec_for_test();
12501    }
12502    #[cfg(feature = "test-support")]
12503    if let Some(context) = &prepared.config_context {
12504        context
12505            .0
12506            .pause_spawn_before_exec()
12507            .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(context)))?;
12508    }
12509    let mut command = prepared.command;
12510    let carrying = prepared.carrying;
12511    let carried = prepared.carried;
12512    #[cfg(unix)]
12513    let launch_identity = prepared.launch_identity;
12514    #[cfg(unix)]
12515    let group_target = prepared.group_target;
12516    #[cfg(target_os = "linux")]
12517    let cgroup_placement = prepared.cgroup_placement.as_ref();
12518    #[cfg(target_os = "linux")]
12519    let cgroup_name = prepared.cgroup_name;
12520    #[cfg(target_os = "linux")]
12521    let mut cgroup_cleanup = prepared.cgroup_cleanup;
12522    #[cfg(target_os = "linux")]
12523    let _ = cgroup_cleanup.0.take();
12524    let cgroup_path = prepared.cgroup_path;
12525    #[cfg(unix)]
12526    let admission_gate = prepared.admission_gate;
12527    let output_sink = prepared.output_sink;
12528    #[cfg(target_os = "macos")]
12529    let mut privacy_exec = prepared.privacy_exec;
12530    #[cfg(target_os = "macos")]
12531    if let Some(pending) = &mut privacy_exec {
12532        pending.admission_binding = binding_publication.clone();
12533        if expected_image.is_some() {
12534            pending.expected = expected_image;
12535        }
12536    }
12537    #[cfg(any(unix, windows))]
12538    let registration_channel = prepared.registration_channel;
12539    let registration_spawn = prepared.registration_spawn;
12540    let registration_handle = prepared.registration_handle;
12541    #[cfg(windows)]
12542    let image_capture = prepared.image_capture;
12543    #[cfg(windows)]
12544    let nonce_handoff = prepared.nonce_handoff;
12545    let spawned_at = prepared.spawned_at;
12546    let config_delivery = prepared.config_delivery;
12547    let delivery_registration = prepared.delivery_registration;
12548    let _program_binding = prepared.program_binding;
12549    #[cfg(all(test, unix))]
12550    containment_tests::record_spawn_attempt(spec, &mut command);
12551    #[cfg(unix)]
12552    let spawned = match admission_gate {
12553        Some(gate) => gate
12554            .spawn(
12555                command,
12556                spec,
12557                roster,
12558                launch_identity,
12559                #[cfg(target_os = "linux")]
12560                cgroup_placement
12561                    .zip(cgroup_path.as_ref())
12562                    .map(|(placement, path)| {
12563                        (placement.clone(), cgroup_name.clone(), path.clone())
12564                    }),
12565            )
12566            .map(|(child, admitted)| (child, Some(admitted))),
12567        None => {
12568            let spawned =
12569                subc_os::child_spawn::tokio_spawn(&mut command).map(|child| (child, None));
12570            // The callback owns all child-only sources. Closing them here is
12571            // necessary for channel EOF and macOS exec acknowledgement EOF.
12572            drop(command);
12573            spawned
12574        }
12575    };
12576    #[cfg(not(unix))]
12577    let spawned = subc_os::child_spawn::tokio_spawn(&mut command);
12578    let spawned = match spawned {
12579        Ok(child) => child,
12580        Err(source) => {
12581            if carried
12582                .as_ref()
12583                .and_then(|container| container.group())
12584                .is_some()
12585            {
12586                containment::terminate_blocking(&roster.containment);
12587            }
12588            #[cfg(target_os = "linux")]
12589            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12590                remove_module_cgroup(placement, &cgroup_name);
12591            }
12592            return Err(SuperviseError::Spawn {
12593                program: spec.program.clone(),
12594                source,
12595                cgroup_path,
12596            });
12597        }
12598    };
12599    #[cfg(unix)]
12600    let (mut child, mut admitted) = spawned;
12601    #[cfg(unix)]
12602    let group_target = admitted
12603        .as_ref()
12604        .map(|attempt| attempt.group_target())
12605        .unwrap_or(group_target);
12606    #[cfg(not(unix))]
12607    let mut child = spawned;
12608    if let Some(delivery) = prepared.script_delivery {
12609        delivery.start(child.id().expect("spawned child has a pid"));
12610    }
12611    #[cfg(test)]
12612    spawn_record_tests::after_create(&child, spec);
12613
12614    // Bind while the child is still suspended, before job assignment resumes it.
12615    // Keeping the executable handle open across CreateProcessW denies rename
12616    // and deletion, so the file cannot be exchanged during spawn. Bind that
12617    // file ID to the process handle returned by spawn, never a reopened PID.
12618    #[cfg(windows)]
12619    let windows_image = match image_capture.bind_tokio(&child) {
12620        Ok(image) => Arc::new(image),
12621        Err(source) => {
12622            kill_and_reap_rejected_child(&mut child);
12623            return Err(SuperviseError::Spawn {
12624                program: spec.program.clone(),
12625                source,
12626                cgroup_path: None,
12627            });
12628        }
12629    };
12630    // Assign the suspended child to its module job; resume only after its
12631    // process identity has been durably recorded for orphan recovery.
12632    #[cfg(windows)]
12633    let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
12634    let spawned_at_ms = unix_ms_now();
12635    #[cfg(not(windows))]
12636    let spawned_from = spec.program.clone();
12637    #[cfg(windows)]
12638    let spawned_from = windows_image.path().to_path_buf();
12639    #[cfg(not(windows))]
12640    let spawned_file_identity = spawned_file_identity(&spawned_from);
12641    #[cfg(windows)]
12642    let spawned_file_identity = Some(SpawnedFileIdentity {
12643        device: 0,
12644        inode: 0,
12645        windows_image: Some(Arc::clone(&windows_image)),
12646    });
12647    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
12648        program: spec.program.clone(),
12649        source: io::Error::other("spawned child exposed no live pid"),
12650        cgroup_path: cgroup_path.clone(),
12651    })?;
12652    #[cfg(windows)]
12653    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
12654    #[cfg(not(windows))]
12655    let process_start_time = admitted
12656        .as_ref()
12657        .map(|admitted| admitted.birth)
12658        .or_else(|| crate::provenance::process_start_time(pid));
12659    #[cfg(windows)]
12660    let process_start_time = Some(windows_image.start_time());
12661    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
12662    #[cfg(all(test, target_os = "macos"))]
12663    privacy_exec_boundary_tests::before_image_sample(spec, pid);
12664    #[cfg(all(test, target_os = "linux"))]
12665    assert!(roster.admission_event_for_test(&spec.module_id, "before_image_binding", Some(pid)));
12666    // Record the committed kernel image, not the configured pathname: PATH
12667    // lookup and shebang interpretation can select a different executable.
12668    // Linux's posix_spawn can return before that image is installed, so the
12669    // observer also waits for the kernel's first-exec flag to clear.
12670    #[cfg(not(windows))]
12671    let recorded_image = observe_spawned_image(pid);
12672    #[cfg(all(test, target_os = "linux"))]
12673    let recorded_image = if roster.admission_event_for_test(
12674        &spec.module_id,
12675        "after_spawn_image_sample",
12676        Some(pid),
12677    ) {
12678        recorded_image
12679    } else {
12680        None
12681    };
12682    #[cfg(target_os = "linux")]
12683    // An exited child cannot receive HELLO service and has no readable /proc/exe.
12684    // Keep its real exit for the supervisor instead of turning a crash into a hold.
12685    let exited_before_binding = expected_image.is_some()
12686        && recorded_image.and_then(|image| image.executable) != expected_image
12687        && child.try_wait().ok().flatten().is_some();
12688    #[cfg(target_os = "linux")]
12689    let unavailable_initial_image = expected_image.is_some()
12690        && recorded_image.and_then(|image| image.executable).is_none()
12691        && !exited_before_binding;
12692    #[cfg(not(target_os = "linux"))]
12693    let unavailable_initial_image = false;
12694    #[cfg(target_os = "linux")]
12695    if expected_image
12696        .is_some_and(|expected| recorded_image.and_then(|image| image.executable) != Some(expected))
12697        && !exited_before_binding
12698        && !unavailable_initial_image
12699    {
12700        kill_module_cgroup(cgroup_placement, &cgroup_name);
12701        kill_and_reap_rejected_child(&mut child, group_target.clone());
12702        return Err(SuperviseError::ConfigHeld {
12703            module_id: spec.module_id.clone(),
12704            code: "program_refused".into(),
12705            detail: format!(
12706                "spawned executable {:?} does not match hashed program {:?}",
12707                recorded_image.and_then(|image| image.executable),
12708                expected_image
12709            ),
12710        });
12711    }
12712    #[cfg(not(target_os = "linux"))]
12713    let _ = expected_image;
12714    #[cfg(target_os = "macos")]
12715    let binding_ready = privacy_exec.is_none();
12716    #[cfg(target_os = "linux")]
12717    let binding_ready = !exited_before_binding;
12718    #[cfg(not(any(target_os = "macos", target_os = "linux")))]
12719    let binding_ready = cfg!(windows);
12720    // spawn() confirms only the first exec, into the trampoline. Never persist
12721    // the trampoline image; the asynchronous acknowledgement publishes the
12722    // module image once the trampoline has replaced itself with the module.
12723    #[cfg(target_os = "macos")]
12724    let recorded_image = if privacy_exec.is_some() {
12725        None
12726    } else {
12727        recorded_image
12728    };
12729    #[cfg(target_os = "linux")]
12730    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
12731    #[cfg(not(target_os = "linux"))]
12732    let recorded_cgroup_name = None;
12733    #[cfg(unix)]
12734    let preadmitted = admitted
12735        .as_mut()
12736        .and_then(|admitted| admitted.leader.take());
12737    #[cfg(not(unix))]
12738    let preadmitted: Option<crate::child_roster::RosterGuard> = None;
12739    let roster_guard = if let Some(guard) = preadmitted {
12740        Ok(guard)
12741    } else {
12742        roster.admit(
12743            spec.module_id.clone(),
12744            pid,
12745            spec.protocol,
12746            process_start_time,
12747            crate::child_roster::RecordedIdentity {
12748                container: None,
12749                #[cfg(unix)]
12750                carried: None,
12751                #[cfg(not(windows))]
12752                start_time: recorded_image.map(|image| image.start_time),
12753                #[cfg(windows)]
12754                start_time: process_start_time,
12755                #[cfg(not(windows))]
12756                executable: recorded_image
12757                    .and_then(|image| image.executable)
12758                    .map(crate::live_children::ExecutableIdentity::from),
12759                #[cfg(windows)]
12760                executable: Some(crate::live_children::ExecutableIdentity::from(
12761                    windows_image.identity(),
12762                )),
12763                cgroup_name: recorded_cgroup_name,
12764                #[cfg(target_os = "linux")]
12765                cgroup_placement: cgroup_placement.cloned(),
12766            },
12767        )
12768    };
12769    let roster_guard = match roster_guard {
12770        Ok(guard) => guard,
12771        Err(failure) => {
12772            return failure.with_cleanup(|source| {
12773                #[cfg(windows)]
12774                if let Some(job) = &job {
12775                    let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12776                }
12777                #[cfg(target_os = "linux")]
12778                kill_module_cgroup(cgroup_placement, &cgroup_name);
12779                #[cfg(unix)]
12780                let rejected_group = group_target.clone();
12781                kill_and_reap_rejected_child(
12782                    &mut child,
12783                    #[cfg(unix)]
12784                    rejected_group,
12785                );
12786                #[cfg(target_os = "linux")]
12787                if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12788                    remove_module_cgroup(placement, &cgroup_name);
12789                }
12790                Err(SuperviseError::Spawn {
12791                    program: spec.program.clone(),
12792                    source: io::Error::other(format!(
12793                        "could not durably record module child: {source}"
12794                    )),
12795                    cgroup_path,
12796                })
12797            });
12798        }
12799    };
12800    #[cfg(unix)]
12801    if admitted.is_some() {
12802        if let Some(image) = recorded_image {
12803            if let Err(source) = roster_guard.confirm_executable(image) {
12804                kill_and_reap_rejected_child(&mut child, group_target.clone());
12805                return Err(SuperviseError::Spawn {
12806                    program: spec.program.clone(),
12807                    source,
12808                    cgroup_path,
12809                });
12810            }
12811        }
12812    }
12813    // The pre-spawn check can pass just before shutdown begins. Ownership is
12814    // published at the start of admission, even while a record write is queued.
12815    // If shutdown already saw an empty roster before that publication, this
12816    // check sees the roster closed and ends the child before starting a module.
12817    if roster.is_closed() {
12818        if carrying {
12819            #[cfg(unix)]
12820            let rejected_group = admitted
12821                .as_ref()
12822                .map(|admitted| admitted.rejection_target())
12823                .unwrap_or_else(|| group_target.clone());
12824            #[cfg(windows)]
12825            if let Some(job) = &job {
12826                let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12827            }
12828            kill_and_reap_rejected_child(
12829                &mut child,
12830                #[cfg(unix)]
12831                rejected_group,
12832            );
12833            #[cfg(all(test, unix))]
12834            let _ =
12835                roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12836            drop(roster_guard);
12837            return Err(SuperviseError::Spawn {
12838                program: spec.program.clone(),
12839                source: io::Error::other(
12840                    "the daemon began shutting down during carried admission; ended the container",
12841                ),
12842                cgroup_path,
12843            });
12844        }
12845        // This child was never admitted, so there is no module protocol shutdown to wait for.
12846        #[cfg(target_os = "linux")]
12847        kill_module_cgroup(cgroup_placement, &cgroup_name);
12848        if let Err(error) = child.start_kill() {
12849            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
12850        }
12851        #[cfg(target_os = "linux")]
12852        if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12853            // This spawn was never admitted, so shutdown has no roster entry
12854            // to await. Do not detach its cleanup: the runtime could exit
12855            // before that task reaps the rejected child and removes its group.
12856            while matches!(child.try_wait(), Ok(None)) {
12857                std::thread::yield_now();
12858            }
12859            if matches!(
12860                subc_cgroup::kill_module(Some(placement), &cgroup_name),
12861                subc_cgroup::KillOutcome::Killed
12862            ) {
12863                if let Ok(path) = placement.module_path(&cgroup_name) {
12864                    while std::fs::read_to_string(path.join("cgroup.events"))
12865                        .ok()
12866                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
12867                    {
12868                        std::thread::yield_now();
12869                    }
12870                }
12871            }
12872            remove_module_cgroup(placement, &cgroup_name);
12873        }
12874        drop(roster_guard);
12875        return Err(SuperviseError::Spawn {
12876            program: spec.program.clone(),
12877            source: io::Error::other(
12878                "the daemon began shutting down while this process was starting; ended it",
12879            ),
12880            cgroup_path,
12881        });
12882    }
12883
12884    let stdout_pump = match child.stdout.take() {
12885        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
12886        None => {
12887            warn!(
12888                module_id = %spec.module_id,
12889                "spawned child exposed no stdout pipe; file capture will be incomplete"
12890            );
12891            None
12892        }
12893    };
12894    let stderr_pump = match child.stderr.take() {
12895        Some(stderr) => {
12896            let generation = ring
12897                .lock()
12898                .unwrap_or_else(|poisoned| poisoned.into_inner())
12899                .begin_process();
12900            Some(StderrPump {
12901                task: tokio::spawn(pump_stderr_to(
12902                    stderr,
12903                    Arc::clone(ring),
12904                    generation,
12905                    output_sink,
12906                )),
12907                generation,
12908            })
12909        }
12910        None => {
12911            // Spawning succeeded but the pipe did not materialise. Recording it as
12912            // uncaptured keeps the tail honest: the alternative is an empty tail
12913            // that reads as a module which printed nothing.
12914            ring.lock()
12915                .unwrap_or_else(|poisoned| poisoned.into_inner())
12916                .mark_not_captured("stderr pipe was not available on spawn");
12917            warn!(
12918                module_id = %spec.module_id,
12919                "spawned child exposed no stderr pipe; tail will be unavailable"
12920            );
12921            None
12922        }
12923    };
12924
12925    let exec = Arc::new(crate::activity::ExecOutcome::new());
12926    #[cfg(target_os = "macos")]
12927    let pending_exec = privacy_exec.is_some();
12928    #[cfg(not(target_os = "macos"))]
12929    let pending_exec = false;
12930    if !pending_exec {
12931        exec.finish(crate::activity::ExecState::CommittedExec);
12932    }
12933    #[cfg(windows)]
12934    let subtree_target = job
12935        .as_ref()
12936        .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
12937        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
12938    #[cfg(target_os = "linux")]
12939    let subtree_target = cgroup_path
12940        .map(crate::subtree_resources::Target::Cgroup)
12941        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
12942    #[cfg(not(any(target_os = "linux", windows)))]
12943    let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
12944    let mut supervised = SupervisedChild {
12945        registration_spawn,
12946        #[cfg(any(unix, windows))]
12947        registration_channel,
12948        live_program_binding: None,
12949        registered_config_connection: None,
12950        admission_base,
12951        registration_declaration,
12952        _program_binding: binding_publication,
12953        subtree_target,
12954        child: containment::ProcessChild::plain(child, pid),
12955        exec: crate::activity::ExecOwner(exec),
12956        #[cfg(windows)]
12957        nonce_delivery,
12958        protocol: spec.protocol,
12959        #[cfg(target_os = "linux")]
12960        module_id: cgroup_name,
12961        #[cfg(target_os = "linux")]
12962        cgroup_placement: cgroup_placement.cloned(),
12963        #[cfg(windows)]
12964        job,
12965        stdout_pump,
12966        stderr_pump,
12967        stderr_ring: Arc::clone(ring),
12968        spawned_at,
12969        config_delivery,
12970        registration_deadline: None,
12971        registration_watch: None,
12972        spawned_at_ms,
12973        spawned_from,
12974        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
12975        spawned_file_identity,
12976        process_start_time,
12977        process_identity,
12978        pid,
12979        roster_guard: Some(roster_guard),
12980        #[cfg(target_os = "macos")]
12981        privacy_exec,
12982        #[cfg(target_os = "macos")]
12983        report_ready: Arc::new(OnceLock::new()),
12984        spawn_failure: None,
12985        admission_kill_sent: false,
12986    };
12987    if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
12988        supervised.child.reject(
12989            #[cfg(unix)]
12990            group_target,
12991        );
12992        #[cfg(all(test, unix))]
12993        let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12994        return Err(error);
12995    }
12996    if let (Some(launch), Some(owner), Some(base)) = (
12997        program_launch,
12998        config_owner,
12999        supervised.admission_base.clone(),
13000    ) {
13001        supervised.live_program_binding =
13002            content_binding::LiveProgramBinding::new(launch, &supervised, owner, base);
13003        if let (Some(publication), Some(live)) = (
13004            &supervised._program_binding,
13005            &supervised.live_program_binding,
13006        ) {
13007            publication.attach(live);
13008        }
13009    }
13010    #[cfg(unix)]
13011    if let Some(admitted) = admitted.take() {
13012        admitted.disarm();
13013    }
13014    #[cfg(windows)]
13015    if let Err(error) = resume_suspended_child(pid, spec) {
13016        containment::terminate_blocking(&roster.containment);
13017        let _ = supervised.child.start_kill();
13018        return Err(error);
13019    }
13020    if binding_ready {
13021        // An unreadable initial image is not a definite mismatch. Keep the
13022        // owned child and delivery registration until its reaped status decides
13023        // whether it crashed on its own or was stopped by admission's kill.
13024        let checked = if unavailable_initial_image {
13025            Err(crate::config_gate::GateError::new(
13026                "config_delivery_deferred",
13027                "initial process image is unavailable",
13028            ))
13029        } else {
13030            match (
13031                &supervised._program_binding,
13032                &supervised.live_program_binding,
13033            ) {
13034                (Some(binding), _) => binding.verify(),
13035                (None, Some(live)) => live.check(),
13036                _ if supervised.admission_base.is_some() => {
13037                    Err(crate::config_gate::GateError::new(
13038                        "config_delivery_deferred",
13039                        "live program witness is unavailable",
13040                    ))
13041                }
13042                _ => Ok(()),
13043            }
13044        };
13045        #[cfg(all(test, target_os = "linux"))]
13046        let checked = if roster.admission_event_for_test(
13047            &spec.module_id,
13048            "after_content_witness_check",
13049            Some(pid),
13050        ) {
13051            checked
13052        } else {
13053            Err(crate::config_gate::GateError::new(
13054                "config_delivery_deferred",
13055                "current process image is unavailable",
13056            ))
13057        };
13058        if let Err(error) = checked {
13059            if let Some(binding) = &mut supervised._program_binding {
13060                binding.reject();
13061            }
13062            // Reap through the normal child-reaping path, retaining the real exit record and
13063            // its roster ownership. A refusal must also end any helpers.
13064            if error.code == "program_refused" || supervised.try_wait().ok().flatten().is_none() {
13065                supervised.spawn_failure = Some(format!("{}: {}", error.code, error.detail));
13066                #[cfg(all(test, target_os = "linux"))]
13067                assert!(roster.admission_event_for_test(
13068                    &spec.module_id,
13069                    if unavailable_initial_image {
13070                        "before_initial_image_kill"
13071                    } else {
13072                        "before_content_witness_kill"
13073                    },
13074                    Some(pid),
13075                ));
13076                let _ = supervised.start_admission_kill();
13077                #[cfg(all(test, target_os = "linux"))]
13078                if supervised.admission_kill_sent {
13079                    assert!(roster.admission_event_for_test(
13080                        &spec.module_id,
13081                        "after_admission_kill_sent",
13082                        Some(pid),
13083                    ));
13084                }
13085            }
13086        }
13087    }
13088    if let Some(registration) = delivery_registration {
13089        registration.commit();
13090    }
13091    if let Some(spawn) = &supervised.registration_spawn {
13092        if let Some(identity) = supervised.process_identity {
13093            let _ = spawn.identity.set(identity);
13094        }
13095        if supervised.spawn_failure.is_some() {
13096            spawn.refuse();
13097        } else if binding_ready {
13098            spawn.ready();
13099        }
13100        #[cfg(any(unix, windows))]
13101        if let (Some(handle), Some(channel)) =
13102            (registration_handle, supervised.registration_channel.take())
13103        {
13104            handle.serve_registration(channel, spawn.clone(), pid, spawned_at + register_timeout);
13105        }
13106    }
13107    Ok(supervised)
13108}
13109
13110#[cfg(test)]
13111mod spawn_record_tests {
13112    use super::*;
13113
13114    thread_local! {
13115        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
13116    }
13117
13118    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
13119        LAST_PID.set(child.id());
13120        #[cfg(windows)]
13121        if let Some((_, address)) = _spec
13122            .env
13123            .iter()
13124            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
13125        {
13126            use std::io::{Read, Write};
13127            let mut stream = std::net::TcpStream::connect(address).unwrap();
13128            stream
13129                .write_all(&child.id().unwrap().to_ne_bytes())
13130                .unwrap();
13131            let mut release = [0];
13132            stream.read_exact(&mut release).unwrap();
13133        }
13134    }
13135
13136    #[cfg(windows)]
13137    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
13138        if spec
13139            .env
13140            .iter()
13141            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
13142        {
13143            Err(io::Error::other("injected module job assignment failure"))
13144        } else {
13145            Ok(())
13146        }
13147    }
13148
13149    fn spec() -> ModuleSpec {
13150        ModuleSpec {
13151            script: None,
13152            module_id: "spawn-record-worker".into(),
13153            #[cfg(unix)]
13154            program: "/bin/sleep".into(),
13155            #[cfg(windows)]
13156            program: std::env::current_exe().unwrap(),
13157            #[cfg(unix)]
13158            args: vec!["60".into()],
13159            #[cfg(windows)]
13160            args: vec![
13161                "--exact".into(),
13162                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
13163                "--nocapture".into(),
13164            ],
13165            env: vec![],
13166            reserved: false,
13167            reserved_prefixes: vec![],
13168            protocol: ModuleProtocol::None,
13169            overlap: Default::default(),
13170            carry_containment: false,
13171        }
13172    }
13173
13174    #[tokio::test]
13175    async fn record_sync_failure_fails_spawn_and_reaps_child() {
13176        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
13177        let supervisor =
13178            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
13179                .with_live_children_record(home.join("live-children.json"));
13180        crate::live_children::fail_next_file_sync();
13181        let result = supervisor.spawn(spec());
13182        let failed = result.is_err();
13183        if let Ok(module) = result {
13184            module.stop().await.unwrap();
13185        }
13186        assert!(failed, "record failure must fail spawn");
13187        let pid = LAST_PID.get().expect("the real OS child was created");
13188        assert!(
13189            subc_os::Process::open(pid).unwrap().is_none(),
13190            "child {pid} survived rejected admission or was not reaped"
13191        );
13192    }
13193
13194    #[cfg(target_os = "macos")]
13195    #[test]
13196    #[ignore = "manual on-device latency measurement"]
13197    fn durable_record_write_latency_50() {
13198        let home = cortexkit_test_support::ScratchDir::new("record-latency");
13199        let path = home.join("live-children.json");
13200        let children = vec![crate::live_children::LiveChild {
13201            module_id: "latency-worker".into(),
13202            pid: std::process::id(),
13203            protocol: ModuleProtocol::None,
13204            start_time: Some(1),
13205            executable: None,
13206            cgroup_name: None,
13207            container: None,
13208        }];
13209        let mut samples = Vec::new();
13210        for _ in 0..50 {
13211            let start = std::time::Instant::now();
13212            crate::live_children::write_record(&path, &children).unwrap();
13213            samples.push(start.elapsed());
13214        }
13215        samples.sort();
13216        eprintln!(
13217            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
13218            (samples[24] + samples[25]) / 2,
13219            samples[47]
13220        );
13221    }
13222
13223    #[cfg(windows)]
13224    #[test]
13225    fn windows_store_touch_fixture() {
13226        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
13227            std::fs::write(path, b"module ran").unwrap();
13228        }
13229    }
13230
13231    #[cfg(windows)]
13232    #[tokio::test]
13233    async fn windows_daemon_fixture() {
13234        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
13235            return;
13236        };
13237        let mut spec = spec();
13238        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
13239        let supervisor =
13240            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13241        let _module = supervisor.spawn(spec).unwrap();
13242    }
13243
13244    #[cfg(windows)]
13245    #[test]
13246    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
13247        use std::io::Read;
13248        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13249        // An outer job models a scheduler/service host. The daemon's root and
13250        // root job must nest without requiring breakaway from this job.
13251        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13252        command
13253            .args([
13254                "--exact",
13255                "supervise::spawn_record_tests::windows_daemon_fixture",
13256                "--nocapture",
13257            ])
13258            .env(
13259                "SUBC_TEST_DAEMON_BARRIER",
13260                listener.local_addr().unwrap().to_string(),
13261            );
13262        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13263        listener.set_nonblocking(true).unwrap();
13264        let deadline = std::time::Instant::now() + Duration::from_secs(30);
13265        let mut stream = loop {
13266            if let Ok((stream, _)) = listener.accept() {
13267                break stream;
13268            }
13269            assert!(
13270                daemon.child.try_wait().unwrap().is_none(),
13271                "daemon failed before rendezvous"
13272            );
13273            assert!(
13274                std::time::Instant::now() < deadline,
13275                "daemon did not reach CreateProcess failpoint"
13276            );
13277            std::thread::yield_now();
13278        };
13279        stream
13280            .set_read_timeout(Some(Duration::from_secs(30)))
13281            .unwrap();
13282        let mut pid = [0; 4];
13283        stream.read_exact(&mut pid).unwrap();
13284        let pid = u32::from_ne_bytes(pid);
13285        assert!(subc_jobobject::process_exists(pid));
13286        // Terminate only the daemon, not its outer job: otherwise the host job
13287        // would also kill a child whose daemon never joined its root job.
13288        daemon.child.kill().unwrap();
13289        daemon.child.wait().unwrap();
13290        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
13291        if !exited {
13292            daemon.job.terminate().unwrap();
13293        }
13294        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
13295    }
13296
13297    #[cfg(windows)]
13298    #[tokio::test]
13299    async fn windows_containment_failure_never_resumes() {
13300        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
13301        let sentinel = home.join("store-touched");
13302        let mut spec = spec();
13303        spec.env
13304            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
13305        spec.env.push((
13306            "SUBC_TEST_STORE_TOUCH".into(),
13307            sentinel.display().to_string(),
13308        ));
13309        let supervisor =
13310            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13311        let result = supervisor.spawn(spec);
13312        // A version that resumes despite assignment failure must have time to
13313        // execute its sentinel before dropping the returned module kills it.
13314        if let Ok(module) = result {
13315            let _ = tokio::time::timeout(Duration::from_secs(10), async {
13316                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
13317                    tokio::task::yield_now().await;
13318                }
13319            })
13320            .await;
13321            let _ = module.stop().await;
13322            panic!(
13323                "assignment failure resumed a module; store touched={}",
13324                sentinel.exists()
13325            );
13326        }
13327        assert!(
13328            !sentinel.exists(),
13329            "refused module executed store-touch sentinel"
13330        );
13331        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
13332    }
13333
13334    #[cfg(windows)]
13335    #[tokio::test]
13336    async fn windows_nested_daemon_fixture() {
13337        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
13338            return;
13339        };
13340        let mut spec = spec();
13341        spec.env.push((
13342            "SUBC_TEST_STORE_TOUCH".into(),
13343            sentinel.to_str().unwrap().into(),
13344        ));
13345        let supervisor =
13346            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13347        let module = supervisor
13348            .spawn(spec)
13349            .expect("per-module job must nest below daemon and host jobs");
13350        tokio::time::timeout(Duration::from_secs(10), async {
13351            while !PathBuf::from(&sentinel).exists() {
13352                tokio::task::yield_now().await;
13353            }
13354        })
13355        .await
13356        .unwrap();
13357        module.stop().await.unwrap();
13358    }
13359
13360    #[cfg(windows)]
13361    #[test]
13362    fn windows_module_job_nests_under_host_job() {
13363        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
13364        let sentinel = home.join("nested-store-touched");
13365        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13366        command
13367            .args([
13368                "--exact",
13369                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
13370                "--nocapture",
13371            ])
13372            .env("SUBC_TEST_NESTED_STORE", &sentinel);
13373        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13374        assert!(daemon.child.wait().unwrap().success());
13375        assert!(sentinel.exists(), "nested protocol:none module never ran");
13376    }
13377}
13378
13379#[cfg(target_os = "linux")]
13380pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
13381    use subc_cgroup::KillOutcome;
13382    match subc_cgroup::kill_module(placement, module_id) {
13383        KillOutcome::Killed => {}
13384        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
13385            debug!(
13386                module_id,
13387                "cgroup tree kill unavailable; using direct-child kill"
13388            );
13389        }
13390        KillOutcome::IoError { path, error } => {
13391            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
13392        }
13393    }
13394}
13395
13396/// Assign a freshly spawned Windows child without ever resuming on failure.
13397///
13398/// Steps 2 and 3 of the suspended-create contract: the job is created and the
13399/// child assigned **while it is still suspended** (step 1 is
13400/// `suspend_on_create_async` at the spawn site). The caller resumes only after
13401/// the child's roster admission is durable. Failed containment kills and reaps
13402/// the suspended child instead of running an unprotected module.
13403#[cfg(windows)]
13404fn contain_spawned_child(
13405    child: &mut Child,
13406    spec: &ModuleSpec,
13407    carried: Option<&Arc<containment::CarriedContainment>>,
13408) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
13409    let assignment = || {
13410        let job = match carried {
13411            Some(container) => container.job(),
13412            None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
13413        };
13414        #[cfg(test)]
13415        spawn_record_tests::assignment_failpoint(spec)?;
13416        job.lock()
13417            .unwrap_or_else(|p| p.into_inner())
13418            .assign(child)?;
13419        Ok::<_, io::Error>(job)
13420    };
13421    match assignment() {
13422        Ok(job) => Ok(Some(job)),
13423        Err(source) => {
13424            kill_and_reap_rejected_child(child);
13425            Err(SuperviseError::Spawn {
13426                program: spec.program.clone(),
13427                source: io::Error::other(format!(
13428                    "Windows module job containment failed: {source}"
13429                )),
13430                cgroup_path: None,
13431            })
13432        }
13433    }
13434}
13435
13436/// Rejection is synchronous: do not return a spawn error while the child can
13437/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
13438fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
13439    #[cfg(unix)]
13440    group.signal_rejected(child);
13441    let _ = child.start_kill();
13442    while matches!(child.try_wait(), Ok(None)) {
13443        std::thread::yield_now();
13444    }
13445}
13446
13447/// Resume a suspended child, killing it if it cannot be started.
13448///
13449/// A suspended process holds a pid and does nothing, so there is no useful
13450/// state to return: the caller gets an error and the spawn fails.
13451#[cfg(windows)]
13452fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
13453    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
13454        // Kill it here rather than leaving a suspended process for the caller
13455        // to notice; `kill_on_drop` would eventually do this, but the module
13456        // would have been reported as running in between.
13457        let _ = subc_os::child_spawn::status(
13458            std::process::Command::new("taskkill.exe")
13459                .args(["/PID", &pid.to_string(), "/T", "/F"])
13460                .stdin(Stdio::null())
13461                .stdout(Stdio::null())
13462                .stderr(Stdio::null()),
13463        );
13464        return Err(SuperviseError::Spawn {
13465            program: spec.program.clone(),
13466            source,
13467            cgroup_path: None,
13468        });
13469    }
13470    Ok(())
13471}
13472
13473#[cfg(target_os = "linux")]
13474fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13475    match placement.remove_module(module_id) {
13476        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
13477        Err(error) => warn!(
13478            module_id,
13479            error = %error,
13480            "could not remove module cgroup after process exit; continuing teardown"
13481        ),
13482    }
13483}
13484
13485#[cfg(target_os = "linux")]
13486const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
13487#[cfg(target_os = "linux")]
13488const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
13489
13490#[cfg(target_os = "linux")]
13491async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13492    let path = placement.module_path(module_id).ok();
13493    if path.as_ref().is_some_and(|path| {
13494        std::fs::read_to_string(path.join("cgroup.events"))
13495            .ok()
13496            .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
13497    }) {
13498        remove_module_cgroup(placement, module_id);
13499        return;
13500    }
13501
13502    // Freeze before enumerating: a frozen task cannot exit and release its PID
13503    // for reuse by a process outside this module while SIGTERM is being sent.
13504    if terminate_reaped_cgroup(placement, module_id).await {
13505        if let Some(path) = &path {
13506            let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
13507            while cgroup_is_populated(path) && Instant::now() < deadline {
13508                sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
13509            }
13510        }
13511    }
13512
13513    // Reaping the direct child is not proof its descendants exited. Kill any
13514    // survivors; when `cgroup.kill` succeeds, wait for the kernel to report the
13515    // cgroup empty before rmdir.
13516    if matches!(
13517        subc_cgroup::kill_module(Some(placement), module_id),
13518        subc_cgroup::KillOutcome::Killed
13519    ) {
13520        if let Some(path) = &path {
13521            while cgroup_is_populated(path) {
13522                sleep(Duration::from_millis(1)).await;
13523            }
13524        }
13525    }
13526    remove_module_cgroup(placement, module_id);
13527}
13528
13529#[cfg(target_os = "linux")]
13530fn cgroup_is_populated(path: &std::path::Path) -> bool {
13531    std::fs::read_to_string(path.join("cgroup.events"))
13532        .ok()
13533        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
13534}
13535
13536#[cfg(target_os = "linux")]
13537async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
13538    let frozen =
13539        match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
13540            .await
13541        {
13542            subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
13543            outcome => {
13544                tracing::debug!(
13545                    module_id,
13546                    ?outcome,
13547                    "cgroup freeze failed; skipping SIGTERM grace"
13548                );
13549                return false;
13550            }
13551        };
13552    let listed = match frozen.processes() {
13553        subc_cgroup::ProcsOutcome::Listed(pids) => {
13554            for pid in pids.into_iter().filter(|pid| *pid > 1) {
13555                if let Some(pid) = rustix::process::Pid::from_raw(pid) {
13556                    if let Err(error) =
13557                        rustix::process::kill_process(pid, rustix::process::Signal::TERM)
13558                    {
13559                        tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
13560                    }
13561                }
13562            }
13563            true
13564        }
13565        outcome => {
13566            tracing::debug!(
13567                module_id,
13568                ?outcome,
13569                "cgroup process listing failed; skipping SIGTERM grace"
13570            );
13571            false
13572        }
13573    };
13574    match frozen.thaw() {
13575        subc_cgroup::ThawOutcome::Thawed => listed,
13576        outcome => {
13577            tracing::debug!(
13578                module_id,
13579                ?outcome,
13580                "cgroup thaw failed; proceeding to cgroup.kill"
13581            );
13582            false
13583        }
13584    }
13585}
13586
13587#[cfg(target_os = "linux")]
13588fn apply_cgroup_placement(
13589    command: &mut Command,
13590    spec: &ModuleSpec,
13591    path: &std::path::Path,
13592) -> Result<(), SuperviseError> {
13593    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
13594        module_id: spec.module_id.clone(),
13595        source,
13596    })
13597}
13598
13599fn capture_retention(spec: &ModuleSpec) -> Retention {
13600    let defaults = Retention::default();
13601    let value = |name: &str| {
13602        spec.env
13603            .iter()
13604            .rev()
13605            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
13606    };
13607    Retention {
13608        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
13609            .and_then(|value| value.parse().ok())
13610            .unwrap_or(defaults.max_file_mb),
13611        keep: value(CAPTURE_KEEP_ENV)
13612            .and_then(|value| value.parse().ok())
13613            .unwrap_or(defaults.keep),
13614        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
13615            .and_then(|value| value.parse().ok())
13616            .unwrap_or(defaults.max_age_days),
13617    }
13618}
13619
13620/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
13621/// module's registration to the exact process the supervisor spawned.
13622fn generate_launch_nonce() -> Result<String, SuperviseError> {
13623    let mut bytes = [0u8; 32];
13624    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
13625        reason: source.to_string(),
13626    })?;
13627    let mut hex = String::with_capacity(64);
13628    for b in bytes {
13629        use std::fmt::Write;
13630        let _ = write!(hex, "{b:02x}");
13631    }
13632    Ok(hex)
13633}
13634
13635/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
13636/// signal about how many leading bytes matched.
13637fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
13638    if a.len() != b.len() {
13639        return false;
13640    }
13641    let mut diff = 0u8;
13642    for (x, y) in a.iter().zip(b.iter()) {
13643        diff |= x ^ y;
13644    }
13645    diff == 0
13646}
13647
13648/// The kernel's image after an acknowledged exec, shared by ordinary launches
13649/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
13650/// not identities inferred from a configured pathname.
13651#[cfg(not(windows))]
13652fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
13653    let process = subc_os::Process::open(pid).ok().flatten()?;
13654    #[cfg(target_os = "linux")]
13655    return sample_after_linux_exec(
13656        || {
13657            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
13658            let committed = linux_exec_committed(&stat);
13659            #[cfg(test)]
13660            spawn_image_tests::report_exec_state(committed);
13661            committed
13662        },
13663        || process.observe(),
13664    );
13665    #[cfg(not(target_os = "linux"))]
13666    process.observe()
13667}
13668
13669/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
13670/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
13671/// installation. Until it clears, /proc/pid/exe can still name the daemon.
13672#[cfg(target_os = "linux")]
13673fn linux_exec_committed(stat: &str) -> Option<bool> {
13674    const PF_FORKNOEXEC: u64 = 0x40;
13675    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
13676    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
13677}
13678
13679#[cfg(target_os = "linux")]
13680fn sample_after_linux_exec(
13681    mut exec_committed: impl FnMut() -> Option<bool>,
13682    observe: impl FnOnce() -> Option<subc_os::Observation>,
13683) -> Option<subc_os::Observation> {
13684    let deadline = std::time::Instant::now() + Duration::from_secs(1);
13685    while !exec_committed()? {
13686        // A failed or indefinitely stalled exec must not publish the inherited
13687        // image as authority to signal a future orphan. Missing identity keeps
13688        // the existing conservative orphan-sweep behavior.
13689        if std::time::Instant::now() >= deadline {
13690            return None;
13691        }
13692        std::thread::sleep(Duration::from_micros(50));
13693    }
13694    observe()
13695}
13696
13697#[cfg(all(test, target_os = "linux"))]
13698mod spawn_image_tests {
13699    use super::*;
13700    use std::cell::{Cell, RefCell};
13701
13702    enum ExecEvent {
13703        Uncommitted,
13704        Image(Option<subc_os::Observation>),
13705    }
13706
13707    thread_local! {
13708        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
13709    }
13710
13711    pub(super) fn report_exec_state(committed: Option<bool>) {
13712        if committed == Some(false) {
13713            EXEC_EVENTS.with(|events| {
13714                if let Some(events) = events.borrow_mut().take() {
13715                    let _ = events.send(ExecEvent::Uncommitted);
13716                }
13717            });
13718        }
13719    }
13720
13721    #[test]
13722    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
13723        use std::io::{Read, Write};
13724        use std::os::fd::AsRawFd;
13725
13726        let descriptor = std::fs::File::open("/dev/null").unwrap();
13727        let mut command = std::process::Command::new("/bin/sleep");
13728        command.arg("30");
13729        let (mut ready, mut release) =
13730            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
13731                .unwrap();
13732        let spawn = std::thread::spawn(move || command.spawn());
13733        let mut message = [0u8; 8];
13734        ready.read_exact(&mut message).unwrap();
13735        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
13736        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
13737        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
13738        let (events, received) = std::sync::mpsc::channel();
13739        let observer = std::thread::spawn(move || {
13740            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
13741            let image = observe_spawned_image(pid);
13742            let _ = events.send(ExecEvent::Image(image));
13743        });
13744        // Release only once the production observer has seen the uncommitted
13745        // exec, or has incorrectly returned the inherited image. Either way,
13746        // release and reap before asserting so a regression cannot leak a child.
13747        let first = received.recv_timeout(Duration::from_secs(5));
13748        release.write_all(&[1]).unwrap();
13749        let mut child = spawn.join().unwrap().unwrap();
13750        let after = match first {
13751            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
13752                Ok(ExecEvent::Image(image)) => image,
13753                _ => None,
13754            },
13755            Ok(ExecEvent::Image(image)) => image,
13756            Err(_) => None,
13757        };
13758        observer.join().unwrap();
13759        child.kill().unwrap();
13760        child.wait().unwrap();
13761
13762        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
13763        assert_eq!(
13764            inherited.unwrap().executable,
13765            subc_os::file_identity(&std::env::current_exe().unwrap())
13766        );
13767        assert_eq!(
13768            after.unwrap().executable,
13769            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
13770        );
13771    }
13772
13773    #[test]
13774    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
13775        let committed = Cell::new(false);
13776        let inherited = subc_os::Observation {
13777            start_time: 23,
13778            executable: Some(subc_os::FileIdentity {
13779                device: 29,
13780                inode: 31,
13781            }),
13782        };
13783        let module = subc_os::Observation {
13784            start_time: 23,
13785            executable: Some(subc_os::FileIdentity {
13786                device: 29,
13787                inode: 37,
13788            }),
13789        };
13790        let polls = Cell::new(0);
13791        let image = sample_after_linux_exec(
13792            || {
13793                polls.set(polls.get() + 1);
13794                // Hold the inherited image through the first poll, as in the
13795                // posix_spawn parent-wake window, then commit the module image.
13796                committed.set(polls.get() == 2);
13797                Some(committed.get())
13798            },
13799            || Some(if committed.get() { module } else { inherited }),
13800        );
13801        assert_eq!(image, Some(module));
13802        assert_eq!(polls.get(), 2);
13803    }
13804
13805    #[test]
13806    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
13807        assert_eq!(
13808            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
13809            Some(false)
13810        );
13811        assert_eq!(
13812            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
13813            Some(true)
13814        );
13815        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
13816        assert_eq!(linux_exec_committed("17 (a) R"), None);
13817    }
13818
13819    #[test]
13820    fn unreadable_exec_state_never_samples_an_inherited_image() {
13821        assert_eq!(
13822            sample_after_linux_exec(
13823                || None,
13824                || panic!("must not sample before exec confirmation")
13825            ),
13826            None
13827        );
13828    }
13829
13830    #[test]
13831    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
13832        assert_eq!(
13833            sample_after_linux_exec(
13834                || Some(false),
13835                || panic!("must not sample after an exec timeout")
13836            ),
13837            None
13838        );
13839    }
13840}
13841
13842/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
13843/// different executable after the no-delete capture was taken.
13844#[cfg(windows)]
13845pub(crate) fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
13846    if let Err(error) = require_program_search_path(spec) {
13847        return Err(match error {
13848            SuperviseError::Spawn { source, .. } => source,
13849            error => io::Error::other(error),
13850        });
13851    }
13852    let mut program = spec.program.clone();
13853    if program.is_absolute() || program.components().count() > 1 {
13854        // An explicit path can name a PE image with no extension. Try that
13855        // exact file first, as Command does, before falling back to .exe.
13856        match subc_os::ExecutableCapture::open(&program) {
13857            Ok(capture) => return Ok(capture),
13858            Err(error)
13859                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
13860            Err(error) => return Err(error),
13861        }
13862        program.set_extension("exe");
13863        return subc_os::ExecutableCapture::open(&program);
13864    }
13865    if program.extension().is_none() {
13866        program.set_extension("exe");
13867    }
13868    let custom_path = spec
13869        .env
13870        .iter()
13871        .rev()
13872        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
13873    let mut directories = Vec::new();
13874    // These are Windows' default executable search locations. A configured
13875    // PATH still overrides the inherited PATH, not the system directories.
13876    if let Some(parent) = std::env::current_exe()?.parent() {
13877        directories.push(parent.to_path_buf());
13878    }
13879    directories.push(std::env::current_dir()?);
13880    if let Some(windows) = std::env::var_os("SystemRoot") {
13881        let windows = PathBuf::from(windows);
13882        directories.push(windows.join("System32"));
13883        directories.push(windows.clone());
13884    }
13885    let path = custom_path
13886        .map(|(_, value)| std::ffi::OsString::from(value))
13887        .or_else(|| std::env::var_os("PATH"));
13888    if let Some(path) = path {
13889        directories.extend(std::env::split_paths(&path));
13890    }
13891    for directory in directories {
13892        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
13893            Ok(capture) => return Ok(capture),
13894            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
13895            Err(error) => return Err(error),
13896        }
13897    }
13898    Err(io::Error::new(
13899        io::ErrorKind::NotFound,
13900        "resolved executable not found",
13901    ))
13902}
13903
13904async fn spawn_and_mark_running_async(
13905    spec: &ModuleSpec,
13906    runtime: &SupervisorRuntimeConfig,
13907    snapshot: &SharedSnapshot,
13908) -> Result<SupervisedChild, SuperviseError> {
13909    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13910    let registration_watch = watch_registration(&runtime.registry, spec);
13911    #[cfg(unix)]
13912    let joining_group = containment::current(&runtime.child_roster.containment)
13913        .is_some_and(|container| container.group().is_some());
13914    let result = spawn_child_async(spec, runtime).await;
13915    #[cfg(unix)]
13916    let result = if result.is_err()
13917        && joining_group
13918        && !runtime.child_roster.is_closed()
13919        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13920        && runtime
13921            .child_roster
13922            .previous_instance(&spec.module_id)
13923            .is_none()
13924    {
13925        // A failed group join has already ended the old container. Its single
13926        // fresh-container retry belongs to the same restart budget entry.
13927        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13928        spawn_child_async(spec, runtime).await
13929    } else {
13930        result
13931    };
13932    let mut child = result?;
13933    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13934    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13935        containment::terminate_blocking(&runtime.child_roster.containment);
13936        return Err(error);
13937    }
13938    if let Some(spawn) = activity_spawn {
13939        spawn.complete(snapshot);
13940    }
13941    Ok(child)
13942}
13943
13944#[cfg(test)]
13945fn spawn_and_mark_running(
13946    spec: &ModuleSpec,
13947    runtime: &SupervisorRuntimeConfig,
13948    snapshot: &SharedSnapshot,
13949) -> Result<SupervisedChild, SuperviseError> {
13950    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13951    let registration_watch = watch_registration(&runtime.registry, spec);
13952    #[cfg(unix)]
13953    let joining_group = containment::current(&runtime.child_roster.containment)
13954        .is_some_and(|container| container.group().is_some());
13955    let spawn = || {
13956        spawn_child(
13957            spec,
13958            runtime.connection_file_path.as_deref(),
13959            runtime.supervisor_handle.as_ref(),
13960            &runtime.stderr_ring,
13961            runtime.capture_logs_dir.as_deref(),
13962            &runtime.child_roster,
13963            runtime.spawn_delivery(),
13964            #[cfg(target_os = "linux")]
13965            runtime.cgroup_placement.as_ref(),
13966            runtime.register_timeout,
13967        )
13968    };
13969    let result = spawn();
13970    #[cfg(unix)]
13971    let result = if result.is_err()
13972        && joining_group
13973        && !runtime.child_roster.is_closed()
13974        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13975        && runtime
13976            .child_roster
13977            .previous_instance(&spec.module_id)
13978            .is_none()
13979    {
13980        // The failed join already destroyed and verified the old container.
13981        // Try one fresh group as part of this same restart, without spending
13982        // another crash-budget entry. Cgroups and Windows jobs do not retry.
13983        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13984        spawn()
13985    } else {
13986        result
13987    };
13988    let child = result?;
13989    let mut child = child;
13990    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13991    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13992        containment::terminate_blocking(&runtime.child_roster.containment);
13993        return Err(error);
13994    }
13995    if let Some(spawn) = activity_spawn {
13996        spawn.complete(snapshot);
13997    }
13998    Ok(child)
13999}
14000
14001struct RegistrationWatch {
14002    changes: watch::Receiver<u64>,
14003    baseline: u64,
14004}
14005
14006// Capture before spawning: a fast child's accepted HELLO can disappear from the
14007// active registry before the monitor first runs. A pre-existing registration
14008// must not satisfy a new process's deadline either.
14009fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
14010    if spec.protocol != ModuleProtocol::Subc {
14011        return None;
14012    }
14013    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
14014    let baseline = *changes.borrow();
14015    Some(RegistrationWatch { changes, baseline })
14016}
14017
14018fn arm_registration_deadline(
14019    child: &mut SupervisedChild,
14020    budget: Duration,
14021    registration_watch: Option<RegistrationWatch>,
14022) {
14023    child.registration_watch = registration_watch;
14024    child.registration_deadline = (child.protocol == ModuleProtocol::Subc
14025        && child.awaiting_registration())
14026    .then_some(child.spawned_at + budget);
14027}
14028
14029async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
14030    child.retire_registration();
14031    if let Some(container) = child.child.container() {
14032        container.stop_members().await;
14033        child.kill_leader()?;
14034        child.wait().await?;
14035        return Ok(());
14036    }
14037    #[cfg(unix)]
14038    {
14039        // Children lead their own process groups. Signal the whole group so a
14040        // helper stuck before HELLO cannot survive its supervised parent.
14041        let group = unreaped_registration_group(child)?;
14042        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
14043        // Process-group safety rule: only signal a group while its leader is
14044        // our unreaped child. Even a zombie leader reserves its pid/group id;
14045        // reaping it before KILL would let that id name an unrelated group.
14046        // Sleep without waiting so TERM-ignoring helpers get the full grace.
14047        sleep(crate::child_roster::TERM_TO_KILL).await;
14048        let group = unreaped_registration_group(child)?;
14049        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
14050    }
14051    // Windows terminates the containing Job Object as well as the direct
14052    // child. Linux also kills the child's cgroup when placement is enabled.
14053    child.start_kill()?;
14054    child.wait().await?;
14055    Ok(())
14056}
14057
14058#[cfg(unix)]
14059fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
14060    if let Some(container) = child.child.container() {
14061        return container
14062            .group()
14063            .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
14064            .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
14065    }
14066    // The group id comes only from the pid recorded at spawn, when the child
14067    // created its own group. Never admit init's group or a negative group id.
14068    let group = i32::try_from(child.pid)
14069        .ok()
14070        .filter(|pid| *pid > 1)
14071        .and_then(rustix::process::Pid::from_raw)
14072        .ok_or_else(|| {
14073            io::Error::new(
14074                io::ErrorKind::InvalidInput,
14075                "invalid supervised process group",
14076            )
14077        })?;
14078    if child.child.id() != Some(child.pid) {
14079        return Err(io::Error::new(
14080            io::ErrorKind::InvalidInput,
14081            "refusing process-group signal after supervised child was reaped",
14082        ));
14083    }
14084    Ok(group)
14085}
14086
14087enum RegistrationWaitOutcome {
14088    Registered,
14089    Exited(ExitReport),
14090    TimedOut,
14091}
14092
14093struct ReloadRegistrationFailure {
14094    exit_report: ExitReport,
14095    reason: String,
14096}
14097
14098#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14099enum BusyGaugeObservation {
14100    Quiescent,
14101    Busy,
14102    Omitted,
14103}
14104
14105/// What one health answer said about one declared busy gauge.
14106#[derive(Debug, Clone, PartialEq, Eq)]
14107enum GaugeVerdict {
14108    /// The gauge read 0: this gauge is not what holds the drain.
14109    Zero,
14110    /// The gauge read a positive count.
14111    NonZero(u64),
14112    /// The gauge was present but not a non-negative integer (`0.0`, a string,
14113    /// a negative number). It counts as busy, because the daemon cannot tell
14114    /// it is idle. Carries the value as JSON text, cut to a short prefix.
14115    NotACount(String),
14116    /// The health answer had no metrics object, or one without this gauge.
14117    Missing,
14118    /// The probe produced no health answer to read; carries the probe error's
14119    /// evidence label (`no-answer` for a probe that timed out).
14120    ProbeFailed(&'static str),
14121}
14122
14123impl GaugeVerdict {
14124    fn reads_quiet(&self) -> bool {
14125        matches!(self, Self::Zero)
14126    }
14127}
14128
14129impl fmt::Display for GaugeVerdict {
14130    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14131        match self {
14132            Self::Zero => f.write_str("zero"),
14133            Self::NonZero(value) => write!(f, "non_zero({value})"),
14134            Self::NotACount(value) => write!(f, "not_a_count({value})"),
14135            Self::Missing => f.write_str("missing_from_health_body"),
14136            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
14137        }
14138    }
14139}
14140
14141/// The longest JSON text kept for a gauge value that is not a count, so a
14142/// module reporting a large object as a gauge cannot flood the drain log.
14143const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
14144
14145/// Read each declared gauge out of one health answer's metrics.
14146fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
14147    let metrics = metrics.and_then(Value::as_object);
14148    gauges
14149        .iter()
14150        .map(
14151            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
14152                None => GaugeVerdict::Missing,
14153                Some(value) => match value.as_u64() {
14154                    Some(0) => GaugeVerdict::Zero,
14155                    Some(count) => GaugeVerdict::NonZero(count),
14156                    None => {
14157                        let mut text = value.to_string();
14158                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
14159                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
14160                            while !text.is_char_boundary(end) {
14161                                end -= 1;
14162                            }
14163                            text.truncate(end);
14164                            text.push_str("...");
14165                        }
14166                        GaugeVerdict::NotACount(text)
14167                    }
14168                },
14169            },
14170        )
14171        .collect()
14172}
14173
14174/// The drain's overall reading of one health answer: omitted when any
14175/// declared gauge is missing, quiescent only when every gauge reads 0.
14176fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
14177    if verdicts
14178        .iter()
14179        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
14180    {
14181        BusyGaugeObservation::Omitted
14182    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
14183        BusyGaugeObservation::Quiescent
14184    } else {
14185        BusyGaugeObservation::Busy
14186    }
14187}
14188
14189/// One declared gauge's history over a drain: the latest verdict on it, and
14190/// the latest value actually read for it (which a later failed probe or
14191/// missing gauge does not erase). Times are offsets from the start of the
14192/// drain's wait.
14193#[derive(Debug, Clone, Default, PartialEq, Eq)]
14194struct GaugeTrace {
14195    latest: Option<(GaugeVerdict, Duration)>,
14196    last_value: Option<(GaugeVerdict, Duration)>,
14197}
14198
14199/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
14200/// its ceiling can say in its timeout line which declared gauge held it and
14201/// why. Without it, a gauge that read busy, a gauge the module never
14202/// reported, and a probe that never got an answer all looked the same: a
14203/// drain that ended at its ceiling with nothing in flight.
14204#[derive(Debug, Clone, Default, PartialEq, Eq)]
14205struct DrainGaugeTrace {
14206    /// Health probes the drain sent.
14207    probes: u32,
14208    /// Each declared gauge, in declaration order.
14209    gauges: Vec<(String, GaugeTrace)>,
14210}
14211
14212impl DrainGaugeTrace {
14213    fn new(gauges: &[String]) -> Self {
14214        Self {
14215            probes: 0,
14216            gauges: gauges
14217                .iter()
14218                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
14219                .collect(),
14220        }
14221    }
14222
14223    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
14224    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
14225        self.probes = self.probes.saturating_add(1);
14226        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
14227            trace.latest = Some((verdict.clone(), at));
14228            if matches!(
14229                verdict,
14230                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
14231            ) {
14232                trace.last_value = Some((verdict.clone(), at));
14233            }
14234        }
14235    }
14236}
14237
14238/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
14239/// declared gauge, separated by ` | `. A value is shown as its verdict
14240/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
14241/// and whether that counted as busy.
14242impl fmt::Display for DrainGaugeTrace {
14243    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14244        if self.gauges.is_empty() {
14245            return f.write_str("none declared");
14246        }
14247        for (index, (name, trace)) in self.gauges.iter().enumerate() {
14248            if index > 0 {
14249                f.write_str(" | ")?;
14250            }
14251            write!(f, "{name}: ")?;
14252            match &trace.latest {
14253                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
14254                None => f.write_str("never probed")?,
14255            }
14256            match &trace.last_value {
14257                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
14258                None => f.write_str("; no value ever read")?,
14259            }
14260        }
14261        Ok(())
14262    }
14263}
14264
14265pub(crate) fn declared_busy_gauges(
14266    registry: &Registry,
14267    module_id: &str,
14268) -> Result<Vec<String>, SuperviseError> {
14269    busy_gauges_of(
14270        registry
14271            .get_module(module_id)
14272            .map_err(SuperviseError::Registry)?,
14273    )
14274}
14275
14276/// [`declared_busy_gauges`] for the registration a connection holds, in any
14277/// slot: after cutover the incumbent is no longer the id's active
14278/// registration, and its own manifest is the one that names its gauges.
14279fn declared_busy_gauges_for_connection(
14280    registry: &Registry,
14281    connection_id: ConnectionId,
14282) -> Result<Vec<String>, SuperviseError> {
14283    busy_gauges_of(
14284        registry
14285            .get_module_by_connection(connection_id)
14286            .map_err(SuperviseError::Registry)?,
14287    )
14288}
14289
14290fn busy_gauges_of(
14291    registration: Option<crate::registry::ModuleRegistration>,
14292) -> Result<Vec<String>, SuperviseError> {
14293    let Some(registration) = registration else {
14294        return Ok(Vec::new());
14295    };
14296    let Some(self_signals) = registration.manifest.self_signals else {
14297        return Ok(Vec::new());
14298    };
14299
14300    let mut gauges = Vec::new();
14301    for declaration in self_signals {
14302        if declaration.kind != SelfSignalKind::Busy {
14303            continue;
14304        }
14305        match declaration.anchored_to {
14306            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
14307                gauges.extend(declared)
14308            }
14309            _ => {
14310                // An invalid Busy anchor is fail-safe: the empty name cannot be
14311                // present in a conforming health report, so this drain stays busy.
14312                gauges.push(String::new());
14313            }
14314        }
14315    }
14316    Ok(gauges)
14317}
14318
14319/// Wait for `endpoint` to have nothing in flight and, when the module declares
14320/// busy gauges, for a health probe to report them quiet. The probe is addressed
14321/// by `scope`: a swap's superseded incumbent must be asked about its own
14322/// gauges, and by module id the probe would reach the promoted candidate.
14323/// While the gauges read busy, the probe repeats every
14324/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
14325/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
14326/// timeout line.
14327#[allow(clippy::too_many_arguments)]
14328async fn wait_for_forwarding_quiescence(
14329    forwarding: &ForwardingTable,
14330    module_id: &str,
14331    runtime: &SupervisorRuntimeConfig,
14332    endpoint: crate::ModuleEndpointId,
14333    deadline: Instant,
14334    busy_gauges: &[String],
14335    scope: DrainScope,
14336    trace: &mut DrainGaugeTrace,
14337) -> Result<bool, SuperviseError> {
14338    let wait_started = Instant::now();
14339    let mut gauges_quiescent = busy_gauges.is_empty();
14340    let mut next_probe_at = wait_started;
14341    let mut omission_reported = false;
14342
14343    loop {
14344        let now = Instant::now();
14345        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
14346            let report = match scope {
14347                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
14348                DrainScope::Endpoint(endpoint) => {
14349                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
14350                }
14351            };
14352            let verdicts = match &report {
14353                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
14354                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
14355            };
14356            trace.record(&verdicts, wait_started.elapsed());
14357            gauges_quiescent = match report {
14358                Ok(_) => match busy_gauge_observation(&verdicts) {
14359                    BusyGaugeObservation::Quiescent => true,
14360                    BusyGaugeObservation::Busy => false,
14361                    BusyGaugeObservation::Omitted => {
14362                        // Once per drain: a module that never reports a gauge
14363                        // it declared would otherwise say so on every probe.
14364                        if !omission_reported {
14365                            forwarding
14366                                .counters()
14367                                .increment_drains_with_undeclared_gauge();
14368                            let missing = busy_gauges
14369                                .iter()
14370                                .zip(&verdicts)
14371                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
14372                                .map(|(gauge, _)| gauge.as_str())
14373                                .collect::<Vec<_>>()
14374                                .join(",");
14375                            warn!(
14376                                module_id,
14377                                missing_gauges = %missing,
14378                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
14379                            );
14380                            omission_reported = true;
14381                        }
14382                        false
14383                    }
14384                },
14385                Err(err) => {
14386                    warn!(
14387                        module_id,
14388                        error = %err,
14389                        "drain health.check did not produce declared busy gauges; treating module as busy"
14390                    );
14391                    false
14392                }
14393            };
14394            // Counted from the probe's completion: the probe above was awaited,
14395            // so no second probe can start while one is still outstanding.
14396            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
14397        }
14398
14399        let in_flight = forwarding
14400            .endpoint_in_flight_count(endpoint)
14401            .map_err(SuperviseError::Forwarding)?;
14402        if in_flight == 0 && gauges_quiescent {
14403            return Ok(true);
14404        }
14405
14406        let now = Instant::now();
14407        if now >= deadline {
14408            return Ok(false);
14409        }
14410        let mut wait = deadline
14411            .saturating_duration_since(now)
14412            .min(REGISTRY_RELEASE_POLL);
14413        if !busy_gauges.is_empty() {
14414            wait = wait.min(next_probe_at.saturating_duration_since(now));
14415        }
14416        sleep(wait).await;
14417    }
14418}
14419
14420/// The wait between drain busy-gauge probes: the larger of
14421/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
14422/// drain's own in-flight poll interval.
14423fn drain_busy_reprobe_interval() -> Duration {
14424    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
14425}
14426
14427/// Logs why a drain gave up: the drain waited its full timeout without the
14428/// module going quiet, and teardown is being forced.
14429///
14430/// The line names two kinds of evidence, because a bare "timed out" hides
14431/// causes that need different fixes:
14432/// - `holdouts`: the requests and routes still open when the timeout hit,
14433///   and on which connections. One stuck call is a module bug; a held stream
14434///   on every session is a module that should end its streams when it
14435///   receives `module.draining`.
14436/// - `gauges`: for each busy gauge the module declares, its latest reading
14437///   and how many probes ran. This separates a module that kept reporting
14438///   itself busy, one whose health answer never contained the gauge, and one
14439///   that never answered the probes, which otherwise all look like a timeout
14440///   with nothing in flight.
14441fn warn_drain_timed_out(
14442    module_id: &str,
14443    drain_timeout: Duration,
14444    reason: RouteCloseReason,
14445    holdouts: &crate::forwarding::DrainHoldouts,
14446    gauges: &DrainGaugeTrace,
14447) {
14448    warn!(
14449        module_id = %module_id,
14450        waited = ?drain_timeout,
14451        ?reason,
14452        held_requests = holdouts.requests,
14453        held_routes = holdouts.routes,
14454        total_routes = holdouts.total_routes,
14455        top_connections = ?holdouts.top_connections,
14456        // `module_channel:corr`, so the module can find each held request
14457        // in its own log; capped, so `held_requests` is the full count.
14458        held = %holdouts
14459            .held
14460            .iter()
14461            .map(|(channel, corr)| format!("{channel}:{corr}"))
14462            .collect::<Vec<_>>()
14463            .join(","),
14464        gauge_probes = gauges.probes,
14465        busy_gauges = %gauges,
14466        "route drain timed out before request quiescence; forcing teardown"
14467    );
14468}
14469
14470/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
14471///
14472/// `Ok` is always honest and passed straight through -- the wait actually measured
14473/// in-flight state. `Err` means the wait produced no measurement at all (the
14474/// forwarding table's lock was poisoned), so `false` is reported as the one honest
14475/// constant: the drain did not complete. Never recomputed from route state, never a
14476/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
14477fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
14478    match wait_result {
14479        Ok(drained) => *drained,
14480        Err(_) => false,
14481    }
14482}
14483
14484fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
14485    for released in released_routes {
14486        let frame = match Frame::build_with_version(
14487            released.negotiated_ver,
14488            FrameType::Goodbye,
14489            control_flags(),
14490            released.channel,
14491            released.epoch,
14492            0,
14493            Vec::new(),
14494        ) {
14495            Ok(frame) => frame,
14496            Err(err) => {
14497                warn!(
14498                    route_channel = released.channel,
14499                    error = %err,
14500                    "failed to build supervisor drain route GOODBYE frame"
14501                );
14502                continue;
14503            }
14504        };
14505        if !released.close_on_delivery_failure() {
14506            crate::forwarding::send_module_route_goodbye(
14507                &forwarding.counters(),
14508                &released.sink,
14509                frame,
14510                released.module_id.as_deref(),
14511                "supervisor drain",
14512            );
14513            continue;
14514        }
14515        if let Err(err) = released.sink.try_send(frame) {
14516            warn!(
14517                target_connection_id = released.connection_id.get(),
14518                route_channel = released.channel,
14519                error = %err,
14520                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
14521            );
14522            let _ = forwarding.escalate_client_delivery_failure(
14523                released.connection_id,
14524                released.channel,
14525                released.epoch,
14526                CloseReason::new(
14527                    "route_goodbye_delivery_failed",
14528                    format!(
14529                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
14530                        released.channel
14531                    ),
14532                ),
14533                crate::forwarding::UndeliveredFrame {
14534                    module_id: released.module_id.as_deref(),
14535                    sink: &released.sink,
14536                },
14537            );
14538        }
14539    }
14540}
14541
14542async fn send_module_draining(
14543    module_id: &str,
14544    reason: RouteCloseReason,
14545    deadline_ms: u64,
14546    target: &ModuleDrainTarget,
14547    _wait_for_flush: bool,
14548) -> bool {
14549    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
14550        reason,
14551        deadline_ms,
14552    }) {
14553        Ok(body) => body,
14554        Err(err) => {
14555            warn!(
14556                module_id,
14557                error = %err,
14558                "failed to encode module draining command"
14559            );
14560            return false;
14561        }
14562    };
14563    let frame = match Frame::build_with_version(
14564        target.negotiated_ver,
14565        FrameType::Push,
14566        control_flags(),
14567        0,
14568        0,
14569        0,
14570        body,
14571    ) {
14572        Ok(frame) => frame,
14573        Err(err) => {
14574            warn!(
14575                module_id,
14576                error = %err,
14577                "failed to build module draining command frame"
14578            );
14579            return false;
14580        }
14581    };
14582    #[cfg(unix)]
14583    if _wait_for_flush {
14584        return target.sink.send_flushed(frame).await.is_ok();
14585    }
14586    if let Err(err) = target.sink.try_send(frame) {
14587        warn!(
14588            module_id,
14589            target_connection_id = target.endpoint.connection_id.get(),
14590            error = %err,
14591            "module draining command was not delivered to peer"
14592        );
14593        return false;
14594    }
14595    true
14596}
14597
14598/// The channel-0 GOODBYE that tells a module its stop is planned.
14599fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
14600    match Frame::build_with_version(
14601        negotiated_ver,
14602        FrameType::Goodbye,
14603        control_flags(),
14604        0,
14605        0,
14606        0,
14607        Vec::new(),
14608    ) {
14609        Ok(frame) => Some(frame),
14610        Err(err) => {
14611            warn!(
14612                module_id,
14613                error = %err,
14614                "failed to build module GOODBYE frame"
14615            );
14616            None
14617        }
14618    }
14619}
14620
14621/// Send every registered module connection its module GOODBYE at daemon
14622/// shutdown, then request that connection's close.
14623///
14624/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
14625/// before EOF, so the GOODBYE must reach the socket before the close. A close
14626/// request does not wait for the connection's queued frames: its writer gets a
14627/// bounded grace after the close, is aborted if it overruns it, and the daemon
14628/// process may exit before that grace ends. So with `wait_for_flush`, each
14629/// connection is closed only after its writer has acknowledged writing the
14630/// GOODBYE, or once a short shared budget runs out, so one module that is not
14631/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
14632/// are only queued, for a shutdown the operator has told to stop waiting.
14633/// A connection that is already gone is skipped.
14634#[cfg(unix)]
14635async fn send_module_goodbyes_for_daemon_shutdown(
14636    forwarding: &Arc<ForwardingTable>,
14637    reason: &CloseReason,
14638    wait_for_flush: bool,
14639) {
14640    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
14641    let targets = match forwarding.module_connections() {
14642        Ok(targets) => targets,
14643        Err(err) => {
14644            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
14645            return;
14646        }
14647    };
14648    let deadline = Instant::now() + GOODBYE_BUDGET;
14649    let mut sends = tokio::task::JoinSet::new();
14650    for target in targets {
14651        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
14652            continue;
14653        };
14654        if !wait_for_flush {
14655            if let Err(err) = target.sink.try_send(frame) {
14656                debug!(
14657                    module_id = %target.module_id,
14658                    error = %err,
14659                    "shutdown module GOODBYE was not queued"
14660                );
14661            }
14662            continue;
14663        }
14664        let forwarding = Arc::clone(forwarding);
14665        let reason = reason.clone();
14666        sends.spawn(async move {
14667            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
14668                Ok(Ok(())) => {}
14669                Ok(Err(err)) => debug!(
14670                    module_id = %target.module_id,
14671                    error = %err,
14672                    "module connection closed before its shutdown GOODBYE was written"
14673                ),
14674                Err(_) => warn!(
14675                    module_id = %target.module_id,
14676                    budget = ?GOODBYE_BUDGET,
14677                    "shutdown module GOODBYE was not written within its budget; closing anyway"
14678                ),
14679            }
14680            forwarding.request_connection_close(target.endpoint.connection_id, reason);
14681        });
14682    }
14683    // Every task ends by the shared deadline, so this wait is bounded too.
14684    while sends.join_next().await.is_some() {}
14685}
14686
14687async fn send_module_goodbye(
14688    module_id: &str,
14689    forwarding: &ForwardingTable,
14690    target: &ModuleDrainTarget,
14691    wait_for_flush: bool,
14692) {
14693    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
14694        return;
14695    };
14696    // A channel is shut down synchronously after its drain. Its final frame
14697    // must reach the socket writer first; queue admission alone can lose it.
14698    let sent = if wait_for_flush {
14699        timeout(Duration::from_millis(500), target.sink.send_flushed(frame))
14700            .await
14701            .map_err(|_| "module GOODBYE flush exceeded 500ms".to_owned())
14702            .and_then(|result| result.map_err(|err| err.to_string()))
14703    } else {
14704        target.sink.try_send(frame).map_err(|err| err.to_string())
14705    };
14706    if let Err(err) = sent {
14707        warn!(
14708            module_id,
14709            target_connection_id = target.endpoint.connection_id.get(),
14710            error = %err,
14711            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
14712        );
14713        forwarding.request_connection_close(
14714            target.endpoint.connection_id,
14715            CloseReason::new(
14716                "module_goodbye_delivery_failed",
14717                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
14718            ),
14719        );
14720    }
14721}
14722
14723#[derive(Clone, Copy)]
14724struct ForwardingDrainContext<'a> {
14725    spec: &'a ModuleSpec,
14726    runtime: &'a SupervisorRuntimeConfig,
14727    registry: &'a Registry,
14728    scope: DrainScope,
14729}
14730
14731/// Which process a forwarding drain addresses.
14732#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14733enum DrainScope {
14734    /// Whatever endpoint is active for the module id: every plain stop,
14735    /// restart and reload. Also moves the module's state to `Draining`.
14736    Active,
14737    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
14738    /// module id would resolve to the promoted candidate and leave neither
14739    /// process routable. The module's state is left alone, since the promoted
14740    /// candidate is what it describes and that process is running.
14741    Endpoint(crate::ModuleEndpointId),
14742}
14743
14744/// Whether a child being drained has already been asked to stop by the time
14745/// its drain wait starts.
14746///
14747/// The drain wait is the same budget whatever this says. What it decides is
14748/// whether the supervisor must ask by signal before that wait begins: a child
14749/// that nobody asked will sit out the whole budget and then be SIGKILLed,
14750/// healthy or not.
14751#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14752enum StopNotice {
14753    /// The module was sent `module.draining` and a module GOODBYE over its own
14754    /// registered connection, and stops itself.
14755    SentOverConnection,
14756    /// The forwarding drain found no registered connection for the module: a
14757    /// subc child spawned moments ago that has not sent HELLO yet, or a
14758    /// `protocol: "none"` child, which never registers.
14759    NoConnection,
14760    /// This path sends nothing over the module's connection: the supervisor has
14761    /// no forwarding table, or the caller stops the child without a forwarding
14762    /// drain.
14763    NotSent,
14764}
14765
14766async fn begin_forwarding_drain(
14767    spec: &ModuleSpec,
14768    runtime: &SupervisorRuntimeConfig,
14769    registry: &Registry,
14770    snapshot: &SharedSnapshot,
14771    enabled: Option<bool>,
14772    reason: RouteCloseReason,
14773) -> Result<StopNotice, SuperviseError> {
14774    let Some(forwarding) = runtime.forwarding.as_ref() else {
14775        return Err(SuperviseError::ReloadUnavailable {
14776            module_id: spec.module_id.clone(),
14777            reason: "supervisor was not configured with a forwarding table".to_string(),
14778        });
14779    };
14780
14781    begin_forwarding_drain_with(
14782        forwarding,
14783        ForwardingDrainContext {
14784            spec,
14785            runtime,
14786            registry,
14787            scope: DrainScope::Active,
14788        },
14789        snapshot,
14790        enabled,
14791        reason,
14792        runtime.drain_timeout,
14793    )
14794    .await
14795}
14796
14797async fn begin_forwarding_drain_if_configured(
14798    spec: &ModuleSpec,
14799    runtime: &SupervisorRuntimeConfig,
14800    registry: &Registry,
14801    snapshot: &SharedSnapshot,
14802    enabled: Option<bool>,
14803    reason: RouteCloseReason,
14804) -> Result<StopNotice, SuperviseError> {
14805    begin_forwarding_drain_with_timeout(
14806        spec,
14807        runtime,
14808        registry,
14809        snapshot,
14810        enabled,
14811        reason,
14812        runtime.drain_timeout,
14813    )
14814    .await
14815}
14816
14817/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
14818/// budget, for paths where the operator overrides the module's configured one
14819/// (`supervisor.restart{drain_timeout_ms}`).
14820async fn begin_forwarding_drain_with_timeout(
14821    spec: &ModuleSpec,
14822    runtime: &SupervisorRuntimeConfig,
14823    registry: &Registry,
14824    snapshot: &SharedSnapshot,
14825    enabled: Option<bool>,
14826    reason: RouteCloseReason,
14827    drain_timeout: Duration,
14828) -> Result<StopNotice, SuperviseError> {
14829    let Some(forwarding) = runtime.forwarding.as_ref() else {
14830        return Ok(StopNotice::NotSent);
14831    };
14832
14833    begin_forwarding_drain_with(
14834        forwarding,
14835        ForwardingDrainContext {
14836            spec,
14837            runtime,
14838            registry,
14839            scope: DrainScope::Active,
14840        },
14841        snapshot,
14842        enabled,
14843        reason,
14844        drain_timeout,
14845    )
14846    .await
14847}
14848
14849async fn begin_forwarding_drain_with(
14850    forwarding: &ForwardingTable,
14851    context: ForwardingDrainContext<'_>,
14852    snapshot: &SharedSnapshot,
14853    enabled: Option<bool>,
14854    reason: RouteCloseReason,
14855    drain_timeout: Duration,
14856) -> Result<StopNotice, SuperviseError> {
14857    let ForwardingDrainContext {
14858        spec,
14859        runtime,
14860        registry,
14861        scope,
14862    } = context;
14863    debug_assert_ne!(reason, RouteCloseReason::Crash);
14864    let terminal = matches!(reason, RouteCloseReason::Disable);
14865    let drain_started_at = Instant::now();
14866    let drain_deadline = drain_started_at + drain_timeout;
14867    let deadline_ms =
14868        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
14869    let busy_gauges = match scope {
14870        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
14871        DrainScope::Endpoint(endpoint) => {
14872            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
14873        }
14874    };
14875
14876    // Admission gate first: route.open/commit and route REQUEST admission are closed
14877    // before the first quiescence check, so the outstanding count can only fall.
14878    let gate_started = Instant::now();
14879    let drain_target = match scope {
14880        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
14881        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
14882    }
14883    .map_err(SuperviseError::Forwarding)?;
14884    // The instant admission closed, and how long taking the forwarding write
14885    // lock to close it took. The timeout line reports only the quiescence
14886    // wait, so without this a drain that started late looked like one that
14887    // started on time.
14888    info!(
14889        module_id = %spec.module_id,
14890        ?reason,
14891        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
14892        connected = drain_target.is_some(),
14893        "module drain began; route admission closed"
14894    );
14895    if scope == DrainScope::Active {
14896        update_snapshot(snapshot, Some(&spec.module_id), |state| {
14897            state.state = ModuleState::Draining;
14898            state.draining_to_replace =
14899                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
14900            if let Some(enabled) = enabled {
14901                state.enabled = enabled;
14902            }
14903        })?;
14904    }
14905
14906    let Some(target) = drain_target.as_ref() else {
14907        // Nothing was sent: the module has no registered connection to carry
14908        // `module.draining` or a GOODBYE. The caller must not assume the child
14909        // was asked to stop.
14910        return Ok(StopNotice::NoConnection);
14911    };
14912    let flush_goodbye = runtime.supervisor_handle.as_ref().is_some_and(|handle| {
14913        handle
14914            .registration_spawn(target.endpoint.connection_id)
14915            .is_some()
14916    });
14917    {
14918        send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
14919        #[cfg(feature = "test-support")]
14920        forwarding.wait_before_route_closing_for_test().await;
14921        let routes = forwarding
14922            .endpoint_routes(target.endpoint)
14923            .map_err(SuperviseError::Forwarding)?;
14924        let routes_notified = routes.len();
14925        crate::control::send_route_control_pushes(
14926            forwarding,
14927            routes.clone(),
14928            ClientControlPush::RouteClosing {
14929                module_id: spec.module_id.clone(),
14930                channels: Vec::new(),
14931                reason,
14932            },
14933        );
14934        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
14935
14936        // `route.closing` was just sent above: from here on every return path,
14937        // including an early one, MUST send `route.closed` before propagating
14938        // anything else. A client holds `closing` as a promise that a verdict is
14939        // coming; leaving early without `closed` strands it waiting forever, since
14940        // `closing` carries no timeout of its own.
14941        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
14942        let wait_result = wait_for_forwarding_quiescence(
14943            forwarding,
14944            &spec.module_id,
14945            runtime,
14946            target.endpoint,
14947            drain_deadline,
14948            &busy_gauges,
14949            scope,
14950            &mut gauge_trace,
14951        )
14952        .await;
14953        let drained = drained_after_quiescence_wait(&wait_result);
14954        if let Err(err) = &wait_result {
14955            error!(
14956                module_id = %spec.module_id,
14957                ?reason,
14958                error = %err,
14959                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
14960            );
14961        } else if !drained {
14962            // Read before teardown releases the routes.
14963            let holdouts = forwarding
14964                .endpoint_drain_holdouts(target.endpoint)
14965                .unwrap_or_default();
14966            warn_drain_timed_out(
14967                &spec.module_id,
14968                drain_timeout,
14969                reason,
14970                &holdouts,
14971                &gauge_trace,
14972            );
14973        }
14974        crate::control::send_route_control_pushes(
14975            forwarding,
14976            routes,
14977            ClientControlPush::RouteClosed {
14978                module_id: spec.module_id.clone(),
14979                channels: Vec::new(),
14980                reason,
14981                drained,
14982                abandoned: target.abandoned_bindings.len() as u32,
14983                excluded_subscriptions: target.excluded_subscriptions,
14984                terminal: Some(terminal),
14985            },
14986        );
14987        wait_result?;
14988
14989        // `route.closed` has now been sent unconditionally above. From here the
14990        // remaining steps are cleanup (route + module GOODBYE) rather than a
14991        // promise the client is waiting on, but a lock-poisoned
14992        // `release_module_endpoint_routes` would otherwise skip the module
14993        // GOODBYE silently too -- send it before propagating the error.
14994        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
14995            Ok(routes) => routes,
14996            Err(err) => {
14997                warn!(
14998                    module_id = %spec.module_id,
14999                    ?reason,
15000                    error = %err,
15001                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
15002                );
15003                send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
15004                return Err(SuperviseError::Forwarding(err));
15005            }
15006        };
15007        let route_goodbye_count = released_routes.len();
15008        send_route_goodbyes(forwarding, released_routes);
15009        send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
15010
15011        // The drain's happy path was previously silent: every emission above is
15012        // best-effort with only its failure arm logged, so "were consumers told"
15013        // was unprovable from the daemon log (surfaced by a 30-minute consumer
15014        // hang where the open question was exactly whether teardown notice went
15015        // out). One summary line makes that class decidable in one grep.
15016        info!(
15017            module_id = %spec.module_id,
15018            ?reason,
15019            routes_notified,
15020            route_goodbyes = route_goodbye_count,
15021            abandoned_reservations = target.abandoned_bindings.len(),
15022            excluded_subscriptions = target.excluded_subscriptions,
15023            drained,
15024            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
15025        );
15026    }
15027
15028    Ok(StopNotice::SentOverConnection)
15029}
15030
15031/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
15032/// the only slot a plain (non-swap) spawn can register into.
15033async fn wait_for_registration_after_reload(
15034    registry: &Registry,
15035    module_id: &str,
15036    snapshot: &SharedSnapshot,
15037    child: &mut SupervisedChild,
15038    wait: Duration,
15039) -> Result<RegistrationWaitOutcome, SuperviseError> {
15040    wait_for_slot_registration(
15041        registry,
15042        crate::registry::RegistrationSlot::Active(module_id),
15043        module_id,
15044        snapshot,
15045        child,
15046        wait,
15047    )
15048    .await
15049}
15050
15051/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
15052///
15053/// Keyed on the slot rather than the bare module id because during a swap the
15054/// id's active slot is already held by the incumbent: an id-keyed wait would
15055/// report the incumbent's registration as the candidate's and a candidate that
15056/// never registers would look registered. A swap candidate waits on
15057/// `crate::registry::RegistrationSlot::Candidate`.
15058async fn wait_for_slot_registration(
15059    registry: &Registry,
15060    slot: crate::registry::RegistrationSlot<'_>,
15061    module_id: &str,
15062    snapshot: &SharedSnapshot,
15063    child: &mut SupervisedChild,
15064    wait: Duration,
15065) -> Result<RegistrationWaitOutcome, SuperviseError> {
15066    let deadline = Instant::now() + wait;
15067    loop {
15068        if registry
15069            .registration(slot)
15070            .map_err(SuperviseError::Registry)?
15071            .is_some()
15072        {
15073            return Ok(RegistrationWaitOutcome::Registered);
15074        }
15075
15076        let now = Instant::now();
15077        if now >= deadline {
15078            return Ok(RegistrationWaitOutcome::TimedOut);
15079        }
15080        let remaining = deadline.saturating_duration_since(now);
15081        let poll = remaining.min(REGISTRY_RELEASE_POLL);
15082
15083        tokio::select! {
15084            wait_result = child.wait() => {
15085                let status = wait_result.map_err(|source| SuperviseError::Wait {
15086                    module_id: module_id.to_string(),
15087                    source,
15088                })?;
15089                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
15090                    snapshot,
15091                    child,
15092                    &status,
15093                )));
15094            }
15095            _ = sleep(poll) => {}
15096        }
15097    }
15098}
15099
15100fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
15101    // A process stopped for missing its first HELLO did not provide service,
15102    // even if its TERM handler exits zero. Reload also uses this classification
15103    // when a replacement exits before registering.
15104    if exit_report.kind != ExitKind::DeliberateSeverance {
15105        exit_report.kind = ExitKind::Crash;
15106    }
15107    exit_report
15108}
15109
15110async fn handle_reload_child_registration_failure(
15111    spec: &ModuleSpec,
15112    runtime: &SupervisorRuntimeConfig,
15113    registry: &Registry,
15114    process_liveness: &SupervisorProcessLiveness,
15115    snapshot: &SharedSnapshot,
15116    _child: &mut Option<SupervisedChild>,
15117    failure: ReloadRegistrationFailure,
15118) -> Result<(), SuperviseError> {
15119    let ReloadRegistrationFailure {
15120        exit_report,
15121        reason,
15122    } = failure;
15123    match on_child_exit(
15124        spec,
15125        runtime.restart_policy,
15126        registry,
15127        snapshot,
15128        &runtime.terminal_ring,
15129        &runtime.spawn_events,
15130        &runtime.child_roster,
15131        exit_report,
15132        runtime.recovery_is_daemon,
15133    )
15134    .await
15135    {
15136        NextAction::Stop {
15137            registration_released,
15138        } => {
15139            if registration_released {
15140                process_liveness.untrack_if_current(&spec.module_id, snapshot);
15141            }
15142        }
15143        NextAction::Restart { schedule } => {
15144            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
15145                schedule.delay
15146            });
15147            if let Some(schedule) = schedule {
15148                log_crash_respawn(&spec.module_id, schedule);
15149            }
15150            schedule_respawn(
15151                runtime,
15152                snapshot,
15153                &spec.module_id,
15154                delay,
15155                RespawnKind::Spawn,
15156                true,
15157            )?;
15158        }
15159    }
15160    Err(SuperviseError::ReloadFailed {
15161        module_id: spec.module_id.clone(),
15162        reason,
15163    })
15164}
15165
15166async fn handle_reload_spawn_failure(
15167    spec: &ModuleSpec,
15168    runtime: &SupervisorRuntimeConfig,
15169    process_liveness: &SupervisorProcessLiveness,
15170    snapshot: &SharedSnapshot,
15171    _child: &mut Option<SupervisedChild>,
15172    reason: String,
15173) -> Result<(), SuperviseError> {
15174    let now = Instant::now();
15175    let mut schedule = None;
15176    update_snapshot(snapshot, Some(&spec.module_id), |state| {
15177        clear_current_process_facts(state);
15178        if state.enabled {
15179            schedule = state.next_crash_restart(&runtime.restart_policy, now);
15180            state.state = if schedule.is_some() {
15181                ModuleState::Restarting
15182            } else {
15183                if runtime
15184                    .failure_context(runtime.recovery_is_daemon)
15185                    .may_retry()
15186                {
15187                    state.failed_retry.schedule(&spec.module_id);
15188                }
15189                ModuleState::Failed
15190            };
15191        } else {
15192            state.state = ModuleState::Disabled;
15193        }
15194    })?;
15195    if let Some(schedule) = schedule {
15196        schedule_respawn(
15197            runtime,
15198            snapshot,
15199            &spec.module_id,
15200            schedule.delay,
15201            RespawnKind::Spawn,
15202            true,
15203        )?;
15204    } else {
15205        process_liveness.untrack_if_current(&spec.module_id, snapshot);
15206    }
15207    Err(SuperviseError::ReloadFailed {
15208        module_id: spec.module_id.clone(),
15209        reason,
15210    })
15211}
15212
15213fn control_flags() -> Flags {
15214    Flags::new(false, Priority::Passive, false)
15215}
15216
15217#[allow(clippy::too_many_arguments)]
15218async fn drain_optional_child(
15219    module_id: &str,
15220    protocol: ModuleProtocol,
15221    stop_notice: StopNotice,
15222    registry: &Registry,
15223    forwarding: Option<&ForwardingTable>,
15224    snapshot: &SharedSnapshot,
15225    terminal_ring: &Arc<Mutex<TerminalRing>>,
15226    spawn_events: &SpawnEventFeed,
15227    child: &mut Option<SupervisedChild>,
15228    drain_timeout: Duration,
15229    final_state: ModuleState,
15230    enabled: Option<bool>,
15231    failure: &FailureContext,
15232) -> Result<(), SuperviseError> {
15233    if let Some(child) = child.take() {
15234        drain_child_to_state(
15235            module_id,
15236            protocol,
15237            stop_notice,
15238            registry,
15239            forwarding,
15240            snapshot,
15241            terminal_ring,
15242            spawn_events,
15243            child,
15244            drain_timeout,
15245            final_state,
15246            enabled,
15247            failure,
15248        )
15249        .await
15250    } else {
15251        update_snapshot(snapshot, Some(module_id), |state| {
15252            state.state = final_state;
15253            if final_state == ModuleState::Failed {
15254                state.failed_retry.failed_at = Some(Instant::now());
15255            }
15256            if let Some(enabled) = enabled {
15257                state.enabled = enabled;
15258            }
15259            clear_current_process_facts(state);
15260        })?;
15261        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
15262    }
15263}
15264
15265#[allow(clippy::too_many_arguments)]
15266async fn drain_child_to_state(
15267    module_id: &str,
15268    _protocol: ModuleProtocol,
15269    stop_notice: StopNotice,
15270    registry: &Registry,
15271    forwarding: Option<&ForwardingTable>,
15272    snapshot: &SharedSnapshot,
15273    terminal_ring: &Arc<Mutex<TerminalRing>>,
15274    spawn_events: &SpawnEventFeed,
15275    mut child: SupervisedChild,
15276    drain_timeout: Duration,
15277    final_state: ModuleState,
15278    enabled: Option<bool>,
15279    failure: &FailureContext,
15280) -> Result<(), SuperviseError> {
15281    let protocol = child.protocol;
15282    child.retire_registration();
15283    update_snapshot(snapshot, Some(module_id), |state| {
15284        state.state = ModuleState::Draining;
15285        state.draining_to_replace = final_state == ModuleState::Restarting;
15286        if let Some(enabled) = enabled {
15287            state.enabled = enabled;
15288        }
15289    })?;
15290
15291    // The wait below is the same budget in every case; what differs is
15292    // whether anything has ASKED the child to stop before it starts. Only a
15293    // forwarding drain that reached the module's registered connection has
15294    // (`module.draining`, then a module GOODBYE). Every other child was told
15295    // nothing: a `protocol: "none"` module, which never registers; a subc
15296    // module spawned moments ago that has not sent HELLO yet; or a stop that
15297    // runs no forwarding drain. Without a signal the budget is only a delay
15298    // in front of SIGKILL -- and the not-yet-registered child is the worst
15299    // case, because it registers into a module that is already draining,
15300    // is never told, and is killed while healthy.
15301    if stop_notice != StopNotice::SentOverConnection {
15302        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
15303            info!(
15304                module_id,
15305                pid = child.pid,
15306                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15307                "module has no connection yet; requesting stop by signal"
15308            );
15309        }
15310        request_graceful_stop(module_id, &child);
15311    }
15312
15313    let exit_report = match timeout(drain_timeout, child.wait()).await {
15314        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
15315        Ok(Err(source)) => {
15316            fail_after_wait_error(
15317                module_id,
15318                snapshot,
15319                terminal_ring,
15320                spawn_events,
15321                format!("failed to wait during drain: {source}"),
15322                failure,
15323            );
15324            return Err(SuperviseError::Wait {
15325                module_id: module_id.to_string(),
15326                source,
15327            });
15328        }
15329        Err(_) => {
15330            // Mirror the sibling arm above: state is already `Draining`, and an
15331            // error propagated from here would strand it there -- a state
15332            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
15333            // `Failed | Stopped`), leaving an operator Restart as the only exit.
15334            // `Failed` before `?` keeps the module operator-visible and
15335            // revivable. Trigger is an ESRCH race (process exits between the
15336            // drain timeout firing and the kill) or a post-kill wait failure
15337            // (issue #34).
15338            //
15339            // Logged because the kill is otherwise visible only as signal 9 in
15340            // the terminal ring, and the budget it follows can be long enough
15341            // that consumers see a stretch of refusals with no stated cause.
15342            warn!(
15343                module_id,
15344                pid = child.pid,
15345                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15346                reason = ?final_state,
15347                ?stop_notice,
15348                "drain budget expired before the module exited; killing it"
15349            );
15350            let kill =
15351                if final_state == ModuleState::Restarting && child.child.container().is_some() {
15352                    child.kill_leader()
15353                } else {
15354                    child.start_kill()
15355                };
15356            kill.map_err(|source| {
15357                fail_after_wait_error(
15358                    module_id,
15359                    snapshot,
15360                    terminal_ring,
15361                    spawn_events,
15362                    format!("failed to kill during drain: {source}"),
15363                    failure,
15364                );
15365                SuperviseError::Kill {
15366                    module_id: module_id.to_string(),
15367                    source,
15368                }
15369            })?;
15370            let status = child.wait().await.map_err(|source| {
15371                fail_after_wait_error(
15372                    module_id,
15373                    snapshot,
15374                    terminal_ring,
15375                    spawn_events,
15376                    format!("failed to wait after drain kill: {source}"),
15377                    failure,
15378                );
15379                SuperviseError::Wait {
15380                    module_id: module_id.to_string(),
15381                    source,
15382                }
15383            })?;
15384            classify_reaped_child_exit(snapshot, &child, &status)
15385        }
15386    };
15387
15388    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
15389    publish_after_terminal_record(
15390        module_id,
15391        || {
15392            record_terminal_with_detail(
15393                module_id,
15394                terminal_ring,
15395                spawn_events,
15396                &exit_report,
15397                terminal_disposition(final_state),
15398                detail,
15399            )
15400        },
15401        || {
15402            update_snapshot(snapshot, Some(module_id), |state| {
15403                state.state = final_state;
15404                if final_state == ModuleState::Failed {
15405                    state.failed_retry.failed_at = Some(Instant::now());
15406                }
15407                if let Some(enabled) = enabled {
15408                    state.enabled = enabled;
15409                }
15410                clear_current_process_facts(state);
15411                state.last_exit = Some(exit_report.clone());
15412                if exit_report.kind == ExitKind::DeliberateSeverance {
15413                    state.lifetime_restarts += 1;
15414                }
15415            })
15416        },
15417    )?;
15418    child.drain_stderr(module_id).await;
15419
15420    let released =
15421        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
15422    child
15423        .finish_exit(
15424            released.is_ok()
15425                && final_state == ModuleState::Restarting
15426                && child.spawn_failure.is_none(),
15427        )
15428        .await
15429        .map_err(|source| SuperviseError::Wait {
15430            module_id: module_id.to_owned(),
15431            source,
15432        })?;
15433    released
15434}
15435
15436/// Ask a child that nothing else has asked to stop, by signal.
15437///
15438/// A registered subc module is asked over its own connection: the drain sends
15439/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
15440/// module GOODBYE, and the module stops itself. A module that speaks no subc
15441/// wire receives none of that, and neither does a subc module that has not
15442/// registered yet, so for them the drain budget would be pure delay in front of
15443/// a SIGKILL -- and for a process with a store to flush (JetStream is the
15444/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
15445/// into a recovery on the next start.
15446///
15447/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
15448/// rule rather than an optimisation: that module's graceful stop is already
15449/// running by the time its child is drained, and a signal would race it.
15450///
15451/// Best-effort by construction. A child that has already exited is the ordinary
15452/// case rather than an error (the kill lands on a reaped or exiting pid), so a
15453/// failure is logged at debug and the wait-then-kill below still decides the
15454/// outcome.
15455#[cfg(unix)]
15456fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
15457    let Some(pid) = child
15458        .id()
15459        .and_then(|pid| i32::try_from(pid).ok())
15460        .and_then(rustix::process::Pid::from_raw)
15461    else {
15462        debug!(
15463            module_id,
15464            "no pid to signal for teardown; falling through to the drain wait"
15465        );
15466        return;
15467    };
15468    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
15469        Ok(()) => debug!(
15470            module_id,
15471            "sent SIGTERM to a module nothing else asked to stop"
15472        ),
15473        Err(err) => debug!(
15474            module_id,
15475            error = %err,
15476            "SIGTERM to module failed; the drain wait and kill still apply"
15477        ),
15478    }
15479}
15480
15481/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
15482/// Windows does offer need cooperation this supervisor cannot assume: a console
15483/// control event requires sharing a console with the child, and `WM_CLOSE`
15484/// requires the child to pump a message loop. A supervised server process does
15485/// neither, so there is nothing to send and teardown is the wait followed by the
15486/// kill. Emulating a signal here would mean inventing a stop protocol, which is
15487/// the thing `protocol: "none"` exists to avoid.
15488#[cfg(not(unix))]
15489fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
15490    debug!(
15491        module_id,
15492        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
15493    );
15494}
15495
15496fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
15497    match final_state {
15498        ModuleState::Stopped => TerminalDisposition::Stopped,
15499        ModuleState::Disabled => TerminalDisposition::Disabled,
15500        ModuleState::Restarting => TerminalDisposition::Restarting,
15501        ModuleState::Failed => TerminalDisposition::Failed,
15502        ModuleState::Starting
15503        | ModuleState::Running
15504        | ModuleState::Unresponsive
15505        | ModuleState::Draining => {
15506            unreachable!("terminal exits only finish in terminal or restarting states")
15507        }
15508    }
15509}
15510
15511/// Release a reaped child's registration before allowing another spawn.
15512///
15513/// EOF is not a process-lifetime signal: an inherited socket can stay open
15514/// indefinitely, and serial frame dispatch can be waiting on egress instead of
15515/// reading EOF. After the normal release grace, request connection close (which
15516/// cancels both reads and dispatch), then allow one more release grace for the
15517/// connection guard's forwarding cleanup. Never evict a different connection.
15518async fn release_dead_registration(
15519    registry: &Registry,
15520    forwarding: Option<&ForwardingTable>,
15521    snapshot: &SharedSnapshot,
15522    module_id: &str,
15523    failure: &FailureContext,
15524) -> Result<(), SuperviseError> {
15525    let result = async {
15526        let registration = registry
15527            .get_module(module_id)
15528            .map_err(SuperviseError::Registry)?;
15529        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
15530            Ok(()) => return Ok(()),
15531            Err(SuperviseError::RegistrationStillActive { .. }) => {}
15532            Err(err) => return Err(err),
15533        }
15534        let pid = lock_snapshot(snapshot)?.reaped_pid;
15535        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
15536            warn!(
15537                module_id,
15538                pid,
15539                connection_id = registration.connection_id.get(),
15540                "reaped module registration outlived release grace; closing dead connection"
15541            );
15542            forwarding.request_connection_close(
15543                registration.connection_id,
15544                CloseReason::new(
15545                    "supervised_process_reaped",
15546                    format!("module '{module_id}' pid {pid} exited"),
15547                ),
15548            );
15549            wait_for_slot_registration_release(
15550                registry,
15551                crate::registry::RegistrationSlot::Connection(registration.connection_id),
15552                REGISTRY_RELEASE_TIMEOUT,
15553            )
15554            .await?;
15555        }
15556        wait_for_registration_release(registry, module_id, Duration::ZERO).await
15557    }
15558    .await;
15559    if let Err(err) = &result {
15560        fail_snapshot(snapshot, Some(module_id), None, failure);
15561        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
15562    }
15563    result
15564}
15565
15566/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
15567/// plain stop or restart waits for before it spawns a replacement.
15568async fn wait_for_registration_release(
15569    registry: &Registry,
15570    module_id: &str,
15571    wait: Duration,
15572) -> Result<(), SuperviseError> {
15573    wait_for_slot_registration_release(
15574        registry,
15575        crate::registry::RegistrationSlot::Active(module_id),
15576        wait,
15577    )
15578    .await
15579}
15580
15581/// Wait for the registration in `slot` to go away.
15582///
15583/// Keyed on the slot rather than the bare module id because a successful swap
15584/// never empties the id's active slot (the promoted candidate is in it), so an
15585/// id-keyed wait for the incumbent's release would always time out. Draining a
15586/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
15587/// incumbent's connection instead.
15588async fn wait_for_slot_registration_release(
15589    registry: &Registry,
15590    slot: crate::registry::RegistrationSlot<'_>,
15591    wait: Duration,
15592) -> Result<(), SuperviseError> {
15593    let deadline = Instant::now() + wait;
15594    let mut release_events = registration_release_events().subscribe();
15595    let still_active = |registration: &crate::registry::ModuleRegistration| {
15596        SuperviseError::RegistrationStillActive {
15597            module_id: registration.manifest.module_id.clone(),
15598            waited: wait,
15599        }
15600    };
15601    loop {
15602        let _observed_generation = *release_events.borrow_and_update();
15603        let Some(registration) = registry
15604            .registration(slot)
15605            .map_err(SuperviseError::Registry)?
15606        else {
15607            return Ok(());
15608        };
15609
15610        let now = Instant::now();
15611        if now >= deadline {
15612            return Err(still_active(&registration));
15613        }
15614
15615        let remaining = deadline.saturating_duration_since(now);
15616        match timeout(remaining, release_events.changed()).await {
15617            Ok(Ok(())) | Ok(Err(_)) => {}
15618            Err(_) => return Err(still_active(&registration)),
15619        }
15620    }
15621}
15622
15623#[cfg(test)]
15624mod slot_registration_wait_tests {
15625    use super::*;
15626    use crate::registry::{ConnectionId, RegistrationSlot};
15627    use subc_protocol::manifest::ModuleManifest;
15628
15629    #[tokio::test]
15630    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
15631        let registry = Arc::new(Registry::default());
15632        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
15633        let runtime = supervisor.runtime_config();
15634        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
15635        let spec = ModuleSpec {
15636            script: None,
15637            module_id: "enable-stale-registration".to_string(),
15638            program: PathBuf::from("/missing/enable-retry-test"),
15639            args: Vec::new(),
15640            env: Vec::new(),
15641            reserved: false,
15642            reserved_prefixes: Vec::new(),
15643            protocol: ModuleProtocol::Subc,
15644            overlap: Default::default(),
15645            carry_containment: false,
15646        };
15647        let connection = ConnectionId::new(90);
15648        registry
15649            .register_with_control_ops(
15650                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
15651                1,
15652                connection,
15653                Vec::new(),
15654            )
15655            .unwrap();
15656        let mut child = None;
15657        let err = set_child_enabled(
15658            &spec,
15659            &runtime,
15660            &registry,
15661            &supervisor.process_liveness,
15662            &snapshot,
15663            &mut child,
15664            true,
15665            &runtime.failure_context(false),
15666        )
15667        .await
15668        .unwrap_err();
15669        assert!(matches!(
15670            err,
15671            SuperviseError::RegistrationStillActive { .. }
15672        ));
15673        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15674        assert!(child.is_none());
15675        registry.deregister_connection(connection).unwrap();
15676        let err = set_child_enabled(
15677            &spec,
15678            &runtime,
15679            &registry,
15680            &supervisor.process_liveness,
15681            &snapshot,
15682            &mut child,
15683            true,
15684            &runtime.failure_context(false),
15685        )
15686        .await
15687        .unwrap_err();
15688        assert!(
15689            matches!(err, SuperviseError::Spawn { .. }),
15690            "second enable must attempt a spawn: {err}"
15691        );
15692        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15693    }
15694
15695    const INCUMBENT: u64 = 1;
15696    const CANDIDATE: u64 = 2;
15697
15698    fn swapped_registry() -> Arc<Registry> {
15699        let registry = Arc::new(Registry::default());
15700        let manifest = ModuleManifest::builder("m", "0.1.0").build();
15701        registry
15702            .register_with_control_ops(
15703                manifest.clone(),
15704                1,
15705                ConnectionId::new(INCUMBENT),
15706                Vec::new(),
15707            )
15708            .unwrap();
15709        registry
15710            .register_candidate_with_control_ops(
15711                manifest,
15712                1,
15713                ConnectionId::new(CANDIDATE),
15714                Vec::new(),
15715            )
15716            .unwrap();
15717        registry
15718    }
15719
15720    /// After a promotion the id's active slot is held by the new process, so an
15721    /// id-keyed wait for the incumbent's release can never succeed; the
15722    /// connection-keyed wait completes as soon as the incumbent deregisters.
15723    #[tokio::test]
15724    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
15725        let registry = swapped_registry();
15726        registry.promote_candidate("m").unwrap().unwrap();
15727
15728        assert!(matches!(
15729            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
15730            Err(SuperviseError::RegistrationStillActive { .. })
15731        ));
15732
15733        // Still held while the incumbent's connection has not deregistered.
15734        assert!(matches!(
15735            wait_for_slot_registration_release(
15736                &registry,
15737                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15738                Duration::from_millis(50),
15739            )
15740            .await,
15741            Err(SuperviseError::RegistrationStillActive { .. })
15742        ));
15743
15744        let releaser = Arc::clone(&registry);
15745        let release = tokio::spawn(async move {
15746            sleep(Duration::from_millis(20)).await;
15747            releaser
15748                .deregister_connection(ConnectionId::new(INCUMBENT))
15749                .unwrap();
15750            notify_registration_release();
15751        });
15752        wait_for_slot_registration_release(
15753            &registry,
15754            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15755            Duration::from_secs(5),
15756        )
15757        .await
15758        .expect("the incumbent's own registration is released");
15759        release.await.unwrap();
15760        assert!(registry.get_module("m").unwrap().is_some());
15761    }
15762
15763    /// The candidate slot is waited on separately from the active slot: the
15764    /// incumbent's registration neither holds up nor stands in for it.
15765    #[tokio::test]
15766    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
15767        let registry = swapped_registry();
15768        assert!(matches!(
15769            wait_for_slot_registration_release(
15770                &registry,
15771                RegistrationSlot::Candidate("m"),
15772                Duration::from_millis(50),
15773            )
15774            .await,
15775            Err(SuperviseError::RegistrationStillActive { .. })
15776        ));
15777        registry
15778            .deregister_connection(ConnectionId::new(CANDIDATE))
15779            .unwrap();
15780        wait_for_slot_registration_release(
15781            &registry,
15782            RegistrationSlot::Candidate("m"),
15783            Duration::from_millis(50),
15784        )
15785        .await
15786        .expect("a candidate slot with no candidate is released");
15787        assert!(registry
15788            .registration(RegistrationSlot::Active("m"))
15789            .unwrap()
15790            .is_some());
15791    }
15792}
15793
15794fn classify_exit(status: &ExitStatus) -> ExitReport {
15795    ExitReport {
15796        kind: if status.success() {
15797            ExitKind::Clean
15798        } else {
15799            ExitKind::Crash
15800        },
15801        code: status.code(),
15802        signal: exit_signal(status),
15803        at_ms: unix_ms_now(),
15804    }
15805}
15806
15807/// The terminal record for a module whose `wait()` call itself errored (e.g. the
15808/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
15809/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
15810/// disposition still must be `Failed` so the terminal ring is not silently missing
15811/// an entry, matching what `fail_snapshot` records for this same arm.
15812fn wait_error_exit_report() -> ExitReport {
15813    ExitReport {
15814        kind: ExitKind::Crash,
15815        code: None,
15816        signal: None,
15817        at_ms: unix_ms_now(),
15818    }
15819}
15820
15821#[cfg(unix)]
15822fn exit_signal(status: &ExitStatus) -> Option<i32> {
15823    use std::os::unix::process::ExitStatusExt;
15824
15825    status.signal()
15826}
15827
15828#[cfg(not(unix))]
15829fn exit_signal(_status: &ExitStatus) -> Option<i32> {
15830    None
15831}
15832
15833/// Give an operator-touched module its full crash budget back.
15834///
15835/// Named for the counter it used to zero; it now empties the in-window ring,
15836/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
15837/// ledger of what happened survives every operator action.
15838fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
15839    update_snapshot(snapshot, Some(module_id), |state| {
15840        state.clear_crash_restarts();
15841    })
15842}
15843
15844fn set_running(
15845    snapshot: &SharedSnapshot,
15846    child: &SupervisedChild,
15847    module_id: &str,
15848    spawn_events: &SpawnEventFeed,
15849) -> Result<(), SuperviseError> {
15850    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15851        module_id: Some(module_id.to_string()),
15852    })?;
15853    state.spawn_generation = spawn_events.emit_spawned(
15854        module_id,
15855        child
15856            .registration_spawn
15857            .as_ref()
15858            .map(|spawn| spawn.authority.spawn_number()),
15859        child.pid,
15860        child.spawned_at_ms,
15861    );
15862    if std::mem::take(&mut state.coalesced_restart_pending) {
15863        let generation = state.spawn_generation;
15864        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
15865    }
15866    state.drain_disposition_detail = None;
15867    state.spawn_failure = None;
15868    state.admission_refused = None;
15869    state.config_hold_parked = false;
15870    state.config_hold_daemon_initiated = false;
15871    state.config_hold_approval = None;
15872    state.legacy_registration_spawn =
15873        child.registration_spawn.is_some() && child.admission_base.is_none();
15874    state.registration_declaration = child.registration_declaration.clone();
15875    state.stop_reason = None;
15876    state.config_exit_failed_approval = None;
15877    state.waiting_for_previous_pid = None;
15878    state.waiting_for_previous_detail = None;
15879    // Every caller of this is a plain spawn, which always uses the primary key;
15880    // a promoted swap candidate sets the flag itself after this returns.
15881    state.in_alternate_slot = false;
15882    state.configuration_updated_since_spawn = false;
15883    state.spawned_protocol = Some(child.protocol);
15884    state.state = ModuleState::Running;
15885    state.approved_start_failure = None;
15886    state.enabled = true;
15887    state.process_alive = true;
15888    state.pid = child.id();
15889    state.subtree_target = child.subtree_target.clone();
15890    // Probe samples belong to one process. Keep module-level escalation and
15891    // late-answer history, but never charge a new child for its predecessor's misses.
15892    state.health.status = SupervisorHealthStatus::Unknown;
15893    state.last_probe_evidence = None;
15894    state.health.consecutive_failures = 0;
15895    state.health.detail = None;
15896    state.health.metrics = None;
15897    state.health.last_probe_ms = None;
15898    state.failed_retry.healthy_since = None;
15899    state.failed_retry.failed_at = None;
15900    state.failed_retry.running_since = Some(child.spawned_at);
15901    #[cfg(windows)]
15902    {
15903        state.pipe_consumed = child
15904            .nonce_delivery
15905            .as_ref()
15906            .map(|delivery| delivery.consumed());
15907    }
15908    #[cfg(target_os = "macos")]
15909    {
15910        state.report_ready = Some(Arc::clone(&child.report_ready));
15911    }
15912    state.spawned_at_ms = Some(child.spawned_at_ms);
15913    state.exec_outcome = Some(Arc::clone(&child.exec.0));
15914    state.spawned_from = Some(child.spawned_from.clone());
15915    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
15916    state.spawned_file_identity = child.spawned_file_identity.clone();
15917    state.process_start_time = child.process_start_time;
15918    Ok(())
15919}
15920
15921fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
15922    state.legacy_registration_spawn = false;
15923    state.registration_declaration = None;
15924    state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
15925    state.failed_retry.healthy_since = None;
15926    state.failed_retry.running_since = None;
15927    state.waiting_for_previous_pid = None;
15928    state.waiting_for_previous_detail = None;
15929    state.process_alive = false;
15930    state.spawned_protocol = None;
15931    state.pid = None;
15932    #[cfg(windows)]
15933    {
15934        state.pipe_consumed = None;
15935    }
15936    #[cfg(target_os = "macos")]
15937    {
15938        state.report_ready = None;
15939    }
15940    state.spawned_at_ms = None;
15941    state.exec_outcome = None;
15942    state.spawned_from = None;
15943    state.launch_spec_digest = None;
15944    state.spawned_file_identity = None;
15945    state.process_start_time = None;
15946    state.deliberate_severance = None;
15947}
15948
15949#[cfg(test)]
15950fn record_deliberate_severance(
15951    snapshot: &SharedSnapshot,
15952    identity: ProcessIdentity,
15953) -> Result<(), SuperviseError> {
15954    update_snapshot(snapshot, None, |state| {
15955        state.deliberate_severance = Some(identity);
15956    })
15957}
15958
15959fn apply_deliberate_severance_marker(
15960    snapshot: &SharedSnapshot,
15961    exited_identity: Option<ProcessIdentity>,
15962    mut exit_report: ExitReport,
15963) -> ExitReport {
15964    let marker = lock_snapshot(snapshot)
15965        .ok()
15966        .and_then(|mut state| state.deliberate_severance.take());
15967    if marker.is_some() && marker == exited_identity {
15968        exit_report.kind = ExitKind::DeliberateSeverance;
15969    }
15970    exit_report
15971}
15972
15973fn classify_reaped_child_exit(
15974    snapshot: &SharedSnapshot,
15975    child: &SupervisedChild,
15976    status: &ExitStatus,
15977) -> ExitReport {
15978    let _ = update_snapshot(snapshot, None, |state| {
15979        state.reaped_pid = Some(child.pid);
15980        state.spawn_failure = child.spawn_failure.clone();
15981    });
15982    if matches!(
15983        child.spawn_failure.as_deref(),
15984        Some("registration_timeout" | "registration_channel_closed")
15985    ) {
15986        registration_failure_exit_report(classify_exit(status))
15987    } else {
15988        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
15989    }
15990}
15991
15992fn fail_snapshot(
15993    snapshot: &SharedSnapshot,
15994    module_id: Option<&str>,
15995    last_exit: Option<ExitReport>,
15996    failure: &FailureContext,
15997) {
15998    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
15999        state.state = ModuleState::Failed;
16000        state.config_hold_parked = false;
16001        state.config_hold_daemon_initiated = false;
16002        state.config_hold_approval = None;
16003        clear_current_process_facts(state);
16004        if failure.may_retry() && state.enabled {
16005            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
16006        } else if failure.shutdown.is_set() {
16007            state.failed_retry.cancel();
16008        }
16009        if let Some(last_exit) = last_exit {
16010            state.last_exit = Some(last_exit);
16011        }
16012    }) {
16013        error!(error = %err, "failed to mark supervisor state failed");
16014    }
16015}
16016
16017fn update_snapshot(
16018    snapshot: &SharedSnapshot,
16019    module_id: Option<&str>,
16020    update: impl FnOnce(&mut SupervisorSnapshot),
16021) -> Result<(), SuperviseError> {
16022    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
16023        module_id: module_id.map(ToOwned::to_owned),
16024    })?;
16025    update(&mut state);
16026    Ok(())
16027}
16028
16029const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
16030
16031fn lock_snapshot_for_control<'a>(
16032    snapshot: &'a SharedSnapshot,
16033    module_id: &str,
16034    caller: &'static str,
16035) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
16036    let started_at = Instant::now();
16037    let guard = lock_snapshot(snapshot)?;
16038    let waited = started_at.elapsed();
16039    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
16040        warn!(
16041            module_id = %module_id,
16042            waited_ms = waited.as_millis() as u64,
16043            caller = %caller,
16044            "slow snapshot lock"
16045        );
16046    }
16047    Ok(guard)
16048}
16049
16050fn lock_snapshot(
16051    snapshot: &SharedSnapshot,
16052) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
16053    snapshot
16054        .lock()
16055        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
16056}
16057
16058#[cfg(test)]
16059mod terminal_history_tests {
16060    include!("supervise_route_open_recovery_tests.rs");
16061    use std::{
16062        path::PathBuf,
16063        sync::Arc,
16064        time::{Duration, Instant},
16065    };
16066
16067    use tokio::{sync::mpsc, time::sleep};
16068
16069    use super::{
16070        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
16071        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
16072        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
16073        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
16074        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
16075        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
16076        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
16077        SupervisorHealthStatus, SupervisorSnapshot,
16078    };
16079    // The supervisor's clock, distinct from the `std::time::Instant` these tests
16080    // use for their own wall-clock deadlines: crash-restart instants must be on
16081    // the same clock the production code stamps them with, which is tokio's (and
16082    // is what `start_paused` tests can move).
16083    use super::Instant as ClockInstant;
16084    use crate::{
16085        registry::Registry,
16086        terminal_ring::{TerminalRing, TerminalRingConfig},
16087    };
16088    use std::sync::Mutex;
16089    use subc_control::TerminalDisposition;
16090
16091    /// See the twin in `control.rs` for why this derives the path from
16092    /// `current_exe()` and why the existence check is here: `--lib` alone does
16093    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
16094    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
16095    pub(super) fn fake_aft_stub_path() -> PathBuf {
16096        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
16097        assert!(
16098            path.exists(),
16099            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16100             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16101            path.display()
16102        );
16103        path
16104    }
16105
16106    #[test]
16107    fn reserved_never_spawned_refuses_every_hello() {
16108        // The canary hole: a reserved id whose module has never spawned had NO
16109        // gate entry and admitted anyone -- the reservation protected the nonce
16110        // holder, not the NAME. Now the entry is present with no legitimate
16111        // holder and refuses all comers.
16112        let supervisor = SupervisorHandle::default();
16113        supervisor.apply_identity_configuration(&ModuleSpec {
16114            script: None,
16115            module_id: "never-spawned".to_string(),
16116            program: PathBuf::from("/usr/bin/false"),
16117            args: Vec::new(),
16118            env: Vec::new(),
16119            reserved: true,
16120            reserved_prefixes: Vec::new(),
16121            protocol: ModuleProtocol::Subc,
16122            overlap: Default::default(),
16123            carry_containment: false,
16124        });
16125        assert!(
16126            supervisor
16127                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
16128                .is_err(),
16129            "forged nonce must refuse on a reserved never-spawned id"
16130        );
16131        assert!(
16132            supervisor
16133                .reserved_hello_verification("never-spawned", None)
16134                .is_err(),
16135            "absent nonce must refuse on a reserved never-spawned id"
16136        );
16137        // And a real spawn nonce minted later admits exactly that nonce.
16138        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
16139        supervisor.apply_identity_configuration(&ModuleSpec {
16140            script: None,
16141            module_id: "never-spawned".to_string(),
16142            program: PathBuf::from("/usr/bin/false"),
16143            args: Vec::new(),
16144            env: Vec::new(),
16145            reserved: true,
16146            reserved_prefixes: Vec::new(),
16147            protocol: ModuleProtocol::Subc,
16148            overlap: Default::default(),
16149            carry_containment: false,
16150        });
16151        assert!(supervisor
16152            .reserved_hello_verification("never-spawned", Some("minted"))
16153            .is_ok());
16154        assert!(supervisor
16155            .reserved_hello_verification("never-spawned", Some("forged"))
16156            .is_err());
16157    }
16158
16159    /// Put `count` crash restarts on a snapshot's ring as if they had all just
16160    /// happened, which is what "spent budget" looks like to every reader.
16161    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
16162        let now = ClockInstant::now();
16163        for _ in 0..count {
16164            state.crash_restarts.push_back(now);
16165        }
16166    }
16167
16168    /// Age the oldest recorded restart out of `window`, standing in for the hours
16169    /// that would otherwise have to pass. Injecting the instant is the point: a
16170    /// test that slept a real window would take ten minutes and still prove less.
16171    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
16172        let aged = state
16173            .crash_restarts
16174            .front()
16175            .expect("a crash restart must be recorded before it can be aged")
16176            .checked_sub(window + Duration::from_secs(1))
16177            .expect("the test clock is far enough from its origin to age an instant");
16178        state.crash_restarts[0] = aged;
16179    }
16180
16181    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
16182        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
16183        seed_crash_restarts(&mut state, count);
16184        state
16185    }
16186
16187    #[test]
16188    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
16189        let policy = RestartPolicy::new(3, Duration::ZERO);
16190        let now = ClockInstant::now();
16191        assert!(daemon_will_restart(
16192            &mut snapshot_with_restarts(true, 2),
16193            &policy,
16194            now
16195        ));
16196        assert!(!daemon_will_restart(
16197            &mut snapshot_with_restarts(true, 3),
16198            &policy,
16199            now
16200        ));
16201        assert!(!daemon_will_restart(
16202            &mut snapshot_with_restarts(false, 0),
16203            &policy,
16204            now
16205        ));
16206    }
16207
16208    #[test]
16209    fn crash_restart_backoff_escalates_with_in_window_count() {
16210        let policy = RestartPolicy::new(4, Duration::from_millis(100))
16211            .with_max_backoff(Duration::from_secs(30));
16212        let now = ClockInstant::now();
16213        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16214        let schedules = (0..4)
16215            .map(|_| {
16216                state
16217                    .next_crash_restart(&policy, now)
16218                    .expect("the test policy allows four crash restarts")
16219            })
16220            .collect::<Vec<_>>();
16221
16222        assert_eq!(
16223            schedules
16224                .iter()
16225                .map(|schedule| schedule.restart_in_window)
16226                .collect::<Vec<_>>(),
16227            vec![0, 1, 2, 3]
16228        );
16229        assert_eq!(
16230            schedules
16231                .iter()
16232                .map(|schedule| schedule.delay)
16233                .collect::<Vec<_>>(),
16234            vec![
16235                Duration::from_millis(100),
16236                Duration::from_secs(1),
16237                Duration::from_secs(10),
16238                Duration::from_secs(30),
16239            ]
16240        );
16241    }
16242
16243    #[test]
16244    fn crash_restart_backoff_resets_after_ring_clear() {
16245        let policy = RestartPolicy::new(3, Duration::from_millis(100));
16246        let now = ClockInstant::now();
16247        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16248        assert_eq!(
16249            state.next_crash_restart(&policy, now).unwrap().delay,
16250            Duration::from_millis(100)
16251        );
16252        assert_eq!(
16253            state.next_crash_restart(&policy, now).unwrap().delay,
16254            Duration::from_secs(1)
16255        );
16256
16257        state.clear_crash_restarts();
16258        let schedule = state
16259            .next_crash_restart(&policy, now)
16260            .expect("a cleared ring must allow another restart");
16261        assert_eq!(schedule.restart_in_window, 0);
16262        assert_eq!(schedule.delay, Duration::from_millis(100));
16263    }
16264
16265    #[test]
16266    fn crash_restart_backoff_ignores_aged_restarts() {
16267        let policy = RestartPolicy::new(3, Duration::from_millis(100));
16268        let now = ClockInstant::now();
16269        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16270        state
16271            .next_crash_restart(&policy, now)
16272            .expect("the first restart is allowed");
16273        state
16274            .next_crash_restart(&policy, now)
16275            .expect("the second restart is allowed");
16276        state.crash_restarts[0] = now
16277            .checked_sub(policy.window + Duration::from_secs(1))
16278            .expect("the fake clock can age a restart past the window");
16279
16280        let schedule = state
16281            .next_crash_restart(&policy, now)
16282            .expect("an aged restart must release its slot");
16283        assert_eq!(schedule.restart_in_window, 1);
16284        assert_eq!(schedule.delay, Duration::from_secs(1));
16285        assert_eq!(state.crash_restarts.len(), 2);
16286    }
16287
16288    /// The budget is a rate: the same three spent restarts refuse a respawn
16289    /// while they are recent and allow one once they have aged past the window.
16290    /// Nothing about the module changed in between, which is the whole point.
16291    #[test]
16292    fn a_budget_spent_before_the_window_no_longer_refuses() {
16293        let policy = RestartPolicy::new(3, Duration::ZERO);
16294        let mut state = snapshot_with_restarts(true, 3);
16295        let now = ClockInstant::now();
16296        assert!(!daemon_will_restart(&mut state, &policy, now));
16297
16298        assert!(daemon_will_restart(
16299            &mut state,
16300            &policy,
16301            now + policy.window + Duration::from_secs(1)
16302        ));
16303        assert!(
16304            state.crash_restarts.is_empty(),
16305            "reading the budget must drop the instants that left the window"
16306        );
16307    }
16308
16309    fn module_with_recovery_snapshot(
16310        state: ModuleState,
16311        enabled: bool,
16312        restart_count: u32,
16313    ) -> SupervisedModule {
16314        let registry = Arc::new(Registry::default());
16315        let supervisor =
16316            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
16317        let runtime = supervisor.runtime_config();
16318        let spec = ModuleSpec {
16319            script: None,
16320            module_id: "recovery-snapshot".to_string(),
16321            program: fake_aft_stub_path(),
16322            args: Vec::new(),
16323            env: Vec::new(),
16324            reserved: false,
16325            reserved_prefixes: Vec::new(),
16326            protocol: ModuleProtocol::Subc,
16327            overlap: Default::default(),
16328            carry_containment: false,
16329        };
16330        let mut snapshot = SupervisorSnapshot::new(state, enabled);
16331        seed_crash_restarts(&mut snapshot, restart_count);
16332        // These tests read synthetic snapshots. A real child and monitor would
16333        // race those reads by replacing the requested state during startup.
16334        let (commands, _rx) = mpsc::channel(4);
16335        let configured_enabled = snapshot.enabled;
16336        SupervisedModule {
16337            inner: Arc::new(SupervisedModuleInner {
16338                module_id: spec.module_id.clone(),
16339                registry,
16340                snapshot: Arc::new(Mutex::new(snapshot)),
16341                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
16342                    spec,
16343                    enabled: configured_enabled,
16344                    health: runtime.health,
16345                    register_timeout: runtime.register_timeout,
16346                    restart_policy: runtime.restart_policy,
16347                })),
16348                stderr_ring: runtime.stderr_ring,
16349                terminal_ring: runtime.terminal_ring,
16350                commands,
16351                monitor: Mutex::new(None),
16352                effective_drain_timeout: runtime.effective_drain_timeout,
16353                provenance_probe: supervisor.provenance_probe.clone(),
16354                retained: runtime.child_roster.retained_view(),
16355            }),
16356        }
16357    }
16358
16359    #[cfg(target_os = "linux")]
16360    #[tokio::test]
16361    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
16362        let supervisor =
16363            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16364                .with_cgroup_placement(None);
16365        let result = supervisor.spawn(ModuleSpec {
16366            script: None,
16367            module_id: "no-cgroup-placement".to_string(),
16368            program: fake_aft_stub_path(),
16369            args: Vec::new(),
16370            env: Vec::new(),
16371            reserved: false,
16372            reserved_prefixes: Vec::new(),
16373            protocol: ModuleProtocol::Subc,
16374            overlap: Default::default(),
16375            carry_containment: false,
16376        });
16377
16378        assert!(
16379            result.is_ok(),
16380            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
16381        );
16382    }
16383
16384    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16385    async fn undecided_snapshot_uses_shared_restart_predicate() {
16386        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
16387            .will_recover_after_connection_loss()
16388            .unwrap());
16389        assert!(
16390            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
16391                .will_recover_after_connection_loss()
16392                .unwrap()
16393        );
16394    }
16395
16396    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16397    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
16398        assert!(
16399            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
16400                .will_recover_after_connection_loss()
16401                .unwrap()
16402        );
16403    }
16404
16405    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16406    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
16407        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
16408            .will_recover_after_connection_loss()
16409            .unwrap());
16410        assert!(
16411            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
16412                .will_recover_after_connection_loss()
16413                .unwrap()
16414        );
16415    }
16416
16417    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16418    async fn warming_snapshot_is_limited_to_startup_phases() {
16419        for state in [
16420            ModuleState::Starting,
16421            ModuleState::Running,
16422            ModuleState::Restarting,
16423        ] {
16424            assert!(
16425                module_with_recovery_snapshot(state, true, 0)
16426                    .is_warming()
16427                    .unwrap(),
16428                "{state:?} should be warming"
16429            );
16430        }
16431        for state in [
16432            ModuleState::Unresponsive,
16433            ModuleState::Draining,
16434            ModuleState::Stopped,
16435            ModuleState::Failed,
16436            ModuleState::Disabled,
16437        ] {
16438            assert!(
16439                !module_with_recovery_snapshot(state, true, 0)
16440                    .is_warming()
16441                    .unwrap(),
16442                "{state:?} should not be warming"
16443            );
16444        }
16445    }
16446
16447    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16448    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
16449        let registry = Arc::new(Registry::default());
16450        let supervisor =
16451            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
16452        let module = supervisor
16453            .spawn(ModuleSpec {
16454                script: None,
16455                module_id: "terminal-history".to_string(),
16456                program: fake_aft_stub_path(),
16457                args: Vec::new(),
16458                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16459                reserved: false,
16460                reserved_prefixes: Vec::new(),
16461                protocol: ModuleProtocol::Subc,
16462                overlap: Default::default(),
16463                carry_containment: false,
16464            })
16465            .unwrap();
16466
16467        let deadline = Instant::now() + Duration::from_secs(5);
16468        loop {
16469            let history = module.terminal_history();
16470            if history.entries.len() == 2 {
16471                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
16472                assert_eq!(history.dropped, 0);
16473                assert_eq!(
16474                    history
16475                        .entries
16476                        .iter()
16477                        .map(|entry| entry.exit_code)
16478                        .collect::<Vec<_>>(),
16479                    vec![Some(23), Some(23)]
16480                );
16481                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
16482                return;
16483            }
16484            assert!(
16485                Instant::now() < deadline,
16486                "module did not retain two terminal exits: {history:?}"
16487            );
16488            sleep(Duration::from_millis(10)).await;
16489        }
16490    }
16491
16492    /// A disable issued while a crash respawn is still backing off must preempt
16493    /// that respawn: the operator's stop wins, the disable must not queue behind
16494    /// the backoff, and the module must never come back up afterwards.
16495    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16496    async fn disable_during_crash_backoff_cancels_pending_respawn() {
16497        let backoff = Duration::from_secs(2);
16498        let supervisor = Supervisor::new_for_test(
16499            Arc::new(Registry::default()),
16500            RestartPolicy::new(10, backoff),
16501        );
16502        let module = supervisor
16503            .spawn(ModuleSpec {
16504                script: None,
16505                module_id: "disable-during-backoff".to_string(),
16506                program: fake_aft_stub_path(),
16507                args: Vec::new(),
16508                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16509                reserved: false,
16510                reserved_prefixes: Vec::new(),
16511                protocol: ModuleProtocol::Subc,
16512                overlap: Default::default(),
16513                carry_containment: false,
16514            })
16515            .unwrap();
16516
16517        // Wait for the first crash to put the module into its backoff window.
16518        let deadline = Instant::now() + Duration::from_secs(5);
16519        loop {
16520            if module.status().unwrap().state == ModuleState::Restarting {
16521                break;
16522            }
16523            assert!(
16524                Instant::now() < deadline,
16525                "module never entered the crash backoff"
16526            );
16527            sleep(Duration::from_millis(10)).await;
16528        }
16529
16530        let started = Instant::now();
16531        module.set_enabled(false).await.unwrap();
16532        let waited = started.elapsed();
16533
16534        assert!(
16535            waited < backoff / 2,
16536            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
16537        );
16538        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
16539
16540        // Outlast the backoff: the respawn it was counting down to must never run.
16541        sleep(backoff + Duration::from_millis(500)).await;
16542        let status = module.status().unwrap();
16543        assert_eq!(status.state, ModuleState::Disabled);
16544        assert_eq!(
16545            status.spawn_generation, 1,
16546            "module respawned after the operator disabled it"
16547        );
16548    }
16549
16550    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
16551    /// the shape of nats-server, the program this rule exists for.
16552    #[cfg(unix)]
16553    fn protocol_none_sigterm_exits_clean_spec(
16554        module_id: &str,
16555        dir: &std::path::Path,
16556    ) -> (ModuleSpec, PathBuf, PathBuf) {
16557        let ready = dir.join("ready");
16558        let marker = dir.join("sigterm");
16559        let spec = ModuleSpec {
16560            script: None,
16561            module_id: module_id.to_string(),
16562            program: fake_aft_stub_path(),
16563            args: Vec::new(),
16564            env: vec![
16565                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16566                (
16567                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
16568                    marker.display().to_string(),
16569                ),
16570                (
16571                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
16572                    ready.display().to_string(),
16573                ),
16574            ],
16575            reserved: false,
16576            reserved_prefixes: Vec::new(),
16577            protocol: ModuleProtocol::None,
16578            overlap: Default::default(),
16579            carry_containment: false,
16580        };
16581        (spec, ready, marker)
16582    }
16583
16584    /// Wait for a file the child writes, so a signal is never sent before the
16585    /// child's SIGTERM handler is installed (the default disposition would
16586    /// kill it by signal and the exit would not be clean).
16587    #[cfg(unix)]
16588    async fn wait_for_file(path: &std::path::Path) {
16589        let deadline = Instant::now() + Duration::from_secs(10);
16590        while !path.exists() {
16591            assert!(
16592                Instant::now() < deadline,
16593                "{} never appeared",
16594                path.display()
16595            );
16596            sleep(Duration::from_millis(10)).await;
16597        }
16598    }
16599
16600    /// A protocol-none module that exits 0 because something OUTSIDE the
16601    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
16602    /// the crash-path disposition rather than `stopped`.
16603    #[cfg(unix)]
16604    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16605    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
16606        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
16607        let (spec, ready, marker) =
16608            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
16609        let supervisor = Supervisor::new_for_test(
16610            Arc::new(Registry::default()),
16611            RestartPolicy::new(3, Duration::ZERO),
16612        );
16613        let module = supervisor.spawn(spec).unwrap();
16614        wait_for_file(&ready).await;
16615        // The ready file proves the child installed its SIGTERM handler, not
16616        // that the supervisor has processed the privacy trampoline's exec
16617        // acknowledgement. On macOS status withholds the pid until then.
16618        let deadline = Instant::now() + Duration::from_secs(10);
16619        let first_pid = loop {
16620            let status = module.status().unwrap();
16621            if status.state == ModuleState::Running {
16622                if let Some(pid) = status.pid {
16623                    break pid;
16624                }
16625            }
16626            assert!(
16627                Instant::now() < deadline,
16628                "a running module must report its pid after exec confirmation: {status:?}"
16629            );
16630            sleep(Duration::from_millis(10)).await;
16631        };
16632
16633        rustix::process::kill_process(
16634            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
16635            rustix::process::Signal::TERM,
16636        )
16637        .unwrap();
16638
16639        let deadline = Instant::now() + Duration::from_secs(10);
16640        let respawned = loop {
16641            let status = module.status().unwrap();
16642            if status.state == ModuleState::Running
16643                && status.pid.is_some_and(|pid| pid != first_pid)
16644            {
16645                break status;
16646            }
16647            assert!(
16648                Instant::now() < deadline,
16649                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
16650            );
16651            sleep(Duration::from_millis(10)).await;
16652        };
16653        assert_eq!(respawned.spawn_generation, 2);
16654        assert!(
16655            marker.exists(),
16656            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
16657        );
16658
16659        let history = module.terminal_history();
16660        assert_eq!(history.entries.len(), 1, "{history:?}");
16661        let entry = &history.entries[0];
16662        assert_eq!(entry.exit_code, Some(0));
16663        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
16664        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
16665
16666        module.stop().await.unwrap();
16667    }
16668
16669    /// Repeated unrequested clean exits of a protocol-none module spend the
16670    /// restart budget exactly as crashes do, and the module ends `failed` with
16671    /// the budget named.
16672    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16673    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
16674        let supervisor = Supervisor::new_for_test(
16675            Arc::new(Registry::default()),
16676            RestartPolicy::new(1, Duration::ZERO),
16677        );
16678        let module = supervisor
16679            .spawn(ModuleSpec {
16680                script: None,
16681                module_id: "none-clean-exit-budget".to_string(),
16682                program: fake_aft_stub_path(),
16683                args: Vec::new(),
16684                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
16685                reserved: false,
16686                reserved_prefixes: Vec::new(),
16687                protocol: ModuleProtocol::None,
16688                overlap: Default::default(),
16689                carry_containment: false,
16690            })
16691            .unwrap();
16692
16693        // Failed follows the terminal write; this deadline only bounds a hang,
16694        // not an assumed duration for the two launches or their exit recording.
16695        let deadline = Instant::now() + Duration::from_secs(10);
16696        loop {
16697            let status = module.status().unwrap();
16698            if status.state == ModuleState::Failed {
16699                break;
16700            }
16701            assert!(
16702                Instant::now() < deadline,
16703                "module never exhausted its budget: {status:?} {:?}",
16704                module.terminal_history()
16705            );
16706            sleep(Duration::from_millis(10)).await;
16707        }
16708        let history = module.terminal_history();
16709        assert_eq!(
16710            history
16711                .entries
16712                .iter()
16713                .map(|entry| (entry.exit_code, entry.disposition.clone()))
16714                .collect::<Vec<_>>(),
16715            vec![
16716                (Some(0), TerminalDisposition::Restarting),
16717                (Some(0), TerminalDisposition::Failed),
16718            ]
16719        );
16720        let detail = history.entries[1]
16721            .disposition_detail
16722            .as_deref()
16723            .expect("a budget failure names the budget");
16724        assert!(detail.contains("max_restarts=1"), "{detail}");
16725        assert_eq!(module.status().unwrap().spawn_generation, 2);
16726    }
16727
16728    #[test]
16729    fn restart_budget_failure_is_published_after_its_terminal_record() {
16730        use std::io::{Read, Write};
16731        let supervisor = Supervisor::new_for_test(
16732            Arc::new(Registry::default()),
16733            RestartPolicy::new(0, Duration::ZERO),
16734        );
16735        let runtime = supervisor.runtime_config();
16736        let spec = ModuleSpec {
16737            script: None,
16738            protocol: ModuleProtocol::None,
16739            ..windowed_crash_spec("budget-publication-order")
16740        };
16741        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16742            ModuleState::Running,
16743            true,
16744        )));
16745        lock_snapshot(&snapshot).unwrap().process_alive = true;
16746        let reaped_snapshot = snapshot.clone();
16747        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16748        let history = runtime.terminal_ring.clone();
16749        let reap = std::thread::spawn(move || {
16750            tokio::runtime::Builder::new_current_thread()
16751                .enable_all()
16752                .build()
16753                .unwrap()
16754                .block_on(on_child_exit(
16755                    &spec,
16756                    runtime.restart_policy,
16757                    &supervisor.registry,
16758                    &reaped_snapshot,
16759                    &runtime.terminal_ring,
16760                    &runtime.spawn_events,
16761                    &runtime.child_roster,
16762                    ExitReport {
16763                        kind: ExitKind::Clean,
16764                        code: Some(0),
16765                        signal: None,
16766                        at_ms: 1,
16767                    },
16768                    true,
16769                ))
16770        });
16771        let mut reached = [0];
16772        barrier.read_exact(&mut reached).unwrap();
16773        assert_eq!(&reached, b"R");
16774        let (before_state, before_alive) = {
16775            let state = lock_snapshot(&snapshot).unwrap();
16776            (state.state, state.process_alive)
16777        };
16778        let before_history = history.lock().unwrap().snapshot();
16779        barrier.write_all(b"X").unwrap();
16780        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
16781        assert!(before_history.entries.is_empty());
16782        assert_ne!(
16783            before_state,
16784            ModuleState::Failed,
16785            "Failed was visible before its terminal record could be written"
16786        );
16787        assert!(before_alive);
16788        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16789        let history = history.lock().unwrap().snapshot();
16790        assert_eq!(history.entries.len(), 1);
16791        assert_eq!(history.entries[0].exit_code, Some(0));
16792        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16793    }
16794
16795    #[test]
16796    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
16797        use std::io::{Read, Write};
16798
16799        let supervisor = Supervisor::new_for_test(
16800            Arc::new(Registry::default()),
16801            RestartPolicy::new(3, Duration::ZERO),
16802        );
16803        let mut runtime = supervisor.runtime_config();
16804        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
16805        let snapshot = module.inner.snapshot.clone();
16806        lock_snapshot(&snapshot).unwrap().process_alive = true;
16807        runtime.terminal_ring = module.inner.terminal_ring.clone();
16808        let spec = windowed_crash_spec(module.module_id());
16809        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16810        let reaped_snapshot = snapshot.clone();
16811        let reap = std::thread::spawn(move || {
16812            tokio::runtime::Builder::new_current_thread()
16813                .enable_all()
16814                .build()
16815                .unwrap()
16816                .block_on(on_child_exit(
16817                    &spec,
16818                    runtime.restart_policy,
16819                    &supervisor.registry,
16820                    &reaped_snapshot,
16821                    &runtime.terminal_ring,
16822                    &runtime.spawn_events,
16823                    &runtime.child_roster,
16824                    ExitReport {
16825                        kind: ExitKind::Crash,
16826                        code: Some(2),
16827                        signal: None,
16828                        at_ms: 1,
16829                    },
16830                    true,
16831                ))
16832        });
16833
16834        // Hold the crash handler between its budget decision and publication.
16835        // Connection cleanup must still see the recovery that this crash chose,
16836        // even when that recovery consumes the last available restart slot.
16837        let mut reached = [0];
16838        barrier.read_exact(&mut reached).unwrap();
16839        assert_eq!(&reached, b"R");
16840        let before_status = module.status().unwrap();
16841        let before_history = module.terminal_history();
16842        let before_recovers = module.will_recover_after_connection_loss().unwrap();
16843        barrier.write_all(b"X").unwrap();
16844        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
16845
16846        assert_eq!(before_status.state, ModuleState::Running);
16847        assert!(before_status.process_alive);
16848        assert_eq!(before_status.restart_count, 3);
16849        assert!(before_history.entries.is_empty());
16850        assert!(
16851            before_recovers,
16852            "connection cleanup would report a terminal close for a scheduled crash restart"
16853        );
16854        let restarting = module.status().unwrap();
16855        assert_eq!(restarting.state, ModuleState::Restarting);
16856        assert_eq!(restarting.restart_count, 3);
16857        assert!(module.will_recover_after_connection_loss().unwrap());
16858        let history = module.terminal_history();
16859        assert_eq!(history.entries.len(), 1);
16860        assert_eq!(
16861            history.entries[0].disposition,
16862            TerminalDisposition::Restarting
16863        );
16864
16865        // Once the replacement is running, its next crash must be judged
16866        // against the exhausted budget rather than the previous recovery.
16867        update_snapshot(&snapshot, Some(module.module_id()), |state| {
16868            state.state = ModuleState::Running;
16869            state.respawn_pending = false;
16870        })
16871        .unwrap();
16872        assert!(!module.will_recover_after_connection_loss().unwrap());
16873        update_snapshot(&snapshot, Some(module.module_id()), |state| {
16874            state.enabled = false;
16875            state.respawn_pending = true;
16876        })
16877        .unwrap();
16878        assert!(!module.will_recover_after_connection_loss().unwrap());
16879    }
16880
16881    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
16882        use std::net::{TcpListener, TcpStream};
16883        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
16884        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
16885        let (barrier, _) = listener.accept().unwrap();
16886        for stream in [&writer, &barrier] {
16887            stream
16888                .set_read_timeout(Some(Duration::from_secs(10)))
16889                .unwrap();
16890            stream
16891                .set_write_timeout(Some(Duration::from_secs(10)))
16892                .unwrap();
16893        }
16894        *events.1.lock().unwrap() = Some(writer);
16895        barrier
16896    }
16897
16898    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
16899        use std::io::{Read, Write};
16900        let barrier = events.1.lock().unwrap().take();
16901        if let Some(mut barrier) = barrier {
16902            barrier.write_all(b"R").unwrap();
16903            let mut release = [0];
16904            barrier.read_exact(&mut release).unwrap();
16905            assert_eq!(&release, b"X");
16906        }
16907    }
16908
16909    #[test]
16910    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
16911        use std::io::{Read, Write};
16912
16913        let supervisor = Supervisor::new_for_test(
16914            Arc::new(Registry::default()),
16915            RestartPolicy::new(0, Duration::ZERO),
16916        );
16917        let mut runtime = supervisor.runtime_config();
16918        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
16919        let snapshot = module.inner.snapshot.clone();
16920        runtime.terminal_ring = module.inner.terminal_ring.clone();
16921        let spec = ModuleSpec {
16922            script: None,
16923            module_id: module.module_id().into(),
16924            program: fake_aft_stub_path(),
16925            protocol: ModuleProtocol::None,
16926            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16927            ..windowed_crash_spec(module.module_id())
16928        };
16929        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16930        let reap = std::thread::spawn(move || {
16931            tokio::runtime::Builder::new_current_thread()
16932                .enable_all()
16933                .build()
16934                .unwrap()
16935                .block_on(async {
16936                    let mut child =
16937                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16938                    health_restart_child(
16939                        &spec,
16940                        &runtime,
16941                        &supervisor.registry,
16942                        &supervisor.process_liveness,
16943                        &snapshot,
16944                        &mut child,
16945                        SupervisorHealthStatus::Failing,
16946                        Some("health probe did not answer"),
16947                        1,
16948                    )
16949                    .await
16950                    .unwrap();
16951                    assert!(child.is_none());
16952                });
16953        });
16954        // The loopback barrier acknowledges entry into terminal recording by
16955        // health_restart_child's drain, then waits for the test's release byte.
16956        // Status and history are observed while that writer is held, not after
16957        // an elapsed-time guess about when the health action has completed.
16958        let mut reached = [0];
16959        barrier.read_exact(&mut reached).unwrap();
16960        assert_eq!(&reached, b"R");
16961        let before_status = module.status().unwrap();
16962        let before_history = module.terminal_history();
16963        barrier.write_all(b"X").unwrap();
16964        reap.join().unwrap();
16965        assert!(before_history.entries.is_empty());
16966        assert_ne!(
16967            before_status.state,
16968            ModuleState::Failed,
16969            "Failed was visible before its terminal record could be written"
16970        );
16971        assert!(
16972            before_status.process_alive,
16973            "process death was visible before its terminal record could be written"
16974        );
16975        let failed = module.status().unwrap();
16976        assert_eq!(failed.state, ModuleState::Failed);
16977        assert!(!failed.process_alive);
16978        let history = module.terminal_history();
16979        assert_eq!(history.entries.len(), 1);
16980        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16981        assert_eq!(
16982            history.entries[0].disposition_detail.as_deref(),
16983            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
16984        );
16985    }
16986
16987    #[test]
16988    fn observed_exits_publish_process_death_after_terminal_history() {
16989        use std::io::{Read, Write};
16990        for (kind, enabled, shutdown, wait_error, expected_state) in [
16991            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
16992            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
16993            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
16994            (
16995                ExitKind::DeliberateSeverance,
16996                true,
16997                false,
16998                false,
16999                ModuleState::Restarting,
17000            ),
17001            #[cfg(unix)]
17002            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
17003            (ExitKind::Crash, true, false, true, ModuleState::Failed),
17004        ] {
17005            let supervisor = Supervisor::new_for_test(
17006                Arc::new(Registry::default()),
17007                RestartPolicy::new(1, Duration::ZERO),
17008            );
17009            let runtime = supervisor.runtime_config();
17010            #[cfg(unix)]
17011            if shutdown {
17012                runtime.child_roster.close();
17013            }
17014            #[cfg(not(unix))]
17015            let _ = shutdown;
17016            let spec = windowed_crash_spec("exit-publication-order");
17017            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
17018                ModuleState::Running,
17019                enabled,
17020            )));
17021            lock_snapshot(&snapshot).unwrap().process_alive = true;
17022            let reaped_snapshot = snapshot.clone();
17023            let history = runtime.terminal_ring.clone();
17024            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
17025            let reap = std::thread::spawn(move || {
17026                if wait_error {
17027                    super::fail_after_wait_error(
17028                        &spec.module_id,
17029                        &reaped_snapshot,
17030                        &runtime.terminal_ring,
17031                        &runtime.spawn_events,
17032                        "wait failed".into(),
17033                        &runtime.failure_context(true),
17034                    );
17035                } else {
17036                    tokio::runtime::Builder::new_current_thread()
17037                        .enable_all()
17038                        .build()
17039                        .unwrap()
17040                        .block_on(on_child_exit(
17041                            &spec,
17042                            runtime.restart_policy,
17043                            &supervisor.registry,
17044                            &reaped_snapshot,
17045                            &runtime.terminal_ring,
17046                            &runtime.spawn_events,
17047                            &runtime.child_roster,
17048                            ExitReport {
17049                                kind,
17050                                code: Some(0),
17051                                signal: None,
17052                                at_ms: 1,
17053                            },
17054                            true,
17055                        ));
17056                }
17057            });
17058            let mut reached = [0];
17059            barrier.read_exact(&mut reached).unwrap();
17060            assert_eq!(&reached, b"R");
17061            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
17062            let before_history = history.lock().unwrap().snapshot();
17063            barrier.write_all(b"X").unwrap();
17064            reap.join().unwrap();
17065            assert!(
17066                before_alive,
17067                "{kind:?}: death was published before recording"
17068            );
17069            assert!(before_history.entries.is_empty());
17070            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
17071            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17072            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
17073        }
17074    }
17075
17076    #[tokio::test]
17077    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
17078        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
17079        let path = dir.join("terminals.jsonl");
17080        std::fs::create_dir(&path).unwrap();
17081        let supervisor = Supervisor::new_for_test(
17082            Arc::new(Registry::default()),
17083            RestartPolicy::new(0, Duration::ZERO),
17084        )
17085        .with_terminal_journal(path, "health-budget-journal-failure".into());
17086        let runtime = supervisor.runtime_config();
17087        let spec = ModuleSpec {
17088            script: None,
17089            program: fake_aft_stub_path(),
17090            protocol: ModuleProtocol::None,
17091            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17092            ..windowed_crash_spec("health-budget-journal-failure")
17093        };
17094        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17095        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17096        health_restart_child(
17097            &spec,
17098            &runtime,
17099            &supervisor.registry,
17100            &supervisor.process_liveness,
17101            &snapshot,
17102            &mut child,
17103            SupervisorHealthStatus::Failing,
17104            None,
17105            1,
17106        )
17107        .await
17108        .unwrap();
17109        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17110        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17111        let history = runtime
17112            .terminal_ring
17113            .lock()
17114            .unwrap()
17115            .durable_history(&spec.module_id);
17116        assert!(history.journal_write_failures > 0);
17117        assert_eq!(history.entries.len(), 1);
17118        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17119    }
17120
17121    #[test]
17122    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
17123        let supervisor = Supervisor::new_for_test(
17124            Arc::new(Registry::default()),
17125            RestartPolicy::new(0, Duration::ZERO),
17126        );
17127        let runtime = supervisor.runtime_config();
17128        let spec = ModuleSpec {
17129            script: None,
17130            program: fake_aft_stub_path(),
17131            protocol: ModuleProtocol::None,
17132            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17133            ..windowed_crash_spec("health-budget-recording-panic")
17134        };
17135        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17136        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17137            tokio::runtime::Builder::new_current_thread()
17138                .enable_all()
17139                .build()
17140                .unwrap()
17141                .block_on(async {
17142                    let mut child =
17143                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17144                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17145                    health_restart_child(
17146                        &spec,
17147                        &runtime,
17148                        &supervisor.registry,
17149                        &supervisor.process_liveness,
17150                        &snapshot,
17151                        &mut child,
17152                        SupervisorHealthStatus::Failing,
17153                        None,
17154                        1,
17155                    )
17156                    .await
17157                    .unwrap();
17158                });
17159        }));
17160        let panic = result.expect_err("recording must still unwind");
17161        assert_eq!(
17162            panic.downcast_ref::<String>().map(String::as_str),
17163            Some("spawn event sequence exhausted")
17164        );
17165        let state = lock_snapshot(&snapshot).unwrap();
17166        assert_eq!(state.state, ModuleState::Failed);
17167        assert!(!state.process_alive);
17168        assert!(state.last_exit.is_some());
17169    }
17170
17171    #[tokio::test]
17172    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
17173        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
17174        let path = dir.join("terminals.jsonl");
17175        std::fs::create_dir(&path).unwrap();
17176        let supervisor = Supervisor::new_for_test(
17177            Arc::new(Registry::default()),
17178            RestartPolicy::new(0, Duration::ZERO),
17179        )
17180        .with_terminal_journal(path, "budget-journal-failure".into());
17181        let runtime = supervisor.runtime_config();
17182        let spec = windowed_crash_spec("budget-journal-failure");
17183        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17184        assert!(matches!(
17185            on_child_exit(
17186                &spec,
17187                runtime.restart_policy,
17188                &supervisor.registry,
17189                &snapshot,
17190                &runtime.terminal_ring,
17191                &runtime.spawn_events,
17192                &runtime.child_roster,
17193                crash_exit_report(1),
17194                true,
17195            )
17196            .await,
17197            NextAction::Stop { .. }
17198        ));
17199        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17200        let history = runtime
17201            .terminal_ring
17202            .lock()
17203            .unwrap()
17204            .durable_history(&spec.module_id);
17205        assert!(history.journal_write_failures > 0);
17206        assert_eq!(history.entries.len(), 1);
17207        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17208    }
17209
17210    #[test]
17211    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
17212        let supervisor = Supervisor::new_for_test(
17213            Arc::new(Registry::default()),
17214            RestartPolicy::new(0, Duration::ZERO),
17215        );
17216        let runtime = supervisor.runtime_config();
17217        let spec = windowed_crash_spec("budget-recording-panic");
17218        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17219        runtime
17220            .spawn_events
17221            .emit_spawned(&spec.module_id, None, 1, 1);
17222        // Exhausting the event sequence makes emit_exited panic before the
17223        // terminal write, exercising publication on the recording unwind.
17224        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17225        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17226            tokio::runtime::Builder::new_current_thread()
17227                .enable_all()
17228                .build()
17229                .unwrap()
17230                .block_on(on_child_exit(
17231                    &spec,
17232                    runtime.restart_policy,
17233                    &supervisor.registry,
17234                    &snapshot,
17235                    &runtime.terminal_ring,
17236                    &runtime.spawn_events,
17237                    &runtime.child_roster,
17238                    crash_exit_report(1),
17239                    true,
17240                ))
17241        }));
17242        let panic = result.err().expect("recording must still unwind");
17243        assert_eq!(
17244            panic.downcast_ref::<String>().map(String::as_str),
17245            Some("spawn event sequence exhausted")
17246        );
17247        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17248    }
17249
17250    /// A stop the supervisor itself requests still stops a protocol-none
17251    /// module, even though the child answers the SIGTERM with exit 0.
17252    #[cfg(unix)]
17253    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17254    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
17255        for disable in [false, true] {
17256            let label = if disable {
17257                "none-requested-disable"
17258            } else {
17259                "none-requested-stop"
17260            };
17261            let dir = cortexkit_test_support::ScratchDir::new(label);
17262            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
17263            let supervisor = Supervisor::new_for_test(
17264                Arc::new(Registry::default()),
17265                RestartPolicy::new(3, Duration::ZERO),
17266            );
17267            let module = supervisor.spawn(spec).unwrap();
17268            wait_for_file(&ready).await;
17269
17270            if disable {
17271                module.set_enabled(false).await.unwrap();
17272            } else {
17273                module.stop().await.unwrap();
17274            }
17275            assert!(
17276                marker.exists(),
17277                "{label}: the child must have left through its SIGTERM handler with exit 0"
17278            );
17279
17280            // Long enough for a zero-backoff respawn to have happened if the
17281            // exit had been treated as a crash.
17282            sleep(Duration::from_millis(500)).await;
17283            let status = module.status().unwrap();
17284            let expected = if disable {
17285                ModuleState::Disabled
17286            } else {
17287                ModuleState::Stopped
17288            };
17289            assert_eq!(status.state, expected, "{label}");
17290            assert_eq!(
17291                status.spawn_generation, 1,
17292                "{label}: respawned after a requested stop"
17293            );
17294            let history = module.terminal_history();
17295            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
17296            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
17297            assert_ne!(
17298                history.entries[0].disposition,
17299                TerminalDisposition::Restarting,
17300                "{label}"
17301            );
17302        }
17303    }
17304
17305    /// A subc-wire module that exits 0 on its own is still a stop: the
17306    /// protocol-none rule must not reach it.
17307    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17308    async fn subc_wire_clean_exit_is_still_a_stop() {
17309        let supervisor = Supervisor::new_for_test(
17310            Arc::new(Registry::default()),
17311            RestartPolicy::new(3, Duration::ZERO),
17312        );
17313        let module = supervisor
17314            .spawn(ModuleSpec {
17315                script: None,
17316                module_id: "wire-clean-exit".to_string(),
17317                program: fake_aft_stub_path(),
17318                args: Vec::new(),
17319                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
17320                reserved: false,
17321                reserved_prefixes: Vec::new(),
17322                protocol: ModuleProtocol::Subc,
17323                overlap: Default::default(),
17324                carry_containment: false,
17325            })
17326            .unwrap();
17327
17328        let deadline = Instant::now() + Duration::from_secs(10);
17329        while module.terminal_history().entries.is_empty() {
17330            assert!(Instant::now() < deadline, "module never exited");
17331            sleep(Duration::from_millis(10)).await;
17332        }
17333        // Long enough for a zero-backoff respawn to have happened.
17334        sleep(Duration::from_millis(500)).await;
17335        let status = module.status().unwrap();
17336        assert_eq!(status.state, ModuleState::Stopped);
17337        assert_eq!(status.spawn_generation, 1);
17338        let history = module.terminal_history();
17339        assert_eq!(history.entries.len(), 1, "{history:?}");
17340        assert_eq!(history.entries[0].exit_code, Some(0));
17341        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
17342    }
17343
17344    #[cfg(unix)]
17345    #[tokio::test]
17346    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
17347        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
17348        let record = dir.join("live-children.json");
17349        let supervisor = Supervisor::new_for_test(
17350            Arc::new(Registry::default()),
17351            RestartPolicy::new(0, Duration::ZERO),
17352        );
17353        let mut runtime = supervisor.runtime_config();
17354        runtime.child_roster.record_to(record.clone());
17355        let gate = Arc::new(super::ReloadExitRecordGate::default());
17356        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
17357        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17358        let spec = ModuleSpec {
17359            script: None,
17360            module_id: "reload-exit-roster".into(),
17361            program: fake_aft_stub_path(),
17362            args: Vec::new(),
17363            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
17364            reserved: false,
17365            reserved_prefixes: Vec::new(),
17366            protocol: ModuleProtocol::Subc,
17367            overlap: Default::default(),
17368            carry_containment: false,
17369        };
17370        let mut child = None;
17371        let reload = super::finish_reload_child(
17372            &spec,
17373            &runtime,
17374            &supervisor.registry,
17375            &supervisor.process_liveness,
17376            &snapshot,
17377            &mut child,
17378        );
17379        tokio::pin!(reload);
17380        tokio::select! {
17381            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
17382            _ = gate.reached.notified() => {}
17383        }
17384        assert!(runtime
17385            .terminal_ring
17386            .lock()
17387            .unwrap()
17388            .snapshot()
17389            .entries
17390            .is_empty());
17391        assert_eq!(
17392            crate::live_children::read_record(&record).unwrap().len(),
17393            1,
17394            "shutdown must still wait for the reaped child until its terminal record exists"
17395        );
17396        runtime.child_roster.close();
17397        gate.resume.notify_one();
17398        assert!(reload.await.is_err());
17399        assert!(crate::live_children::read_record(&record)
17400            .unwrap()
17401            .is_empty());
17402        let history = runtime.terminal_ring.lock().unwrap().snapshot();
17403        assert_eq!(history.entries.len(), 1);
17404        assert_eq!(
17405            history.entries[0].disposition,
17406            TerminalDisposition::DaemonShutdown
17407        );
17408    }
17409
17410    /// Each restart-producing arm has its own state transition. Keeping their
17411    /// lifetime count assertions adjacent prevents a later new arm from silently
17412    /// spending budget without recording the historical restart.
17413    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17414    async fn every_restart_increment_path_advances_lifetime_count() {
17415        let supervisor = Supervisor::new_for_test(
17416            Arc::new(Registry::default()),
17417            RestartPolicy::new(1, Duration::ZERO),
17418        );
17419        let runtime = supervisor.runtime_config();
17420        let spec = ModuleSpec {
17421            script: None,
17422            module_id: "lifetime-increment-path".to_string(),
17423            program: PathBuf::from("/unused/lifetime-increment-path"),
17424            args: Vec::new(),
17425            env: Vec::new(),
17426            reserved: false,
17427            reserved_prefixes: Vec::new(),
17428            protocol: ModuleProtocol::Subc,
17429            overlap: Default::default(),
17430            carry_containment: false,
17431        };
17432
17433        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17434        assert!(matches!(
17435            on_child_exit(
17436                &spec,
17437                runtime.restart_policy,
17438                &supervisor.registry,
17439                &crash_snapshot,
17440                &runtime.terminal_ring,
17441                &runtime.spawn_events,
17442                &runtime.child_roster,
17443                ExitReport {
17444                    kind: ExitKind::Crash,
17445                    code: Some(1),
17446                    signal: None,
17447                    at_ms: 1,
17448                },
17449                true,
17450            )
17451            .await,
17452            NextAction::Restart { schedule: _ }
17453        ));
17454        let (crash_restarts, crash_lifetime) = {
17455            let state = lock_snapshot(&crash_snapshot).unwrap();
17456            (state.crash_restarts.len(), state.lifetime_restarts)
17457        };
17458        assert_eq!(crash_restarts, 1);
17459        assert_eq!(crash_lifetime, 1);
17460
17461        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17462        let mut health_child = None;
17463        assert!(matches!(
17464            health_restart_child(
17465                &spec,
17466                &runtime,
17467                &supervisor.registry,
17468                &supervisor.process_liveness,
17469                &health_snapshot,
17470                &mut health_child,
17471                SupervisorHealthStatus::Failing,
17472                None,
17473                2,
17474            )
17475            .await,
17476            Ok(())
17477        ));
17478        assert!(health_child.is_none());
17479        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
17480        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
17481        let (health_restarts, health_lifetime) = {
17482            let state = lock_snapshot(&health_snapshot).unwrap();
17483            (state.crash_restarts.len(), state.lifetime_restarts)
17484        };
17485        assert_eq!(health_restarts, 1);
17486        assert_eq!(health_lifetime, 1);
17487
17488        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17489        let mut reload_child = None;
17490        assert!(matches!(
17491            handle_reload_spawn_failure(
17492                &spec,
17493                &runtime,
17494                &supervisor.process_liveness,
17495                &reload_snapshot,
17496                &mut reload_child,
17497                "forced reload spawn failure".to_string(),
17498            )
17499            .await,
17500            Err(SuperviseError::ReloadFailed { .. })
17501        ));
17502        let (reload_restarts, reload_lifetime) = {
17503            let state = lock_snapshot(&reload_snapshot).unwrap();
17504            (state.crash_restarts.len(), state.lifetime_restarts)
17505        };
17506        assert_eq!(reload_restarts, 1);
17507        assert_eq!(reload_lifetime, 1);
17508    }
17509
17510    #[tokio::test]
17511    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
17512        let supervisor = Supervisor::new_for_test(
17513            Arc::new(Registry::default()),
17514            RestartPolicy::new(3, Duration::ZERO),
17515        );
17516        let runtime = supervisor.runtime_config();
17517        let spec = ModuleSpec {
17518            script: None,
17519            module_id: "deliberately-severed".to_string(),
17520            program: PathBuf::from("/unused/deliberately-severed"),
17521            args: Vec::new(),
17522            env: Vec::new(),
17523            reserved: false,
17524            reserved_prefixes: Vec::new(),
17525            protocol: ModuleProtocol::Subc,
17526            overlap: Default::default(),
17527            carry_containment: false,
17528        };
17529        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17530        let process = ProcessIdentity {
17531            pid: 41,
17532            start_time: 101,
17533        };
17534        record_deliberate_severance(&snapshot, process).unwrap();
17535        let exit_report = apply_deliberate_severance_marker(
17536            &snapshot,
17537            Some(process),
17538            ExitReport {
17539                kind: ExitKind::Crash,
17540                code: Some(1),
17541                signal: None,
17542                at_ms: 1,
17543            },
17544        );
17545        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
17546
17547        assert!(matches!(
17548            on_child_exit(
17549                &spec,
17550                runtime.restart_policy,
17551                &supervisor.registry,
17552                &snapshot,
17553                &runtime.terminal_ring,
17554                &runtime.spawn_events,
17555                &runtime.child_roster,
17556                exit_report,
17557                true,
17558            )
17559            .await,
17560            NextAction::Restart { schedule: _ }
17561        ));
17562        let state = lock_snapshot(&snapshot).unwrap();
17563        assert_eq!(state.lifetime_restarts, 1);
17564        assert_eq!(state.crash_restarts.len(), 0);
17565    }
17566
17567    #[tokio::test]
17568    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
17569        let supervisor = Supervisor::new_for_test(
17570            Arc::new(Registry::default()),
17571            RestartPolicy::new(3, Duration::ZERO),
17572        );
17573        let runtime = supervisor.runtime_config();
17574        let spec = ModuleSpec {
17575            script: None,
17576            module_id: "genuine-crash".to_string(),
17577            program: PathBuf::from("/unused/genuine-crash"),
17578            args: Vec::new(),
17579            env: Vec::new(),
17580            reserved: false,
17581            reserved_prefixes: Vec::new(),
17582            protocol: ModuleProtocol::Subc,
17583            overlap: Default::default(),
17584            carry_containment: false,
17585        };
17586        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17587
17588        assert!(matches!(
17589            on_child_exit(
17590                &spec,
17591                runtime.restart_policy,
17592                &supervisor.registry,
17593                &snapshot,
17594                &runtime.terminal_ring,
17595                &runtime.spawn_events,
17596                &runtime.child_roster,
17597                ExitReport {
17598                    kind: ExitKind::Crash,
17599                    code: Some(1),
17600                    signal: None,
17601                    at_ms: 1,
17602                },
17603                true,
17604            )
17605            .await,
17606            NextAction::Restart { schedule: _ }
17607        ));
17608        let state = lock_snapshot(&snapshot).unwrap();
17609        assert_eq!(state.lifetime_restarts, 1);
17610        assert_eq!(state.crash_restarts.len(), 1);
17611    }
17612
17613    fn crash_exit_report(at_ms: u64) -> ExitReport {
17614        ExitReport {
17615            kind: ExitKind::Crash,
17616            code: Some(1),
17617            signal: None,
17618            at_ms,
17619        }
17620    }
17621
17622    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
17623        ModuleSpec {
17624            script: None,
17625            module_id: module_id.to_string(),
17626            program: PathBuf::from("/unused").join(module_id),
17627            args: Vec::new(),
17628            env: Vec::new(),
17629            reserved: false,
17630            reserved_prefixes: Vec::new(),
17631            protocol: ModuleProtocol::Subc,
17632            overlap: Default::default(),
17633            carry_containment: false,
17634        }
17635    }
17636
17637    /// A real crash loop still stops. Three crashes with nothing aging out spend
17638    /// a budget of two and the third respawn is refused, and both surfaces an
17639    /// operator has -- the log line and the retained terminal record -- name the
17640    /// window rather than only the cap, because `max_restarts=2` alone is what
17641    /// this budget used to mean.
17642    #[tokio::test]
17643    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
17644        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
17645        let supervisor = Supervisor::new_for_test(
17646            Arc::new(Registry::default()),
17647            RestartPolicy::new(2, Duration::ZERO),
17648        );
17649        let runtime = supervisor.runtime_config();
17650        let spec = windowed_crash_spec("crash-loop-in-window");
17651        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17652
17653        for attempt in 1..=2 {
17654            assert!(
17655                matches!(
17656                    on_child_exit(
17657                        &spec,
17658                        runtime.restart_policy,
17659                        &supervisor.registry,
17660                        &snapshot,
17661                        &runtime.terminal_ring,
17662                        &runtime.spawn_events,
17663                        &runtime.child_roster,
17664                        crash_exit_report(attempt),
17665                        true,
17666                    )
17667                    .await,
17668                    NextAction::Restart { schedule: _ }
17669                ),
17670                "crash {attempt} is inside the budget and must respawn"
17671            );
17672        }
17673
17674        assert!(matches!(
17675            on_child_exit(
17676                &spec,
17677                runtime.restart_policy,
17678                &supervisor.registry,
17679                &snapshot,
17680                &runtime.terminal_ring,
17681                &runtime.spawn_events,
17682                &runtime.child_roster,
17683                crash_exit_report(3),
17684                true,
17685            )
17686            .await,
17687            NextAction::Stop { .. }
17688        ));
17689
17690        {
17691            let state = lock_snapshot(&snapshot).unwrap();
17692            assert_eq!(state.state, ModuleState::Failed);
17693            assert_eq!(state.crash_restarts.len(), 2);
17694            assert_eq!(state.lifetime_restarts, 2);
17695        }
17696
17697        let history = runtime
17698            .terminal_ring
17699            .lock()
17700            .expect("terminal ring is not poisoned")
17701            .snapshot();
17702        let last = history
17703            .entries
17704            .last()
17705            .expect("the refused crash is retained");
17706        assert_eq!(last.disposition, TerminalDisposition::Failed);
17707        assert_eq!(
17708            last.disposition_detail.as_deref(),
17709            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
17710        );
17711
17712        let captured = crate::router::test_log::captured_logs(&logs);
17713        assert!(
17714            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
17715            "the stop must be logged with its window: {captured}"
17716        );
17717    }
17718
17719    /// The rate, stated as a test: three crashes where the first has aged past
17720    /// the window are two crashes as far as the budget is concerned, so the
17721    /// third respawn is allowed and the ring holds only the two recent ones.
17722    ///
17723    /// This is the case a lifetime counter got wrong -- and the case the daemon
17724    /// now hits routinely, since a module exits non-zero every time its
17725    /// connection to the daemon drops.
17726    #[tokio::test]
17727    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
17728        let supervisor = Supervisor::new_for_test(
17729            Arc::new(Registry::default()),
17730            RestartPolicy::new(2, Duration::ZERO),
17731        );
17732        let runtime = supervisor.runtime_config();
17733        let spec = windowed_crash_spec("crash-across-windows");
17734        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17735
17736        for attempt in 1..=2 {
17737            assert!(matches!(
17738                on_child_exit(
17739                    &spec,
17740                    runtime.restart_policy,
17741                    &supervisor.registry,
17742                    &snapshot,
17743                    &runtime.terminal_ring,
17744                    &runtime.spawn_events,
17745                    &runtime.child_roster,
17746                    crash_exit_report(attempt),
17747                    true,
17748                )
17749                .await,
17750                NextAction::Restart { schedule: _ }
17751            ));
17752        }
17753
17754        // The oldest crash moves out of the window; nothing else about the
17755        // module changes.
17756        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17757            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
17758        })
17759        .unwrap();
17760
17761        assert!(
17762            matches!(
17763                on_child_exit(
17764                    &spec,
17765                    runtime.restart_policy,
17766                    &supervisor.registry,
17767                    &snapshot,
17768                    &runtime.terminal_ring,
17769                    &runtime.spawn_events,
17770                    &runtime.child_roster,
17771                    crash_exit_report(3),
17772                    true,
17773                )
17774                .await,
17775                NextAction::Restart { schedule: _ }
17776            ),
17777            "a crash older than the window must not hold a budget slot"
17778        );
17779
17780        let state = lock_snapshot(&snapshot).unwrap();
17781        assert_eq!(state.state, ModuleState::Restarting);
17782        assert_eq!(
17783            state.crash_restarts.len(),
17784            2,
17785            "the aged instant is dropped and the new one takes its place"
17786        );
17787        assert_eq!(
17788            state.lifetime_restarts, 3,
17789            "the ledger counts every restart, including the ones the window forgot"
17790        );
17791    }
17792
17793    /// An operator restart hands the budget back whole, and the ledger keeps
17794    /// counting. Those are different questions -- "how close is this module to
17795    /// being stopped" and "how many times has it been replaced" -- and the
17796    /// operator action answers only the first.
17797    #[tokio::test]
17798    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
17799        let supervisor = Supervisor::new_for_test(
17800            Arc::new(Registry::default()),
17801            RestartPolicy::new(2, Duration::ZERO),
17802        );
17803        let runtime = supervisor.runtime_config();
17804        let spec = windowed_crash_spec("operator-cleared-budget");
17805        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17806
17807        for attempt in 1..=2 {
17808            assert!(matches!(
17809                on_child_exit(
17810                    &spec,
17811                    runtime.restart_policy,
17812                    &supervisor.registry,
17813                    &snapshot,
17814                    &runtime.terminal_ring,
17815                    &runtime.spawn_events,
17816                    &runtime.child_roster,
17817                    crash_exit_report(attempt),
17818                    true,
17819                )
17820                .await,
17821                NextAction::Restart { schedule: _ }
17822            ));
17823        }
17824
17825        reset_restart_count(&snapshot, &spec.module_id).unwrap();
17826        {
17827            let state = lock_snapshot(&snapshot).unwrap();
17828            assert!(
17829                state.crash_restarts.is_empty(),
17830                "an operator restart returns the full budget"
17831            );
17832            assert_eq!(
17833                state.lifetime_restarts, 2,
17834                "clearing the budget must not unmake the crashes"
17835            );
17836        }
17837
17838        assert!(
17839            matches!(
17840                on_child_exit(
17841                    &spec,
17842                    runtime.restart_policy,
17843                    &supervisor.registry,
17844                    &snapshot,
17845                    &runtime.terminal_ring,
17846                    &runtime.spawn_events,
17847                    &runtime.child_roster,
17848                    crash_exit_report(3),
17849                    true,
17850                )
17851                .await,
17852                NextAction::Restart { schedule: _ }
17853            ),
17854            "the cleared budget must be spendable again"
17855        );
17856        let state = lock_snapshot(&snapshot).unwrap();
17857        assert_eq!(state.crash_restarts.len(), 1);
17858        assert_eq!(state.lifetime_restarts, 3);
17859    }
17860
17861    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17862    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
17863        let severed = ProcessIdentity {
17864            pid: 41,
17865            start_time: 101,
17866        };
17867        let successor = ProcessIdentity {
17868            pid: 41,
17869            start_time: 202,
17870        };
17871        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
17872        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
17873            state.pid = Some(successor.pid);
17874            state.process_start_time = Some(successor.start_time);
17875        })
17876        .unwrap();
17877        assert!(!module.record_deliberate_severance(severed).unwrap());
17878
17879        let exit_report = apply_deliberate_severance_marker(
17880            &module.inner.snapshot,
17881            Some(successor),
17882            ExitReport {
17883                kind: ExitKind::Crash,
17884                code: Some(1),
17885                signal: None,
17886                at_ms: 1,
17887            },
17888        );
17889
17890        assert_eq!(exit_report.kind, ExitKind::Crash);
17891    }
17892
17893    #[tokio::test]
17894    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
17895        let registry = Registry::default();
17896        let supervisor = Supervisor::new_for_test(
17897            Arc::new(Registry::default()),
17898            RestartPolicy::new(3, Duration::ZERO),
17899        );
17900        let runtime = supervisor.runtime_config();
17901        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17902        let spec = ModuleSpec {
17903            script: None,
17904            module_id: "drain-deliberate-severance".to_string(),
17905            program: fake_aft_stub_path(),
17906            args: Vec::new(),
17907            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17908            reserved: false,
17909            reserved_prefixes: Vec::new(),
17910            protocol: ModuleProtocol::Subc,
17911            overlap: Default::default(),
17912            carry_containment: false,
17913        };
17914        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17915        let process = ProcessIdentity {
17916            pid: 41,
17917            start_time: 101,
17918        };
17919        child.process_identity = Some(process);
17920        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17921            state.pid = Some(process.pid);
17922            state.process_start_time = Some(process.start_time);
17923        })
17924        .unwrap();
17925        record_deliberate_severance(&snapshot, process).unwrap();
17926
17927        drain_child_to_state(
17928            &spec.module_id,
17929            spec.protocol,
17930            // The child exits on its own; no signal may change the exit this
17931            // test classifies.
17932            StopNotice::SentOverConnection,
17933            &registry,
17934            None,
17935            &snapshot,
17936            &runtime.terminal_ring,
17937            &runtime.spawn_events,
17938            child,
17939            Duration::from_secs(1),
17940            ModuleState::Stopped,
17941            Some(false),
17942            &super::FailureContext::operator(),
17943        )
17944        .await
17945        .unwrap();
17946
17947        let state = lock_snapshot(&snapshot).unwrap();
17948        assert_eq!(
17949            state.last_exit.as_ref().map(|exit| exit.kind),
17950            Some(ExitKind::DeliberateSeverance)
17951        );
17952        assert_eq!(state.lifetime_restarts, 1);
17953        assert_eq!(state.crash_restarts.len(), 0);
17954        drop(state);
17955        let history = runtime.terminal_ring.lock().unwrap().snapshot();
17956        assert_eq!(
17957            history.entries[0].exit_kind,
17958            subc_control::TerminalExitKind::DeliberateSeverance
17959        );
17960    }
17961
17962    #[tokio::test]
17963    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
17964        let registry = Registry::default();
17965        let supervisor = Supervisor::new_for_test(
17966            Arc::new(Registry::default()),
17967            RestartPolicy::new(3, Duration::ZERO),
17968        );
17969        let runtime = supervisor.runtime_config();
17970        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17971        let spec = ModuleSpec {
17972            script: None,
17973            module_id: "ordinary-drain".to_string(),
17974            program: fake_aft_stub_path(),
17975            args: Vec::new(),
17976            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17977            reserved: false,
17978            reserved_prefixes: Vec::new(),
17979            protocol: ModuleProtocol::Subc,
17980            overlap: Default::default(),
17981            carry_containment: false,
17982        };
17983        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17984
17985        drain_child_to_state(
17986            &spec.module_id,
17987            spec.protocol,
17988            // The child exits on its own; no signal may change the exit this
17989            // test classifies.
17990            StopNotice::SentOverConnection,
17991            &registry,
17992            None,
17993            &snapshot,
17994            &runtime.terminal_ring,
17995            &runtime.spawn_events,
17996            child,
17997            Duration::from_secs(1),
17998            ModuleState::Stopped,
17999            Some(false),
18000            &super::FailureContext::operator(),
18001        )
18002        .await
18003        .unwrap();
18004
18005        let state = lock_snapshot(&snapshot).unwrap();
18006        assert_eq!(
18007            state.last_exit.as_ref().map(|exit| exit.kind),
18008            Some(ExitKind::Crash)
18009        );
18010        assert_eq!(state.lifetime_restarts, 0);
18011        assert_eq!(state.crash_restarts.len(), 0);
18012    }
18013
18014    #[test]
18015    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
18016        // The server's generic fatal-routing branch only knows that the
18017        // connection failed; it does not know that the daemon deliberately
18018        // initiated a process-killing severance. Keep this seam explicit so a
18019        // future connection error path cannot silently reintroduce the stale
18020        // exemption that mislabels a later genuine crash.
18021        assert!(!include_str!("server.rs")
18022            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
18023    }
18024
18025    /// The `route.closed` `drained` value must be the quiescence wait's own
18026    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
18027    /// measurement at all and `false` is the one honest constant. This is the exact
18028    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
18029    /// on every return path, including the one that used to return early via `?`
18030    /// with `route.closing` already sent and no `route.closed` ever following.
18031    #[test]
18032    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
18033        assert!(drained_after_quiescence_wait(&Ok(true)));
18034        assert!(!drained_after_quiescence_wait(&Ok(false)));
18035        assert!(!drained_after_quiescence_wait(&Err(
18036            SuperviseError::StatePoisoned { module_id: None }
18037        )));
18038    }
18039
18040    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
18041    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
18042    /// already reaped out-of-band) still leaves a terminal record rather than none
18043    /// at all. Triggering the real `wait()` I/O error from an integration test would
18044    /// need a genuine already-reaped-child race, which is OS-specific and not
18045    /// something this suite attempts elsewhere; this test instead verifies the
18046    /// record produced for that arm end-to-end through the real `TerminalRing`, and
18047    /// the call site itself is verified by inspection to sit in that exact arm.
18048    #[test]
18049    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
18050        let ring = Arc::new(Mutex::new(TerminalRing::new(
18051            TerminalRingConfig::default(),
18052            0,
18053        )));
18054        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
18055
18056        let snapshot = ring.lock().unwrap().snapshot();
18057        assert_eq!(snapshot.entries.len(), 1);
18058        let entry = &snapshot.entries[0];
18059        assert_eq!(entry.exit_code, None);
18060        assert_eq!(entry.exit_signal, None);
18061        assert_eq!(entry.disposition, TerminalDisposition::Failed);
18062    }
18063
18064    /// A launch is numbered before it starts. One that never runs keeps its
18065    /// number, and the next launch of the module is numbered after it.
18066    #[test]
18067    fn spawn_generation_of_a_failed_launch_is_never_reused() {
18068        let feed = super::SpawnEventFeed::default();
18069        feed.configure_incarnation("launch-numbering".to_string());
18070        let (incarnation, failed) = feed.allocate_generation("relaunched");
18071        assert_eq!(incarnation, "launch-numbering");
18072        let (_, next) = feed.allocate_generation("relaunched");
18073        assert!(next > failed);
18074        assert_eq!(feed.emit_spawned("relaunched", Some(next), 7, 1), next);
18075        // A process with no pre-allocated number is numbered after both.
18076        assert!(feed.emit_spawned("relaunched", None, 8, 2) > next);
18077        assert_eq!(
18078            feed.allocate_generation("other").1,
18079            1,
18080            "numbering is per module"
18081        );
18082    }
18083
18084    #[test]
18085    fn wait_error_exit_path_preserves_spawn_event_density() {
18086        let feed = super::SpawnEventFeed::default();
18087        feed.configure_incarnation("wait-error-density".to_string());
18088        feed.emit_spawned("wait-error", None, 41, 1);
18089        let ring = Arc::new(Mutex::new(TerminalRing::new(
18090            TerminalRingConfig::default(),
18091            0,
18092        )));
18093
18094        record_wait_error_terminal("wait-error", &ring, &feed);
18095        feed.emit_spawned("after-wait-error", None, 42, 2);
18096
18097        let state = feed.0.lock().unwrap();
18098        let sequences = state
18099            .events
18100            .iter()
18101            .map(|event| event.cursor.seq)
18102            .collect::<Vec<_>>();
18103        assert_eq!(sequences, vec![1, 2, 3]);
18104        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
18105        assert_eq!(state.events[1].exit_code, None);
18106        assert_eq!(state.events[1].exit_signal, None);
18107    }
18108
18109    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
18110    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
18111    /// not a clean exit it never actually observed.
18112    #[test]
18113    fn wait_error_exit_report_is_classified_as_a_crash() {
18114        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
18115    }
18116}
18117
18118#[cfg(all(test, unix))]
18119mod restart_ordering_tests {
18120    use super::*;
18121    use std::io::Write;
18122    use tokio::io::AsyncReadExt;
18123
18124    #[test]
18125    #[ignore = "child process fixture"]
18126    fn serving_child() {
18127        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
18128            return;
18129        };
18130        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
18131        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
18132        for stream in listener.incoming() {
18133            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
18134        }
18135    }
18136
18137    async fn restart_before_publication(requests: usize) {
18138        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
18139        let address_file = home.path().join("address");
18140        let spec = ModuleSpec {
18141            script: None,
18142            module_id: "restart-serving-child".into(),
18143            program: std::env::current_exe().unwrap(),
18144            args: vec![
18145                "--exact".into(),
18146                "supervise::restart_ordering_tests::serving_child".into(),
18147                "--ignored".into(),
18148                "--nocapture".into(),
18149            ],
18150            env: vec![(
18151                "RESTART_TEST_ADDRESS_FILE".into(),
18152                address_file.display().to_string(),
18153            )],
18154            reserved: false,
18155            reserved_prefixes: vec![],
18156            protocol: ModuleProtocol::None,
18157            overlap: Default::default(),
18158            carry_containment: false,
18159        };
18160        let mut supervisor =
18161            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18162        let (spawned_tx, spawned_rx) = oneshot::channel();
18163        let spawned_tx = Mutex::new(Some(spawned_tx));
18164        let (release_tx, release_rx) = std::sync::mpsc::channel();
18165        let release_rx = Mutex::new(release_rx);
18166        // Hold publication, not the child: it must answer a real request before
18167        // the restart enters the queue, even though status still has generation 0.
18168        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
18169            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
18170            release_rx
18171                .lock()
18172                .unwrap()
18173                .recv_timeout(Duration::from_secs(20))
18174                .unwrap();
18175        })));
18176        let registered = supervisor
18177            .register_configured_with_health(
18178                spec.clone(),
18179                true,
18180                HealthConfig::default(),
18181                None,
18182                RestartPolicy::new(3, Duration::ZERO),
18183                DEFAULT_REGISTER_TIMEOUT,
18184            )
18185            .unwrap();
18186        let module = registered.module.clone();
18187        let startup = tokio::task::spawn_blocking(move || {
18188            supervisor.start_registered_configured(registered, None, true)
18189        });
18190        spawned_rx.await.unwrap();
18191        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18192            let address = loop {
18193                if let Ok(address) = std::fs::read_to_string(&address_file) {
18194                    if !address.is_empty() {
18195                        break address;
18196                    }
18197                }
18198                tokio::time::sleep(Duration::from_millis(1)).await;
18199            };
18200            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
18201            let mut reply = String::new();
18202            stream.read_to_string(&mut reply).await.unwrap();
18203            reply.trim().parse::<u32>().unwrap()
18204        })
18205        .await
18206        .expect("child must serve before generation publication");
18207        assert_eq!(
18208            lock_snapshot(&module.inner.snapshot)
18209                .unwrap()
18210                .spawn_generation,
18211            0
18212        );
18213        let mut replies = Vec::new();
18214        for _ in 0..requests {
18215            let mut restart = Box::pin(module.restart(Some(0)));
18216            // Poll the public API through queue admission, leaving only its ACK
18217            // pending. The monitor cannot consume the command until released.
18218            std::future::poll_fn(|cx| {
18219                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18220                std::task::Poll::Ready(())
18221            })
18222            .await;
18223            replies.push(restart);
18224        }
18225        release_tx.send(()).unwrap();
18226        startup.await.unwrap().unwrap();
18227        for reply in replies {
18228            reply.await.unwrap();
18229        }
18230        // Restart schedules its replacement after completing the handler. Wait
18231        // for that separate actor turn before placing the command barrier.
18232        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
18233            loop {
18234                if lock_snapshot(&module.inner.snapshot)
18235                    .unwrap()
18236                    .spawn_generation
18237                    > 1
18238                {
18239                    break;
18240                }
18241                tokio::time::sleep(Duration::from_millis(1)).await;
18242            }
18243        })
18244        .await;
18245        // Commands are handled serially. The update-spec reply proves every
18246        // restart handler has finished before we inspect the snapshot; restart's
18247        // own ACK only acknowledges initiation, not replacement completion.
18248        module.update_spec_for_test(spec).await.unwrap();
18249        let (generation, pid) = {
18250            let state = lock_snapshot(&module.inner.snapshot).unwrap();
18251            (state.spawn_generation, state.pid)
18252        };
18253        module.stop().await.unwrap();
18254        replacement.expect("restart must publish a replacement for the already-serving child");
18255        assert_ne!(
18256            pid,
18257            Some(old_pid),
18258            "a serving child predating the restart must be replaced"
18259        );
18260        assert_eq!(generation, 2, "queued restarts must share one replacement");
18261    }
18262
18263    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18264    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
18265        restart_before_publication(1).await;
18266    }
18267
18268    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18269    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
18270        restart_before_publication(2).await;
18271    }
18272
18273    #[tokio::test]
18274    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
18275        let supervisor =
18276            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18277        let registered = supervisor
18278            .register_configured_with_health(
18279                ModuleSpec {
18280                    script: None,
18281                    module_id: "queued-before-replacement".into(),
18282                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
18283                    // macOS a module whose running image is not the configured
18284                    // program is refused once its exec is confirmed (see
18285                    // `record_write_stall_tests::sleeping_spec`).
18286                    program: "/bin/sleep".into(),
18287                    args: vec!["600".into()],
18288                    env: vec![],
18289                    reserved: false,
18290                    reserved_prefixes: vec![],
18291                    protocol: ModuleProtocol::None,
18292                    overlap: Default::default(),
18293                    carry_containment: false,
18294                },
18295                true,
18296                HealthConfig::default(),
18297                None,
18298                RestartPolicy::default(),
18299                DEFAULT_REGISTER_TIMEOUT,
18300            )
18301            .unwrap();
18302        let RegisteredConfiguredModule {
18303            mut spec,
18304            mut runtime,
18305            module,
18306            mut commands,
18307        } = registered;
18308        let snapshot = &module.inner.snapshot;
18309        let mut restart = Box::pin(module.restart(Some(0)));
18310        std::future::poll_fn(|cx| {
18311            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18312            std::task::Poll::Ready(())
18313        })
18314        .await;
18315        // Drive the actor explicitly so a replacement is definitely spawned
18316        // after receipt, but before this queued command is dequeued.
18317        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
18318        let replacement_pid = child.as_ref().unwrap().id();
18319        let command = commands.recv().await.unwrap();
18320        assert!(
18321            handle_supervisor_command(
18322                command,
18323                &mut spec,
18324                &mut runtime,
18325                &supervisor.registry,
18326                &supervisor.process_liveness,
18327                snapshot,
18328                &mut child,
18329                &mut commands,
18330                &mut VecDeque::new(),
18331            )
18332            .await
18333        );
18334        restart.await.unwrap();
18335        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
18336        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
18337        if let Some(mut child) = child {
18338            child.start_kill().unwrap();
18339            child.wait().await.unwrap();
18340        }
18341        assert_eq!(
18342            kept_pid, replacement_pid,
18343            "a newer replacement must survive"
18344        );
18345        assert!(
18346            !pending,
18347            "a satisfied restart must not schedule another spawn"
18348        );
18349    }
18350}
18351
18352#[cfg(all(test, unix))]
18353mod health_event_tests {
18354    use super::*;
18355    use cortexkit_test_support::ScratchDir;
18356    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
18357
18358    struct Actor {
18359        module: SupervisedModule,
18360        registry: Arc<Registry>,
18361        forwarding: Arc<ForwardingTable>,
18362        spec: ModuleSpec,
18363        health: HealthConfig,
18364        rx: mpsc::Receiver<crate::router::OutboundFrame>,
18365        roster: ChildRoster,
18366        _home: ScratchDir,
18367    }
18368
18369    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
18370        // OS exec and socket readiness are real I/O. Stay runnable while waiting
18371        // for an observed condition, rather than letting paused time auto-advance
18372        // deadlines or assuming a fixed number of yields finishes the work.
18373        let deadline = std::time::Instant::now() + Duration::from_secs(10);
18374        let tick = Instant::now();
18375        loop {
18376            if let Some(value) = observe() {
18377                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
18378                return value;
18379            }
18380            assert!(
18381                std::time::Instant::now() < deadline,
18382                "timed out waiting for {reason}"
18383            );
18384            tokio::task::yield_now().await;
18385        }
18386    }
18387
18388    impl Actor {
18389        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
18390            Self::start_with_policy(
18391                protocol,
18392                cadence,
18393                DEFAULT_REGISTER_TIMEOUT,
18394                RestartPolicy::new(3, Duration::ZERO),
18395            )
18396            .await
18397        }
18398
18399        async fn start_with_policy(
18400            protocol: ModuleProtocol,
18401            cadence: Duration,
18402            register_timeout: Duration,
18403            policy: RestartPolicy,
18404        ) -> Self {
18405            let home = ScratchDir::new("health-events");
18406            let registry = Arc::new(Registry::default());
18407            let forwarding = Arc::new(ForwardingTable::default());
18408            let health = HealthConfig {
18409                cadence,
18410                deadline: Duration::from_secs(3600),
18411                ..HealthConfig::default()
18412            };
18413            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
18414                .with_forwarding(forwarding.clone())
18415                .with_health_config(health.clone());
18416            let spec = ModuleSpec {
18417                script: None,
18418                module_id: "event-health-child".into(),
18419                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
18420                // the privacy trampoline refuses a module whose running image
18421                // is not the configured program, and `start` below waits for
18422                // the process to be confirmed (`reported_pid`). The shell form
18423                // passed only while the image check ran before the shell had
18424                // replaced itself with `sleep` (see
18425                // `record_write_stall_tests::sleeping_spec`).
18426                program: PathBuf::from("/bin/sleep"),
18427                // The process stays alive but has no real bus peer. Registrations
18428                // below exercise the same registry writes as accepted HELLOs.
18429                args: vec!["600".into()],
18430                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18431                    .into_iter()
18432                    .map(|key| (key.into(), home.path().display().to_string()))
18433                    .collect(),
18434                reserved: false,
18435                reserved_prefixes: vec![],
18436                protocol,
18437                overlap: Default::default(),
18438                carry_containment: false,
18439            };
18440            let module = supervisor
18441                .supervise_configured_requested(
18442                    spec.clone(),
18443                    true,
18444                    health.clone(),
18445                    Some(0),
18446                    policy,
18447                    register_timeout,
18448                    None,
18449                )
18450                .unwrap();
18451            let (_, rx) = mpsc::channel(8);
18452            let actor = Self {
18453                module,
18454                registry,
18455                forwarding,
18456                spec,
18457                health,
18458                rx,
18459                roster: supervisor.child_roster.clone(),
18460                _home: home,
18461            };
18462            actor
18463                .wait_for_select(|checkpoint| checkpoint.generation > 0)
18464                .await;
18465            actor
18466        }
18467
18468        async fn expire_registration_wait(&self, remaining: Duration) {
18469            tokio::time::advance(remaining).await;
18470            wait_for("registration timeout TERM grace", || {
18471                lock_snapshot(&self.module.inner.snapshot)
18472                    .unwrap()
18473                    .actor_select
18474                    .is_none()
18475                    .then_some(())
18476            })
18477            .await;
18478            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
18479        }
18480
18481        fn turns(&self) -> u64 {
18482            lock_snapshot(&self.module.inner.snapshot)
18483                .unwrap()
18484                .actor_turns
18485        }
18486
18487        async fn wait_for_select(
18488            &self,
18489            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
18490        ) -> ActorSelectCheckpoint {
18491            wait_for("actor select checkpoint", || {
18492                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
18493                state.actor_select.filter(|checkpoint| {
18494                    checkpoint.generation == state.spawn_generation
18495                        && state.state == ModuleState::Running
18496                        && state.process_alive
18497                        && state.reported_pid().is_some()
18498                        && expected(checkpoint)
18499                })
18500            })
18501            .await
18502        }
18503
18504        fn hello(&mut self, connection: u64, health: bool) {
18505            let manifest =
18506                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
18507                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
18508                    .build();
18509            let connection = ConnectionId::new(connection);
18510            let (tx, rx) = mpsc::channel(8);
18511            self.rx = rx;
18512            self.forwarding
18513                .register_module_connection(
18514                    connection,
18515                    self.spec.module_id.clone(),
18516                    subc_protocol::PROTOCOL_VERSION,
18517                    Concurrency::ModuleManaged,
18518                    FrameSink::new(tx),
18519                )
18520                .unwrap();
18521            self.registry
18522                .register_with_control_ops(
18523                    manifest,
18524                    subc_protocol::PROTOCOL_VERSION,
18525                    connection,
18526                    if health {
18527                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
18528                    } else {
18529                        vec![]
18530                    },
18531                )
18532                .unwrap();
18533        }
18534
18535        async fn probe(&mut self) -> crate::router::OutboundFrame {
18536            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
18537                Ok(frame) => Some(frame),
18538                Err(mpsc::error::TryRecvError::Empty) => None,
18539                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
18540            })
18541            .await;
18542            let body: Value = serde_json::from_slice(&frame.body).unwrap();
18543            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
18544            frame
18545        }
18546
18547        fn no_probe(&mut self) {
18548            assert!(matches!(
18549                self.rx.try_recv(),
18550                Err(mpsc::error::TryRecvError::Empty)
18551            ));
18552        }
18553
18554        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
18555            self.forwarding
18556                .complete_module_control_rpc(
18557                    ConnectionId::new(connection),
18558                    frame.header.corr,
18559                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
18560                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
18561                        status: HealthStatus::Ok,
18562                        detail: None,
18563                        metrics: None,
18564                    }),
18565                )
18566                .unwrap();
18567        }
18568    }
18569
18570    #[tokio::test(start_paused = true)]
18571    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
18572        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18573        let initial = actor.module.status().unwrap();
18574        tokio::time::advance(Duration::from_secs(119)).await;
18575        tokio::task::yield_now().await;
18576        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
18577        assert!(actor.module.terminal_history().entries.is_empty());
18578        actor.expire_registration_wait(Duration::from_secs(1)).await;
18579        actor
18580            .wait_for_select(|at| at.generation > initial.spawn_generation)
18581            .await;
18582        let status = actor.module.status().unwrap();
18583        assert_ne!(status.pid, initial.pid);
18584        assert!(!cortexkit_test_support::process_alive(
18585            initial.pid.unwrap() as i32
18586        ));
18587        assert_eq!(status.restart_count, 1);
18588        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
18589        let entries = actor.module.terminal_history().entries;
18590        assert_eq!(entries.len(), 1);
18591        assert_eq!(
18592            entries[0].disposition_detail.as_deref(),
18593            Some("registration_timeout")
18594        );
18595        actor.module.stop().await.unwrap();
18596    }
18597
18598    #[tokio::test(start_paused = true)]
18599    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
18600        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18601        let pid = actor.module.status().unwrap().pid;
18602        tokio::time::advance(Duration::from_secs(119)).await;
18603        actor.hello(1, false);
18604        let registered = actor
18605            .wait_for_select(|at| at.registered_connection.is_some())
18606            .await;
18607        assert_eq!(
18608            registered.wake_after, None,
18609            "HELLO must cancel the deadline"
18610        );
18611        tokio::time::advance(Duration::from_secs(300)).await;
18612        tokio::task::yield_now().await;
18613        assert_eq!(actor.module.status().unwrap().pid, pid);
18614        assert!(actor.module.terminal_history().entries.is_empty());
18615        // A disconnect after the first HELLO must not re-arm a startup timer.
18616        actor
18617            .registry
18618            .deregister_connection(ConnectionId::new(1))
18619            .unwrap();
18620        actor
18621            .wait_for_select(|at| at.registered_connection.is_none())
18622            .await;
18623        tokio::time::advance(Duration::from_secs(300)).await;
18624        tokio::task::yield_now().await;
18625        assert_eq!(actor.module.status().unwrap().pid, pid);
18626        assert!(actor.module.terminal_history().entries.is_empty());
18627        actor.module.stop().await.unwrap();
18628
18629        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18630        let pid = transient.module.status().unwrap().pid;
18631        tokio::time::advance(Duration::from_secs(119)).await;
18632        let turns = transient.turns();
18633        // Both writes happen before the supervisor can refresh the registry.
18634        // An accepted HELLO must remain evidence after its connection is gone.
18635        transient.hello(1, false);
18636        transient
18637            .registry
18638            .deregister_connection(ConnectionId::new(1))
18639            .unwrap();
18640        let parked = transient.wait_for_select(|at| at.turn > turns).await;
18641        assert_eq!(
18642            parked.wake_after, None,
18643            "transient HELLO must cancel the deadline"
18644        );
18645        tokio::time::advance(Duration::from_secs(300)).await;
18646        tokio::task::yield_now().await;
18647        assert_eq!(transient.module.status().unwrap().pid, pid);
18648        assert!(transient.module.terminal_history().entries.is_empty());
18649        transient.module.stop().await.unwrap();
18650    }
18651
18652    #[tokio::test(start_paused = true)]
18653    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
18654        let actor = Actor::start_with_policy(
18655            ModuleProtocol::Subc,
18656            Duration::from_secs(30),
18657            Duration::from_secs(10),
18658            RestartPolicy::new(1, Duration::ZERO),
18659        )
18660        .await;
18661        let generation = actor.module.status().unwrap().spawn_generation;
18662        actor
18663            .expire_registration_wait(Duration::from_secs(10))
18664            .await;
18665        actor.wait_for_select(|at| at.generation > generation).await;
18666        actor
18667            .expire_registration_wait(Duration::from_secs(10))
18668            .await;
18669        let retry = wait_for("failed retry schedule", || {
18670            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
18671            (state.state == ModuleState::Failed)
18672                .then_some(state.failed_retry.pending)
18673                .flatten()
18674        })
18675        .await;
18676        assert_eq!(retry.delay, Duration::from_secs(600));
18677        assert_eq!(retry.attempt, 1);
18678        let entries = actor.module.terminal_history().entries;
18679        assert_eq!(entries.len(), 2);
18680        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
18681        assert!(entries[1]
18682            .disposition_detail
18683            .as_deref()
18684            .unwrap()
18685            .contains("registration_timeout"));
18686        tokio::time::advance(Duration::from_secs(599)).await;
18687        tokio::task::yield_now().await;
18688        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
18689        tokio::time::advance(Duration::from_secs(1)).await;
18690        actor
18691            .wait_for_select(|at| at.generation > generation + 1)
18692            .await;
18693        // The slow retry is another ordinary spawn, with its own HELLO budget.
18694        actor
18695            .expire_registration_wait(Duration::from_secs(10))
18696            .await;
18697        wait_for("slow retry registration timeout", || {
18698            (actor.module.terminal_history().entries.len() == 3).then_some(())
18699        })
18700        .await;
18701        actor.module.stop().await.unwrap();
18702    }
18703
18704    #[tokio::test(start_paused = true)]
18705    async fn registration_timeout_honours_ten_second_override() {
18706        let actor = Actor::start_with_policy(
18707            ModuleProtocol::Subc,
18708            Duration::from_secs(30),
18709            Duration::from_secs(10),
18710            RestartPolicy::new(3, Duration::ZERO),
18711        )
18712        .await;
18713        let generation = actor.module.status().unwrap().spawn_generation;
18714        tokio::time::advance(Duration::from_secs(9)).await;
18715        tokio::task::yield_now().await;
18716        assert!(actor.module.terminal_history().entries.is_empty());
18717        actor.expire_registration_wait(Duration::from_secs(1)).await;
18718        actor.wait_for_select(|at| at.generation > generation).await;
18719        assert_eq!(actor.module.terminal_history().entries.len(), 1);
18720        actor.module.stop().await.unwrap();
18721    }
18722
18723    #[tokio::test]
18724    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
18725        use std::os::unix::process::{CommandExt, ExitStatusExt};
18726
18727        struct Helper(std::process::Child);
18728        impl Drop for Helper {
18729            fn drop(&mut self) {
18730                // This handle still owns the helper until wait returns. Never
18731                // clean up its old process group after the leader is reaped.
18732                if matches!(self.0.try_wait(), Ok(None)) {
18733                    let _ = self.0.kill();
18734                    let _ = self.0.wait();
18735                }
18736            }
18737        }
18738
18739        let home = ScratchDir::new("registration-timeout-helper");
18740        let supervisor = Supervisor::new_for_test(
18741            Arc::new(Registry::default()),
18742            RestartPolicy::new(0, Duration::ZERO),
18743        );
18744        let spec = ModuleSpec {
18745            script: None,
18746            module_id: "registration-timeout-group".into(),
18747            program: "/bin/sleep".into(),
18748            args: vec!["120".into()],
18749            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18750                .into_iter()
18751                .map(|key| (key.into(), home.path().display().to_string()))
18752                .collect(),
18753            reserved: false,
18754            reserved_prefixes: vec![],
18755            protocol: ModuleProtocol::Subc,
18756            overlap: Default::default(),
18757            carry_containment: false,
18758        };
18759        let module = supervisor
18760            .supervise_configured_requested(
18761                spec,
18762                true,
18763                HealthConfig::default(),
18764                None,
18765                RestartPolicy::new(0, Duration::ZERO),
18766                Duration::from_secs(10),
18767                None,
18768            )
18769            .unwrap();
18770        let leader = timeout(Duration::from_secs(5), async {
18771            loop {
18772                if let Some(pid) = module.status().unwrap().pid {
18773                    break pid;
18774                }
18775                sleep(Duration::from_millis(5)).await;
18776            }
18777        })
18778        .await
18779        .expect("supervised group leader must be reported before the deadline");
18780        assert!(leader > 1);
18781        let ready = home.join("helper.ready");
18782        // Own the helper separately so even a broken group kill can be cleaned
18783        // up by its own unreaped child handle, without signalling a stale pgid.
18784        let mut helper = Helper(
18785            std::process::Command::new("/bin/sh")
18786                .args([
18787                    "-c",
18788                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
18789                    "helper",
18790                ])
18791                .arg(&ready)
18792                .process_group(i32::try_from(leader).unwrap())
18793                .stdout(std::process::Stdio::null())
18794                .stderr(std::process::Stdio::null())
18795                .spawn()
18796                .unwrap(),
18797        );
18798        timeout(Duration::from_secs(5), async {
18799            while !ready.exists() {
18800                assert!(helper.0.try_wait().unwrap().is_none());
18801                sleep(Duration::from_millis(5)).await;
18802            }
18803        })
18804        .await
18805        .expect("helper must install TERM ignore before the registration deadline");
18806        let helper_pid =
18807            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
18808        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
18809        sleep(Duration::from_millis(20)).await;
18810        assert!(
18811            helper.0.try_wait().unwrap().is_none(),
18812            "helper must ignore TERM"
18813        );
18814
18815        timeout(Duration::from_secs(20), async {
18816            while module.state().unwrap() != ModuleState::Failed {
18817                sleep(Duration::from_millis(5)).await;
18818            }
18819        })
18820        .await
18821        .expect("registration timeout must stop and reap the group leader");
18822        let helper_exit = timeout(Duration::from_secs(3), async {
18823            loop {
18824                if let Some(status) = helper.0.try_wait().unwrap() {
18825                    break status;
18826                }
18827                sleep(Duration::from_millis(5)).await;
18828            }
18829        })
18830        .await
18831        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
18832        assert_eq!(helper_exit.signal(), Some(9));
18833        assert!(!cortexkit_test_support::process_alive(
18834            i32::try_from(leader).unwrap()
18835        ));
18836        assert_eq!(
18837            module.status().unwrap().last_exit.unwrap().kind,
18838            ExitKind::Crash
18839        );
18840        assert!(module.terminal_history().entries[0]
18841            .disposition_detail
18842            .as_deref()
18843            .unwrap()
18844            .contains("registration_timeout"));
18845        module.stop().await.unwrap();
18846    }
18847
18848    #[tokio::test(start_paused = true)]
18849    async fn registration_timeout_does_not_count_an_earlier_exit() {
18850        let actor = Actor::start_with_policy(
18851            ModuleProtocol::Subc,
18852            Duration::from_secs(30),
18853            DEFAULT_REGISTER_TIMEOUT,
18854            RestartPolicy::new(0, Duration::ZERO),
18855        )
18856        .await;
18857        let pid = actor.module.status().unwrap().pid.unwrap();
18858        rustix::process::kill_process(
18859            rustix::process::Pid::from_raw(pid as i32).unwrap(),
18860            rustix::process::Signal::TERM,
18861        )
18862        .unwrap();
18863        wait_for("early exit", || {
18864            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
18865        })
18866        .await;
18867        tokio::time::advance(Duration::from_secs(121)).await;
18868        tokio::task::yield_now().await;
18869        let entries = actor.module.terminal_history().entries;
18870        assert_eq!(entries.len(), 1);
18871        assert!(!entries[0]
18872            .disposition_detail
18873            .as_deref()
18874            .unwrap()
18875            .contains("registration_timeout"));
18876        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
18877        actor.module.stop().await.unwrap();
18878    }
18879
18880    #[tokio::test(start_paused = true)]
18881    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
18882        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18883        let pid = actor.module.status().unwrap().pid;
18884        tokio::time::advance(Duration::from_secs(119)).await;
18885        actor.roster.close();
18886        let turns = actor.turns();
18887        tokio::time::advance(Duration::from_secs(300)).await;
18888        let parked = actor.wait_for_select(|at| at.turn > turns).await;
18889        assert_eq!(parked.wake_after, None);
18890        assert_eq!(actor.module.status().unwrap().pid, pid);
18891        assert!(actor.module.terminal_history().entries.is_empty());
18892        actor.module.stop().await.unwrap();
18893        let status = actor.module.status().unwrap();
18894        assert_eq!(status.restart_count, 0);
18895        assert_eq!(
18896            actor.module.terminal_history().entries[0].disposition,
18897            TerminalDisposition::DaemonShutdown
18898        );
18899    }
18900
18901    #[tokio::test(start_paused = true)]
18902    async fn no_health_children_park_for_sixty_seconds() {
18903        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18904        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18905        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18906        unadvertised.hello(1, false);
18907        // Include any one-off registration wake in the budget; receiving a
18908        // no-health HELLO must not turn a parked child into a polling child.
18909        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
18910        // a single 60 s jump would only observe one expired timer.
18911        for _ in 0..6000 {
18912            tokio::time::advance(Duration::from_millis(10)).await;
18913            tokio::task::yield_now().await;
18914        }
18915        for actor in [&none, &unregistered, &unadvertised] {
18916            assert!(
18917                actor.turns() <= 5,
18918                "no-health actor woke {} times",
18919                actor.turns()
18920            );
18921            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
18922        }
18923    }
18924
18925    #[tokio::test(start_paused = true)]
18926    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
18927        // With a zero cadence the first probe is due at once, so this measures
18928        // only how fast the registration notification starts health probing,
18929        // not the normal 30-33 s wait before a first probe (checked below).
18930        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
18931        let tick = Instant::now();
18932        actor.hello(1, true);
18933        actor.probe().await;
18934        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
18935    }
18936
18937    #[tokio::test(start_paused = true)]
18938    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
18939        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18940        let hello_tick = Instant::now();
18941        actor.hello(1, true);
18942        let armed = actor
18943            .wait_for_select(|checkpoint| {
18944                checkpoint.registered_connection == Some(ConnectionId::new(1))
18945                    && checkpoint.next_probe_at.is_some()
18946            })
18947            .await;
18948        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18949        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
18950        let first_deadline = hello_tick + delay;
18951        assert_eq!(armed.next_probe_at, Some(first_deadline));
18952        tokio::time::advance(Duration::from_secs(29)).await;
18953        let turns = actor.turns();
18954        actor
18955            .registry
18956            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
18957            .unwrap();
18958        let updated = actor
18959            .wait_for_select(|checkpoint| checkpoint.turn > turns)
18960            .await;
18961        assert_eq!(
18962            updated.next_probe_at,
18963            Some(first_deadline),
18964            "catalog update must not reset cadence"
18965        );
18966        actor.no_probe();
18967        tokio::time::advance(first_deadline - Instant::now()).await;
18968        let frame = actor.probe().await;
18969        actor.answer(1, frame);
18970        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
18971        let next_deadline = Instant::now() + next;
18972        let rearmed = actor
18973            .wait_for_select(|checkpoint| {
18974                checkpoint
18975                    .next_probe_at
18976                    .is_some_and(|deadline| deadline > first_deadline)
18977            })
18978            .await;
18979        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
18980        tokio::time::advance(next - Duration::from_nanos(1)).await;
18981        actor.no_probe();
18982        tokio::time::advance(Duration::from_nanos(1)).await;
18983        actor.probe().await;
18984    }
18985
18986    #[tokio::test(start_paused = true)]
18987    async fn disconnect_disarms_and_reconnect_rearms_health() {
18988        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18989        actor.hello(1, true);
18990        actor
18991            .wait_for_select(|checkpoint| {
18992                checkpoint.registered_connection == Some(ConnectionId::new(1))
18993                    && checkpoint.next_probe_at.is_some()
18994            })
18995            .await;
18996        let turns = actor.turns();
18997        let tick = Instant::now();
18998        actor
18999            .registry
19000            .deregister_connection(ConnectionId::new(1))
19001            .unwrap();
19002        let disarmed = actor
19003            .wait_for_select(|checkpoint| {
19004                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
19005            })
19006            .await;
19007        assert_eq!(disarmed.next_probe_at, None);
19008        assert_eq!(disarmed.wake_after, None);
19009        assert_eq!(
19010            Instant::now(),
19011            tick,
19012            "disconnect must disarm in the notification tick"
19013        );
19014        tokio::time::advance(Duration::from_secs(60)).await;
19015        actor.no_probe();
19016        let reconnect_tick = Instant::now();
19017        actor.hello(2, true);
19018        let rearmed = actor
19019            .wait_for_select(|checkpoint| {
19020                checkpoint.registered_connection == Some(ConnectionId::new(2))
19021                    && checkpoint.next_probe_at.is_some()
19022            })
19023            .await;
19024        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
19025        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
19026        tokio::time::advance(delay).await;
19027        actor.probe().await;
19028    }
19029
19030    #[tokio::test(start_paused = true)]
19031    async fn reregistration_without_health_disarms_the_old_schedule() {
19032        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
19033        actor.hello(1, true);
19034        actor
19035            .wait_for_select(|checkpoint| {
19036                checkpoint.registered_connection == Some(ConnectionId::new(1))
19037                    && checkpoint.next_probe_at.is_some()
19038            })
19039            .await;
19040        actor
19041            .registry
19042            .deregister_connection(ConnectionId::new(1))
19043            .unwrap();
19044        actor.hello(2, false);
19045        let disarmed = actor
19046            .wait_for_select(|checkpoint| {
19047                checkpoint.registered_connection == Some(ConnectionId::new(2))
19048            })
19049            .await;
19050        assert_eq!(disarmed.next_probe_at, None);
19051        assert_eq!(disarmed.wake_after, None);
19052        tokio::time::advance(Duration::from_secs(60)).await;
19053        actor.no_probe();
19054    }
19055
19056    #[tokio::test(start_paused = true)]
19057    async fn restart_waits_for_its_own_hello_before_rearming_health() {
19058        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
19059        actor.hello(1, true);
19060        let armed = actor
19061            .wait_for_select(|checkpoint| {
19062                checkpoint.registered_connection == Some(ConnectionId::new(1))
19063                    && checkpoint.next_probe_at.is_some()
19064            })
19065            .await;
19066        // Simulate the old process's connection closing before the restart
19067        // tears the process down. The replacement process has started but has
19068        // not sent its HELLO, so it is not registered.
19069        actor
19070            .registry
19071            .deregister_connection(ConnectionId::new(1))
19072            .unwrap();
19073        actor.module.restart(Some(0)).await.unwrap();
19074        let parked = actor
19075            .wait_for_select(|checkpoint| {
19076                checkpoint.generation > armed.generation
19077                    && checkpoint.registered_connection.is_none()
19078                    && checkpoint.next_probe_at.is_none()
19079            })
19080            .await;
19081        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
19082        actor.rx.close();
19083        // Drain the stop notice the restart queued for the old process's connection,
19084        // so it isn't mistaken for traffic to the replacement.
19085        while actor.rx.try_recv().is_ok() {}
19086        let turns = actor.turns();
19087        tokio::time::advance(Duration::from_secs(60)).await;
19088        let after = actor.turns();
19089        eprintln!(
19090            "restart park turns: before={turns} after={after} delta={}",
19091            after - turns
19092        );
19093        assert_eq!(after, turns, "replacement must park before HELLO");
19094        let hello_tick = Instant::now();
19095        actor.hello(2, true);
19096        let rearmed = actor
19097            .wait_for_select(|checkpoint| {
19098                checkpoint.registered_connection == Some(ConnectionId::new(2))
19099                    && checkpoint.next_probe_at.is_some()
19100            })
19101            .await;
19102        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
19103        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
19104        tokio::time::advance(delay).await;
19105        actor.probe().await;
19106    }
19107
19108    #[tokio::test(start_paused = true)]
19109    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
19110        use tokio::{
19111            io::{AsyncReadExt, AsyncWriteExt},
19112            net::TcpListener,
19113        };
19114        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19115        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19116        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
19117        let (started_tx, mut started_rx) = mpsc::channel(8);
19118        let peer = tokio::spawn(async move {
19119            loop {
19120                let (mut socket, _) = listener.accept().await.unwrap();
19121                let mut request = [0; 1024];
19122                assert!(socket.read(&mut request).await.unwrap() > 0);
19123                started_tx.send(Instant::now()).await.unwrap();
19124                socket
19125                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19126                    .await
19127                    .unwrap();
19128            }
19129        });
19130        actor
19131            .module
19132            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19133            .await
19134            .unwrap();
19135        let first_tick = Instant::now();
19136        let first = actor
19137            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
19138            .await;
19139        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
19140        assert!(started_rx.try_recv().is_err());
19141
19142        // Restart just before the old deadline: reusing it would give the new
19143        // process only one second to start, rather than a full cadence.
19144        tokio::time::advance(
19145            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
19146        )
19147        .await;
19148        actor.module.restart(Some(0)).await.unwrap();
19149        let spawn_tick = Instant::now();
19150        let replacement = actor
19151            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
19152            .await;
19153        let deadline = replacement.next_probe_at.unwrap();
19154        assert!(
19155            deadline >= spawn_tick + actor.health.cadence,
19156            "a replacement must not inherit its predecessor's probe deadline"
19157        );
19158        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
19159        assert!(
19160            started_rx.try_recv().is_err(),
19161            "no HTTP probe during startup grace"
19162        );
19163        tokio::time::advance(deadline - Instant::now()).await;
19164        let landed = wait_for("first replacement HTTP probe", || {
19165            started_rx.try_recv().ok()
19166        })
19167        .await;
19168        assert!(landed >= spawn_tick + actor.health.cadence);
19169        wait_for("healthy replacement", || {
19170            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19171                .then_some(())
19172        })
19173        .await;
19174        peer.abort();
19175    }
19176
19177    #[tokio::test(start_paused = true)]
19178    async fn rescan_adds_and_removes_http_health_without_polling() {
19179        use tokio::{
19180            io::{AsyncReadExt, AsyncWriteExt},
19181            net::TcpListener,
19182        };
19183        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19184        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19185        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
19186        // One millisecond separates successive probes without a polling delay.
19187        actor.health.cadence = Duration::from_millis(1);
19188        let (started_tx, mut started_rx) = mpsc::channel(8);
19189        let peer = tokio::spawn(async move {
19190            loop {
19191                let (mut socket, _) = listener.accept().await.unwrap();
19192                let mut request = [0; 1024];
19193                assert!(socket.read(&mut request).await.unwrap() > 0);
19194                started_tx.send(()).await.unwrap();
19195                socket
19196                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19197                    .await
19198                    .unwrap();
19199            }
19200        });
19201        let tick = Instant::now();
19202        let turns = actor.turns();
19203        actor
19204            .module
19205            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19206            .await
19207            .unwrap();
19208        let armed = actor
19209            .wait_for_select(|checkpoint| {
19210                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
19211            })
19212            .await;
19213        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
19214        assert_eq!(Instant::now(), tick);
19215        tokio::time::advance(Duration::from_millis(1)).await;
19216        let mut started = false;
19217        wait_for("successful HTTP probe", || {
19218            started |= started_rx.try_recv().is_ok();
19219            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19220                .then_some(())
19221        })
19222        .await;
19223        assert!(started, "added HTTP check must start at its first deadline");
19224        assert_eq!(
19225            actor.module.status().unwrap().health.status,
19226            SupervisorHealthStatus::Ok
19227        );
19228        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
19229        actor.health.http = None;
19230        let turns = actor.turns();
19231        actor
19232            .module
19233            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19234            .await
19235            .unwrap();
19236        let parked = actor
19237            .wait_for_select(|checkpoint| {
19238                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
19239            })
19240            .await;
19241        assert_eq!(parked.wake_after, None);
19242        let turns = actor.turns();
19243        tokio::time::advance(Duration::from_secs(60)).await;
19244        assert_eq!(
19245            actor.turns(),
19246            turns,
19247            "removed HTTP check must park the actor"
19248        );
19249        assert!(
19250            started_rx.try_recv().is_err(),
19251            "removed HTTP check must stay disarmed"
19252        );
19253        assert_eq!(
19254            actor.module.status().unwrap().health,
19255            ModuleHealthStatus::default()
19256        );
19257        peer.abort();
19258    }
19259}
19260
19261#[cfg(test)]
19262mod drain_busy_reprobe_tests {
19263    //! The drain's busy-gauge wait, driven against an in-process module
19264    //! connection whose health answers the test scripts. Time is paused, so
19265    //! every elapsed duration below is exact virtual time.
19266    use super::*;
19267    use std::collections::BTreeMap;
19268    use std::sync::Mutex as StdMutex;
19269    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
19270
19271    const MODULE_ID: &str = "drain-reprobe-module";
19272    const CONNECTION: u64 = 7;
19273    const GAUGE: &str = "runs_in_flight";
19274    /// The production default for both the health cadence and the drain
19275    /// ceiling, which is the pairing that used to cost a full ceiling.
19276    const CADENCE: Duration = Duration::from_secs(30);
19277    const CEILING: Duration = Duration::from_secs(30);
19278
19279    /// How the scripted module answers each `health.check`.
19280    #[derive(Clone, Copy)]
19281    enum Answer {
19282        /// Report the gauge busy until this instant, quiet from then on, after
19283        /// taking `latency` to answer.
19284        BusyUntil { until: Instant, latency: Duration },
19285        /// Answer at once, with metrics that leave the declared gauge out.
19286        OmitsGauge,
19287        /// Never answer.
19288        Silent,
19289    }
19290
19291    struct Drain {
19292        forwarding: Arc<ForwardingTable>,
19293        runtime: SupervisorRuntimeConfig,
19294        endpoint: crate::ModuleEndpointId,
19295        /// When each probe reached the module.
19296        probes: Arc<StdMutex<Vec<Instant>>>,
19297        responder: tokio::task::JoinHandle<()>,
19298    }
19299
19300    impl Drain {
19301        fn start(answer: Answer) -> Self {
19302            let registry = Arc::new(Registry::default());
19303            let forwarding = Arc::new(ForwardingTable::default());
19304            let health = HealthConfig {
19305                cadence: CADENCE,
19306                ..HealthConfig::default()
19307            };
19308            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
19309                .with_forwarding(forwarding.clone())
19310                .with_health_config(health)
19311                .runtime_config();
19312            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
19313            let endpoint = forwarding
19314                .register_module_connection(
19315                    ConnectionId::new(CONNECTION),
19316                    MODULE_ID.to_string(),
19317                    subc_protocol::PROTOCOL_VERSION,
19318                    Concurrency::ModuleManaged,
19319                    FrameSink::new(tx),
19320                )
19321                .unwrap();
19322            let probes = Arc::new(StdMutex::new(Vec::new()));
19323            let responder = tokio::spawn({
19324                let forwarding = forwarding.clone();
19325                let probes = probes.clone();
19326                async move {
19327                    while let Some(frame) = rx.recv().await {
19328                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
19329                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
19330                        probes.lock().unwrap().push(Instant::now());
19331                        let metrics = match answer {
19332                            Answer::BusyUntil { until, latency } => {
19333                                sleep(latency).await;
19334                                let busy = u64::from(Instant::now() < until);
19335                                serde_json::json!({ GAUGE: busy })
19336                            }
19337                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
19338                            Answer::Silent => continue,
19339                        };
19340                        // A probe abandoned at its deadline has no RPC left to
19341                        // complete, so a late answer may be refused.
19342                        let _ = forwarding.complete_module_control_rpc(
19343                            ConnectionId::new(CONNECTION),
19344                            frame.header.corr,
19345                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
19346                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19347                                status: HealthStatus::Ok,
19348                                detail: None,
19349                                metrics: Some(metrics),
19350                            }),
19351                        );
19352                    }
19353                }
19354            });
19355            Self {
19356                forwarding,
19357                runtime,
19358                endpoint,
19359                probes,
19360                responder,
19361            }
19362        }
19363
19364        /// Run one by-id drain wait with `ceiling`, returning its verdict and
19365        /// how long it took.
19366        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
19367            let (drained, elapsed, _) = self.run_traced(ceiling).await;
19368            (drained, elapsed)
19369        }
19370
19371        /// [`Self::run`], also returning what the drain's probes read.
19372        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
19373            let started = Instant::now();
19374            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
19375            let drained = wait_for_forwarding_quiescence(
19376                &self.forwarding,
19377                MODULE_ID,
19378                &self.runtime,
19379                self.endpoint,
19380                started + ceiling,
19381                &[GAUGE.to_string()],
19382                DrainScope::Active,
19383                &mut trace,
19384            )
19385            .await
19386            .unwrap();
19387            (drained, started.elapsed(), trace)
19388        }
19389
19390        fn probes(&self) -> Vec<Instant> {
19391            self.probes.lock().unwrap().clone()
19392        }
19393    }
19394
19395    impl Drop for Drain {
19396        fn drop(&mut self) {
19397            self.responder.abort();
19398        }
19399    }
19400
19401    #[tokio::test(start_paused = true)]
19402    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
19403        let drain = Drain::start(Answer::BusyUntil {
19404            until: Instant::now() + Duration::from_secs(1),
19405            latency: Duration::ZERO,
19406        });
19407        let (drained, elapsed) = drain.run(CEILING).await;
19408        assert!(
19409            drained,
19410            "gauges that read zero during the drain must let it finish drained, \
19411             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
19412        );
19413        assert!(
19414            elapsed < Duration::from_secs(3),
19415            "drain took {elapsed:?} after the gauges went quiet at 1s"
19416        );
19417        let probes = drain.probes();
19418        assert!(
19419            probes.len() >= 2,
19420            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
19421        );
19422    }
19423
19424    /// The same shape in real time on a multi-threaded runtime: the first
19425    /// probe is answered busy at once, within a millisecond of the drain
19426    /// starting, and later probes read idle. The drain must keep probing
19427    /// after that first instant answer and end soon after the gauge falls.
19428    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19429    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
19430        let drain = Drain::start(Answer::BusyUntil {
19431            until: Instant::now() + Duration::from_millis(600),
19432            latency: Duration::ZERO,
19433        });
19434        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
19435        assert!(drained, "the gauge fell at 600ms: {trace}");
19436        assert!(
19437            elapsed < Duration::from_secs(3),
19438            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
19439        );
19440        let probes = drain.probes();
19441        assert!(
19442            probes.len() >= 2 && trace.probes as usize == probes.len(),
19443            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
19444        );
19445    }
19446
19447    #[tokio::test(start_paused = true)]
19448    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
19449        let started = Instant::now();
19450        let drain = Drain::start(Answer::BusyUntil {
19451            until: started + Duration::from_secs(3600),
19452            latency: Duration::ZERO,
19453        });
19454        let (drained, elapsed) = drain.run(CEILING).await;
19455        assert!(!drained, "a module busy throughout must not drain");
19456        assert!(
19457            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
19458            "a busy module is held until the ceiling and no longer: {elapsed:?}"
19459        );
19460        // Re-probing is paced by the interval, not a busy loop: at most one
19461        // probe per interval across the ceiling, plus the first at time zero.
19462        let probes = drain.probes();
19463        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
19464        assert!(
19465            probes.len() as u128 <= max_probes,
19466            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
19467            probes.len()
19468        );
19469        for pair in probes.windows(2) {
19470            assert!(
19471                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
19472                "consecutive probes {:?} apart",
19473                pair[1] - pair[0]
19474            );
19475        }
19476    }
19477
19478    #[tokio::test(start_paused = true)]
19479    async fn a_slow_probe_is_never_overlapped_by_the_next() {
19480        // Each answer takes longer than the re-probe interval; the next probe
19481        // must wait for the previous answer and then the interval.
19482        let latency = Duration::from_secs(1);
19483        let drain = Drain::start(Answer::BusyUntil {
19484            until: Instant::now() + Duration::from_secs(3600),
19485            latency,
19486        });
19487        let (drained, _) = drain.run(Duration::from_secs(5)).await;
19488        assert!(!drained);
19489        let probes = drain.probes();
19490        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
19491        for pair in probes.windows(2) {
19492            assert!(
19493                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
19494                "a probe started {:?} after the previous one, before it could finish",
19495                pair[1] - pair[0]
19496            );
19497        }
19498    }
19499
19500    #[tokio::test(start_paused = true)]
19501    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
19502        let drain = Drain::start(Answer::Silent);
19503        let ceiling = Duration::from_secs(2);
19504        let (drained, elapsed) = drain.run(ceiling).await;
19505        assert!(!drained, "an unanswered probe counts as busy");
19506        assert!(
19507            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
19508            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
19509        );
19510        assert_eq!(
19511            drain.probes().len(),
19512            1,
19513            "the unanswered probe holds the only slot until the ceiling"
19514        );
19515    }
19516
19517    /// Every event logged while it is installed, as `message` plus fields.
19518    #[derive(Clone, Default)]
19519    struct LogCapture {
19520        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
19521    }
19522
19523    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
19524        fn on_event(
19525            &self,
19526            event: &tracing::Event<'_>,
19527            _context: tracing_subscriber::layer::Context<'_, S>,
19528        ) {
19529            struct Fields(BTreeMap<String, String>);
19530            impl tracing::field::Visit for Fields {
19531                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
19532                    self.0
19533                        .insert(field.name().to_string(), format!("{value:?}"));
19534                }
19535            }
19536            let mut fields = Fields(BTreeMap::new());
19537            event.record(&mut fields);
19538            self.events.lock().unwrap().push(fields.0);
19539        }
19540    }
19541
19542    impl LogCapture {
19543        fn install(&self) -> tracing::subscriber::DefaultGuard {
19544            use tracing_subscriber::layer::SubscriberExt;
19545            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
19546        }
19547
19548        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
19549            self.events
19550                .lock()
19551                .unwrap()
19552                .iter()
19553                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
19554                .cloned()
19555                .collect()
19556        }
19557    }
19558
19559    /// A module that declares a busy gauge but leaves it out of its health
19560    /// answers holds a drain to its ceiling. The timeout line must say so by
19561    /// name, and the drain must warn about the missing gauge once, not on
19562    /// every probe.
19563    #[tokio::test(start_paused = true)]
19564    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
19565        let capture = LogCapture::default();
19566        let _guard = capture.install();
19567        let drain = Drain::start(Answer::OmitsGauge);
19568        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
19569        assert!(!drained, "a gauge never reported cannot read quiet");
19570        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
19571
19572        let missing = capture.with_message("lacks a declared busy gauge");
19573        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
19574        assert_eq!(
19575            missing[0].get("missing_gauges").map(String::as_str),
19576            Some(GAUGE)
19577        );
19578
19579        warn_drain_timed_out(
19580            MODULE_ID,
19581            Duration::from_secs(2),
19582            RouteCloseReason::Restart,
19583            &Default::default(),
19584            &trace,
19585        );
19586        let timeouts = capture.with_message("route drain timed out");
19587        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
19588        let gauges = &timeouts[0]["busy_gauges"];
19589        assert!(
19590            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
19591            "the timeout line must name the gauge and why it read busy: {gauges}"
19592        );
19593        assert!(
19594            gauges.ends_with("; no value ever read"),
19595            "no value was ever read for the gauge: {gauges}"
19596        );
19597        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
19598    }
19599
19600    /// The trace names a busy value and when it was read. For a probe that
19601    /// never got an answer, it names the probe's error kind and says that no
19602    /// value was ever read.
19603    #[tokio::test(start_paused = true)]
19604    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
19605        let busy = Drain::start(Answer::BusyUntil {
19606            until: Instant::now() + Duration::from_secs(3600),
19607            latency: Duration::ZERO,
19608        });
19609        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
19610        let rendered = trace.to_string();
19611        assert!(
19612            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
19613                && rendered.contains("; last value non_zero(1) @"),
19614            "{rendered}"
19615        );
19616
19617        let silent = Drain::start(Answer::Silent);
19618        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
19619        // The probe is capped at the 1 s ceiling, and its reading is stamped
19620        // when it gave up.
19621        assert_eq!(
19622            trace.to_string(),
19623            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
19624        );
19625        assert_eq!(trace.probes, 1);
19626    }
19627
19628    /// A gauge value that is not a count reads busy and is shown as sent.
19629    #[test]
19630    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
19631        let gauges = [GAUGE.to_string()];
19632        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
19633        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
19634        assert_eq!(
19635            busy_gauge_observation(&verdicts),
19636            BusyGaugeObservation::Busy
19637        );
19638        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
19639        assert_eq!(
19640            busy_gauge_observation(&zero),
19641            BusyGaugeObservation::Quiescent
19642        );
19643        let none = gauge_verdicts(None, &gauges);
19644        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
19645    }
19646}
19647
19648#[cfg(test)]
19649mod health_evidence_tests {
19650    use super::{HealthProbeError, HealthProbeEvidence};
19651    use std::collections::HashSet;
19652
19653    /// The evidential asymmetry, asserted rather than described.
19654    ///
19655    /// Exactly ONE observation is proof a module cannot serve, and the one that
19656    /// fires under CPU starvation is not it. Before the split, all fifteen
19657    /// construction sites collapsed into a single String, so a timeout carried the
19658    /// same weight as a dead lane -- which is how a healthy module was restarted
19659    /// three times in one day.
19660    #[test]
19661    fn only_a_dead_lane_is_proof_of_death() {
19662        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
19663        // Three non-proof classes, each for a different reason: silence is
19664        // consistent with health, a bad answer proves the module ALIVE, and a
19665        // daemon-side fault never reached the module at all.
19666        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
19667        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
19668        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
19669    }
19670
19671    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
19672    ///
19673    /// A shared label renders two different observations identically in the line an
19674    /// operator reads after an unexplained restart -- the exact confusion this
19675    /// change removes.
19676    #[test]
19677    fn every_evidence_class_has_a_distinct_label() {
19678        let labels = [
19679            HealthProbeError::lane_dead("").label(),
19680            HealthProbeError::no_answer("").label(),
19681            HealthProbeError::bad_answer("").label(),
19682            HealthProbeError::misconfigured("").label(),
19683        ];
19684        let unique: HashSet<_> = labels.iter().collect();
19685        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
19686    }
19687
19688    /// The class is additional information, not a replacement.
19689    ///
19690    /// An operator needs both "this was silence" and the specific text saying how
19691    /// long we waited; a classification that swallowed the message would trade one
19692    /// missing distinction for another.
19693    #[test]
19694    fn classification_preserves_the_original_message() {
19695        let err = HealthProbeError::no_answer("module did not answer within 5s");
19696        assert_eq!(err.to_string(), "module did not answer within 5s");
19697        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19698    }
19699}
19700
19701#[cfg(test)]
19702mod health_tombstone_tests {
19703    use std::{path::PathBuf, sync::Arc, time::Duration};
19704
19705    use subc_protocol::{
19706        manifest::Concurrency,
19707        session::{HealthStatus, ModuleControlResponse},
19708    };
19709    use tokio::sync::mpsc;
19710
19711    use super::{
19712        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
19713        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
19714    };
19715    use crate::{
19716        control::ControlHandler,
19717        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
19718        registry::{ConnectionId, Registry},
19719        router::FrameSink,
19720    };
19721
19722    struct ProbeHarness {
19723        spec: ModuleSpec,
19724        runtime: SupervisorRuntimeConfig,
19725        forwarding: Arc<ForwardingTable>,
19726        module_connection: ConnectionId,
19727        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19728        handler: ControlHandler,
19729        module: super::SupervisedModule,
19730    }
19731
19732    fn probe_harness() -> ProbeHarness {
19733        let registry = Arc::new(Registry::default());
19734        let forwarding = Arc::new(ForwardingTable::default());
19735        let supervisor_handle = super::SupervisorHandle::new();
19736        let health = HealthConfig {
19737            http: None,
19738            cadence: Duration::from_secs(30),
19739            deadline: Duration::from_secs(5),
19740            failure_threshold: 3,
19741            on_degraded: HealthAction::Report,
19742            on_failing: HealthAction::Report,
19743            critical: false,
19744        };
19745        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
19746            .with_forwarding(Arc::clone(&forwarding))
19747            .with_handle(supervisor_handle.clone())
19748            .with_health_config(health);
19749        let spec = ModuleSpec {
19750            script: None,
19751            module_id: "late-health-module".to_string(),
19752            program: PathBuf::from("disabled-module"),
19753            args: Vec::new(),
19754            env: Vec::new(),
19755            reserved: false,
19756            reserved_prefixes: Vec::new(),
19757            protocol: ModuleProtocol::Subc,
19758            overlap: Default::default(),
19759            carry_containment: false,
19760        };
19761        let module = supervisor
19762            .supervise_configured(spec.clone(), false)
19763            .unwrap();
19764        let runtime = supervisor.runtime_config();
19765        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
19766            .with_supervisor(supervisor_handle);
19767        let module_connection = ConnectionId::new(700);
19768        let (module_tx, module_rx) = mpsc::channel(8);
19769        forwarding
19770            .register_module_connection(
19771                module_connection,
19772                spec.module_id.clone(),
19773                subc_protocol::PROTOCOL_VERSION,
19774                Concurrency::ModuleManaged,
19775                FrameSink::new(module_tx),
19776            )
19777            .unwrap();
19778
19779        ProbeHarness {
19780            spec,
19781            runtime,
19782            forwarding,
19783            module_connection,
19784            module_rx,
19785            handler,
19786            module,
19787        }
19788    }
19789
19790    async fn finish_after(
19791        harness: &mut ProbeHarness,
19792        stall: Duration,
19793    ) -> ModuleControlRpcCompletion {
19794        assert!(stall > harness.runtime.health.deadline);
19795        let deadline = harness.runtime.health.deadline;
19796        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19797        let answer = async {
19798            let frame = harness.module_rx.recv().await.expect("health.check frame");
19799            tokio::time::advance(deadline).await;
19800            tokio::task::yield_now().await;
19801            tokio::time::advance(stall - deadline).await;
19802            harness
19803                .forwarding
19804                .complete_module_control_rpc(
19805                    harness.module_connection,
19806                    frame.header.corr,
19807                    Some("health.check"),
19808                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19809                        status: HealthStatus::Ok,
19810                        detail: None,
19811                        metrics: None,
19812                    }),
19813                )
19814                .unwrap()
19815        };
19816        let (probe_result, completion) = tokio::join!(probe, answer);
19817        let err = probe_result.expect_err("probe must miss its deadline");
19818        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19819        completion
19820    }
19821
19822    async fn time_out_without_answer(harness: &mut ProbeHarness) {
19823        let deadline = harness.runtime.health.deadline;
19824        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19825        let exhaust_deadline = async {
19826            let _frame = harness.module_rx.recv().await.expect("health.check frame");
19827            tokio::time::advance(deadline).await;
19828            tokio::task::yield_now().await;
19829        };
19830        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
19831        let err = probe_result.expect_err("probe must miss its deadline");
19832        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19833    }
19834
19835    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
19836        let registry = Arc::clone(&harness.module.inner.registry);
19837        let snapshot = Arc::clone(&harness.module.inner.snapshot);
19838        let process_liveness = super::SupervisorProcessLiveness::default();
19839        let mut child = None;
19840        let cycle = super::run_health_probe_cycle(
19841            &harness.spec,
19842            &harness.runtime,
19843            &registry,
19844            &process_liveness,
19845            &snapshot,
19846            &mut child,
19847        );
19848        let peer = async {
19849            let frame = harness.module_rx.recv().await.expect("health.check frame");
19850            if answer {
19851                harness
19852                    .forwarding
19853                    .complete_module_control_rpc(
19854                        harness.module_connection,
19855                        frame.header.corr,
19856                        Some("health.check"),
19857                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19858                            status: HealthStatus::Ok,
19859                            detail: None,
19860                            metrics: Some(serde_json::json!({"ready": true})),
19861                        }),
19862                    )
19863                    .unwrap();
19864            } else {
19865                tokio::time::advance(harness.runtime.health.deadline).await;
19866                tokio::task::yield_now().await;
19867            }
19868        };
19869        tokio::join!(cycle, peer);
19870    }
19871
19872    #[tokio::test(start_paused = true)]
19873    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
19874        let mut harness = probe_harness();
19875        // Drive the probe cycle directly with an in-memory wire peer. Stop the
19876        // disabled module's monitor so only this test owns lifecycle transitions;
19877        // no OS process is launched, and a restart is observed at scheduling.
19878        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
19879        monitor.abort();
19880        let _ = monitor.await;
19881        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
19882            state.enabled = true;
19883            state.state = super::ModuleState::Running;
19884            state.process_alive = true;
19885        })
19886        .unwrap();
19887
19888        run_probe_cycle(&mut harness, true).await;
19889        assert_eq!(
19890            harness.module.status().unwrap().health.status,
19891            super::SupervisorHealthStatus::Ok
19892        );
19893
19894        for failures in 1..harness.runtime.health.failure_threshold {
19895            run_probe_cycle(&mut harness, false).await;
19896            let status = harness.module.status().unwrap();
19897            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19898            assert_eq!(status.health.consecutive_failures, failures);
19899            assert!(status.health.last_probe_ms.is_some());
19900            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
19901            assert!(status.health.metrics.is_none());
19902            assert_eq!(status.state, super::ModuleState::Running);
19903            assert!(status.process_alive);
19904            assert_eq!(status.restart_count, 0);
19905            assert_eq!(status.lifetime_restarts, 0);
19906            assert!(status.health.last_action.is_none());
19907            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19908        }
19909
19910        run_probe_cycle(&mut harness, true).await;
19911        let recovered = harness.module.status().unwrap();
19912        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
19913        assert_eq!(recovered.health.consecutive_failures, 0);
19914        assert!(recovered.health.detail.is_none());
19915        assert_eq!(
19916            recovered.health.metrics,
19917            Some(serde_json::json!({"ready": true}))
19918        );
19919        assert_eq!(recovered.lifetime_restarts, 0);
19920
19921        for failures in 1..=harness.runtime.health.failure_threshold {
19922            run_probe_cycle(&mut harness, false).await;
19923            let status = harness.module.status().unwrap();
19924            assert_eq!(status.health.consecutive_failures, failures);
19925            if failures < harness.runtime.health.failure_threshold {
19926                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19927                assert_eq!(status.state, super::ModuleState::Running);
19928                assert_eq!(status.lifetime_restarts, 0);
19929                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19930            } else {
19931                assert_eq!(
19932                    status.health.status,
19933                    super::SupervisorHealthStatus::Unresponsive
19934                );
19935                assert_eq!(status.state, super::ModuleState::Restarting);
19936                assert_eq!(status.restart_count, 1);
19937                assert_eq!(status.lifetime_restarts, 1);
19938                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
19939                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
19940            }
19941        }
19942    }
19943
19944    #[tokio::test(start_paused = true)]
19945    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
19946        let mut harness = probe_harness();
19947
19948        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
19949        let first_latency = match &first {
19950            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19951            other => panic!("late answer was not retained: {other:?}"),
19952        };
19953        assert!(harness.handler.observe_module_control_completion(first));
19954
19955        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
19956        let second_latency = match &second {
19957            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19958            other => panic!("late answer was not retained: {other:?}"),
19959        };
19960        assert!(harness.handler.observe_module_control_completion(second));
19961
19962        assert_eq!(first_latency, Duration::from_secs(8));
19963        assert_eq!(
19964            second_latency - first_latency,
19965            Duration::from_secs(3),
19966            "latency must grow linearly with the additional stall"
19967        );
19968        let health = harness.module.status().unwrap().health;
19969        assert_eq!(health.late_answer_count, 2);
19970        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
19971    }
19972
19973    /// A module that answers every probe late must never march to the kill
19974    /// threshold: the late answer proves it is alive, so it must clear the miss
19975    /// streak the timeout recorded. Without the reset, a CPU-starved module
19976    /// that serves every probe seconds past the deadline accumulates
19977    /// `consecutive_failures` to the threshold and is killed — the exact
19978    /// sequence from the 2026-08-14 aft disable, where the daemon logged
19979    /// "proves the module is alive" five times while counting five misses.
19980    #[tokio::test(start_paused = true)]
19981    async fn late_answer_clears_the_consecutive_failure_streak() {
19982        let mut harness = probe_harness();
19983
19984        // Timeout recorded first: the probe path saw no answer in time.
19985        time_out_without_answer(&mut harness).await;
19986        harness
19987            .module
19988            .record_health_probe_failure_for_test("[no-answer] test miss")
19989            .unwrap();
19990        assert_eq!(
19991            harness.module.status().unwrap().health.consecutive_failures,
19992            1,
19993            "precondition: the miss must be on the streak before the late answer"
19994        );
19995
19996        // The stalled reply then lands: proof of life.
19997        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
19998        assert!(matches!(
19999            late,
20000            ModuleControlRpcCompletion::LateHealthAnswer { .. }
20001        ));
20002        assert!(harness.handler.observe_module_control_completion(late));
20003
20004        let health = harness.module.status().unwrap().health;
20005        assert_eq!(
20006            health.consecutive_failures, 0,
20007            "a late answer is an answer: the streak must reset"
20008        );
20009        assert_eq!(health.late_answer_count, 1);
20010    }
20011
20012    #[tokio::test(start_paused = true)]
20013    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
20014        let mut harness = probe_harness();
20015
20016        for _ in 0..20 {
20017            time_out_without_answer(&mut harness).await;
20018            assert_eq!(
20019                harness.forwarding.health_probe_tombstone_count().unwrap(),
20020                1
20021            );
20022        }
20023    }
20024}
20025
20026#[cfg(test)]
20027mod child_env_tests {
20028    #[cfg(windows)]
20029    use super::{
20030        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
20031    };
20032    #[cfg(windows)]
20033    use std::sync::{Arc, OnceLock};
20034    #[cfg(windows)]
20035    #[test]
20036    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
20037        let spec = spec(Vec::new());
20038        let mut command = Command::new("unused");
20039        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
20040            .unwrap()
20041            .unwrap();
20042        let name = command
20043            .as_std()
20044            .get_envs()
20045            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
20046            .and_then(|(_, value)| value)
20047            .unwrap();
20048        assert_eq!(name, OsStr::new(handoff.name()));
20049        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
20050        assert!(command.as_std().get_envs().any(|(key, value)| key
20051            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20052            && value == Some(OsStr::new("env"))));
20053        assert!(command
20054            .as_std()
20055            .get_envs()
20056            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
20057    }
20058
20059    #[cfg(windows)]
20060    #[test]
20061    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
20062        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
20063        snapshot.process_alive = true;
20064        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
20065        assert_eq!(
20066            observed_launch_nonce_source(&snapshot).as_deref(),
20067            Some("env")
20068        );
20069        let consumed = Arc::new(OnceLock::new());
20070        snapshot.pipe_consumed = Some(consumed.clone());
20071        assert_eq!(
20072            observed_launch_nonce_source(&snapshot).as_deref(),
20073            Some("env")
20074        );
20075        consumed.set(()).unwrap();
20076        assert_eq!(
20077            observed_launch_nonce_source(&snapshot).as_deref(),
20078            Some("pipe")
20079        );
20080        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
20081        assert_eq!(
20082            observed_launch_nonce_source(&snapshot).as_deref(),
20083            Some("env")
20084        );
20085        clear_current_process_facts(&mut snapshot);
20086        assert_eq!(observed_launch_nonce_source(&snapshot), None);
20087    }
20088    use super::{
20089        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
20090        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
20091        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
20092    };
20093    use std::{ffi::OsStr, path::PathBuf};
20094    use tokio::process::Command;
20095
20096    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
20097        ModuleSpec {
20098            script: None,
20099            module_id: "env-plan".to_string(),
20100            program: PathBuf::from("/nonexistent"),
20101            args: Vec::new(),
20102            env,
20103            reserved: false,
20104            reserved_prefixes: Vec::new(),
20105            protocol: ModuleProtocol::Subc,
20106            overlap: Default::default(),
20107            carry_containment: false,
20108        }
20109    }
20110
20111    #[test]
20112    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
20113        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
20114        let digest = LaunchSpecDigest::from_spec(&base);
20115        let mut reordered = base.clone();
20116        reordered.env.reverse();
20117        reordered.module_id = "different-id".into();
20118        reordered.reserved = true;
20119        reordered.overlap = Default::default();
20120        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
20121        let mut delimiter_collision = base.clone();
20122        delimiter_collision.env[0] = ("A=B".into(), "C".into());
20123        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
20124        let mut changed = base.clone();
20125        changed.protocol = ModuleProtocol::None;
20126        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20127        changed = base.clone();
20128        changed.args.push("--flag".into());
20129        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20130        changed = base.clone();
20131        changed.program = PathBuf::from("/other");
20132        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20133        changed = base.clone();
20134        changed.env[1].1 = "other".into();
20135        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20136        assert!(!format!("{digest:?}").contains("B=C"));
20137        assert!(!format!("{digest:?}").contains("[u8"));
20138    }
20139
20140    #[cfg(unix)]
20141    #[test]
20142    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
20143        use std::os::unix::ffi::OsStringExt;
20144        let mut left = spec(Vec::new());
20145        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
20146        let mut right = left.clone();
20147        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
20148        assert_eq!(
20149            left.program.to_string_lossy(),
20150            right.program.to_string_lossy()
20151        );
20152        assert_ne!(
20153            LaunchSpecDigest::from_spec(&left),
20154            LaunchSpecDigest::from_spec(&right)
20155        );
20156    }
20157
20158    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
20159    /// one still gets its own.
20160    ///
20161    /// This is the narrow goal `env_clear()` was reached for, and the reason the
20162    /// fix is `env_remove` rather than deleting the line: an operator's ambient
20163    /// filter silently becoming an unconfigured module's log level is a real
20164    /// defect, just a much smaller one than clearing the environment.
20165    ///
20166    /// Asserted on the command plan rather than a spawned child because proving
20167    /// the ABSENCE of an inherited variable needs the parent's environment
20168    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
20169    /// removal as `(key, None)`, which is exactly the distinction wanted: not
20170    /// "absent because nobody set it" but "explicitly unset for the child".
20171    #[test]
20172    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
20173        let mut command = Command::new("/nonexistent");
20174        apply_child_env(&mut command, &spec(Vec::new()));
20175        let removed = command
20176            .as_std()
20177            .get_envs()
20178            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
20179        assert!(
20180            removed,
20181            "ambient CK_LOG must be explicitly removed for an unconfigured module"
20182        );
20183
20184        let mut configured = Command::new("/nonexistent");
20185        apply_child_env(
20186            &mut configured,
20187            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
20188        );
20189        let effective = configured
20190            .as_std()
20191            .get_envs()
20192            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
20193            .last()
20194            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20195        assert_eq!(
20196            effective,
20197            Some(Some("debug".to_string())),
20198            "a module's configured CK_LOG must survive the ambient removal"
20199        );
20200    }
20201
20202    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
20203    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
20204    /// the same reason as the CK_LOG test above.
20205    ///
20206    /// The argument is the load-bearing half: a stock binary exits on an
20207    /// unknown flag before it listens, so with `--subc` appended the mode
20208    /// could not supervise the one process it exists for. Found by the first
20209    /// conformance run (nats-server: `flag provided but not defined: -subc`).
20210    #[test]
20211    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
20212        let connection_file = std::path::Path::new("/run/subc-connection.json");
20213        let handle = SupervisorHandle::new();
20214
20215        let mut none_spec = spec(Vec::new());
20216        none_spec.protocol = ModuleProtocol::None;
20217        let mut none = Command::new("/nonexistent");
20218        let none_handoff =
20219            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
20220                .expect("protocol-none spawn args apply");
20221        assert!(
20222            none_handoff.is_none(),
20223            "protocol:none spawn must not receive a nonce descriptor"
20224        );
20225        #[cfg(windows)]
20226        assert!(!none.as_std().get_envs().any(|(key, value)| key
20227            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20228            && value.is_some()));
20229        assert!(
20230            !none.as_std().get_envs().any(|(key, value)| key
20231                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
20232                && value.is_some()),
20233            "protocol:none spawn must not name a nonce descriptor"
20234        );
20235        let none_args: Vec<String> = none
20236            .as_std()
20237            .get_args()
20238            .map(|a| a.to_string_lossy().into_owned())
20239            .collect();
20240        assert!(
20241            !none_args.iter().any(|a| a == SUBC_ARG),
20242            "protocol:none argv must not carry --subc; got {none_args:?}"
20243        );
20244        let none_has_nonce = none
20245            .as_std()
20246            .get_envs()
20247            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
20248        assert!(
20249            !none_has_nonce,
20250            "protocol:none spawn must not receive a launch nonce"
20251        );
20252        let none_has_module_id = none
20253            .as_std()
20254            .get_envs()
20255            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
20256        assert!(
20257            none_has_module_id,
20258            "SUBC_MODULE_ID is inert and stays on every path"
20259        );
20260        assert!(
20261            handle.spawn_nonce(&none_spec.module_id).is_none(),
20262            "no nonce record for a process that will never present one"
20263        );
20264
20265        // Control: the subc-wire path is unchanged by the branch above.
20266        let wire_spec = spec(Vec::new());
20267        let mut wire = Command::new("/nonexistent");
20268        let wire_handoff =
20269            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
20270                .expect("subc-wire spawn args apply");
20271        let wire_fd_env = wire
20272            .as_std()
20273            .get_envs()
20274            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
20275            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20276        #[cfg(unix)]
20277        assert_eq!(
20278            wire_fd_env,
20279            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
20280            "a subc-wire spawn names the pipe it will receive at descriptor 3"
20281        );
20282        #[cfg(windows)]
20283        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
20284        #[cfg(not(any(unix, windows)))]
20285        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
20286        let wire_args: Vec<String> = wire
20287            .as_std()
20288            .get_args()
20289            .map(|a| a.to_string_lossy().into_owned())
20290            .collect();
20291        assert_eq!(
20292            wire_args,
20293            vec![
20294                SUBC_ARG.to_string(),
20295                connection_file.to_string_lossy().into_owned()
20296            ],
20297            "a subc-wire spawn still carries --subc <path>"
20298        );
20299        assert_eq!(
20300            wire.as_std()
20301                .get_envs()
20302                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
20303            !cfg!(unix),
20304            "only Windows supplies the environment nonce"
20305        );
20306        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
20307    }
20308
20309    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
20310    /// spec tries to set it; only a swap candidate carries it.
20311    ///
20312    /// "Set it only on candidates" is not enough, because spawn applies the
20313    /// spec's env verbatim and the daemon's own environment is inherited: either
20314    /// could hand a plain restart the swap role, and a module reading it would
20315    /// warm on its long swap budget while callers wait. Asserted as an explicit
20316    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
20317    /// test above gives.
20318    #[test]
20319    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
20320        let role = |command: &Command| {
20321            command
20322                .as_std()
20323                .get_envs()
20324                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
20325                .last()
20326                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
20327        };
20328        let forged = spec(vec![(
20329            SUBC_SPAWN_ROLE_ENV.to_string(),
20330            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
20331        )]);
20332
20333        let mut plain = Command::new("/nonexistent");
20334        apply_child_env(&mut plain, &forged);
20335        apply_spawn_role(&mut plain, SpawnRole::Plain);
20336        assert_eq!(
20337            role(&plain),
20338            Some(None),
20339            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
20340        );
20341
20342        let mut candidate = Command::new("/nonexistent");
20343        apply_child_env(&mut candidate, &spec(Vec::new()));
20344        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
20345        assert_eq!(
20346            role(&candidate),
20347            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
20348        );
20349    }
20350
20351    /// Daemon-private capture retention keys never reach the child.
20352    ///
20353    /// cortexkit-log exposes retention as a Rust struct with no environment
20354    /// names, so these entries are supervisor metadata. Passing them through
20355    /// would invent a public child-process contract by accident.
20356    #[test]
20357    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
20358        let mut command = Command::new("/nonexistent");
20359        apply_child_env(
20360            &mut command,
20361            &spec(vec![
20362                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
20363                ("KEPT".to_string(), "yes".to_string()),
20364            ]),
20365        );
20366        let keys: Vec<String> = command
20367            .as_std()
20368            .get_envs()
20369            .filter(|(_, value)| value.is_some())
20370            .map(|(key, _)| key.to_string_lossy().into_owned())
20371            .collect();
20372        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
20373        assert!(
20374            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
20375            "daemon-private capture key leaked to the child: {keys:?}"
20376        );
20377    }
20378}
20379
20380#[cfg(test)]
20381mod jitter_tests {
20382    use super::jittered_health_delay;
20383    use std::{collections::HashSet, time::Duration};
20384
20385    /// Module ids drawn from a real fleet, so the dispersal claim is about names
20386    /// that actually occur rather than invented ones.
20387    ///
20388    /// This is a SAMPLE, not a registry: the property under test is that distinct
20389    /// ids disperse, which holds for any set of distinct strings. Several entries
20390    /// are already historical (modules get renamed), and that costs nothing here --
20391    /// but it means a reader must not mistake this for the live module set, and a
20392    /// rename sweep will match it without there being anything to change.
20393    const FLEET: [&str; 14] = [
20394        "aft",
20395        "alfonso-core",
20396        "magic-context",
20397        "broca",
20398        "thalamus",
20399        "quota",
20400        "engram",
20401        "plexus",
20402        "cerebellum",
20403        "astrocyte",
20404        "synapse",
20405        "subc-mcp",
20406        "cortexkit-credentials",
20407        "subc-federation",
20408    ];
20409
20410    /// Probes must not converge after a fleet-wide restart.
20411    ///
20412    /// This is the property the jitter exists for: every module reconnects at
20413    /// once, and without dispersal all fourteen would then probe on the same
20414    /// tick forever. Nothing failed visibly when this went untested -- a
20415    /// convergent fleet still probes correctly, just in a burst, so the symptom
20416    /// is a periodic load spike that looks like whatever else is running.
20417    #[test]
20418    fn probe_delays_disperse_across_the_fleet() {
20419        let cadence = Duration::from_secs(30);
20420        let delays: HashSet<Duration> = FLEET
20421            .iter()
20422            .map(|id| jittered_health_delay(id, 0, cadence))
20423            .collect();
20424        assert_eq!(
20425            delays.len(),
20426            FLEET.len(),
20427            "every supervised module must land on its own probe offset"
20428        );
20429    }
20430
20431    /// The offset may only ever DELAY a probe, never bring it forward.
20432    ///
20433    /// A delay below the cadence would probe a module more often than
20434    /// configured, which is the opposite of what an operator asked for and
20435    /// would tighten the failure budget without anyone changing it.
20436    #[test]
20437    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
20438        let cadence = Duration::from_secs(30);
20439        let span = cadence / 10;
20440        for id in FLEET {
20441            for probe_index in 0..8 {
20442                let delay = jittered_health_delay(id, probe_index, cadence);
20443                assert!(
20444                    delay >= cadence,
20445                    "{id}#{probe_index}: jitter must not shorten the cadence"
20446                );
20447                assert!(
20448                    delay < cadence + span,
20449                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
20450                );
20451            }
20452        }
20453    }
20454
20455    /// A module keeps its offset across daemon restarts.
20456    ///
20457    /// The delay is derived rather than randomised precisely so a restart does
20458    /// not re-roll every module into a fresh chance of collision. A random
20459    /// source would satisfy the dispersal test above and quietly lose this.
20460    #[test]
20461    fn a_module_offset_is_stable_across_restarts() {
20462        let cadence = Duration::from_secs(30);
20463        for id in FLEET {
20464            assert_eq!(
20465                jittered_health_delay(id, 0, cadence),
20466                jittered_health_delay(id, 0, cadence),
20467                "{id}: the same module and probe index must produce the same offset"
20468            );
20469        }
20470    }
20471
20472    /// A zero cadence disables probing rather than producing a busy loop.
20473    #[test]
20474    fn zero_cadence_yields_zero_delay() {
20475        assert_eq!(
20476            jittered_health_delay("aft", 0, Duration::ZERO),
20477            Duration::ZERO
20478        );
20479    }
20480}
20481
20482#[cfg(all(test, target_os = "linux"))]
20483mod cgroup_placement_tests {
20484    use super::{
20485        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
20486        SuperviseError, SupervisedChild,
20487    };
20488    use crate::stderr_tail::{StderrRing, StderrTailConfig};
20489    use cortexkit_test_support::ScratchDir;
20490    use std::{
20491        fs, io,
20492        path::{Path, PathBuf},
20493        sync::{Arc, Mutex},
20494    };
20495    use tokio::process::Command;
20496
20497    #[tokio::test]
20498    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
20499        use super::*;
20500        let dir = ScratchDir::new("unique-spawn-cgroups");
20501        let root = PathBuf::from(format!(
20502            "/sys/fs/cgroup/subc-unique-test-{}-{}",
20503            std::process::id(),
20504            unix_ms_now()
20505        ));
20506        if let Err(error) = fs::create_dir(&root) {
20507            assert!(
20508                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
20509                "required cgroup test cannot execute: {error}"
20510            );
20511            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
20512            return;
20513        }
20514        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
20515        let group_count = || {
20516            fs::read_dir(root.join("subc-modules"))
20517                .unwrap()
20518                .map(|entry| entry.unwrap().file_type().unwrap())
20519                .filter(|kind| kind.is_dir())
20520                .count()
20521        };
20522        let supervisor =
20523            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
20524                .with_cgroup_placement(Some(placement.clone()));
20525        let runtime = supervisor.runtime_config();
20526        let mut spec = ModuleSpec {
20527            script: None,
20528            module_id: "unique-spawn".into(),
20529            program: PathBuf::from("/bin/sleep"),
20530            args: vec!["60".into()],
20531            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
20532                .into_iter()
20533                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
20534                .collect(),
20535            reserved: false,
20536            reserved_prefixes: vec![],
20537            protocol: ModuleProtocol::None,
20538            overlap: Default::default(),
20539            carry_containment: false,
20540        };
20541        let spawn = |spec: &ModuleSpec| {
20542            spawn_child(
20543                spec,
20544                None,
20545                None,
20546                &runtime.stderr_ring,
20547                None,
20548                &runtime.child_roster,
20549                runtime.spawn_delivery(),
20550                Some(&placement),
20551                runtime.register_timeout,
20552            )
20553            .unwrap()
20554        };
20555        let mut live = spawn(&spec);
20556        for _ in 0..3 {
20557            // A new process can enter the old slot while retirement is pending.
20558            let next = spawn(&spec);
20559            assert_ne!(live.module_id, next.module_id);
20560            live.start_kill().unwrap();
20561            live.wait().await.unwrap();
20562            live = next;
20563            assert!(
20564                live.child.try_wait().unwrap().is_none(),
20565                "retiring the old slot must not kill the replacement"
20566            );
20567            assert_eq!(
20568                group_count(),
20569                1,
20570                "only the live spawn's cgroup should remain"
20571            );
20572        }
20573        supervisor.begin_daemon_shutdown();
20574        let reap = tokio::spawn(async move {
20575            live.wait().await.unwrap();
20576        });
20577        supervisor
20578            .end_children_for_daemon_shutdown(false, std::future::pending())
20579            .await;
20580        reap.await.unwrap();
20581        assert_eq!(group_count(), 0);
20582        // A normal exit uses the same tree-cleanup path as a killed spawn.
20583        spec.program = PathBuf::from("/bin/true");
20584        spec.args.clear();
20585        let fresh_roster = ChildRoster::default();
20586        let mut short = spawn_child(
20587            &spec,
20588            None,
20589            None,
20590            &runtime.stderr_ring,
20591            None,
20592            &fresh_roster,
20593            None,
20594            Some(&placement),
20595            runtime.register_timeout,
20596        )
20597        .unwrap();
20598        short.wait().await.unwrap();
20599        assert_eq!(group_count(), 0);
20600        spec.module_id = "_".repeat(255);
20601        let mut long_id = spawn_child(
20602            &spec,
20603            None,
20604            None,
20605            &runtime.stderr_ring,
20606            None,
20607            &fresh_roster,
20608            None,
20609            Some(&placement),
20610            runtime.register_timeout,
20611        )
20612        .unwrap();
20613        long_id.wait().await.unwrap();
20614        assert_eq!(
20615            group_count(),
20616            0,
20617            "valid long module IDs must not exceed cgroup NAME_MAX"
20618        );
20619        fs::remove_dir(root.join("subc-modules")).unwrap();
20620        fs::remove_dir(root).unwrap();
20621        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
20622    }
20623
20624    #[test]
20625    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
20626        let path = Path::new("/definitely-missing-subc-cgroup");
20627        let mut command = Command::new("true");
20628        let error = apply_cgroup_placement(
20629            &mut command,
20630            &ModuleSpec {
20631                script: None,
20632                module_id: "broken-cgroup".to_string(),
20633                program: PathBuf::from("true"),
20634                args: Vec::new(),
20635                env: Vec::new(),
20636                reserved: false,
20637                reserved_prefixes: Vec::new(),
20638                protocol: ModuleProtocol::Subc,
20639                overlap: Default::default(),
20640                carry_containment: false,
20641            },
20642            path,
20643        )
20644        .expect_err("a parent cgroup open failure must reject the supervised spawn");
20645        let reason = error.to_string();
20646
20647        assert!(
20648            matches!(error, SuperviseError::Cgroup { .. }),
20649            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
20650        );
20651        assert!(
20652            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
20653            "parent cgroup open failure must name cgroup.procs: {reason}"
20654        );
20655    }
20656
20657    #[tokio::test]
20658    async fn reaping_a_child_removes_its_empty_module_cgroup() {
20659        let root = ScratchDir::new("supervisor-reap-cgroup");
20660        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20661        let placement = subc_cgroup::prepare_at(&root)
20662            .expect("prepare scratch cgroup root")
20663            .expect("scratch root has a cgroup.procs marker");
20664        let module_id = "reaped-module";
20665        let module = placement
20666            .module_path(module_id)
20667            .expect("create scratch module cgroup");
20668        let child = Command::new("true")
20669            .env("XDG_DATA_HOME", root.path())
20670            .env("XDG_RUNTIME_DIR", root.path())
20671            .env("XDG_CONFIG_HOME", root.path())
20672            .spawn()
20673            .expect("spawn short-lived child");
20674        let pid = child.id().expect("spawned child has pid");
20675        let mut child = SupervisedChild {
20676            registration_channel: None,
20677            registration_spawn: None,
20678            live_program_binding: None,
20679            registered_config_connection: None,
20680            admission_base: None,
20681            registration_declaration: None,
20682            _program_binding: None,
20683            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
20684            child: crate::supervise::containment::ProcessChild::plain(child, pid),
20685            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
20686            protocol: ModuleProtocol::Subc,
20687            module_id: module_id.to_string(),
20688            cgroup_placement: Some(placement),
20689            stdout_pump: None,
20690            stderr_pump: None,
20691            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
20692            spawned_at: tokio::time::Instant::now(),
20693            config_delivery: None,
20694            registration_deadline: None,
20695            registration_watch: None,
20696            spawned_at_ms: 0,
20697            spawned_from: PathBuf::from("true"),
20698            launch_spec_digest: LaunchSpecDigest([0; 32]),
20699            spawned_file_identity: None,
20700            process_start_time: None,
20701            process_identity: None,
20702            pid,
20703            roster_guard: None,
20704            #[cfg(target_os = "macos")]
20705            privacy_exec: None,
20706            spawn_failure: None,
20707            admission_kill_sent: false,
20708        };
20709
20710        child.wait().await.expect("reap short-lived child");
20711
20712        assert!(
20713            !module.exists(),
20714            "waiting for a non-carried child must remove its empty cgroup before returning"
20715        );
20716        child.finish_exit(false).await.unwrap();
20717    }
20718
20719    #[test]
20720    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
20721        let root = ScratchDir::new("supervisor-non-empty-cgroup");
20722        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20723        let placement = subc_cgroup::prepare_at(&root)
20724            .expect("prepare scratch cgroup root")
20725            .expect("scratch root has a cgroup.procs marker");
20726        let module = placement
20727            .module_path("surviving-module")
20728            .expect("create scratch module cgroup");
20729        fs::write(module.join("surviving-process"), b"still present")
20730            .expect("make scratch cgroup non-empty");
20731        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
20732
20733        remove_module_cgroup(&placement, "surviving-module");
20734
20735        let logs = crate::router::test_log::captured_logs(&logs);
20736        assert!(
20737            module.exists(),
20738            "failed removal must leave the cgroup intact"
20739        );
20740        assert!(
20741            logs.contains("could not remove module cgroup after process exit; continuing teardown")
20742                && logs.contains("surviving-module"),
20743            "best-effort removal must report the failure without returning it: {logs}"
20744        );
20745    }
20746
20747    #[test]
20748    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
20749        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
20750        let reason = SuperviseError::Spawn {
20751            program: PathBuf::from("/bin/true"),
20752            source: io::Error::from_raw_os_error(13),
20753            cgroup_path: Some(cgroup_path.clone()),
20754        }
20755        .to_string();
20756
20757        assert!(
20758            reason.contains(&cgroup_path.display().to_string()),
20759            "a pre_exec spawn failure must name the cgroup path: {reason}"
20760        );
20761    }
20762}
20763
20764#[cfg(test)]
20765mod spawn_subscriber_lag_tests {
20766    use super::*;
20767
20768    /// A subscriber whose connection stops draining is dropped once its frame
20769    /// channel fills. The client must learn that from a terminal Error frame
20770    /// after the frames already queued for it, not from a stream that simply
20771    /// goes quiet.
20772    #[tokio::test]
20773    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
20774        let feed = SpawnEventFeed::default();
20775        feed.configure_incarnation("lag-incarnation".to_string());
20776        // A one-slot connection queue that nobody reads until the emits are
20777        // done: the forwarder parks on it and the subscriber channel fills.
20778        let (tx, mut rx) = mpsc::channel(1);
20779        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
20780            .expect("subscribe");
20781        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
20782        for index in 0..emitted {
20783            feed.emit_spawned(&format!("lag-module-{index}"), None, 1000, 0);
20784            // Let the forwarder take what it can so the fill point is the
20785            // subscriber channel, not a scheduling accident.
20786            tokio::task::yield_now().await;
20787        }
20788        assert_eq!(
20789            feed.subscriber_count(),
20790            0,
20791            "the lagged subscriber must be removed"
20792        );
20793
20794        let mut data = Vec::new();
20795        let mut last = None;
20796        loop {
20797            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
20798                .await
20799                .expect("the forwarder must finish once the subscriber is dropped");
20800            let Some(outbound) = next else { break };
20801            let frame = outbound.frame;
20802            if frame.header.ty == FrameType::StreamData {
20803                assert!(last.is_none(), "no data may follow the terminal frame");
20804                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
20805                data.push(event.cursor.seq);
20806            } else {
20807                assert!(last.is_none(), "exactly one terminal frame");
20808                last = Some(frame);
20809            }
20810        }
20811        assert!(!data.is_empty(), "queued frames drain before the terminal");
20812        for pair in data.windows(2) {
20813            assert_eq!(
20814                pair[1],
20815                pair[0] + 1,
20816                "queued frames arrive dense and in order"
20817            );
20818        }
20819        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
20820        assert_eq!(terminal.header.ty, FrameType::Error);
20821        assert_eq!(terminal.header.corr, 7);
20822        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
20823        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
20824        let detail = body.detail.expect("lagged error carries detail");
20825        assert_eq!(
20826            detail["first_undelivered_cursor"]["seq"],
20827            data.last().unwrap() + 1,
20828            "the named cursor is the first event the subscriber did not receive"
20829        );
20830        assert_eq!(
20831            detail["first_undelivered_cursor"]["daemon_incarnation"],
20832            "lag-incarnation"
20833        );
20834    }
20835}
20836
20837#[cfg(test)]
20838mod terminal_history_read_concurrency_tests {
20839    use super::*;
20840    use crate::terminal_journal::read_pause;
20841    use cortexkit_test_support::ScratchDir;
20842    use std::sync::mpsc as std_mpsc;
20843
20844    fn journaled_ring(
20845        journal: &Arc<crate::terminal_journal::TerminalJournal>,
20846    ) -> Arc<Mutex<TerminalRing>> {
20847        Arc::new(Mutex::new(
20848            TerminalRing::new(TerminalRingConfig::default(), 1)
20849                .with_journal(Some(Arc::clone(journal))),
20850        ))
20851    }
20852
20853    fn crash(at_ms: u64) -> ExitReport {
20854        ExitReport {
20855            kind: ExitKind::Crash,
20856            code: Some(1),
20857            signal: None,
20858            at_ms,
20859        }
20860    }
20861
20862    /// Record an exit on another thread and report whether it finished within
20863    /// `bound`. The recorder thread is left running if it did not.
20864    fn record_within(
20865        module_id: &'static str,
20866        ring: &Arc<Mutex<TerminalRing>>,
20867        at_ms: u64,
20868        bound: Duration,
20869    ) -> bool {
20870        let ring = Arc::clone(ring);
20871        let (done, done_rx) = std_mpsc::channel();
20872        std::thread::spawn(move || {
20873            record_terminal(
20874                module_id,
20875                &ring,
20876                &SpawnEventFeed::default(),
20877                &crash(at_ms),
20878                TerminalDisposition::Restarting,
20879            );
20880            let _ = done.send(());
20881        });
20882        done_rx.recv_timeout(bound).is_ok()
20883    }
20884
20885    /// A history read in progress must not hold the journal writer (which every
20886    /// module's exit recording needs) or the module's own ring. Exits recorded
20887    /// while the read is paused complete promptly; the paused read answers as of
20888    /// the moment it started, and the next read has each exit exactly once.
20889    #[test]
20890    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
20891        let dir = ScratchDir::new("terminal-history-concurrent-read");
20892        let path = dir.join("terminals.jsonl");
20893        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
20894            path.clone(),
20895            "daemon".into(),
20896        ));
20897        let reader_ring = journaled_ring(&journal);
20898        let other_ring = journaled_ring(&journal);
20899        assert!(record_within(
20900            "reader-module",
20901            &reader_ring,
20902            10,
20903            Duration::from_secs(5)
20904        ));
20905
20906        let (started, release) = read_pause::install(&path);
20907        let reading = {
20908            let ring = Arc::clone(&reader_ring);
20909            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
20910        };
20911        started
20912            .recv_timeout(Duration::from_secs(5))
20913            .expect("the history read reached its pause");
20914
20915        let bound = Duration::from_secs(1);
20916        assert!(
20917            record_within("other-module", &other_ring, 20, bound),
20918            "another module's exit waited on a history read (journal writer held)"
20919        );
20920        assert!(
20921            record_within("reader-module", &reader_ring, 30, bound),
20922            "the read module's own exit waited on its history read (ring held)"
20923        );
20924
20925        drop(release);
20926        let paused = reading.join().unwrap();
20927        assert_eq!(
20928            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20929            vec![10],
20930            "an exit recorded after the read began lands in neither half of it"
20931        );
20932        assert_eq!(paused.journal_skipped_lines, 0);
20933        assert_eq!(paused.journal_read_errors, 0);
20934
20935        let after = durable_terminal_history_of(&reader_ring, "reader-module");
20936        assert_eq!(
20937            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20938            vec![10, 30],
20939            "the next read merges ring and journal with no duplicate"
20940        );
20941        assert_eq!(after.journal_skipped_lines, 0);
20942    }
20943}
20944
20945/// What a restart does with the exited process's output readers. These drive
20946/// the same settle functions the supervisor calls, with a reader the test
20947/// holds, so a reader that has not been scheduled by the bound is a controlled
20948/// input rather than something only a loaded machine produces.
20949#[cfg(test)]
20950mod stderr_settle_tests {
20951    use std::{
20952        future::Future,
20953        io,
20954        pin::Pin,
20955        sync::{Arc, Mutex},
20956        task::{Context, Poll},
20957        time::Duration,
20958    };
20959
20960    use tokio::{
20961        io::{AsyncRead, ReadBuf},
20962        sync::oneshot,
20963        time::Instant,
20964    };
20965
20966    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
20967    use crate::stderr_tail::{
20968        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
20969        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
20970    };
20971
20972    const BOUND: Duration = Duration::from_millis(250);
20973
20974    /// Yields `before`, then stays pending until the gate is released, then
20975    /// yields `after` and reaches EOF. The bytes after the gate were written
20976    /// by a process that has already exited; only the reader is behind.
20977    struct HeldReader {
20978        before: Option<Vec<u8>>,
20979        gate: Option<oneshot::Receiver<()>>,
20980        after: io::Cursor<Vec<u8>>,
20981    }
20982
20983    impl AsyncRead for HeldReader {
20984        fn poll_read(
20985            mut self: Pin<&mut Self>,
20986            cx: &mut Context<'_>,
20987            buf: &mut ReadBuf<'_>,
20988        ) -> Poll<io::Result<()>> {
20989            if let Some(bytes) = self.before.take() {
20990                buf.put_slice(&bytes);
20991                return Poll::Ready(Ok(()));
20992            }
20993            if let Some(gate) = self.gate.as_mut() {
20994                match Pin::new(gate).poll(cx) {
20995                    Poll::Pending => return Poll::Pending,
20996                    Poll::Ready(_) => self.gate = None,
20997                }
20998            }
20999            Pin::new(&mut self.after).poll_read(cx, buf)
21000        }
21001    }
21002
21003    struct DiscardSink;
21004
21005    impl OutputSink for DiscardSink {
21006        fn write_line(&mut self, _line: &[u8]) {}
21007    }
21008
21009    fn line(text: &str) -> TailEntry {
21010        TailEntry::Line {
21011            text: text.to_string(),
21012            truncated: false,
21013            at_ms: None,
21014        }
21015    }
21016
21017    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
21018        ring.lock().unwrap()
21019    }
21020
21021    /// Start a reader for a new process generation that delivers `before`
21022    /// immediately and `after` only once the returned sender fires (or is
21023    /// dropped).
21024    fn held_pump(
21025        ring: &Arc<Mutex<StderrRing>>,
21026        before: &str,
21027        after: &str,
21028    ) -> (StderrPump, oneshot::Sender<()>) {
21029        let generation = lock(ring).begin_process();
21030        let (release, gate) = oneshot::channel();
21031        let reader = HeldReader {
21032            before: Some(before.as_bytes().to_vec()),
21033            gate: Some(gate),
21034            after: io::Cursor::new(after.as_bytes().to_vec()),
21035        };
21036        let task = tokio::spawn(pump_stderr_to(
21037            reader,
21038            Arc::clone(ring),
21039            generation,
21040            DiscardSink,
21041        ));
21042        (StderrPump { task, generation }, release)
21043    }
21044
21045    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
21046        for _ in 0..1000 {
21047            if done(&lock(ring)) {
21048                return;
21049            }
21050            tokio::time::sleep(Duration::from_millis(1)).await;
21051        }
21052        panic!(
21053            "ring never reached the expected state: {:?}",
21054            lock(ring).snapshot(None, None)
21055        );
21056    }
21057
21058    #[tokio::test(start_paused = true)]
21059    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
21060        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21061        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
21062
21063        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
21064        let before_release = lock(&ring).snapshot(None, None);
21065        assert!(
21066            matches!(before_release.capture, CaptureState::Incomplete { .. }),
21067            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
21068        );
21069
21070        // The restart: the next process starts and writes before the old
21071        // reader catches up.
21072        let next = lock(&ring).begin_process();
21073        lock(&ring).push_line_from(next, "next process booting");
21074        release.send(()).unwrap();
21075        wait_until(&ring, |ring| {
21076            ring.snapshot(None, None).capture == CaptureState::Captured
21077        })
21078        .await;
21079
21080        assert_eq!(
21081            untimed(lock(&ring).snapshot(None, None).entries),
21082            vec![
21083                line("booting"),
21084                line("config error: missing storage"),
21085                TailEntry::ProcessStart,
21086                line("next process booting"),
21087            ],
21088            "the crash's last line must survive a slow reader and stay in the crashed process's section"
21089        );
21090    }
21091
21092    #[tokio::test(start_paused = true)]
21093    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
21094    ) {
21095        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21096        // `_held` is never fired: a descendant keeps the pipe open for the
21097        // whole test.
21098        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
21099
21100        let started = Instant::now();
21101        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
21102        assert_eq!(
21103            started.elapsed(),
21104            BOUND,
21105            "the restart must wait exactly the bound for a pipe that stays open, no longer"
21106        );
21107
21108        let next = lock(&ring).begin_process();
21109        lock(&ring).push_line_from(next, "next process booting");
21110        tokio::time::sleep(Duration::from_secs(60)).await;
21111
21112        let snapshot = lock(&ring).snapshot(None, None);
21113        match &snapshot.capture {
21114            CaptureState::Incomplete { reason } => assert!(
21115                reason.contains("had not reached EOF") && reason.contains("250ms"),
21116                "the reason must say what is missing and after how long: {reason}"
21117            ),
21118            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
21119        }
21120        assert_eq!(
21121            untimed(snapshot.entries),
21122            vec![
21123                line("parent exiting"),
21124                TailEntry::ProcessStart,
21125                line("next process booting"),
21126            ]
21127        );
21128    }
21129
21130    #[tokio::test(start_paused = true)]
21131    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
21132        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21133        let (pump, release) = held_pump(&ring, "one\n", "two\n");
21134        release.send(()).unwrap();
21135
21136        settle_stderr_pump("clean", &ring, pump, BOUND).await;
21137
21138        let snapshot = lock(&ring).snapshot(None, None);
21139        assert_eq!(snapshot.capture, CaptureState::Captured);
21140        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
21141    }
21142
21143    #[tokio::test(start_paused = true)]
21144    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
21145        const BURST: usize = 150;
21146        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
21147        let path = capture.join("two-pipe.stderr.log");
21148        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
21149        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21150        let generation = lock(&ring).begin_process();
21151        let stdout = (0..BURST)
21152            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
21153            .collect::<String>();
21154        let stderr = (0..BURST)
21155            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
21156            .collect::<String>();
21157        // Model a child that has written both bursts and exited. Hold the stdout
21158        // reader after one line and a partial prefix, as if pipe readiness or
21159        // capture-file writes had delayed it. The final line also needs EOF to
21160        // flush: neither complete nor partial buffered output may be discarded.
21161        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
21162        let (release, gate) = oneshot::channel();
21163        let reader = HeldReader {
21164            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
21165            gate: Some(gate),
21166            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
21167        };
21168        let (finished, completion) = oneshot::channel();
21169        let stdout_sink = sink.clone();
21170        let pump = tokio::spawn(async move {
21171            pump_stdout_to(reader, stdout_sink).await;
21172            finished.send(()).unwrap();
21173        });
21174        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
21175
21176        let started = Instant::now();
21177        settle_stdout_pump("two-pipe", pump, BOUND).await;
21178        assert_eq!(
21179            started.elapsed(),
21180            BOUND,
21181            "a late reader must not hold up restart"
21182        );
21183        assert_eq!(
21184            std::fs::read_to_string(&path).unwrap().lines().count(),
21185            BURST + 1
21186        );
21187
21188        release
21189            .send(())
21190            .expect("stdout reader must survive the exit drain bound");
21191        tokio::time::timeout(Duration::from_secs(1), completion)
21192            .await
21193            .expect("released stdout reader must reach EOF")
21194            .expect("stdout capture must finish at EOF, not be cancelled");
21195        let contents = std::fs::read_to_string(&path).unwrap();
21196        let mut lines = contents
21197            .lines()
21198            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
21199            .collect::<Vec<_>>();
21200        let mut expected = (0..BURST)
21201            .flat_map(|i| {
21202                [
21203                    format!("out-{i:04}-{}", "o".repeat(64)),
21204                    format!("err-{i:04}-{}", "e".repeat(64)),
21205                ]
21206            })
21207            .collect::<Vec<_>>();
21208        lines.sort_unstable();
21209        expected.sort_unstable();
21210        assert_eq!(
21211            lines, expected,
21212            "both pipes must retain every complete line exactly once"
21213        );
21214    }
21215}
21216
21217/// Containment of a module's process tree (issue #109).
21218///
21219/// The behaviour these defend against is a module helper surviving its module:
21220/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
21221/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
21222/// compounds it.
21223///
21224/// They run against the SUPERVISOR rather than the job-object crate because the
21225/// claim is about teardown: a crate-level test proves a job can reap a tree, not
21226/// that the daemon's drain path reaches it.
21227///
21228/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
21229/// lane there is a separate containment path with its own tests.
21230#[cfg(all(test, windows))]
21231mod job_containment_tests {
21232    use super::*;
21233    use cortexkit_test_support::ScratchDir;
21234    use std::{
21235        path::{Path, PathBuf},
21236        sync::{Arc, Mutex},
21237        time::{Duration, Instant},
21238    };
21239
21240    /// The stub, expected beside this test executable.
21241    ///
21242    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
21243    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
21244    /// failure then reads as a broken test rather than an unbuilt dependency.
21245    fn stub_path() -> PathBuf {
21246        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
21247        assert!(
21248            path.exists(),
21249            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
21250             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
21251            path.display()
21252        );
21253        path
21254    }
21255
21256    /// Poll for the grandchild pid the stub records, and parse it.
21257    fn read_grandchild_pid(path: &Path) -> u32 {
21258        let deadline = Instant::now() + Duration::from_secs(10);
21259        loop {
21260            if let Ok(contents) = std::fs::read_to_string(path) {
21261                if let Ok(pid) = contents.trim().parse() {
21262                    return pid;
21263                }
21264            }
21265            assert!(
21266                Instant::now() < deadline,
21267                "the stub never recorded a grandchild pid at {}",
21268                path.display()
21269            );
21270            std::thread::sleep(Duration::from_millis(10));
21271        }
21272    }
21273
21274    /// Everything one fixture run needs, so the two tests below differ in exactly
21275    /// one place: whether the child is contained.
21276    struct Fixture {
21277        _dir: ScratchDir,
21278        _roster: ChildRoster,
21279        module_id: String,
21280        grandchild: u32,
21281        child: Option<SupervisedChild>,
21282        registry: Arc<Registry>,
21283        snapshot: Arc<Mutex<SupervisorSnapshot>>,
21284        terminal_ring: Arc<Mutex<TerminalRing>>,
21285        spawn_events: SpawnEventFeed,
21286    }
21287
21288    fn fixture(label: &str, module_id: &str) -> Fixture {
21289        fixture_with_carry(label, module_id, false)
21290    }
21291
21292    fn fixture_with_carry(label: &str, module_id: &str, carry_containment: bool) -> Fixture {
21293        let dir = ScratchDir::new(label);
21294        let pid_file = dir.join("grandchild.pid");
21295        let supervisor = Supervisor::new_for_test(
21296            Arc::new(Registry::default()),
21297            RestartPolicy::new(3, Duration::ZERO),
21298        );
21299        let runtime = supervisor.runtime_config();
21300        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21301        let spec = ModuleSpec {
21302            script: None,
21303            module_id: module_id.to_string(),
21304            program: stub_path(),
21305            // Zero args deliberately: a `--subc` argument would make the stub dial
21306            // a daemon that is not there, and the failure would land in the same
21307            // stderr ring this fixture exists to keep quiet.
21308            args: Vec::new(),
21309            env: vec![
21310                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
21311                (
21312                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
21313                    pid_file.display().to_string(),
21314                ),
21315            ],
21316            reserved: false,
21317            reserved_prefixes: Vec::new(),
21318            protocol: ModuleProtocol::Subc,
21319            overlap: Default::default(),
21320            carry_containment,
21321        };
21322        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
21323            .expect("spawn the supervised fixture");
21324        let grandchild = read_grandchild_pid(&pid_file);
21325        Fixture {
21326            _dir: dir,
21327            _roster: runtime.child_roster.clone(),
21328            module_id: module_id.to_string(),
21329            grandchild,
21330            child: Some(child),
21331            registry: Arc::new(Registry::default()),
21332            snapshot,
21333            terminal_ring: Arc::clone(&runtime.terminal_ring),
21334            spawn_events: SpawnEventFeed::default(),
21335        }
21336    }
21337
21338    async fn assert_admission_kill_reaps_grandchild(carry: bool) {
21339        let label = if carry {
21340            "admission-kill-carried-tree"
21341        } else {
21342            "admission-kill-plain-tree"
21343        };
21344        let mut fixture = fixture_with_carry(label, label, carry);
21345        let descendant = subc_os::Process::open(fixture.grandchild)
21346            .unwrap()
21347            .expect("the stub descendant must be alive before admission kill");
21348        assert!(descendant.observe().is_some());
21349        let child = fixture.child.as_mut().unwrap();
21350        assert_eq!(child.child.container().is_some(), carry);
21351        assert_eq!(child.job.is_some(), !carry);
21352
21353        child.start_admission_kill().unwrap();
21354        assert!(child.admission_kill_sent);
21355        let status = timeout(Duration::from_secs(10), child.wait())
21356            .await
21357            .unwrap()
21358            .unwrap();
21359        assert_eq!(
21360            status.code(),
21361            Some(admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE),
21362            "containment teardown must preserve the leader's distinctive status"
21363        );
21364        // Keep the child and its job/container alive: later exit finalization or
21365        // handle drop must not be what makes the descendant-death claim pass.
21366        assert!(
21367            descendant.wait_for_exit(Duration::from_secs(10)).unwrap(),
21368            "admission kill left a descendant running before exit finalization"
21369        );
21370        assert_eq!(child.child.container().is_some(), carry);
21371        assert_eq!(child.job.is_some(), !carry);
21372        child.finish_exit(false).await.unwrap();
21373    }
21374
21375    #[tokio::test]
21376    async fn admission_kill_reaps_grandchild_with_plain_job() {
21377        assert_admission_kill_reaps_grandchild(false).await;
21378    }
21379
21380    #[tokio::test]
21381    async fn admission_kill_reaps_grandchild_with_carried_job() {
21382        assert_admission_kill_reaps_grandchild(true).await;
21383    }
21384
21385    #[tokio::test]
21386    async fn admission_kill_retires_registration_before_wait() {
21387        let mut fixture = fixture("admission-registration", "admission-registration");
21388        let handle = SupervisorHandle::new();
21389        let spawn =
21390            RegistrationSpawn::new(&handle, "admission-registration", SpawnRole::Plain, None);
21391        spawn.ready();
21392        let child = fixture.child.as_mut().unwrap();
21393        child.registration_spawn = Some(spawn.clone());
21394        child.start_admission_kill().unwrap();
21395        assert!(
21396            !spawn.launch_ready(),
21397            "admission kill must retire before its caller can await the child"
21398        );
21399        timeout(Duration::from_secs(10), child.wait())
21400            .await
21401            .unwrap()
21402            .unwrap();
21403        child.finish_exit(false).await.unwrap();
21404    }
21405
21406    impl Fixture {
21407        /// Drain through the supervisor's own teardown path.
21408        async fn drain(&mut self) {
21409            let child = self
21410                .child
21411                .take()
21412                .expect("the fixture child is still present");
21413            drain_child_to_state(
21414                &self.module_id,
21415                ModuleProtocol::Subc,
21416                // No forwarding table in this fixture, so nothing reaches the
21417                // child over a connection.
21418                StopNotice::NotSent,
21419                &self.registry,
21420                None,
21421                &self.snapshot,
21422                &self.terminal_ring,
21423                &self.spawn_events,
21424                child,
21425                Duration::from_millis(500),
21426                ModuleState::Stopped,
21427                Some(false),
21428                &FailureContext::operator(),
21429            )
21430            .await
21431            .expect("drain the supervised fixture");
21432        }
21433    }
21434
21435    /// Teardown reaps the grandchild, not merely the direct child.
21436    ///
21437    /// This is the assertion the change exists for. Before containment the
21438    /// grandchild survived: it is a separate process, and `start_kill` is
21439    /// `TerminateProcess` scoped to one pid.
21440    #[tokio::test]
21441    async fn teardown_reaps_the_grandchild() {
21442        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
21443        let grandchild = fixture.grandchild;
21444
21445        assert!(
21446            subc_jobobject::process_exists(grandchild),
21447            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
21448        );
21449
21450        fixture.drain().await;
21451
21452        assert!(
21453            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21454            "grandchild {grandchild} outlived module teardown: the tree was not contained"
21455        );
21456    }
21457
21458    /// The mutation control: with containment withheld, the grandchild survives
21459    /// the same kill.
21460    ///
21461    /// This is the defect reproduction from #109 — a direct-child kill reaches
21462    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
21463    /// supervisor because `spawn_and_mark_running` now always contains on
21464    /// Windows, which is the point: there is no longer a path that spawns
21465    /// uncontained, so the control has to construct one.
21466    ///
21467    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
21468    /// grandchild ever dies here, that test is passing for a reason unrelated to
21469    /// the job object and the containment claim is unproven.
21470    #[test]
21471    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
21472        let dir = ScratchDir::new("teardown-uncontained");
21473        let pid_file = dir.join("grandchild.pid");
21474        let mut child = std::process::Command::new(stub_path())
21475            .env("FAKE_AFT_NEVER_CONNECT", "1")
21476            .env(
21477                "FAKE_AFT_GRANDCHILD_PID_FILE",
21478                pid_file.display().to_string(),
21479            )
21480            .stdin(std::process::Stdio::null())
21481            .stdout(std::process::Stdio::null())
21482            .stderr(std::process::Stdio::null())
21483            .spawn()
21484            .expect("spawn the uncontained fixture");
21485        let grandchild = read_grandchild_pid(&pid_file);
21486
21487        // Exactly what the pre-fix teardown did: kill the direct child.
21488        child.kill().expect("kill the direct child");
21489        let _ = child.wait();
21490
21491        assert!(
21492            subc_jobobject::process_exists(grandchild),
21493            "grandchild {grandchild} died with the direct child, so this control no longer \
21494             distinguishes contained from uncontained teardown and the regression test is \
21495             passing vacuously"
21496        );
21497
21498        // The orphan this control demonstrates is the leak the fix prevents, so
21499        // the control must not leave one behind.
21500        kill_tree(grandchild);
21501    }
21502
21503    /// Crash durability: closing the containment handle reaps the tree with no
21504    /// teardown code running at all.
21505    ///
21506    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
21507    /// call anything — and it is why containment is a kernel property of the
21508    /// handle rather than a step in the drain. Discovered by getting the
21509    /// mutation control wrong: clearing `job` to "disable" containment instead
21510    /// killed the tree, which is the guarantee, not a mistake.
21511    #[tokio::test]
21512    async fn dropping_containment_reaps_the_grandchild() {
21513        let mut fixture = fixture("drop-containment", "tree-drop");
21514        let grandchild = fixture.grandchild;
21515
21516        assert!(subc_jobobject::process_exists(grandchild));
21517
21518        // No `drain` call, no kill: dropping the handle is the entire mechanism.
21519        fixture.child.as_mut().expect("child present").job = None;
21520
21521        assert!(
21522            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21523            "grandchild {grandchild} survived the containment handle closing, so a daemon \
21524             crash would leave the tree behind"
21525        );
21526    }
21527
21528    /// Kill a pid and its tree, then confirm it is gone.
21529    fn kill_tree(pid: u32) {
21530        let _ = std::process::Command::new("taskkill.exe")
21531            .args(["/PID", &pid.to_string(), "/T", "/F"])
21532            .stdin(std::process::Stdio::null())
21533            .stdout(std::process::Stdio::null())
21534            .stderr(std::process::Stdio::null())
21535            .status();
21536        assert!(
21537            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
21538            "could not clean up grandchild {pid}"
21539        );
21540    }
21541}
21542
21543#[cfg(test)]
21544mod privacy_trampoline_configuration_tests {
21545    #[tokio::test]
21546    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21547    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
21548        #[cfg(target_os = "macos")]
21549        {
21550            let supervisor = super::Supervisor::new(
21551                std::sync::Arc::new(crate::Registry::default()),
21552                super::RestartPolicy::default(),
21553            );
21554            let error = supervisor.spawn(spec()).unwrap_err();
21555            assert!(
21556                error
21557                    .to_string()
21558                    .contains("no privacy trampoline configured"),
21559                "{error}"
21560            );
21561        }
21562    }
21563
21564    #[tokio::test]
21565    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21566    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
21567        #[cfg(target_os = "macos")]
21568        {
21569            let supervisor = super::Supervisor::new(
21570                std::sync::Arc::new(crate::Registry::default()),
21571                super::RestartPolicy::default(),
21572            )
21573            .with_privacy_trampoline(std::env::current_exe().unwrap());
21574            let error = supervisor.spawn(spec()).unwrap_err();
21575            assert!(
21576                error
21577                    .to_string()
21578                    .contains("binary does not implement the privacy trampoline protocol"),
21579                "{error}"
21580            );
21581        }
21582    }
21583
21584    #[cfg(target_os = "macos")]
21585    fn spec() -> super::ModuleSpec {
21586        super::ModuleSpec {
21587            module_id: "privacy-configuration".into(),
21588            program: "/bin/sleep".into(),
21589            args: vec!["30".into()],
21590            env: vec![],
21591            reserved: false,
21592            reserved_prefixes: vec![],
21593            protocol: subc_control::ModuleProtocol::None,
21594            overlap: super::ModuleOverlap::Exclusive,
21595            carry_containment: false,
21596            script: None,
21597        }
21598    }
21599}
21600
21601#[cfg(test)]
21602mod privacy_exec_boundary_tests {
21603    #[cfg(target_os = "macos")]
21604    use super::*;
21605    #[cfg(target_os = "macos")]
21606    use std::{
21607        io::{Read, Write},
21608        net::{TcpListener, TcpStream},
21609    };
21610
21611    /// Unit-test-only pause at the actual early image read, not at a later
21612    /// status read. Production supervisors never inspect this environment key.
21613    #[cfg(target_os = "macos")]
21614    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
21615        if let Some((_, path)) = spec
21616            .env
21617            .iter()
21618            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
21619        {
21620            let mut barrier = TcpStream::connect(path).unwrap();
21621            barrier
21622                .set_read_timeout(Some(Duration::from_secs(30)))
21623                .unwrap();
21624            barrier.write_all(&pid.to_ne_bytes()).unwrap();
21625            let mut release = [0];
21626            barrier.read_exact(&mut release).unwrap();
21627            assert_eq!(&release, b"X");
21628        }
21629    }
21630
21631    #[cfg(target_os = "macos")]
21632    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
21633        ModuleSpec {
21634            script: None,
21635            module_id: "privacy-boundary".into(),
21636            program: program.into(),
21637            args: args.iter().map(|arg| (*arg).into()).collect(),
21638            env: vec![],
21639            reserved: false,
21640            reserved_prefixes: vec![],
21641            protocol: ModuleProtocol::None,
21642            overlap: ModuleOverlap::Exclusive,
21643            carry_containment: false,
21644        }
21645    }
21646
21647    #[cfg(target_os = "macos")]
21648    async fn accept(listener: TcpListener) -> TcpStream {
21649        // Socket readiness, not elapsed time, establishes both pause points.
21650        let listener = tokio::net::TcpListener::from_std({
21651            listener.set_nonblocking(true).unwrap();
21652            listener
21653        })
21654        .unwrap();
21655        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
21656            .await
21657            .unwrap()
21658            .unwrap();
21659        let stream = stream.into_std().unwrap();
21660        stream.set_nonblocking(false).unwrap();
21661        stream
21662            .set_read_timeout(Some(Duration::from_secs(30)))
21663            .unwrap();
21664        stream
21665    }
21666
21667    #[tokio::test]
21668    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21669    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
21670        #[cfg(target_os = "macos")]
21671        {
21672            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
21673            // Loopback sockets also work when the replay adapter's TMPDIR is
21674            // longer than Darwin's Unix-domain socket path limit.
21675            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21676            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21677            let record = root.join("live-children.json");
21678            let supervisor =
21679                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21680                    .with_live_children_record(&record);
21681            let runtime = supervisor.runtime_config();
21682            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21683            let mut spec = spec("/bin/sleep", &["30"]);
21684            spec.env = vec![
21685                (
21686                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21687                    exec_listener.local_addr().unwrap().to_string(),
21688                ),
21689                (
21690                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
21691                    sample_listener.local_addr().unwrap().to_string(),
21692                ),
21693            ];
21694            let spawn = tokio::task::spawn_blocking(move || {
21695                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
21696            });
21697            let mut sample = accept(sample_listener).await;
21698            let mut pid = [0; 4];
21699            sample.read_exact(&mut pid).unwrap();
21700            let pid = u32::from_ne_bytes(pid);
21701            let mut exec = accept(exec_listener).await;
21702            let mut ready = [0];
21703            exec.read_exact(&mut ready).unwrap();
21704            assert_eq!(&ready, b"R");
21705            // The early read is guaranteed to see a real, non-null trampoline
21706            // image: the fixture has reached its barrier and cannot exec yet.
21707            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
21708            assert_eq!(
21709                observe_spawned_image(pid).unwrap().executable,
21710                Some(trampoline)
21711            );
21712            sample.write_all(b"X").unwrap();
21713            let mut child = spawn.await.unwrap();
21714            let early = crate::live_children::read_record(&record).unwrap();
21715            assert_eq!(early.len(), 1);
21716            assert_eq!(early[0].pid, pid);
21717            assert_eq!(
21718                early[0].executable, None,
21719                "unconfirmed trampoline image entered the roster"
21720            );
21721            assert!(child.report_ready.get().is_none());
21722            // The barrier's duration is unrelated to the production five-second
21723            // exec budget. Start the test's confirmation budget upon release.
21724            child.privacy_exec.as_mut().unwrap().deadline =
21725                tokio::time::Instant::now() + Duration::from_secs(30);
21726            exec.write_all(b"X").unwrap();
21727            child.confirm_privacy_exec().await;
21728            assert_eq!(child.spawn_failure, None);
21729            assert!(child.report_ready.get().is_some());
21730            let confirmed = crate::live_children::read_record(&record).unwrap();
21731            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
21732            assert_ne!(module, trampoline);
21733            assert_eq!(confirmed[0].executable, Some(module.into()));
21734            child.start_kill().unwrap();
21735            child.wait().await.unwrap();
21736            child.release_roster();
21737        }
21738    }
21739
21740    #[tokio::test]
21741    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21742    async fn macos_unstamped_reexec_records_the_observed_native_image() {
21743        #[cfg(target_os = "macos")]
21744        {
21745            let root = cortexkit_test_support::ScratchDir::new("privacy-reexec");
21746            let record = root.join("live-children.json");
21747            let supervisor =
21748                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21749                    .with_live_children_record(&record);
21750            let runtime = supervisor.runtime_config();
21751            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21752            let fixture = crate::config_gate::program::native_fixture_for_test();
21753            let mut spec = spec(
21754                "/bin/sh",
21755                &["-c", "exec \"$1\" 600", "reexec", fixture.to_str().unwrap()],
21756            );
21757            spec.env
21758                .push(("SUBC_TEST_PRIVACY_IMAGE_DELAY_MS".into(), "5".into()));
21759            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21760            let native_image = subc_os::file_identity(&fixture).unwrap();
21761            // Native image installation, not shell startup speed, establishes
21762            // that this launch has crossed the re-exec boundary before sampling.
21763            let reexec = tokio::time::timeout(Duration::from_secs(5), async {
21764                while observe_spawned_image(child.pid).and_then(|image| image.executable)
21765                    != Some(native_image)
21766                {
21767                    tokio::time::sleep(Duration::from_millis(1)).await;
21768                }
21769            })
21770            .await;
21771            child.confirm_privacy_exec().await;
21772            let failure = child.spawn_failure.clone();
21773            let ready = child.report_ready.get().is_some();
21774            let confirmed = crate::live_children::read_record(&record).unwrap();
21775            child.start_kill().unwrap();
21776            tokio::time::timeout(Duration::from_secs(5), child.wait())
21777                .await
21778                .unwrap()
21779                .unwrap();
21780            child.release_roster();
21781
21782            reexec.expect("the shell must re-exec the native fixture");
21783            assert_eq!(
21784                failure, None,
21785                "an unstamped program may re-exec before observation"
21786            );
21787            assert!(ready);
21788            assert_eq!(confirmed.len(), 1);
21789            assert_eq!(confirmed[0].executable, Some(native_image.into()));
21790        }
21791    }
21792
21793    #[tokio::test]
21794    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21795    async fn macos_unstamped_ack_without_exec_still_refuses_the_trampoline() {
21796        #[cfg(target_os = "macos")]
21797        {
21798            let listener = TcpListener::bind("127.0.0.1:0").unwrap();
21799            let supervisor =
21800                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
21801            let runtime = supervisor.runtime_config();
21802            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21803            let fixture = crate::config_gate::program::native_fixture_for_test();
21804            let mut spec = spec(fixture.to_str().unwrap(), &["600"]);
21805            spec.env.push((
21806                "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21807                listener.local_addr().unwrap().to_string(),
21808            ));
21809            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21810            let mut exec = accept(listener).await;
21811            let mut ready = [0];
21812            exec.read_exact(&mut ready).unwrap();
21813            assert_eq!(&ready, b"R");
21814            assert_eq!(
21815                observe_spawned_image(child.pid).unwrap().executable,
21816                subc_os::file_identity(&test_privacy_trampoline())
21817            );
21818            // Simulate a broken trampoline that acknowledges exec but keeps
21819            // running its own image, held at the fixture's pre-exec barrier.
21820            let (reader, writer) = std::io::pipe().unwrap();
21821            drop(writer);
21822            child.privacy_exec.as_mut().unwrap().reader =
21823                tokio::io::unix::AsyncFd::new(reader).unwrap();
21824            child.confirm_privacy_exec().await;
21825            let failure = child.spawn_failure.clone();
21826            let ready = child.report_ready.get().is_some();
21827            child.start_kill().unwrap();
21828            tokio::time::timeout(Duration::from_secs(5), child.wait())
21829                .await
21830                .unwrap()
21831                .unwrap();
21832            child.release_roster();
21833            assert_eq!(
21834                failure.as_deref(),
21835                Some("privacy identity trampoline did not exec the module")
21836            );
21837            assert!(!ready);
21838        }
21839    }
21840
21841    #[tokio::test]
21842    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21843    async fn macos_resumed_121_is_a_module_exit_not_a_trampoline_refusal() {
21844        #[cfg(target_os = "macos")]
21845        {
21846            let registry = Arc::new(Registry::default());
21847            let policy = RestartPolicy::new(0, Duration::ZERO);
21848            let fixture = crate::config_gate::program::native_fixture_for_test();
21849            let spec = spec(fixture.to_str().unwrap(), &["exit121"]);
21850            let scratch = cortexkit_test_support::ScratchDir::new("resumed-exit");
21851            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy)
21852                .with_config_runtime(suspended_spawn_tests::gated_owner(&scratch, &spec, true));
21853            let runtime = supervisor.runtime_config();
21854            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21855            // A module cannot exit before suspended admission. After resume,
21856            // retain the real exit status even when it resembles a refusal.
21857            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21858            child.confirm_privacy_exec().await;
21859            assert_eq!(child.spawn_failure, None);
21860            assert!(child.report_ready.get().is_some());
21861            let pid = child.pid;
21862            tokio::task::spawn_blocking(move || {
21863                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
21864            })
21865            .await
21866            .unwrap()
21867            .unwrap();
21868            let status = child.wait().await.unwrap();
21869            assert_eq!(status.code(), Some(121));
21870            assert!(child.privacy_exec.is_none());
21871            let report = classify_reaped_child_exit(&snapshot, &child, &status);
21872            on_child_exit(
21873                &spec,
21874                policy,
21875                &registry,
21876                &snapshot,
21877                &runtime.terminal_ring,
21878                &runtime.spawn_events,
21879                &runtime.child_roster,
21880                report,
21881                true,
21882            )
21883            .await;
21884            let state = lock_snapshot(&snapshot).unwrap();
21885            assert_eq!(state.state, ModuleState::Failed);
21886            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
21887            assert_eq!(state.reported_pid(), None);
21888            drop(state);
21889            let history = runtime.terminal_ring.lock().unwrap().snapshot();
21890            assert_eq!(history.entries.len(), 1);
21891            let terminal = &history.entries[0];
21892            assert_eq!(terminal.exit_code, Some(121));
21893            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
21894            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
21895            assert_eq!(
21896                terminal.disposition_detail.as_deref(),
21897                Some(policy.budget_exhausted_detail().as_str()),
21898                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
21899            );
21900            assert_eq!(child.spawn_failure, None);
21901            child.release_roster();
21902        }
21903    }
21904}
21905
21906#[cfg(all(test, target_os = "macos"))]
21907#[path = "supervise/suspended_spawn_tests.rs"]
21908mod suspended_spawn_tests;
21909
21910/// The daemon's real spawn path hands a subc-wire child its launch nonce on
21911/// descriptor 3, without an environment copy. The shell records the nonce
21912/// and its environment after exec so these tests observe the real handover.
21913#[cfg(all(test, unix))]
21914mod launch_nonce_descriptor_tests {
21915    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
21916    use crate::stderr_tail::{StderrRing, StderrTailConfig};
21917    use cortexkit_test_support::ScratchDir;
21918    use std::{
21919        path::PathBuf,
21920        sync::{Arc, Mutex},
21921        time::{Duration, Instant},
21922    };
21923
21924    async fn probe(role: super::SpawnRole) {
21925        let scratch = ScratchDir::new("launch-nonce-descriptor");
21926        let fd_copy = scratch.join("from-descriptor");
21927        let env_copy = scratch.join("environment");
21928        let script = format!(
21929            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
21930            fd = fd_copy.display(), env = env_copy.display(),
21931        );
21932        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
21933        let spec = ModuleSpec {
21934            script: None,
21935            module_id: "nonce-descriptor-probe".to_string(),
21936            program: PathBuf::from("/bin/sh"),
21937            args: vec!["-c".to_string(), script],
21938            env: vec![
21939                xdg("XDG_DATA_HOME"),
21940                xdg("XDG_RUNTIME_DIR"),
21941                xdg("XDG_CONFIG_HOME"),
21942                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
21943            ],
21944            reserved: true,
21945            reserved_prefixes: Vec::new(),
21946            protocol: ModuleProtocol::Subc,
21947            overlap: Default::default(),
21948            carry_containment: false,
21949        };
21950        let handle = SupervisorHandle::new();
21951        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21952        let roster = ChildRoster::default();
21953        #[cfg(target_os = "macos")]
21954        {
21955            let path = super::test_privacy_trampoline();
21956            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
21957        }
21958        let child = super::spawn_child_in_slot(
21959            &spec,
21960            None,
21961            Some(&handle),
21962            &ring,
21963            None,
21964            &roster,
21965            None,
21966            #[cfg(target_os = "linux")]
21967            None,
21968            role,
21969            matches!(role, super::SpawnRole::SwapCandidate),
21970            super::DEFAULT_REGISTER_TIMEOUT,
21971        )
21972        .expect("spawn probe");
21973        let deadline = Instant::now() + Duration::from_secs(10);
21974        while !(fd_copy.exists() && env_copy.exists()) {
21975            assert!(Instant::now() < deadline, "probe never wrote its copies");
21976            tokio::time::sleep(Duration::from_millis(20)).await;
21977        }
21978        let nonce = std::fs::read_to_string(fd_copy).unwrap();
21979        assert!(!nonce.is_empty());
21980        let environment = std::fs::read_to_string(env_copy).unwrap();
21981        assert!(environment
21982            .lines()
21983            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
21984        let copy = environment
21985            .lines()
21986            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
21987        assert_eq!(
21988            copy, None,
21989            "Unix children must never receive the environment nonce"
21990        );
21991        if matches!(role, super::SpawnRole::Plain) {
21992            assert_eq!(
21993                handle.spawn_nonce(&spec.module_id).as_deref(),
21994                Some(nonce.as_str())
21995            );
21996        }
21997        drop(child);
21998    }
21999
22000    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22001    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
22002        probe(super::SpawnRole::Plain).await;
22003    }
22004
22005    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22006    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
22007        probe(super::SpawnRole::SwapCandidate).await;
22008    }
22009}
22010
22011#[cfg(all(test, target_os = "linux"))]
22012#[path = "supervise/cgroup_reap_tests.rs"]
22013mod cgroup_reap_tests;
22014
22015#[cfg(all(test, target_os = "linux"))]
22016mod cgroup_containment_tests {
22017    use super::*;
22018    use cortexkit_test_support::ScratchDir;
22019
22020    #[tokio::test]
22021    async fn linux_subtree_memory_covers_module_children() {
22022        let dir = ScratchDir::new("subtree-memory");
22023        let root = PathBuf::from(format!(
22024            "/sys/fs/cgroup/subc-memory-test-{}-{}",
22025            std::process::id(),
22026            unix_ms_now()
22027        ));
22028        if let Err(error) = std::fs::create_dir(&root) {
22029            assert!(
22030                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22031                "required cgroup test cannot execute: {error}"
22032            );
22033            eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
22034            return;
22035        }
22036        let placement = subc_cgroup::prepare_at(&root)
22037            .unwrap()
22038            .expect("delegated cgroup");
22039        let supervisor = Supervisor::new_for_test(
22040            Arc::new(Registry::default()),
22041            RestartPolicy::new(3, Duration::ZERO),
22042        )
22043        .with_cgroup_placement(Some(placement));
22044        let ready = dir.join("ready");
22045        let module = supervisor
22046            .spawn(ModuleSpec {
22047                script: None,
22048                module_id: "memory-tree".into(),
22049                program: PathBuf::from("/bin/sh"),
22050                args: vec![
22051                    "-c".into(),
22052                    "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
22053                    "fixture".into(),
22054                    ready.display().to_string(),
22055                ],
22056                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22057                    .into_iter()
22058                    .map(|key| (key.into(), dir.display().to_string()))
22059                    .collect(),
22060                reserved: false,
22061                reserved_prefixes: Vec::new(),
22062                protocol: ModuleProtocol::None,
22063                overlap: Default::default(),
22064                carry_containment: false,
22065            })
22066            .unwrap();
22067        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22068        while !ready.exists() {
22069            assert!(
22070                tokio::time::Instant::now() < deadline,
22071                "children never started"
22072            );
22073            tokio::time::sleep(Duration::from_millis(10)).await;
22074        }
22075        let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
22076        let reading = &readings[0];
22077        assert!(
22078            reading.process_count.is_some_and(|count| count >= 4),
22079            "{reading:?}"
22080        );
22081        assert!(
22082            reading.memory_bytes.is_some_and(|bytes| bytes > 0),
22083            "{reading:?}"
22084        );
22085        module.stop().await.unwrap();
22086        drop(module);
22087        std::fs::remove_dir(root.join("subc-modules")).unwrap();
22088        std::fs::remove_dir(&root).unwrap();
22089        eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
22090    }
22091
22092    fn running(pid: u32) -> bool {
22093        // An orphan can remain a zombie until the container init reaps it.
22094        std::fs::read_to_string(format!("/proc/{pid}/stat"))
22095            .ok()
22096            .and_then(|stat| {
22097                stat.rsplit_once(") ")
22098                    .map(|(_, rest)| rest.starts_with('Z'))
22099            })
22100            .is_some_and(|zombie| !zombie)
22101    }
22102
22103    #[tokio::test]
22104    async fn linux_teardown_reaps_the_grandchild() {
22105        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
22106    }
22107
22108    #[tokio::test]
22109    async fn linux_shutdown_straggler_reaps_the_grandchild() {
22110        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
22111    }
22112
22113    async fn teardown_tree(test_name: &str, shutdown: bool) {
22114        let dir = ScratchDir::new(test_name);
22115        let root = PathBuf::from(format!(
22116            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
22117            std::process::id(),
22118            unix_ms_now()
22119        ));
22120        if let Err(error) = std::fs::create_dir(&root) {
22121            assert!(
22122                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22123                "required cgroup test cannot execute: {error}"
22124            );
22125            eprintln!(
22126                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
22127                root.display()
22128            );
22129            return;
22130        }
22131        let placement = subc_cgroup::prepare_at(&root)
22132            .expect("prepare isolated kernel cgroup")
22133            .expect("isolated cgroup is delegated");
22134        let module_id = "tree-teardown";
22135        let module = placement
22136            .module_path(module_id)
22137            .expect("create isolated module cgroup");
22138        if !module.join("cgroup.kill").exists() {
22139            std::fs::remove_dir(&module).unwrap();
22140            std::fs::remove_dir(root.join("subc-modules")).unwrap();
22141            std::fs::remove_dir(&root).unwrap();
22142            assert!(
22143                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22144                "required cgroup.kill interface unavailable"
22145            );
22146            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
22147            return;
22148        }
22149        let supervisor = Supervisor::new_for_test(
22150            Arc::new(Registry::default()),
22151            RestartPolicy::new(3, Duration::ZERO),
22152        )
22153        .with_cgroup_placement(Some(placement));
22154        let mut runtime = supervisor.runtime_config();
22155        runtime.child_roster = runtime
22156            .child_roster
22157            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
22158        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
22159        let pid_file = dir.join("grandchild.pid");
22160        let spec = ModuleSpec {
22161            script: None,
22162            module_id: module_id.to_string(),
22163            program: PathBuf::from("/bin/sh"),
22164            args: vec![
22165                "-c".into(),
22166                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
22167                "fixture".into(),
22168                pid_file.display().to_string(),
22169            ],
22170            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22171                .into_iter()
22172                .map(|key| (key.to_string(), dir.display().to_string()))
22173                .collect(),
22174            reserved: false,
22175            reserved_prefixes: Vec::new(),
22176            protocol: ModuleProtocol::None,
22177            overlap: Default::default(),
22178            carry_containment: false,
22179        };
22180        let child =
22181            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
22182        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22183        let grandchild: u32 = loop {
22184            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
22185                if let Ok(pid) = contents.trim().parse() {
22186                    break pid;
22187                }
22188            }
22189            assert!(
22190                tokio::time::Instant::now() < deadline,
22191                "grandchild pid was not recorded"
22192            );
22193            tokio::time::sleep(Duration::from_millis(10)).await;
22194        };
22195        assert!(
22196            running(grandchild),
22197            "grandchild must be alive before teardown"
22198        );
22199        if shutdown {
22200            let mut child = child;
22201            crate::child_roster::end_children_for_daemon_shutdown(
22202                &runtime.child_roster,
22203                false,
22204                std::future::pending(),
22205            )
22206            .await;
22207            child.wait().await.expect("reap shutdown straggler");
22208        } else {
22209            drain_child_to_state(
22210                module_id,
22211                ModuleProtocol::None,
22212                StopNotice::NotSent,
22213                &Registry::default(),
22214                None,
22215                &snapshot,
22216                &runtime.terminal_ring,
22217                &SpawnEventFeed::default(),
22218                child,
22219                Duration::from_millis(100),
22220                ModuleState::Stopped,
22221                Some(false),
22222                &FailureContext::operator(),
22223            )
22224            .await
22225            .expect("real supervisor teardown");
22226        }
22227        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
22228        while running(grandchild) && tokio::time::Instant::now() < deadline {
22229            tokio::time::sleep(Duration::from_millis(10)).await;
22230        }
22231        let survived = running(grandchild);
22232        // Kill a surviving grandchild so a failed test does not leave it behind.
22233        if survived {
22234            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
22235            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
22236            tokio::time::sleep(Duration::from_millis(100)).await;
22237        }
22238        if module.exists() {
22239            std::fs::remove_dir(&module).expect("remove empty module cgroup");
22240        }
22241        std::fs::remove_dir(root.join("subc-modules")).unwrap();
22242        std::fs::remove_dir(&root).unwrap();
22243        assert!(
22244            !survived,
22245            "grandchild {grandchild} outlived module teardown"
22246        );
22247        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
22248    }
22249}
22250
22251/// A slow live-children record write on the restart path must not stop the
22252/// daemon from answering the replacement module's authentication handshake.
22253#[cfg(all(test, unix))]
22254mod record_write_stall_tests {
22255    use super::*;
22256    use std::sync::atomic::{AtomicBool, Ordering};
22257    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
22258
22259    /// `subc-client-rs` gives the whole handshake this long before the module
22260    /// exits with `Timeout { stage: ServerProof }`.
22261    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
22262    /// How long the replacement's admission write is held: longer than the
22263    /// module's deadline, standing in for the delay a busy disk can add
22264    /// across the two syncs in that write.
22265    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
22266
22267    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
22268    /// On macOS every module is launched through the privacy trampoline, and
22269    /// a module that is not a `#!` script is admitted only if its running
22270    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
22271    /// the shell soon replaces itself with `/bin/sleep`, which does not match
22272    /// the configured `/bin/sh`. The image is read only after the spawn's
22273    /// live-children record write, and here that write lasted long enough for
22274    /// the shell to have exec'd every time, so each process was refused and
22275    /// killed right after spawn, and the supervisor crash-looped it with
22276    /// backoff until its restart budget ran out. The restart this test
22277    /// measures would then never produce a module at all.
22278    fn sleeping_spec() -> ModuleSpec {
22279        ModuleSpec {
22280            script: None,
22281            module_id: "record-write-stall".into(),
22282            program: "/bin/sleep".into(),
22283            args: vec!["600".into()],
22284            env: vec![],
22285            reserved: false,
22286            reserved_prefixes: vec![],
22287            protocol: ModuleProtocol::None,
22288            overlap: Default::default(),
22289            carry_containment: false,
22290        }
22291    }
22292
22293    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22294    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
22295        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
22296        let supervisor =
22297            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
22298                .with_live_children_record(home.join("live-children.json"));
22299        let module = supervisor.spawn(sleeping_spec()).unwrap();
22300        // `reported_pid` names a process only once it is confirmed as the
22301        // module (on macOS, once the trampoline's image check has passed), so
22302        // a process the daemon is about to refuse and kill never counts as
22303        // the running module, here or in the final wait below.
22304        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
22305            loop {
22306                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
22307                    .unwrap()
22308                    .reported_pid()
22309                {
22310                    return pid;
22311                }
22312                tokio::time::sleep(Duration::from_millis(10)).await;
22313            }
22314        })
22315        .await
22316        .expect("the first process was admitted as the module");
22317
22318        // The daemon's listener, served by this runtime's workers like the
22319        // supervisor's own tasks.
22320        let key = vec![0x42; 32];
22321        let daemon_id = [0x24; 16];
22322        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22323        let port = listener.local_addr().unwrap().port();
22324        let server = tokio::spawn(crate::server::serve_listener(
22325            listener,
22326            Arc::new(crate::router::Router::with_default_self_handler()),
22327            crate::server::ServerAuth::with_limits(
22328                key.clone(),
22329                daemon_id,
22330                "test",
22331                crate::server::DEFAULT_AUTH_DEADLINE,
22332                4,
22333            ),
22334        ));
22335
22336        // Hold the write that admits the replacement, once, and say when it
22337        // has started.
22338        let (held_tx, held_rx) = std::sync::mpsc::channel();
22339        let armed = AtomicBool::new(true);
22340        supervisor
22341            .child_roster
22342            .set_record_write_hook_for_test(move |live| {
22343                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
22344                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
22345                    let _ = held_tx.send(());
22346                    std::thread::sleep(HELD_RECORD_WRITE);
22347                }
22348            });
22349
22350        // The replacement module, as a separate process would be: its own
22351        // thread and runtime, so only the daemon's runtime can be stalled.
22352        let module_process = std::thread::spawn(move || {
22353            held_rx
22354                .recv_timeout(Duration::from_secs(30))
22355                .expect("the restart reached the replacement's record write");
22356            let conn = ConnectionInfo {
22357                schema: SCHEMA_VERSION,
22358                wire_version: None,
22359                endpoints: vec![Endpoint {
22360                    host: "127.0.0.1".to_owned(),
22361                    port,
22362                }],
22363                key,
22364                daemon_id,
22365                pid: std::process::id(),
22366                daemon_ver: "test".to_owned(),
22367            };
22368            tokio::runtime::Builder::new_current_thread()
22369                .enable_all()
22370                .build()
22371                .unwrap()
22372                .block_on(async {
22373                    let started = Instant::now();
22374                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
22375                        .await
22376                        .unwrap();
22377                    let result =
22378                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
22379                    (result, started.elapsed())
22380                })
22381        });
22382
22383        module.restart(Some(0)).await.unwrap();
22384        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
22385            .await
22386            .unwrap();
22387        assert!(
22388            auth.is_ok(),
22389            "the daemon must answer a module's handshake while a restart waits on the \
22390             live-children record write; got {auth:?} after {elapsed:?}"
22391        );
22392
22393        tokio::time::timeout(Duration::from_secs(20), async {
22394            while lock_snapshot(&module.inner.snapshot)
22395                .unwrap()
22396                .reported_pid()
22397                .is_none_or(|pid| pid == old_pid)
22398            {
22399                tokio::time::sleep(Duration::from_millis(10)).await;
22400            }
22401        })
22402        .await
22403        .expect("the restart published its replacement");
22404        module.stop().await.unwrap();
22405        server.abort();
22406    }
22407}
22408
22409#[cfg(all(test, unix))]
22410mod retained_status_tests {
22411    use super::*;
22412
22413    #[tokio::test]
22414    async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
22415        use crate::live_children::{ContainerIdentity, RetainedReason};
22416        for (reason, carried, expected) in [
22417            (
22418                RetainedReason::ContainerSurvivors(vec![4242]),
22419                true,
22420                Some("waiting: container survivors (pids [4242])".to_owned()),
22421            ),
22422            (
22423                RetainedReason::ContainerUnverifiable("member inspection denied".into()),
22424                true,
22425                Some(
22426                    "waiting: container survivors could not be verified: member inspection denied"
22427                        .to_owned(),
22428                ),
22429            ),
22430            (RetainedReason::Survived, false, None),
22431        ] {
22432            let supervisor =
22433                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
22434            let spec = ModuleSpec {
22435                script: None,
22436                module_id: "retained-status-fixture".into(),
22437                program: PathBuf::from("/not-spawned"),
22438                args: Vec::new(),
22439                env: Vec::new(),
22440                reserved: false,
22441                reserved_prefixes: Vec::new(),
22442                protocol: ModuleProtocol::None,
22443                overlap: ModuleOverlap::Exclusive,
22444                carry_containment: carried,
22445            };
22446            let module = supervisor
22447                .supervise_configured(spec.clone(), false)
22448                .unwrap();
22449            // No process is launched. Invalid group zero is deliberate: if a
22450            // status read wrongly rechecks, verification changes the reason
22451            // and fails the assertions without signalling any real process.
22452            let guard = supervisor
22453                .child_roster
22454                .admit(
22455                    spec.module_id,
22456                    0,
22457                    ModuleProtocol::None,
22458                    None,
22459                    crate::child_roster::RecordedIdentity {
22460                        container: carried.then_some(ContainerIdentity {
22461                            pgid: Some(0),
22462                            created_start_time: Some(0),
22463                            cgroup_path: None,
22464                            ..ContainerIdentity::test_metadata()
22465                        }),
22466                        ..crate::child_roster::RecordedIdentity::default()
22467                    },
22468                )
22469                .unwrap();
22470            guard.retain(reason);
22471            let status = module.status().unwrap();
22472            assert_eq!(status.waiting_for_previous_detail, expected.clone());
22473            assert_eq!(status.health.detail, expected);
22474            assert!(!status.process_alive);
22475        }
22476    }
22477}
22478
22479fn registration_child_is_running(child: &mut SupervisedChild) -> bool {
22480    child.try_wait().is_ok_and(|status| status.is_none())
22481}