Skip to main content

subc_daemon/
supervise.rs

1//! Identity policy updates read the running-protocol snapshot before taking the
2//! nonce locks. Nested identity locks are taken in this order: spawn nonces,
3//! reserved nonces, then `SupervisorHandle::implicit_protocol_none`.
4
5#[cfg(all(test, target_os = "linux"))]
6#[path = "supervise/audit_identity_tests.rs"]
7mod audit_identity_tests;
8mod resource_launch;
9pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
10
11use std::{
12    collections::{HashMap, HashSet, VecDeque},
13    error::Error,
14    fmt, io,
15    path::PathBuf,
16    process::{ExitStatus, Stdio},
17    sync::{Arc, Mutex, OnceLock},
18    time::{Duration, SystemTime, UNIX_EPOCH},
19};
20
21use cortexkit_log::Retention;
22use serde_json::Value;
23use sha2::{Digest, Sha256};
24use subc_control::{
25    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
26    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
27};
28use subc_protocol::{
29    manifest::{SelfSignalKind, SignalAnchor},
30    session::{
31        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
32        MODULE_CONTROL_OP_HEALTH_CHECK,
33    },
34    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
35};
36use tokio::{
37    process::{Child, Command},
38    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
39    task::JoinHandle,
40    time::{sleep, sleep_until, timeout, timeout_at, Instant},
41};
42use tracing::{debug, error, info, warn};
43
44use crate::{
45    child_roster::ChildRoster,
46    daemon_config::{
47        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
48    },
49    forwarding::{
50        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
51        ModuleDrainTarget, PendingModuleControlRpc,
52    },
53    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
54    registry::{ConnectionId, RegistrationEndReason, RegistryError},
55    stderr_tail::{
56        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
57        StderrTailSnapshot,
58    },
59    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
60    Frame, FrameSink, Registry,
61};
62
63#[cfg(not(windows))]
64use crate::provenance::spawned_file_identity;
65
66#[path = "supervise_swap.rs"]
67mod swap;
68
69#[path = "supervise_activity.rs"]
70mod activity;
71
72#[cfg(unix)]
73#[path = "supervise/admission.rs"]
74mod admission;
75mod admission_exit;
76#[cfg(all(test, target_os = "linux"))]
77mod async_spawn_tests;
78#[cfg(all(test, windows))]
79mod carried_job_tests;
80mod config_delivery;
81mod content_binding;
82mod program_binding;
83mod registration;
84#[cfg(all(test, unix))]
85pub(crate) use registration::RegistrationProof;
86pub(crate) use registration::RegistrationSpawn;
87pub(crate) use registration::REGISTRATION_CHANNEL_CLOSED;
88#[cfg(all(test, unix))]
89pub(crate) mod unix_channel_placement_tests;
90use program_binding::ProgramBindingPublication;
91mod config_exit_failure;
92#[cfg(all(test, any(target_os = "linux", windows)))]
93mod missing_path_tests;
94#[cfg(all(test, unix))]
95mod program_binding_tests;
96#[cfg(all(test, windows))]
97mod program_binding_windows_tests;
98#[cfg(all(test, windows))]
99mod windows_channel_tests;
100pub use config_exit_failure::ModuleStopReason;
101#[cfg(all(test, unix))]
102mod config_exit_hold_tests;
103pub(crate) mod containment;
104#[cfg(all(test, unix))]
105mod containment_tests;
106mod slow_retry;
107use slow_retry::{FailedRetry, FailureContext};
108mod host_health;
109use host_health::HostHealth;
110
111#[cfg(all(test, unix))]
112mod slow_retry_tests;
113
114/// Command-line flag used by supervised modules to find subc.
115///
116/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
117/// The path points at the TCP+key connection file; it is not an ambient signal and
118/// is never inherited by standalone children.
119pub const SUBC_ARG: &str = "--subc";
120
121const DEFAULT_MAX_RESTARTS: u32 = 3;
122const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
123const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
124/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
125/// to contain a real crash loop (which respawns in seconds) and short enough
126/// that unrelated crashes hours apart never accumulate into a permanent stop.
127const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
128/// How long a drain waits for already-dispatched requests to finalize before
129/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
130/// not RPC-scale: the original 2s value silently cut nearly every real tool
131/// call at the fence, making the wait-for-finalize design decorative for the
132/// workloads it existed for. Quiescence short-circuits, so an idle module
133/// restarts immediately regardless of this value; the budget is spent only
134/// when a genuine in-flight request is worth finishing. Per-module override:
135/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
136/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
137/// where a stuck request will never settle).
138pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
139pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
140const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
141const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
142/// How long a drain waits after one busy-gauge probe completes before it asks
143/// again.
144///
145/// A drain re-probes on this short interval of its own rather than on the
146/// module's health cadence. The default cadence (30 s) equals the default
147/// drain ceiling, so a cadence-paced drain that found the module busy at its
148/// first probe never asked a second time: it waited out the whole ceiling and
149/// forced GOODBYE even after the gauges had read zero. The interval is counted
150/// from the moment the previous probe finished, and probes run one at a time,
151/// so a slow module is never asked more often than it can answer. Periodic
152/// health checks outside a drain keep using the configured cadence.
153const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
154/// How long a restart waits for the exited process's output readers.
155///
156/// The restart does not depend on either output reader finishing. A stderr
157/// reader still running at this bound is left running, and its later output goes
158/// into the exited process's own section of the stderr ring (see
159/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
160/// bound no longer decides whether a crash's last lines are kept: under load
161/// the reader may simply not have been scheduled yet, and cutting it there
162/// discarded exactly the lines that explained the crash.
163///
164/// What the bound still decides is when the tail starts reporting
165/// `Incomplete`: a pipe open past it usually means a descendant of the exited
166/// process still holds it, and the tail cannot claim to be whole until that
167/// pipe closes. That is also the only case in which the wait costs the restart
168/// anything, because a pipe with no other holder reaches EOF when the process
169/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
170/// `Captured` at EOF with nothing lost.
171///
172/// The stdout reader carries no ring, only the capture file. It also continues
173/// to EOF after this bound: child exit closes the writer but does not mean the
174/// reader has consumed all buffered bytes. Late output from either pipe can
175/// therefore reach the shared capture file after the successor starts.
176const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
177/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
178pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
179const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
180/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
181/// dropped because the subscriber stopped draining its frames. The detail's
182/// `first_undelivered_cursor` names the first event it did not receive; the
183/// client resubscribes from the last cursor it did receive.
184pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
185
186#[cfg(target_os = "macos")]
187struct PrivacyExec {
188    content_check: Option<tokio::task::JoinHandle<Result<(), crate::config_gate::GateError>>>,
189    gated: bool,
190    admission_binding: Option<ProgramBindingPublication>,
191    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
192    deadline: tokio::time::Instant,
193    expected: Option<subc_os::FileIdentity>,
194    trampoline: Option<subc_os::FileIdentity>,
195    signature: Option<Arc<subc_os::privacy_identity::SuspendedSignature>>,
196    module_id: String,
197    #[cfg(test)]
198    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
199    #[cfg(test)]
200    image_probe: Option<fn(u32) -> Option<subc_os::Observation>>,
201    #[cfg(test)]
202    image_read_delay: Option<Duration>,
203}
204
205#[cfg(all(test, target_os = "macos"))]
206pub(crate) fn test_privacy_trampoline() -> PathBuf {
207    // The fixture bin lives in the profile folder the unit-test executable was
208    // built into. This honors custom CARGO_TARGET_DIR too.
209    let path = cortexkit_test_support::cargo_profile_bin("privacy-trampoline-fixture");
210    // Without the fixture every macOS spawn is refused, and the tests that
211    // spawn fail later as a module in state Failed, which names the wrong
212    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
213    assert!(
214        path.exists(),
215        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
216         --bins --features test-support` or `cargo test -p subc-daemon` first",
217        path.display()
218    );
219    path
220}
221
222#[cfg(target_os = "macos")]
223fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
224    use std::io::Read;
225    let mut probe = subc_os::child_spawn::spawn(
226        std::process::Command::new(path)
227            .args(["__disclaim-exec", "--probe-suspended"])
228            .stdin(Stdio::null())
229            .stdout(Stdio::piped())
230            .stderr(Stdio::piped()),
231    )
232    .map_err(|error| {
233        format!(
234            "privacy trampoline probe failed for {}: {error}",
235            path.display()
236        )
237    })?;
238    let deadline = std::time::Instant::now() + Duration::from_secs(5);
239    let status = loop {
240        match probe.try_wait() {
241            Ok(Some(status)) => break status,
242            Ok(None) if std::time::Instant::now() < deadline => {
243                std::thread::sleep(Duration::from_millis(5))
244            }
245            result => {
246                let _ = probe.kill();
247                let _ = probe.wait();
248                return Err(format!(
249                    "privacy trampoline probe failed or timed out for {}: {result:?}",
250                    path.display()
251                ));
252            }
253        }
254    };
255    let mut answer = String::new();
256    if let Some(stdout) = probe.stdout.take() {
257        let _ = stdout.take(256).read_to_string(&mut answer);
258    }
259    if status.success() && answer.trim() == subc_os::privacy_identity::SUSPENDED_TRAMPOLINE_PROBE {
260        return Ok(());
261    }
262    let mut diagnostic = String::new();
263    if let Some(stderr) = probe.stderr.take() {
264        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
265    }
266    let cause = diagnostic
267        .trim()
268        .strip_prefix("ck-subc: own privacy identity refused: ")
269        .unwrap_or("binary does not implement the privacy trampoline protocol");
270    Err(format!(
271        "{cause}: probe of {} exited {status}",
272        path.display()
273    ))
274}
275
276#[cfg(target_os = "macos")]
277fn privacy_command(
278    spec: &ModuleSpec,
279    binding: &SpawnProgramBinding,
280    roster: &ChildRoster,
281    gated: bool,
282) -> Result<
283    (
284        Command,
285        Option<PrivacyExec>,
286        subc_os::privacy_identity::ExecAcknowledgement,
287    ),
288    SuperviseError,
289> {
290    let failure = |cause: String| {
291        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
292        SuperviseError::Spawn {
293            program: spec.program.clone(),
294            source: io::Error::other(cause),
295            cgroup_path: None,
296        }
297    };
298    let trampoline = roster.privacy_trampoline().map_err(failure)?;
299    let expected = Some(binding.identity);
300    let trampoline_image = subc_os::file_identity(&trampoline);
301    let signature = gated
302        .then(|| {
303            subc_os::privacy_identity::SuspendedSignature::capture(&binding._file, &binding.path)
304                .map(Arc::new)
305        })
306        .transpose()
307        .map_err(|error| failure(error.to_string()))?;
308    if expected == trampoline_image {
309        return Err(failure(
310            "privacy identity module executable is missing or is the trampoline itself".to_string(),
311        ));
312    }
313    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
314        .map_err(|error| failure(error.to_string()))?;
315    let reader =
316        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
317    let mut command = Command::new(&trampoline);
318    command
319        .arg(if gated {
320            "__disclaim-exec-suspended"
321        } else {
322            "__disclaim-exec"
323        })
324        .arg(ack.fd().to_string())
325        .arg(&binding.path);
326    Ok((
327        command,
328        Some(PrivacyExec {
329            gated,
330            admission_binding: None,
331            content_check: None,
332            reader,
333            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
334            expected,
335            trampoline: trampoline_image,
336            signature,
337            module_id: spec.module_id.clone(),
338            #[cfg(test)]
339            image_probe: None,
340            #[cfg(test)]
341            image_read_delay: spec
342                .env
343                .iter()
344                .find(|(key, _)| key == "SUBC_TEST_PRIVACY_IMAGE_DELAY_MS")
345                .map(|(_, millis)| Duration::from_millis(millis.parse().unwrap())),
346            #[cfg(test)]
347            confirmation_pause: spec
348                .env
349                .iter()
350                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
351                .map(|(_, address)| (address.clone(), None)),
352        }),
353        ack,
354    ))
355}
356
357#[cfg(unix)]
358type SpawnRegistrationChannel = std::os::unix::net::UnixStream;
359#[cfg(windows)]
360type SpawnRegistrationChannel = subc_os::registration_channel::RegistrationPipe;
361#[cfg(test)]
362type CapturedRegistrationChannels =
363    Arc<Mutex<Option<Vec<(Arc<RegistrationSpawn>, CapturedRegistrationEndpoint)>>>>;
364#[cfg(test)]
365struct CapturedRegistrationEndpoint {
366    _channel: SpawnRegistrationChannel,
367}
368#[cfg(test)]
369impl fmt::Debug for CapturedRegistrationEndpoint {
370    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
371        f.write_str("CapturedRegistrationEndpoint")
372    }
373}
374
375struct SupervisedChild {
376    registration_spawn: Option<Arc<RegistrationSpawn>>,
377    /// Owned daemon endpoint, retained for the per-spawn registration service.
378    #[cfg(any(unix, windows))]
379    registration_channel: Option<SpawnRegistrationChannel>,
380    live_program_binding: Option<Arc<content_binding::LiveProgramBinding>>,
381    registered_config_connection: Option<ConnectionId>,
382    admission_base: Option<subc_config::Digest>,
383    registration_declaration: Option<String>,
384    _program_binding: Option<ProgramBindingPublication>,
385    subtree_target: crate::subtree_resources::Target,
386    child: containment::ProcessChild,
387    exec: crate::activity::ExecOwner,
388    #[cfg(windows)]
389    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
390    #[cfg(target_os = "macos")]
391    privacy_exec: Option<PrivacyExec>,
392    /// Set once this launch's exec acknowledgement confirms the module image.
393    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
394    /// `subc_os::privacy_identity`), which then replaces itself with the
395    /// module. The supervisor owns and can kill that pid from spawn, but
396    /// status readers report it only after this latch is set, so nothing
397    /// reports the trampoline's image as the module's.
398    #[cfg(target_os = "macos")]
399    report_ready: Arc<OnceLock<()>>,
400    /// Supervisor failure detail retained for the terminal record, including
401    /// image refusals and first-HELLO timeouts after a successful spawn.
402    spawn_failure: Option<String>,
403    /// Set only after a kill requested because launch admission failed was
404    /// accepted, never for an operator, health, or shutdown kill.
405    admission_kill_sent: bool,
406    /// The protocol this process was launched with. A reload can store a new
407    /// launch spec with a different protocol, but that takes effect only at the
408    /// next spawn, so this process keeps being handled by the protocol it
409    /// actually speaks.
410    protocol: ModuleProtocol,
411    /// This container's cgroup name: a bounded module/slot label followed by a
412    /// unique suffix (when cgroup placement is on). Carried replacements reuse
413    /// the name; ordinary launches each get a fresh container. A
414    /// retired process in a slot may still be draining when a later one is
415    /// spawned into that slot, so the suffix keeps the later process out of
416    /// the retired one's cgroup, which is the domain a kill applies to.
417    #[cfg(target_os = "linux")]
418    module_id: String,
419    #[cfg(target_os = "linux")]
420    cgroup_placement: Option<subc_cgroup::Placement>,
421    /// The job for an ordinary, non-carried launch. For carried
422    /// launches the actor owns it and the child holds only a weak container link.
423    ///
424    /// Dropping this handle is what reaps a surviving tree when no supervisor
425    /// code runs — a daemon crash — because the job carries
426    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
427    ///
428    /// That limit is not crash-only, and the difference is worth knowing: a
429    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
430    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
431    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
432    /// module at once. Before this change they survived that, saw EOF on the
433    /// control socket, and ran their own teardown; Unix keeps that path
434    /// deliberately, so a module can seal a WAL or close a capture rather than
435    /// be killed mid-write. So this trades graceful teardown on every Windows
436    /// daemon stop for containment on a crash, which is the right way round
437    /// today: orphaned GPU workers are a reported, recurring problem, and the
438    /// modules that write most heavily do not run on Windows.
439    ///
440    /// The fix is a real Windows stop path — the daemon draining before it
441    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
442    /// reaches only what the drain left behind, which is what it should reach.
443    #[cfg(windows)]
444    job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
445    stdout_pump: Option<JoinHandle<()>>,
446    stderr_pump: Option<StderrPump>,
447    stderr_ring: Arc<Mutex<StderrRing>>,
448    /// Conservative monotonic bound sampled before `command.spawn()`, not when
449    /// identity recording finishes or the spawn generation is published.
450    spawned_at: Instant,
451    config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
452    /// Ordinary spawns must send their first HELLO within this budget. Once
453    /// accepted, a later disconnect must not restart the registration clock.
454    registration_deadline: Option<Instant>,
455    registration_watch: Option<RegistrationWatch>,
456    spawned_at_ms: u64,
457    spawned_from: PathBuf,
458    launch_spec_digest: LaunchSpecDigest,
459    spawned_file_identity: Option<SpawnedFileIdentity>,
460    process_start_time: Option<u64>,
461    process_identity: Option<ProcessIdentity>,
462    pid: u32,
463    /// This process's entry in the daemon's child roster, released when the
464    /// process is reaped or this handle is dropped.
465    roster_guard: Option<crate::child_roster::RosterGuard>,
466}
467
468impl SupervisedChild {
469    fn retire_registration(&self) {
470        // All exit observers enter here before cleanup can await. The retired
471        // program-binding flag and the registration's Dead state make repeated
472        // observations harmless, even when another task observes the group
473        // leader retained unreaped to reserve its process-group id.
474        if let Some(spawn) = &self.registration_spawn {
475            spawn.retire();
476        } else if let Some(binding) = &self._program_binding {
477            binding.retire();
478        }
479        #[cfg(unix)]
480        if let Some(channel) = &self.registration_channel {
481            let _ = channel.shutdown(std::net::Shutdown::Both);
482        }
483        #[cfg(windows)]
484        if let Some(channel) = &self.registration_channel {
485            if let Ok(shutdown) = channel.shutdown_handle() {
486                shutdown.shutdown();
487            }
488        }
489    }
490    fn registered_since_spawn(&self) -> bool {
491        self.registration_watch
492            .as_ref()
493            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
494    }
495
496    fn awaiting_registration(&self) -> bool {
497        // Admission failure retires the channel before its kill becomes
498        // waitable. That closure must not replace the admission failure with
499        // a registration failure and spend crash restart budget.
500        let admission_failed = self.admission_kill_sent && self.spawn_failure.is_some();
501        !admission_failed && !self.registered_since_spawn()
502    }
503
504    fn id(&self) -> Option<u32> {
505        Some(self.pid)
506    }
507
508    fn process_identity(&self) -> Option<ProcessIdentity> {
509        self.process_identity
510    }
511
512    fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
513        let result = self.child.try_wait();
514        if !matches!(result, Ok(None)) {
515            self.retire_registration();
516            #[cfg(any(unix, windows))]
517            self.registration_channel.take();
518            if let Some(binding) = &self._program_binding {
519                binding.retire();
520            }
521        }
522        result
523    }
524
525    async fn wait(&mut self) -> io::Result<ExitStatus> {
526        #[cfg(target_os = "macos")]
527        self.confirm_privacy_exec().await;
528        // The roster entry is NOT released here. A daemon shutdown waits for the
529        // roster to empty and then exits the process, so releasing at the reap
530        // let it exit before the exit handler wrote this child's terminal record
531        // (the stderr drain and snapshot update sit in between), and the
532        // shutdown's own `daemon_shutdown` record was intermittently lost. The
533        // caller releases it after recording the exit (`release_roster`), and
534        // dropping the handle releases it too.
535        let result = self.child.wait().await;
536        self.retire_registration();
537        #[cfg(any(unix, windows))]
538        self.registration_channel.take();
539        // All plain and swap waits pass here. Revoke the reaped child's nonce
540        // before cgroup cleanup, stderr draining or exit observation can await.
541        if let Some(binding) = &self._program_binding {
542            binding.retire();
543        }
544        #[cfg(all(test, target_os = "linux"))]
545        if let Some(binding) = &self._program_binding {
546            binding.pause_after_wait_for_test().await;
547        }
548        #[cfg(target_os = "linux")]
549        if result.is_ok() && self.child.container().is_none() {
550            if let Some(placement) = self.cgroup_placement.as_ref() {
551                // Ordinary children still own a per-spawn cgroup: returning
552                // from wait must leave it empty and removed, as before carry.
553                // Keep cleanup ownership across cancellation until it finishes.
554                cleanup_reaped_cgroup(placement, &self.module_id).await;
555                self.cgroup_placement = None;
556            }
557        }
558        result
559    }
560
561    async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
562        self.retire_registration();
563        // Idempotent fallback for cleanup paths that did not call wait.
564        if let Some(binding) = self._program_binding.take() {
565            binding.retire();
566        }
567        if self.child.container().is_some() {
568            if !carry {
569                self.child.terminate_container().await;
570            }
571        } else {
572            #[cfg(target_os = "linux")]
573            if let Some(placement) = self.cgroup_placement.as_ref() {
574                cleanup_reaped_cgroup(placement, &self.module_id).await;
575                self.cgroup_placement = None;
576            }
577        }
578        self.child.reap().await?;
579        self.release_roster();
580        Ok(())
581    }
582
583    #[cfg(target_os = "macos")]
584    async fn confirm_privacy_exec(&mut self) {
585        if self
586            .privacy_exec
587            .as_ref()
588            .is_some_and(|pending| pending.content_check.is_some())
589        {
590            self.finish_privacy_content_check().await;
591            return;
592        }
593        let Some(pending) = &mut self.privacy_exec else {
594            return;
595        };
596        let result = tokio::time::timeout_at(pending.deadline, async {
597            let mut record = Vec::new();
598            loop {
599                let mut ready = pending.reader.readable().await?;
600                let read = ready.try_io(|reader| {
601                    use std::io::Read;
602                    let mut reader = reader.get_ref();
603                    let mut buffer = [0u8; 256];
604                    reader.read(&mut buffer).map(|count| (count, buffer))
605                });
606                match read {
607                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
608                    Ok(Ok((count, buffer))) => {
609                        if record.len() + count > 1024 {
610                            return Err(io::Error::other(
611                                "privacy exec refusal record is too long",
612                            ));
613                        }
614                        record.extend_from_slice(&buffer[..count]);
615                    }
616                    Ok(Err(error)) => return Err(error),
617                    Err(_) => continue,
618                }
619            }
620        })
621        .await;
622        #[cfg(test)]
623        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
624            use tokio::io::{AsyncReadExt, AsyncWriteExt};
625            if stream.is_none() {
626                *stream = Some(
627                    tokio::net::TcpStream::connect(address.as_str())
628                        .await
629                        .unwrap(),
630                );
631                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
632                stream
633                    .as_mut()
634                    .unwrap()
635                    .write_all(&self.pid.to_ne_bytes())
636                    .await
637                    .unwrap();
638            }
639            let mut reply = [0];
640            stream
641                .as_mut()
642                .unwrap()
643                .read_exact(&mut reply)
644                .await
645                .unwrap();
646            pending.confirmation_pause = None;
647            if &reply == b"F" {
648                Ok(Err(io::Error::other("injected late exec refusal")))
649            } else {
650                assert_eq!(&reply, b"X");
651                result
652            }
653        } else {
654            result
655        };
656        // Keep the reader in self across await: select cancellation must not
657        // discard the handshake or reset its original five-second deadline.
658        // Use try_wait so a child reaped during confirmation has its program
659        // binding retired before another command can use its nonce.
660        let waited = if matches!(&result, Ok(Ok(record)) if record.is_empty()) {
661            Some(self.try_wait())
662        } else {
663            None
664        };
665        let pending = self.privacy_exec.as_ref().expect("pending exec");
666        let cause = match result {
667            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
668            Ok(Err(error)) => Some(format!(
669                "privacy identity exec acknowledgement failed: {error}"
670            )),
671            Ok(Ok(record)) if !record.is_empty() => Some(
672                std::str::from_utf8(&record)
673                    .ok()
674                    .and_then(|record| {
675                        record
676                            .trim()
677                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
678                    })
679                    .filter(|cause| !cause.is_empty())
680                    .unwrap_or("invalid privacy exec refusal record")
681                    .to_string(),
682            ),
683            Ok(Ok(_)) if !pending.gated => match waited.expect("exec EOF checked child") {
684                Ok(Some(_)) => None,
685                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
686                Ok(None) => {
687                    #[cfg(test)]
688                    if let Some(delay) = pending.image_read_delay {
689                        tokio::time::sleep(delay).await;
690                    }
691                    let image = observe_spawned_image(self.pid);
692                    #[cfg(test)]
693                    let image = pending.image_probe.map_or(image, |probe| probe(self.pid));
694                    // An unstamped module can re-exec before we wake to read its
695                    // image. Only the trampoline image proves exec never happened;
696                    // retain the observed image for later orphan identification.
697                    if let Some(image) = image.filter(|image| {
698                        image.executable.is_some() && image.executable != pending.trampoline
699                    }) {
700                        match self
701                            .roster_guard
702                            .as_ref()
703                            .map(|guard| guard.confirm_executable(image))
704                            .transpose()
705                        {
706                            Err(error) => Some(format!(
707                                "could not durably record confirmed module image: {error}"
708                            )),
709                            Ok(_) => {
710                                let _ = self.report_ready.set(());
711                                None
712                            }
713                        }
714                    } else if image.is_none()
715                        || image.is_some_and(|image| image.executable.is_none())
716                    {
717                        // Empty EOF acknowledged exec. An image disappearing
718                        // during observation is an ordinary module exit.
719                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
720                            Ok(result) => {
721                                self.retire_registration();
722                                self.registration_channel.take();
723                                if let Some(binding) = &self._program_binding {
724                                    binding.retire();
725                                }
726                                result.err().map(|error| format!("privacy identity trampoline wait failed: {error}"))
727                            }
728                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
729                        }
730                    } else {
731                        Some("privacy identity trampoline did not exec the module".to_string())
732                    }
733                }
734            },
735            Ok(Ok(_)) => match waited.expect("exec EOF checked child") {
736                Ok(Some(_)) => {
737                    Some("privacy identity target exited before suspended admission".to_string())
738                }
739                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
740                Ok(None) => {
741                    // CLOEXEC closes the ack just before the kernel reports the
742                    // suspended target. EOF alone must never admit a running image.
743                    let stopped = tokio::time::timeout_at(pending.deadline, async {
744                        loop {
745                            if subc_os::privacy_identity::SuspendedSignature::is_suspended(
746                                self.pid,
747                            )? {
748                                return Ok::<_, io::Error>(());
749                            }
750                            tokio::time::sleep(Duration::from_millis(1)).await;
751                        }
752                    })
753                    .await;
754                    match stopped {
755                        Err(_) => {
756                            Some("privacy identity target was not suspended within 5s".to_string())
757                        }
758                        Ok(Err(error)) => Some(format!(
759                            "privacy identity suspended target unavailable: {error}"
760                        )),
761                        Ok(Ok(())) => {
762                            let image = observe_spawned_image(self.pid);
763                            if image.is_some_and(|image| {
764                                image.executable.is_some()
765                                    && image.executable != pending.trampoline
766                                    && image.executable == pending.expected
767                            }) {
768                                None
769                            } else {
770                                Some("privacy identity trampoline executable mismatch: expected the suspended module program".to_string())
771                            }
772                        }
773                    }
774                }
775            },
776        };
777        #[cfg(test)]
778        if let Some(binding) = &self._program_binding {
779            binding.pause_after_wait_for_test().await;
780        }
781        if cause.is_none() && pending.gated {
782            let publication = pending.admission_binding.clone();
783            let live = self.live_program_binding.clone();
784            let signature = Arc::clone(pending.signature.as_ref().expect("gated signature"));
785            let pid = self.pid;
786            let gated = pending.gated;
787            self.privacy_exec
788                .as_mut()
789                .expect("pending exec")
790                .content_check = Some(tokio::task::spawn_blocking(move || {
791                signature.verify(pid).map_err(|error| {
792                    crate::config_gate::GateError::new("program_refused", error)
793                })?;
794                if !gated {
795                    return Ok(());
796                }
797                match (publication, live) {
798                    (Some(publication), _) => publication.verify(),
799                    (None, Some(live)) => live.check(),
800                    _ => Err(crate::config_gate::GateError::new(
801                        "config_delivery_deferred",
802                        "live program witness is unavailable",
803                    )),
804                }
805            }));
806            self.finish_privacy_content_check().await;
807        } else {
808            self.finish_privacy_exec(cause);
809        }
810    }
811
812    #[cfg(target_os = "macos")]
813    async fn finish_privacy_content_check(&mut self) {
814        let result = self
815            .privacy_exec
816            .as_mut()
817            .expect("pending exec")
818            .content_check
819            .as_mut()
820            .expect("pending content check")
821            .await;
822        let cause = match result {
823            Ok(Ok(())) => None,
824            Ok(Err(error)) => Some(format!("{}: {}", error.code, error.detail)),
825            Err(error) => Some(format!(
826                "config_delivery_deferred: content check stopped: {error}"
827            )),
828        };
829        self.finish_privacy_exec(cause);
830    }
831
832    #[cfg(target_os = "macos")]
833    fn finish_privacy_exec(&mut self, mut cause: Option<String>) {
834        let mut pending = self.privacy_exec.take().expect("pending exec");
835        if cause.is_none() && pending.gated {
836            let admission = (|| -> io::Result<()> {
837                let image = observe_spawned_image(self.pid)
838                    .filter(|image| image.executable == pending.expected)
839                    .ok_or_else(|| io::Error::other("suspended executable mismatch"))?;
840                if let Some(guard) = &self.roster_guard {
841                    guard.confirm_executable(image)?;
842                }
843                pending
844                    .signature
845                    .as_ref()
846                    .expect("gated signature")
847                    .resume(self.pid)?;
848                let _ = self.report_ready.set(());
849                info!(module_id = %pending.module_id, pid = self.pid,
850                    "verified suspended module resumed with own privacy identity");
851                Ok(())
852            })();
853            if let Err(error) = admission {
854                cause = Some(format!("program_refused: {error}"));
855            }
856        }
857        if cause.is_some() {
858            if let Some(spawn) = &self.registration_spawn {
859                spawn.refuse();
860            }
861            if let Some(binding) = &mut pending.admission_binding {
862                binding.reject();
863            }
864        } else if let Some(spawn) = &self.registration_spawn {
865            spawn.ready();
866        }
867        // Exec confirmation describes launch success, not roster retirement.
868        // Keep even a reaped child recorded until its terminal history exists.
869        self.exec.0.finish(if cause.is_some() {
870            crate::activity::ExecState::Refused
871        } else {
872            crate::activity::ExecState::CommittedExec
873        });
874        if let Some(cause) = cause {
875            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
876            self.spawn_failure = Some(
877                if pending.gated
878                    && (cause.contains("executable mismatch")
879                        || cause.starts_with("program_refused:"))
880                {
881                    format!("program_refused: {cause}")
882                } else if pending.gated {
883                    format!("config_delivery_deferred: {cause}")
884                } else {
885                    cause
886                },
887            );
888            // No resume signal is sent on failure; kill the entire fresh group
889            // even if a faulty trampoline did not honor the suspended protocol.
890            let _ = self.start_admission_kill();
891        }
892    }
893
894    /// Releases the leader's roster entry after recording and finalizing its
895    /// exit. A carried container retains its separate roster entry and, for a
896    /// Unix process group, its unreaped first leader that reserves the group ID.
897    fn release_roster(&mut self) {
898        self.roster_guard = None;
899    }
900
901    /// Kill the child and, where containment is available, its process tree.
902    ///
903    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
904    /// helper process leaked the helper — the Synapse embedding module's CUDA
905    /// worker holds the GPU allocation, so the leak cost VRAM until the next
906    /// restart of something else. Terminating the job reaches grandchildren that
907    /// a tree walk cannot, including one whose parent has already exited and
908    /// been reparented away.
909    ///
910    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
911    /// direct-child kill still decides the outcome, so containment can never
912    /// change whether a module is reported as stopped.
913    fn start_kill(&mut self) -> io::Result<()> {
914        self.retire_registration();
915        if let Some(container) = self.child.container() {
916            container.hard_kill();
917            return self.kill_leader();
918        }
919        #[cfg(windows)]
920        if let Some(job) = &self.job {
921            if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
922                debug!(
923                    error = %error,
924                    "job termination failed; the direct-child kill still owns the outcome"
925                );
926            }
927        }
928        #[cfg(target_os = "linux")]
929        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
930        self.child.start_kill()
931    }
932
933    fn kill_leader(&mut self) -> io::Result<()> {
934        self.retire_registration();
935        self.child.start_kill()
936    }
937
938    fn start_admission_kill(&mut self) -> io::Result<()> {
939        self.retire_registration();
940        #[cfg(windows)]
941        let result = {
942            // Preserve the leader's distinctive status before ending the whole
943            // container/job; descendants must not wait for exit finalization.
944            let result = self
945                .spawned_file_identity
946                .as_ref()
947                .and_then(|identity| identity.windows_image.as_ref())
948                .ok_or_else(|| io::Error::other("admission child handle unavailable"))
949                .and_then(|image| {
950                    image.process().terminate_with_exit_code(
951                        admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE as u32,
952                    )
953                });
954            if let Some(container) = self.child.container() {
955                container.hard_kill();
956            } else if let Some(job) = &self.job {
957                if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
958                    debug!(%error, "job termination failed after admission leader termination");
959                }
960            }
961            result
962        };
963        #[cfg(not(windows))]
964        let result = self.start_kill().map(|()| true);
965        #[cfg(unix)]
966        let result = {
967            // A leader can exit before its helpers. Still end the fresh group
968            // even when the direct-child kill reports that the leader is gone.
969            let group_sent = self.child.container().is_none()
970                && rustix::process::Pid::from_raw(self.pid as i32).is_some_and(|group| {
971                    rustix::process::kill_process_group(group, rustix::process::Signal::KILL)
972                        .is_ok()
973                });
974            if group_sent {
975                Ok(true)
976            } else {
977                result
978            }
979        };
980        self.admission_kill_sent |= result?;
981        Ok(())
982    }
983
984    async fn drain_stderr(&mut self, module_id: &str) {
985        if let Some(pump) = self.stdout_pump.take() {
986            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
987        }
988
989        let Some(pump) = self.stderr_pump.take() else {
990            return;
991        };
992        settle_stderr_pump(
993            module_id,
994            &self.stderr_ring,
995            pump,
996            STDERR_PUMP_DRAIN_TIMEOUT,
997        )
998        .await;
999    }
1000}
1001
1002/// Bound the restart's wait, not the lifetime of the exited child's stdout
1003/// reader. Slow pipe reads or capture writes must not turn the bound into lost
1004/// output; dropping the handle lets the reader finish at EOF, like stderr.
1005async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
1006    match timeout(bound, &mut pump).await {
1007        Ok(Ok(())) => {}
1008        Ok(Err(error)) => {
1009            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
1010        }
1011        Err(_) => {
1012            drop(pump);
1013            warn!(
1014                module_id,
1015                waited = ?bound,
1016                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
1017            );
1018        }
1019    }
1020}
1021
1022/// The reader task for one process's stderr, with the ring generation its
1023/// lines are attributed to.
1024struct StderrPump {
1025    task: JoinHandle<()>,
1026    generation: u64,
1027}
1028
1029/// Retire an exited process's stderr reader and wait up to `bound` for it to
1030/// reach EOF. A reader still running at the bound is detached, not stopped: it
1031/// keeps filling the exited process's section of the ring until its pipe
1032/// closes, and the tail reads `Incomplete` until then. See
1033/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
1034async fn settle_stderr_pump(
1035    module_id: &str,
1036    ring: &Arc<Mutex<StderrRing>>,
1037    pump: StderrPump,
1038    bound: Duration,
1039) {
1040    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
1041    let StderrPump {
1042        mut task,
1043        generation,
1044    } = pump;
1045    lock().retire_pump(generation);
1046    match timeout(bound, &mut task).await {
1047        Ok(Ok(())) => {}
1048        Ok(Err(err)) => {
1049            let mut ring = lock();
1050            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
1051            ring.finish_pump(generation);
1052            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
1053        }
1054        Err(_) => {
1055            // Dropping the handle detaches the task; it ends at EOF on its pipe.
1056            drop(task);
1057            lock().mark_pump_late(
1058                generation,
1059                format!(
1060                    "stderr of the exited process had not reached EOF {bound:?} after it was \
1061                     retired (a descendant may still hold the pipe open); lines it still \
1062                     writes are kept in that process's section"
1063                ),
1064            );
1065            warn!(
1066                module_id,
1067                waited = ?bound,
1068                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
1069            );
1070        }
1071    }
1072}
1073
1074fn registration_release_events() -> &'static watch::Sender<u64> {
1075    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
1076    EVENTS.get_or_init(|| {
1077        let (sender, _receiver) = watch::channel(0);
1078        sender
1079    })
1080}
1081
1082pub(crate) fn notify_registration_release() {
1083    let events = registration_release_events();
1084    let next_generation = (*events.borrow()).wrapping_add(1);
1085    events.send_replace(next_generation);
1086}
1087
1088/// How to launch one singleton module process.
1089#[derive(Debug, Clone, PartialEq, Eq)]
1090pub struct ModuleSpec {
1091    pub module_id: String,
1092    pub program: PathBuf,
1093    pub args: Vec<String>,
1094    pub env: Vec<(String, String)>,
1095    /// When true this is a reserved module: each spawn gets a fresh one-time launch
1096    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
1097    /// process can register this module_id (a security-boundary module like the
1098    /// credential vault must not be impersonable while it is down/restarting).
1099    pub reserved: bool,
1100    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
1101    /// Prefixes come from daemon config and must end in `:` before they reach the
1102    /// supervisor; the owner module's current spawn nonce authorizes claims under
1103    /// each prefix.
1104    pub reserved_prefixes: Vec<String>,
1105    /// The wire protocol this module speaks, as DECLARED in daemon config.
1106    ///
1107    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
1108    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
1109    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
1110    /// and NO launch nonce, and a clean exit the daemon did not request is
1111    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
1112    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
1113    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
1114    /// because a process ignores an environment variable it does not read.
1115    ///
1116    /// The argument is the part that cannot be "harmless to a process that
1117    /// ignores it": a stock binary exits on an unknown flag before it listens
1118    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
1119    /// first conformance run against this mode found it. The nonce is withheld
1120    /// because a process that will never present it gains nothing from holding
1121    /// it, and a secret in the environment of a process that does not need it is
1122    /// a leak surface for no benefit.
1123    pub protocol: ModuleProtocol,
1124    /// Whether two processes of this module may run at once, which is what a
1125    /// blue/green swap does for the length of its overlap. Declared in daemon
1126    /// config because the daemon must be able to answer it while the module is
1127    /// down, and so a module cannot talk itself into it after registering.
1128    pub overlap: ModuleOverlap,
1129    /// Keep module-started helpers in the same container across replacements.
1130    /// This is a daemon-config policy, fixed for the lifetime of a container;
1131    /// stopping the module or losing its supervisor still destroys the container.
1132    pub carry_containment: bool,
1133}
1134
1135/// Only the configured launch inputs enter this value; spawned connection paths,
1136/// nonces and inherited environment never do. Its bytes remain daemon-private.
1137#[derive(Clone, PartialEq, Eq)]
1138pub(crate) struct LaunchSpecDigest([u8; 32]);
1139
1140impl fmt::Debug for LaunchSpecDigest {
1141    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1142        f.write_str("LaunchSpecDigest([redacted])")
1143    }
1144}
1145
1146impl LaunchSpecDigest {
1147    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
1148        fn field(hash: &mut Sha256, bytes: &[u8]) {
1149            hash.update((bytes.len() as u64).to_le_bytes());
1150            hash.update(bytes);
1151        }
1152        let mut hash = Sha256::new();
1153        if let Some(program) = spec.program.to_str() {
1154            field(&mut hash, b"utf8-program");
1155            field(&mut hash, program.as_bytes());
1156        } else {
1157            // The daemon config is UTF-8, but in-process callers may provide
1158            // native paths: lossy conversion would collapse distinct programs.
1159            #[cfg(unix)]
1160            {
1161                use std::os::unix::ffi::OsStrExt;
1162                field(&mut hash, b"unix-program");
1163                field(&mut hash, spec.program.as_os_str().as_bytes());
1164            }
1165            #[cfg(windows)]
1166            {
1167                use std::os::windows::ffi::OsStrExt;
1168                field(&mut hash, b"windows-program");
1169                let bytes = spec
1170                    .program
1171                    .as_os_str()
1172                    .encode_wide()
1173                    .flat_map(u16::to_le_bytes)
1174                    .collect::<Vec<_>>();
1175                field(&mut hash, &bytes);
1176            }
1177            #[cfg(not(any(unix, windows)))]
1178            field(&mut hash, spec.program.to_string_lossy().as_bytes());
1179        }
1180        hash.update((spec.args.len() as u64).to_le_bytes());
1181        for arg in &spec.args {
1182            field(&mut hash, arg.as_bytes());
1183        }
1184        let mut env = spec.env.iter().collect::<Vec<_>>();
1185        env.sort_unstable();
1186        hash.update((env.len() as u64).to_le_bytes());
1187        for (key, value) in env {
1188            field(&mut hash, key.as_bytes());
1189            field(&mut hash, value.as_bytes());
1190        }
1191        field(
1192            &mut hash,
1193            match spec.protocol {
1194                ModuleProtocol::Subc => b"subc",
1195                ModuleProtocol::None => b"none",
1196            },
1197        );
1198        Self(hash.finalize().into())
1199    }
1200}
1201
1202/// Whether a module tolerates a second process of itself running alongside.
1203///
1204/// Most modules are single-writer on their store (a WAL, a capture log, a
1205/// resident index behind a writer barrier), and two processes on one store
1206/// corrupt it. So a swap, which overlaps the old and new process by design,
1207/// is refused unless the module's config opts in with `overlap: "safe"`.
1208#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1209pub enum ModuleOverlap {
1210    /// Never run two processes of this module at once. The default.
1211    #[default]
1212    Exclusive,
1213    /// The module has said a second process of itself is harmless for the
1214    /// length of a swap.
1215    ///
1216    /// Declare it only if a second instance can run for a few seconds without
1217    /// touching ANY single-writer store: every database, WAL, index, projector
1218    /// and scheduled job the module owns. A lease on part of that state is not
1219    /// enough. broca's session lease guards WAL appends while its run index, its
1220    /// store projector and its archive fold timer (which unlinks live WAL files)
1221    /// stay single-writer, so broca is exclusive despite holding a lease. The
1222    /// refusal only fires after this has been decided, so the decision is the
1223    /// check.
1224    Safe,
1225}
1226
1227impl ModuleOverlap {
1228    pub fn as_str(self) -> &'static str {
1229        match self {
1230            Self::Exclusive => "exclusive",
1231            Self::Safe => "safe",
1232        }
1233    }
1234}
1235
1236/// Environment variable telling a spawned module which case it was started
1237/// for, before it sends HELLO. Only a swap candidate carries it, as
1238/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
1239///
1240/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
1241/// longer because nobody waits on it, while a plain restart must flip ready
1242/// quickly because callers see `module_warming` until it does. Absence means
1243/// plain restart, the safe reading. The daemon trusts nothing about it; the
1244/// candidate is proven by its launch nonce at HELLO.
1245pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
1246/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
1247pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
1248/// How long a swap waits for its candidate to register and declare itself
1249/// ready when the operator does not say. A module warming as a swap candidate
1250/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
1251/// daemon allows that plus time to start the process and send HELLO.
1252pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
1253
1254/// Bounded restart policy for crash exits.
1255///
1256/// `max_restarts` is the number of replacement processes allowed after the
1257/// initial spawn WITHIN `window`. After that many crash restarts inside one
1258/// window the module enters [`ModuleState::Failed`] and the supervisor stops
1259/// the crash loop.
1260///
1261/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
1262/// and that only survived because crashes were rare: a module that crashed
1263/// three times across a week was disabled forever by crashes that had nothing
1264/// to do with each other. That stopped being survivable once modules began
1265/// exiting non-zero whenever the daemon's connection to them drops, because
1266/// then every daemon-side connection drop spends a unit of the same budget and
1267/// one flappy hour permanently stops a healthy module. Restarts older than
1268/// `window` release their slot, so a module that crashed twice yesterday has a
1269/// full budget today, while a genuine crash loop -- which is fast by
1270/// definition -- still reaches the cap and stops.
1271#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1272pub struct RestartPolicy {
1273    pub max_restarts: u32,
1274    /// Base delay before a crash replacement. The actual delay escalates with
1275    /// the number of recent crash replacements and is capped by `max_backoff`.
1276    pub backoff: Duration,
1277    /// Maximum delay before a crash replacement.
1278    pub max_backoff: Duration,
1279    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
1280    /// budget effectively infinite (nothing is ever in-window), which is why
1281    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
1282    pub window: Duration,
1283}
1284
1285impl RestartPolicy {
1286    /// A policy with the default crash window. Callers that care about the
1287    /// window say so with [`Self::with_window`]; the ones that do not are
1288    /// asking for the standard rate limit, not for no limit.
1289    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
1290        Self {
1291            max_restarts,
1292            backoff,
1293            max_backoff: DEFAULT_MAX_BACKOFF,
1294            window: DEFAULT_RESTART_WINDOW,
1295        }
1296    }
1297
1298    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
1299        self.max_backoff = max_backoff;
1300        self
1301    }
1302
1303    pub fn with_window(mut self, window: Duration) -> Self {
1304        self.window = window;
1305        self
1306    }
1307
1308    /// Calculate the capped exponential delay for the next crash replacement.
1309    /// `restart_in_window` is zero for the first replacement after an operator
1310    /// action (restart, reload, re-enable) cleared the crash ring, or after all
1311    /// older crash replacements have aged out of the window.
1312    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
1313        if self.backoff.is_zero() || self.max_backoff.is_zero() {
1314            return Duration::ZERO;
1315        }
1316
1317        let mut delay = self.backoff;
1318        for _ in 0..restart_in_window {
1319            if delay >= self.max_backoff {
1320                return self.max_backoff;
1321            }
1322            delay = delay
1323                .checked_mul(10)
1324                .unwrap_or(self.max_backoff)
1325                .min(self.max_backoff);
1326        }
1327        delay.min(self.max_backoff)
1328    }
1329
1330    /// The one sentence that explains a budget-exhausted stop, used for both the
1331    /// log line and the terminal record so the two cannot drift. It names the
1332    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
1333    /// exactly what this budget is not.
1334    fn budget_exhausted_detail(&self) -> String {
1335        format!(
1336            "crash budget exhausted: max_restarts={} within window_secs={}",
1337            self.max_restarts,
1338            self.window.as_secs()
1339        )
1340    }
1341}
1342
1343impl Default for RestartPolicy {
1344    fn default() -> Self {
1345        Self {
1346            max_restarts: DEFAULT_MAX_RESTARTS,
1347            backoff: DEFAULT_BACKOFF,
1348            max_backoff: DEFAULT_MAX_BACKOFF,
1349            window: DEFAULT_RESTART_WINDOW,
1350        }
1351    }
1352}
1353
1354#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1355struct CrashRestartSchedule {
1356    restart_in_window: u32,
1357    delay: Duration,
1358}
1359
1360/// Whether the daemon itself will bring this module back after the exit being
1361/// handled: it is enabled AND its in-window crash restarts are below the cap.
1362///
1363/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
1364/// the window are dropped here rather than by a timer, so the count is right
1365/// the moment somebody asks and no bookkeeping runs for idle modules.
1366fn daemon_will_restart(
1367    state: &mut SupervisorSnapshot,
1368    policy: &RestartPolicy,
1369    now: Instant,
1370) -> bool {
1371    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1372}
1373
1374const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1375const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1376const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1377const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1378
1379#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1380pub enum HealthAction {
1381    Report,
1382    Restart,
1383    Alert,
1384}
1385
1386impl fmt::Display for HealthAction {
1387    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1388        f.write_str(match self {
1389            Self::Report => "report",
1390            Self::Restart => "restart",
1391            Self::Alert => "alert",
1392        })
1393    }
1394}
1395
1396#[derive(Debug, Clone, PartialEq, Eq)]
1397pub struct HealthConfig {
1398    /// Optional loopback HTTP endpoint for a managed non-wire process.
1399    /// Changing it applies live on rescan; the process protocol changes only
1400    /// at its next spawn.
1401    pub http: Option<String>,
1402    pub cadence: Duration,
1403    pub deadline: Duration,
1404    pub failure_threshold: u32,
1405    pub on_degraded: HealthAction,
1406    pub on_failing: HealthAction,
1407    pub critical: bool,
1408}
1409
1410impl Default for HealthConfig {
1411    fn default() -> Self {
1412        Self {
1413            http: None,
1414            cadence: DEFAULT_HEALTH_CADENCE,
1415            deadline: DEFAULT_HEALTH_DEADLINE,
1416            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1417            on_degraded: HealthAction::Report,
1418            on_failing: HealthAction::Report,
1419            critical: false,
1420        }
1421    }
1422}
1423
1424/// The supervisor's view of one module's health, relayed to clients over
1425/// channel-0 and rendered by `ck health`.
1426///
1427/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1428/// stated here rather than only at the wire type a consumer reads. A reader can
1429/// look up what `None` means; only a writer can silently change it, and the
1430/// writer has no reason to go looking at a downstream contract before editing.
1431///
1432/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1433/// back to `None` on re-registration precisely so a respawned module does not
1434/// carry its predecessor's timestamp — so an old value and an absent one call for
1435/// opposite readings, and anything that defaulted this to a number would make a
1436/// never-probed module indistinguishable from one probed at the epoch.
1437///
1438/// `detail` and `metrics` are `None` when the module published none on this
1439/// probe, which does not mean it reported nothing wrong — it is also the shape
1440/// when the probe never reached it. `last_probe_ms` is what separates those.
1441#[derive(Debug, Clone, PartialEq)]
1442pub struct ModuleHealthStatus {
1443    pub status: SupervisorHealthStatus,
1444    pub last_probe_ms: Option<u64>,
1445    pub detail: Option<String>,
1446    pub metrics: Option<Value>,
1447    pub consecutive_failures: u32,
1448    /// Number of replies received after a recurring health probe's deadline.
1449    /// Unlike a timeout, every increment proves the module was alive.
1450    pub late_answer_count: u64,
1451    /// End-to-end latency of the newest late reply, measured from probe start.
1452    pub last_late_answer_latency_ms: Option<u64>,
1453    pub last_action: Option<String>,
1454    /// Set together with `last_action`; the pair moves as one, and both being
1455    /// absent means no escalation has ever been taken rather than that the last
1456    /// one succeeded.
1457    pub last_action_ms: Option<u64>,
1458}
1459
1460impl Default for ModuleHealthStatus {
1461    fn default() -> Self {
1462        Self {
1463            status: SupervisorHealthStatus::Unknown,
1464            last_probe_ms: None,
1465            detail: None,
1466            metrics: None,
1467            consecutive_failures: 0,
1468            late_answer_count: 0,
1469            last_late_answer_latency_ms: None,
1470            last_action: None,
1471            last_action_ms: None,
1472        }
1473    }
1474}
1475
1476/// Typed lifecycle state for a supervised module.
1477#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1478pub enum ModuleState {
1479    Starting,
1480    Running,
1481    Unresponsive,
1482    Restarting,
1483    Draining,
1484    Stopped,
1485    Failed,
1486    Disabled,
1487}
1488
1489impl fmt::Display for ModuleState {
1490    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1491        f.write_str(match self {
1492            Self::Starting => "starting",
1493            Self::Running => "running",
1494            Self::Unresponsive => "unresponsive",
1495            Self::Restarting => "restarting",
1496            Self::Draining => "draining",
1497            Self::Stopped => "stopped",
1498            Self::Failed => "failed",
1499            Self::Disabled => "disabled",
1500        })
1501    }
1502}
1503
1504/// Supervisor classification of a child-process exit.
1505#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1506pub enum ExitKind {
1507    Clean,
1508    Crash,
1509    DeliberateSeverance,
1510}
1511
1512impl From<ExitKind> for TerminalExitKind {
1513    fn from(kind: ExitKind) -> Self {
1514        match kind {
1515            ExitKind::Clean => Self::Clean,
1516            ExitKind::Crash => Self::Crash,
1517            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1518        }
1519    }
1520}
1521
1522/// Exact process identity retained when a supervised module registers its
1523/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1524#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1525pub(crate) struct ProcessIdentity {
1526    pub(crate) pid: u32,
1527    pub(crate) start_time: u64,
1528}
1529
1530/// Last observed child exit, if any.
1531#[derive(Debug, Clone, PartialEq, Eq)]
1532pub struct ExitReport {
1533    pub kind: ExitKind,
1534    pub code: Option<i32>,
1535    pub signal: Option<i32>,
1536    pub at_ms: u64,
1537}
1538
1539/// Point-in-time module status answerable by subc without forwarding to the
1540/// module process.
1541#[derive(Debug, Clone, PartialEq)]
1542pub struct ModuleStatus {
1543    pub(crate) admission_refused: Option<String>,
1544    pub(crate) legacy_registration_spawn: bool,
1545    pub(crate) registration_declaration: Option<String>,
1546    pub module_id: String,
1547    /// Retained configuration-exit infrastructure failure. Cleared only after a
1548    /// successful launch; an ordinary stop has no infrastructure stop reason.
1549    pub stop_reason: Option<ModuleStopReason>,
1550    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1551    pub waiting_for_previous_pid: Option<u32>,
1552    /// Why the previous instance still prevents a new start.
1553    pub waiting_for_previous_detail: Option<String>,
1554    /// The daemon's Windows per-spawn record: `pipe` after the process it
1555    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1556    /// List/status reports use this record, not the source the module sends
1557    /// in its registration message's manifest.provenance.launch_nonce_source.
1558    pub launch_nonce_source: Option<String>,
1559    pub state: ModuleState,
1560    pub enabled: bool,
1561    pub process_alive: bool,
1562    pub registration_active: bool,
1563    /// The module's declared wire protocol, carried beside `live` because it is
1564    /// what makes `live` readable: the two fields answer one question together.
1565    /// While a process is alive this is its launch declaration, not a later
1566    /// pending-reload edit. When down it is the configured next launch protocol.
1567    pub protocol: ModuleProtocol,
1568    /// Whether the module is serving, under the strongest definition the daemon
1569    /// can assert for its protocol.
1570    ///
1571    /// A subc module must also be REGISTERED: its process being alive says
1572    /// nothing about whether it can take a request. A `protocol: "none"` module
1573    /// never registers, so that term is dropped and this falls back to "enabled,
1574    /// running, and the process the daemon launched is alive" -- which is all
1575    /// the daemon observes about a process that speaks no subc wire. It stays a
1576    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1577    /// rather than printing it bare.
1578    pub live: bool,
1579    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1580    /// restarts have already released their slot, so this count can go down
1581    /// without anybody touching the module.
1582    pub restart_count: u32,
1583    /// Replacement processes spawned over this module's entire supervisor lifetime;
1584    /// unlike `restart_count`, this value is never reset by an operator action
1585    /// and never falls out of a window.
1586    pub lifetime_restarts: u32,
1587    pub spawn_generation: u64,
1588    /// Retained daemon-local outcome; the legacy control response omits it.
1589    pub swap_outcome: Option<SwapOutcome>,
1590    /// The budget `restart_count` is spent against. Carried alongside the count
1591    /// because the count alone does not say how close the module is to being
1592    /// disabled, and reporting one without the other is what makes an
1593    /// about-to-be-retired module look ordinary.
1594    pub max_restarts: u32,
1595    /// The span `restart_count` is counted over. Carried with the pair above for
1596    /// the same reason they are carried together: "2 of 3" means one thing for a
1597    /// ten-minute window and something else entirely for a lifetime.
1598    pub restart_window: Duration,
1599    /// Effective drain and restart timing policy used by this running module.
1600    /// These values are carried together with the restart budget so status
1601    /// readers can compare configured intent with what the supervisor applied.
1602    pub drain_timeout: Duration,
1603    pub restart_backoff: Duration,
1604    pub restart_max_backoff: Duration,
1605    /// The module's process. On macOS this stays absent while the `ck-subc`
1606    /// launch trampoline is still running in that pid, and appears once the
1607    /// exec acknowledgement confirms the module image has replaced it. Launch
1608    /// time and the supervisor's own hold on the process are unaffected.
1609    pub pid: Option<u32>,
1610    pub spawned_at_ms: Option<u64>,
1611    pub spawned_from: Option<PathBuf>,
1612    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1613    pub process_start_time: Option<u64>,
1614    pub last_exit: Option<ExitReport>,
1615    pub health: ModuleHealthStatus,
1616}
1617
1618#[cfg(test)]
1619#[derive(Debug, Clone, Copy, PartialEq)]
1620struct ActorSelectCheckpoint {
1621    generation: u64,
1622    turn: u64,
1623    registered_connection: Option<ConnectionId>,
1624    next_probe_at: Option<Instant>,
1625    wake_after: Option<Duration>,
1626}
1627
1628#[derive(Debug, Clone, PartialEq)]
1629struct SupervisorSnapshot {
1630    admission_refused: Option<String>,
1631    /// Distinguishes an admission-parked actor from an unrelated terminal exit.
1632    config_hold_parked: bool,
1633    /// Preserve daemon attribution for a paused start so a failing release can
1634    /// schedule the next slow retry instead of treating the start as operator work.
1635    config_hold_daemon_initiated: bool,
1636    /// The approval that owns the parked attempt, not whichever approval is latest.
1637    config_hold_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1638    /// Retain an approved launch failure across competing reconciliation passes.
1639    approved_start_failure: Option<(crate::config_gate::runtime::ApprovalIdentity, String)>,
1640    legacy_registration_spawn: bool,
1641    registration_declaration: Option<String>,
1642    stop_reason: Option<ModuleStopReason>,
1643    config_exit_failed_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1644    subtree_target: crate::subtree_resources::Target,
1645    failed_retry: FailedRetry,
1646    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1647    #[cfg(windows)]
1648    pipe_consumed: Option<Arc<OnceLock<()>>>,
1649    /// Counts running-child loop turns, not executor polls of a parked wait.
1650    #[cfg(test)]
1651    actor_turns: u64,
1652    /// Running select after exec confirmation and with no unconsumed registry
1653    /// event. Recording a spawn or starting a loop turn is not this barrier.
1654    #[cfg(test)]
1655    actor_select: Option<ActorSelectCheckpoint>,
1656    state: ModuleState,
1657    enabled: bool,
1658    process_alive: bool,
1659    spawned_protocol: Option<ModuleProtocol>,
1660    spawn_failure: Option<String>,
1661    waiting_for_previous_pid: Option<u32>,
1662    waiting_for_previous_detail: Option<String>,
1663    /// When each crash restart was spent, oldest first. This IS the crash
1664    /// budget: its in-window length is the count an operator sees and the count
1665    /// the restart decision is made against, so there is no second counter that
1666    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1667    /// operator actions that used to zero the old lifetime counter.
1668    crash_restarts: VecDeque<Instant>,
1669    lifetime_restarts: u32,
1670    /// Successful child spawns in this daemon incarnation.
1671    ///
1672    /// `lifetime_restarts` was considered and rejected: it starts at zero
1673    /// (line 640), successful initial/operator spawns in `set_running` do not
1674    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1675    /// increments before a successful replacement exists (lines 604, 3846,
1676    /// and 3921), so a failed spawn can consume it. This counter moves only
1677    /// when a live PID is accepted below.
1678    spawn_generation: u64,
1679    swap_outcome: Option<SwapOutcome>,
1680    pid: Option<u32>,
1681    #[cfg(target_os = "macos")]
1682    report_ready: Option<Arc<OnceLock<()>>>,
1683    /// Last reaped child, retained after current process facts are cleared.
1684    reaped_pid: Option<u32>,
1685    /// Whether a replacement has already been scheduled for the current process.
1686    respawn_pending: bool,
1687    /// A second restart is waiting for the replacement already scheduled.
1688    coalesced_restart_pending: bool,
1689    spawned_at_ms: Option<u64>,
1690    spawned_from: Option<PathBuf>,
1691    launch_spec_digest: Option<LaunchSpecDigest>,
1692    spawned_file_identity: Option<SpawnedFileIdentity>,
1693    process_start_time: Option<u64>,
1694    deliberate_severance: Option<ProcessIdentity>,
1695    last_exit: Option<ExitReport>,
1696    /// Diagnostic attached to the next drain's terminal record, if any.
1697    drain_disposition_detail: Option<String>,
1698    health: ModuleHealthStatus,
1699    last_probe_evidence: Option<HealthProbeEvidence>,
1700    /// Whether the current process was started as a swap candidate and so
1701    /// lives in the module's alternate cgroup. The next swap's candidate takes
1702    /// the other one, so the two processes of a swap never share a cgroup. A
1703    /// plain spawn always uses the primary cgroup.
1704    in_alternate_slot: bool,
1705    /// Whether the current `Draining` state ends in a replacement process
1706    /// (restart, reload, health restart) rather than a stop. Only meaningful
1707    /// while `state` is `Draining`; every entry into that state rewrites it.
1708    /// It is what lets route.open answer the retryable `module_reloading` to a
1709    /// consumer that reaches a still-registered process mid-restart, instead of
1710    /// the `supervisor_not_live` a stop or disable deserves.
1711    draining_to_replace: bool,
1712    /// Whether a configuration update has been applied since the current
1713    /// process was spawned, so that process runs an older spec than the one
1714    /// the supervisor now holds. A queued restart is only coalesced into a
1715    /// fresher process when this is false: a restart requested to pick up a
1716    /// new configuration must not be satisfied by a process that predates it.
1717    configuration_updated_since_spawn: bool,
1718}
1719
1720impl SupervisorSnapshot {
1721    /// The pid that status, provenance and resource readings may report. While
1722    /// the launch trampoline still runs in the pid, reading its executable or
1723    /// resource use would describe `ck-subc`, not the module, so none is
1724    /// reported until the exec acknowledgement confirms the module image.
1725    fn reported_pid(&self) -> Option<u32> {
1726        #[cfg(target_os = "macos")]
1727        if self
1728            .report_ready
1729            .as_ref()
1730            .is_some_and(|ready| ready.get().is_none())
1731        {
1732            return None;
1733        }
1734        self.pid
1735    }
1736
1737    fn starting() -> Self {
1738        Self::new(ModuleState::Starting, true)
1739    }
1740
1741    fn disabled() -> Self {
1742        Self::new(ModuleState::Disabled, false)
1743    }
1744
1745    /// Crash restarts still inside `window`, having dropped the ones that are
1746    /// not. Pruning on read is what makes the budget a rate: an instant older
1747    /// than the window stops holding a slot the moment anybody counts.
1748    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1749        while let Some(oldest) = self.crash_restarts.front() {
1750            if now.duration_since(*oldest) > window {
1751                self.crash_restarts.pop_front();
1752            } else {
1753                break;
1754            }
1755        }
1756        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1757    }
1758
1759    /// Spend one unit of the crash budget and record the restart in the ledger.
1760    ///
1761    /// The ring is bounded by the cap because more than `max_restarts` in-window
1762    /// instants can never be reached (the caller refuses the restart first), so
1763    /// anything beyond that is an unbounded queue waiting to happen.
1764    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1765        self.crash_restarts.push_back(now);
1766        while self.crash_restarts.len() > policy.max_restarts as usize {
1767            self.crash_restarts.pop_front();
1768        }
1769        self.lifetime_restarts += 1;
1770    }
1771
1772    /// Reserve one crash-restart slot and calculate the delay before respawning.
1773    /// The count is captured before recording this restart, so the first retry
1774    /// uses the base delay and each later in-window retry escalates once.
1775    fn next_crash_restart(
1776        &mut self,
1777        policy: &RestartPolicy,
1778        now: Instant,
1779    ) -> Option<CrashRestartSchedule> {
1780        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1781        if restart_in_window >= policy.max_restarts {
1782            return None;
1783        }
1784        self.record_crash_restart(policy, now);
1785        Some(CrashRestartSchedule {
1786            restart_in_window,
1787            delay: policy.delay_for_restart(restart_in_window),
1788        })
1789    }
1790
1791    /// Give the module its full budget back, as an operator restart, reload, or
1792    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1793    /// ledger of what actually happened, and an operator action does not unmake
1794    /// the crashes.
1795    fn clear_crash_restarts(&mut self) {
1796        self.crash_restarts.clear();
1797    }
1798
1799    fn new(state: ModuleState, enabled: bool) -> Self {
1800        Self {
1801            admission_refused: None,
1802            config_hold_parked: false,
1803            config_hold_daemon_initiated: false,
1804            config_hold_approval: None,
1805            approved_start_failure: None,
1806            legacy_registration_spawn: false,
1807            registration_declaration: None,
1808            stop_reason: None,
1809            config_exit_failed_approval: None,
1810            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1811            failed_retry: FailedRetry::default(),
1812            exec_outcome: None,
1813            #[cfg(test)]
1814            actor_turns: 0,
1815            #[cfg(test)]
1816            actor_select: None,
1817            state,
1818            enabled,
1819            process_alive: false,
1820            #[cfg(windows)]
1821            pipe_consumed: None,
1822            spawned_protocol: None,
1823            spawn_failure: None,
1824            waiting_for_previous_pid: None,
1825            waiting_for_previous_detail: None,
1826            crash_restarts: VecDeque::new(),
1827            lifetime_restarts: 0,
1828            spawn_generation: 0,
1829            swap_outcome: None,
1830            pid: None,
1831            #[cfg(target_os = "macos")]
1832            report_ready: None,
1833            reaped_pid: None,
1834            respawn_pending: false,
1835            coalesced_restart_pending: false,
1836            spawned_at_ms: None,
1837            spawned_from: None,
1838            launch_spec_digest: None,
1839            spawned_file_identity: None,
1840            process_start_time: None,
1841            deliberate_severance: None,
1842            last_exit: None,
1843            drain_disposition_detail: None,
1844            health: ModuleHealthStatus::default(),
1845            last_probe_evidence: None,
1846            in_alternate_slot: false,
1847            draining_to_replace: false,
1848            configuration_updated_since_spawn: false,
1849        }
1850    }
1851}
1852
1853type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1854
1855type SpawnSubscriberKey = (ConnectionId, u64);
1856
1857#[derive(Debug)]
1858struct SpawnSubscriber {
1859    version: u8,
1860    frames: mpsc::Sender<Frame>,
1861    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1862    /// from which event. The full frame channel cannot carry that news, so it
1863    /// travels beside it; see `SpawnEventFeed::subscribe`.
1864    lagged: Option<oneshot::Sender<SpawnCursor>>,
1865}
1866
1867#[derive(Debug)]
1868struct SpawnEventState {
1869    daemon_incarnation: String,
1870    seq: u64,
1871    capacity: usize,
1872    live: HashMap<String, LiveSpawn>,
1873    generations: HashMap<String, u64>,
1874    events: VecDeque<SpawnEvent>,
1875    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1876    process_changes: HashMap<String, tokio::sync::watch::Sender<u64>>,
1877}
1878
1879impl Default for SpawnEventState {
1880    fn default() -> Self {
1881        Self {
1882            daemon_incarnation: "unconfigured".to_string(),
1883            seq: 0,
1884            capacity: SPAWN_EVENT_RING_CAPACITY,
1885            live: HashMap::new(),
1886            generations: HashMap::new(),
1887            events: VecDeque::new(),
1888            subscribers: HashMap::new(),
1889            process_changes: HashMap::new(),
1890        }
1891    }
1892}
1893
1894#[derive(Debug, Clone, Default)]
1895struct SpawnEventFeed(
1896    Arc<Mutex<SpawnEventState>>,
1897    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1898);
1899
1900#[derive(Debug, Clone, PartialEq, Eq)]
1901pub(crate) enum SpawnSubscribeRefusal {
1902    ForeignIncarnation { current: String },
1903    TooOld { oldest: SpawnCursor },
1904    Frame(String),
1905}
1906
1907impl SpawnEventFeed {
1908    fn configure_incarnation(&self, daemon_incarnation: String) {
1909        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1910        state.daemon_incarnation = daemon_incarnation;
1911        state.seq = 0;
1912        state.live.clear();
1913        state.generations.clear();
1914        state.events.clear();
1915        state.subscribers.clear();
1916        for changed in state.process_changes.values() {
1917            changed.send_replace(0);
1918        }
1919    }
1920
1921    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1922        SpawnCursor {
1923            daemon_incarnation: state.daemon_incarnation.clone(),
1924            seq: state.seq,
1925        }
1926    }
1927
1928    fn snapshot(&self) -> SpawnSnapshot {
1929        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1930        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1931        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1932        SpawnSnapshot {
1933            cursor: Self::cursor(&state),
1934            ring_bound: state.capacity as u64,
1935            live,
1936        }
1937    }
1938
1939    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1940        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1941        let generation = state
1942            .generations
1943            .get(module_id)
1944            .copied()
1945            .unwrap_or(0)
1946            .checked_add(1)
1947            .expect("spawn generation exhausted");
1948        state.generations.insert(module_id.to_string(), generation);
1949        let live = LiveSpawn {
1950            module_id: module_id.to_string(),
1951            spawn_generation: generation,
1952            pid,
1953            spawned_at_ms,
1954        };
1955        state.live.insert(module_id.to_string(), live);
1956        Self::emit_locked(
1957            &mut state,
1958            SpawnEventKind::Spawned,
1959            module_id.to_string(),
1960            generation,
1961            pid,
1962            None,
1963            None,
1964        );
1965        generation
1966    }
1967
1968    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1969        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1970        let Some(live) = state.live.remove(module_id) else {
1971            warn!(
1972                module_id,
1973                "terminal record had no live spawn event identity"
1974            );
1975            return;
1976        };
1977        Self::emit_locked(
1978            &mut state,
1979            SpawnEventKind::Exited,
1980            module_id.to_string(),
1981            live.spawn_generation,
1982            live.pid,
1983            exit_code,
1984            exit_signal,
1985        );
1986    }
1987
1988    /// Report the exit of a process that a swap has already replaced.
1989    ///
1990    /// `emit_exited` removes the module's live entry, which after a swap's
1991    /// cutover describes the promoted candidate, not the old process now
1992    /// exiting. This emits the old generation's exit and leaves the live entry
1993    /// alone unless it still names that generation.
1994    fn emit_superseded_exited(
1995        &self,
1996        module_id: &str,
1997        spawn_generation: u64,
1998        pid: u32,
1999        exit_code: Option<i32>,
2000        exit_signal: Option<i32>,
2001    ) {
2002        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2003        if state
2004            .live
2005            .get(module_id)
2006            .is_some_and(|live| live.spawn_generation == spawn_generation)
2007        {
2008            state.live.remove(module_id);
2009        }
2010        Self::emit_locked(
2011            &mut state,
2012            SpawnEventKind::Exited,
2013            module_id.to_string(),
2014            spawn_generation,
2015            pid,
2016            exit_code,
2017            exit_signal,
2018        );
2019    }
2020
2021    #[allow(clippy::too_many_arguments)]
2022    fn emit_locked(
2023        state: &mut SpawnEventState,
2024        kind: SpawnEventKind,
2025        module_id: String,
2026        spawn_generation: u64,
2027        pid: u32,
2028        exit_code: Option<i32>,
2029        exit_signal: Option<i32>,
2030    ) {
2031        state.seq = state
2032            .seq
2033            .checked_add(1)
2034            .expect("spawn event sequence exhausted");
2035        let event = SpawnEvent {
2036            cursor: Self::cursor(state),
2037            kind,
2038            module_id,
2039            spawn_generation,
2040            pid,
2041            exit_code,
2042            exit_signal,
2043        };
2044        state
2045            .process_changes
2046            .retain(|_, changed| changed.receiver_count() > 0);
2047        if let Some(changed) = state.process_changes.get(&event.module_id) {
2048            changed.send_replace(state.seq);
2049        }
2050        state.events.push_back(event.clone());
2051        while state.events.len() > state.capacity {
2052            state.events.pop_front();
2053        }
2054        let body = match serde_json::to_vec(&event) {
2055            Ok(body) => body,
2056            Err(error) => {
2057                error!(%error, "failed to serialize supervisor spawn event");
2058                return;
2059            }
2060        };
2061        state.subscribers.retain(|(connection_id, corr), subscriber| {
2062            let frame = Frame::build_with_version(
2063                subscriber.version,
2064                FrameType::StreamData,
2065                control_flags(),
2066                0,
2067                0,
2068                *corr,
2069                body.clone(),
2070            );
2071            match frame {
2072                Ok(frame) => {
2073                    if subscriber.frames.try_send(frame).is_ok() {
2074                        true
2075                    } else {
2076                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
2077                        if let Some(lagged) = subscriber.lagged.take() {
2078                            let _ = lagged.send(event.cursor.clone());
2079                        }
2080                        false
2081                    }
2082                }
2083                Err(error) => {
2084                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
2085                    false
2086                }
2087            }
2088        });
2089    }
2090
2091    fn subscribe(
2092        &self,
2093        connection_id: ConnectionId,
2094        corr: u64,
2095        version: u8,
2096        since: Option<SpawnCursor>,
2097        sink: FrameSink,
2098    ) -> Result<(), SpawnSubscribeRefusal> {
2099        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
2100        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
2101        {
2102            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2103            let replay = if let Some(since) = since {
2104                if since.daemon_incarnation != state.daemon_incarnation {
2105                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
2106                        current: state.daemon_incarnation.clone(),
2107                    });
2108                }
2109                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
2110                    if since.seq < oldest.seq.saturating_sub(1) {
2111                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
2112                    }
2113                }
2114                state
2115                    .events
2116                    .iter()
2117                    .filter(|event| event.cursor.seq > since.seq)
2118                    .cloned()
2119                    .collect::<Vec<_>>()
2120            } else {
2121                Vec::new()
2122            };
2123            for event in replay {
2124                let body = serde_json::to_vec(&event)
2125                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2126                let frame = Frame::build_with_version(
2127                    version,
2128                    FrameType::StreamData,
2129                    control_flags(),
2130                    0,
2131                    0,
2132                    corr,
2133                    body,
2134                )
2135                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2136                frames
2137                    .try_send(frame)
2138                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2139            }
2140            state.subscribers.insert(
2141                (connection_id, corr),
2142                SpawnSubscriber {
2143                    version,
2144                    frames,
2145                    lagged: Some(lagged),
2146                },
2147            );
2148        }
2149        // The lagged terminal is sent here, by the forwarder, rather than by
2150        // the emitter: at the moment of the drop the subscriber's own channel
2151        // is full, and writing to the connection sink directly from the emitter
2152        // would put the Error AHEAD of the events still queued in that channel
2153        // (and the emitter holds the feed lock, so it cannot await the sink).
2154        // Dropping the subscriber drops the only sender, so `recv` drains every
2155        // queued event and then returns `None`; only then is the Error sent, so
2156        // the client sees each event it can keep, then the reason it was cut.
2157        // Cancel and connection removal drop the oneshot unsent, so they end
2158        // the stream with no Error.
2159        tokio::spawn(async move {
2160            while let Some(frame) = receiver.recv().await {
2161                if sink.send(frame).await.is_err() {
2162                    return;
2163                }
2164            }
2165            let Ok(first_undelivered) = lagged_rx.try_recv() else {
2166                return;
2167            };
2168            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
2169                Ok(frame) => {
2170                    let _ = sink.send(frame).await;
2171                }
2172                Err(error) => {
2173                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
2174                }
2175            }
2176        });
2177        Ok(())
2178    }
2179
2180    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
2181        let Some(subscriber) = self
2182            .0
2183            .lock()
2184            .unwrap_or_else(|p| p.into_inner())
2185            .subscribers
2186            .remove(&(connection_id, corr))
2187        else {
2188            return false;
2189        };
2190        if let Ok(frame) = Frame::build_with_version(
2191            subscriber.version,
2192            FrameType::StreamEnd,
2193            control_flags(),
2194            0,
2195            0,
2196            corr,
2197            Vec::new(),
2198        ) {
2199            tokio::spawn(async move {
2200                let _ = subscriber.frames.send(frame).await;
2201            });
2202        }
2203        true
2204    }
2205
2206    fn remove_connection(&self, connection_id: ConnectionId) {
2207        self.0
2208            .lock()
2209            .unwrap_or_else(|p| p.into_inner())
2210            .subscribers
2211            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
2212    }
2213
2214    #[cfg(any(test, feature = "test-support"))]
2215    fn set_capacity(&self, capacity: usize) {
2216        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
2217    }
2218
2219    #[cfg(any(test, feature = "test-support"))]
2220    fn subscriber_count(&self) -> usize {
2221        self.0
2222            .lock()
2223            .unwrap_or_else(|p| p.into_inner())
2224            .subscribers
2225            .len()
2226    }
2227}
2228
2229/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
2230/// The terminal Error a lagged spawn subscriber receives after its queued events.
2231fn spawn_subscriber_lagged_frame(
2232    version: u8,
2233    corr: u64,
2234    first_undelivered: SpawnCursor,
2235) -> Result<Frame, String> {
2236    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
2237    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
2238        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
2239        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
2240            .to_string(),
2241        detail: Some(serde_json::json!({
2242            "first_undelivered_cursor": first_undelivered
2243        })),
2244    })
2245    .map_err(|error| error.to_string())?;
2246    Frame::build_with_version(
2247        version,
2248        FrameType::Error,
2249        crate::router::daemon_error_flags(),
2250        0,
2251        0,
2252        corr,
2253        body,
2254    )
2255    .map_err(|error| error.to_string())
2256}
2257
2258pub trait ModuleProcessLiveness: Send + Sync {
2259    fn process_live(&self, module_id: &str) -> Option<bool>;
2260
2261    /// Whether the supervisor is replacing this module's process right now: an
2262    /// operator restart or reload, a health restart, or a crash respawn whose
2263    /// backoff is running. A module in that state is not live, but a consumer
2264    /// refused now should retry shortly rather than treat the target as gone.
2265    /// Stopped, failed, and disabled modules are not replacing.
2266    fn process_replacing(&self, _module_id: &str) -> bool {
2267        false
2268    }
2269}
2270
2271/// Shared process-liveness registry keyed by supervised `module_id`.
2272#[derive(Debug, Clone, Default)]
2273pub struct SupervisorProcessLiveness {
2274    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
2275}
2276
2277impl SupervisorProcessLiveness {
2278    pub fn new() -> Self {
2279        Self::default()
2280    }
2281
2282    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
2283        let mut snapshots = self
2284            .snapshots
2285            .lock()
2286            .unwrap_or_else(|poisoned| poisoned.into_inner());
2287        snapshots.insert(module_id, snapshot);
2288    }
2289
2290    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
2291        let mut snapshots = self
2292            .snapshots
2293            .lock()
2294            .unwrap_or_else(|poisoned| poisoned.into_inner());
2295        let is_current = snapshots
2296            .get(module_id)
2297            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
2298            .unwrap_or(false);
2299        if is_current {
2300            snapshots.remove(module_id);
2301        }
2302    }
2303}
2304
2305impl ModuleProcessLiveness for SupervisorProcessLiveness {
2306    fn process_live(&self, module_id: &str) -> Option<bool> {
2307        let snapshot = {
2308            let snapshots = self
2309                .snapshots
2310                .lock()
2311                .unwrap_or_else(|poisoned| poisoned.into_inner());
2312            snapshots.get(module_id).cloned()
2313        }?;
2314        let snapshot = snapshot
2315            .lock()
2316            .unwrap_or_else(|poisoned| poisoned.into_inner());
2317        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
2318    }
2319
2320    fn process_replacing(&self, module_id: &str) -> bool {
2321        let Some(snapshot) = self
2322            .snapshots
2323            .lock()
2324            .unwrap_or_else(|poisoned| poisoned.into_inner())
2325            .get(module_id)
2326            .cloned()
2327        else {
2328            return false;
2329        };
2330        let snapshot = snapshot
2331            .lock()
2332            .unwrap_or_else(|poisoned| poisoned.into_inner());
2333        snapshot.enabled
2334            && match snapshot.state {
2335                ModuleState::Restarting => true,
2336                ModuleState::Draining => snapshot.draining_to_replace,
2337                ModuleState::Starting
2338                | ModuleState::Running
2339                | ModuleState::Unresponsive
2340                | ModuleState::Stopped
2341                | ModuleState::Failed
2342                | ModuleState::Disabled => false,
2343            }
2344    }
2345}
2346
2347#[cfg(test)]
2348#[derive(Debug, Default)]
2349struct ReloadExitRecordGate {
2350    reached: tokio::sync::Notify,
2351    resume: tokio::sync::Notify,
2352}
2353
2354#[derive(Debug, Clone, Copy)]
2355enum RespawnKind {
2356    Spawn,
2357    Reload,
2358}
2359
2360#[derive(Debug, Clone)]
2361struct PendingRespawn {
2362    deadline: Instant,
2363    kind: RespawnKind,
2364    daemon_initiated: bool,
2365    approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
2366}
2367
2368impl PendingRespawn {
2369    fn failure_context(&self, runtime: &SupervisorRuntimeConfig) -> FailureContext {
2370        FailureContext {
2371            approval: self.approval.clone(),
2372            ..runtime.failure_context(self.daemon_initiated)
2373        }
2374    }
2375}
2376
2377type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
2378
2379#[derive(Debug, Clone)]
2380struct SupervisorRuntimeConfig {
2381    registry: Arc<Registry>,
2382    config_delivery: Option<config_delivery::ConfigDelivery>,
2383    /// Retained across an operation's deferred backoff, until its child starts.
2384    recovery_is_daemon: bool,
2385    start_requester: Option<Principal>,
2386    pending_activity_restart: bool,
2387    provenance_probe: ExecutableIdentityProbe,
2388    /// Restart operations hand their backoff to the loop, which keeps serving commands.
2389    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2390    /// A reload acknowledges completion only after its replacement registers.
2391    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2392    restart_policy: RestartPolicy,
2393    register_timeout: Duration,
2394    /// This module's RESOLVED drain budget: per-module config when present,
2395    /// else `default_drain_timeout`.
2396    drain_timeout: Duration,
2397    /// Shared with the status handle so the attested value changes atomically
2398    /// when a rescan updates the running drain policy.
2399    effective_drain_timeout: Arc<Mutex<Duration>>,
2400    /// The supervisor-wide fallback, kept so a configuration update that
2401    /// REMOVES the per-module override can re-resolve to it.
2402    default_drain_timeout: Duration,
2403    health: HealthConfig,
2404    host_health: HostHealth,
2405    connection_file_path: Option<PathBuf>,
2406    capture_logs_dir: Option<PathBuf>,
2407    forwarding: Option<Arc<ForwardingTable>>,
2408    /// The shared handle, so every spawn path (initial, restart, reload) records the
2409    /// reserved-module launch nonce the HELLO verifier checks against.
2410    supervisor_handle: Option<SupervisorHandle>,
2411    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
2412    /// status queries.
2413    ///
2414    /// One ring per module, held across every respawn. The lines explaining an exit
2415    /// are written BEFORE that exit, so a ring recreated per process would be empty
2416    /// exactly when it is asked for.
2417    stderr_ring: Arc<Mutex<StderrRing>>,
2418    terminal_ring: Arc<Mutex<TerminalRing>>,
2419    spawn_events: SpawnEventFeed,
2420    child_roster: ChildRoster,
2421    #[cfg(target_os = "linux")]
2422    cgroup_placement: Option<subc_cgroup::Placement>,
2423    #[cfg(test)]
2424    test_seed_stale_facts_before_enable_spawn: bool,
2425    #[cfg(test)]
2426    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2427}
2428
2429impl SupervisorRuntimeConfig {
2430    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2431        FailureContext {
2432            daemon_initiated,
2433            approval: None,
2434            shutdown: self.child_roster.shutdown_flag(),
2435        }
2436    }
2437}
2438
2439#[derive(Debug, Clone, PartialEq, Eq)]
2440struct SupervisedConfiguration {
2441    spec: ModuleSpec,
2442    enabled: bool,
2443    health: HealthConfig,
2444    register_timeout: Duration,
2445    restart_policy: RestartPolicy,
2446}
2447
2448/// Shared daemon lookup table for supervised module handles.
2449///
2450/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2451/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2452/// launch nonces recorded at spawn are checked by the same daemon instance.
2453#[derive(Debug, Clone, Default)]
2454pub struct SupervisorHandle {
2455    #[cfg(test)]
2456    resource_commit_pause: Arc<Mutex<Option<resource_launch::ResourceCommitPause>>>,
2457    #[cfg(test)]
2458    registration_test_channels: CapturedRegistrationChannels,
2459    registration_service: Arc<Mutex<Option<registration::RegistrationService>>>,
2460    registration_connections: Arc<Mutex<HashMap<ConnectionId, Arc<RegistrationSpawn>>>>,
2461    registration_launches: Arc<Mutex<Vec<std::sync::Weak<RegistrationSpawn>>>>,
2462    gate_runtime: Arc<Mutex<Option<std::sync::Weak<crate::config_gate::runtime::RuntimeOwner>>>>,
2463    program_bindings: program_binding::ProgramBindings,
2464    #[cfg(test)]
2465    identity_sample_pause: Arc<Mutex<Option<IdentitySamplePause>>>,
2466    #[cfg(all(test, unix))]
2467    plain_identity_policy_pause: Arc<Mutex<Option<(String, IdentitySamplePause)>>>,
2468    host_health: HostHealth,
2469    restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2470    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2471    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2472    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2473    /// Module ids the supervisor has taken on. An id is added BEFORE the
2474    /// module's first process is spawned and removed only when the module
2475    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2476    /// the keys of `modules`.
2477    ///
2478    /// `modules` cannot answer "is this module configured?" on its own: a
2479    /// [`SupervisedModule`] only exists once its process has been spawned, and
2480    /// a fast child can connect, register, sync its scopes and ask about them
2481    /// before the supervisor has inserted it. Answering "not configured" in that
2482    /// gap makes scope admission refuse with the terminal "will never sync"
2483    /// instead of the retryable "has not synced yet".
2484    configured_ids: Arc<Mutex<HashSet<String>>>,
2485    spawn_events: SpawnEventFeed,
2486    /// The current expected launch nonce for each reserved module_id. Set when the
2487    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2488    /// non-reserved module never has an entry here and is never nonce-checked.
2489    /// Reserved module ids and the nonce that authorizes their next HELLO.
2490    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2491    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2492    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2493    /// had NO entry and admitted anyone: the reservation protected the nonce
2494    /// holder, not the NAME (found live by CKCRED's canary probe registering
2495    /// against a reserved scratch id).
2496    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2497    /// Reservations installed only to prevent HELLO for a protocol:none process.
2498    /// Replacement by a subc process releases them; explicitly reserved ids are
2499    /// never placed in this set and still require an operator release.
2500    implicit_protocol_none: Arc<Mutex<HashSet<String>>>,
2501    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2502    ///
2503    /// This is deliberately in-memory only: subc is state-free across daemon
2504    /// restarts, and the tombstone only explains the hours-after-removal window
2505    /// while this executing daemon is still alive. Do not persist it in a store.
2506    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2507    /// The current launch nonce for every supervised spawn. This is separate from
2508    /// reserved_nonces because consumer route.open attestation applies to all spawned
2509    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2510    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2511    /// Reserved namespace prefixes mapped to the supervised owner module whose
2512    /// current spawn nonce authorizes HELLO claims below the prefix.
2513    ///
2514    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2515    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2516    /// accidental collisions and lower-trust processes from squatting protected
2517    /// namespaces.
2518    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2519    /// Blue/green swaps in progress, by module id. An entry exists from just
2520    /// before the candidate process is spawned until the swap has failed, or
2521    /// has cut over and the old process is gone. While it exists, HELLO for the
2522    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2523    /// consumer attestation accepts both processes' nonces.
2524    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2525    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2526    promotion_observer: PromotionObserverSlot,
2527    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2528    /// this daemon-wide ordering, a rescan could retire or update a module while a
2529    /// concurrent reload still held its old handle and launch specification.
2530    operation_lock: Arc<AsyncMutex<()>>,
2531}
2532
2533#[cfg(test)]
2534#[derive(Debug)]
2535struct IdentitySamplePause {
2536    reached: std::sync::mpsc::Sender<()>,
2537    release: std::sync::mpsc::Receiver<()>,
2538}
2539
2540/// Told when a swap has promoted its candidate to be the module's active
2541/// registration.
2542///
2543/// An ordinary HELLO runs the control plane's registration side effects (the
2544/// capability cache, the deny census, the requirement recompute) as it
2545/// registers. A swap candidate's HELLO does not, because it is not routable;
2546/// promotion is when those must run instead, and promotion happens in the
2547/// supervisor, which has no other way into the control handler.
2548pub(crate) trait SwapPromotionObserver: Send + Sync {
2549    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2550}
2551
2552/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2553/// control handler) owns this handle, so a strong reference back would be a
2554/// cycle that keeps both alive.
2555#[derive(Clone, Default)]
2556struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2557
2558impl fmt::Debug for PromotionObserverSlot {
2559    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2560        f.write_str("PromotionObserverSlot")
2561    }
2562}
2563
2564/// The nonces of one open swap.
2565#[derive(Debug, Clone)]
2566struct OpenSwap {
2567    /// Legacy candidate token. A gated candidate has an empty lifecycle
2568    /// placeholder instead; only its own channel admits its HELLO.
2569    candidate_nonce: String,
2570    /// Incumbent token accepted before cutover, then cleared by promotion even
2571    /// while the incumbent's process is still draining.
2572    incumbent_nonce: Option<String>,
2573    /// Set once a HELLO has been admitted with the swap token, so the token
2574    /// admits one registration and cannot be replayed after cutover empties
2575    /// the candidate slot.
2576    candidate_admitted: bool,
2577}
2578
2579/// What the swap gate says about a HELLO. See
2580/// [`SupervisorHandle::swap_hello_admission`].
2581#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2582pub(crate) enum SwapHelloAdmission {
2583    /// No swap is open for the id (or the HELLO carries the incumbent's own
2584    /// nonce); the ordinary gates decide.
2585    NotSwapping,
2586    /// The HELLO carries the swap token: register it into the candidate slot.
2587    Candidate,
2588    /// A swap is open and the HELLO carries a nonce the supervisor did not
2589    /// mint for this id, no nonce, or a token already used.
2590    Refused,
2591    /// The id belongs to a supervised `protocol: none` module. Such a module
2592    /// never registers, so no HELLO may claim its id, with or without a nonce.
2593    ProtocolNone,
2594    /// The id is gated. Its supervised child registers only over the
2595    /// registration channel it inherited at launch, never by HELLO alone.
2596    ChannelRequired,
2597}
2598
2599#[derive(Debug, Clone, PartialEq, Eq)]
2600pub(crate) enum ReservedHelloRejection {
2601    Exact {
2602        module_id: String,
2603    },
2604    Prefix {
2605        prefix: String,
2606        owner_module_id: String,
2607    },
2608}
2609
2610/// Identity facts established by the reserved-module policy for one HELLO.
2611#[derive(Debug, Default)]
2612pub(crate) struct ReservedHelloVerification {
2613    pub(crate) nonce_verified: bool,
2614    pub(crate) prefix_owner: Option<String>,
2615}
2616
2617#[derive(Debug)]
2618struct DaemonRestartIntent {
2619    expires: Option<Duration>,
2620    window: Duration,
2621}
2622
2623impl Default for DaemonRestartIntent {
2624    fn default() -> Self {
2625        Self {
2626            expires: None,
2627            window: Duration::from_secs(120),
2628        }
2629    }
2630}
2631
2632impl SupervisorHandle {
2633    pub(crate) fn module_is_gated(&self, module: &str) -> bool {
2634        let runtime = {
2635            let Ok(runtime) = self.gate_runtime.lock() else {
2636                return true;
2637            };
2638            runtime.clone()
2639        };
2640        runtime.is_some_and(|runtime| {
2641            runtime
2642                .upgrade()
2643                .is_none_or(|owner| owner.module_is_gated(module))
2644        })
2645    }
2646    pub(crate) fn announce_daemon_restart(&self) -> u64 {
2647        let mut intent = self
2648            .restart_intent
2649            .lock()
2650            .unwrap_or_else(|p| p.into_inner());
2651        intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2652        unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2653    }
2654
2655    #[cfg(feature = "test-support")]
2656    pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2657        self.restart_intent
2658            .lock()
2659            .unwrap_or_else(|p| p.into_inner())
2660            .window = window;
2661    }
2662
2663    #[cfg(unix)]
2664    fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2665        let expiry = self
2666            .restart_intent
2667            .lock()
2668            .unwrap_or_else(|p| p.into_inner())
2669            .expires
2670            .take();
2671        if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2672            RouteCloseReason::Restart
2673        } else {
2674            RouteCloseReason::Disable
2675        }
2676    }
2677
2678    pub fn new() -> Self {
2679        Self::default()
2680    }
2681
2682    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2683        self.spawn_events.snapshot()
2684    }
2685
2686    /// Subscribe before reading the bound process. A coalesced wake is enough:
2687    /// readers compare its exact generation and PID, not the last event kind.
2688    pub(crate) fn subscribe_process_changes(
2689        &self,
2690        module: &str,
2691    ) -> tokio::sync::watch::Receiver<u64> {
2692        let mut state = self
2693            .spawn_events
2694            .0
2695            .lock()
2696            .unwrap_or_else(|p| p.into_inner());
2697        state
2698            .process_changes
2699            .retain(|_, changed| changed.receiver_count() > 0);
2700        let seq = state.seq;
2701        state
2702            .process_changes
2703            .entry(module.into())
2704            .or_insert_with(|| tokio::sync::watch::channel(seq).0)
2705            .subscribe()
2706    }
2707
2708    pub(crate) fn current_spawn(&self, module: &str) -> Option<LiveSpawn> {
2709        self.spawn_events
2710            .0
2711            .lock()
2712            .unwrap_or_else(|p| p.into_inner())
2713            .live
2714            .get(module)
2715            .cloned()
2716    }
2717
2718    pub(crate) fn subscribe_spawns(
2719        &self,
2720        connection_id: ConnectionId,
2721        corr: u64,
2722        version: u8,
2723        since: Option<SpawnCursor>,
2724        sink: FrameSink,
2725    ) -> Result<(), SpawnSubscribeRefusal> {
2726        self.spawn_events
2727            .subscribe(connection_id, corr, version, since, sink)
2728    }
2729
2730    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2731        self.spawn_events.cancel(connection_id, corr)
2732    }
2733
2734    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2735        self.spawn_events.remove_connection(connection_id);
2736    }
2737
2738    #[cfg(any(test, feature = "test-support"))]
2739    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2740        assert!(capacity > 0, "spawn event capacity must be non-zero");
2741        self.spawn_events.set_capacity(capacity);
2742    }
2743
2744    #[cfg(any(test, feature = "test-support"))]
2745    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2746        self.spawn_events.subscriber_count()
2747    }
2748
2749    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2750    /// a respawn invalidates stale consumer identities.
2751    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2752        self.spawn_nonces
2753            .lock()
2754            .unwrap_or_else(|poisoned| poisoned.into_inner())
2755            .insert(module_id.to_string(), nonce);
2756        self.refresh_program_binding_nonces(module_id);
2757    }
2758
2759    /// Record the launch nonce expected from the next HELLO for a reserved module,
2760    /// replacing any prior nonce (a respawn invalidates the previous one).
2761    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2762        {
2763            let mut reserved = self
2764                .reserved_nonces
2765                .lock()
2766                .unwrap_or_else(|p| p.into_inner());
2767            self.implicit_protocol_none
2768                .lock()
2769                .unwrap_or_else(|p| p.into_inner())
2770                .remove(module_id);
2771            reserved.insert(module_id.to_string(), Some(nonce));
2772        }
2773        self.refresh_program_binding_nonces(module_id);
2774    }
2775
2776    /// Record namespace prefixes owned by a supervised module.
2777    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2778        let mut owners = self
2779            .reserved_prefix_owners
2780            .lock()
2781            .unwrap_or_else(|poisoned| poisoned.into_inner());
2782        owners.retain(|_, owner| owner != owner_module_id);
2783        for prefix in prefixes {
2784            owners.insert(prefix.clone(), owner_module_id.to_string());
2785        }
2786    }
2787
2788    /// The launch nonce most recently minted for a module's spawn, if any.
2789    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2790        if self.module_is_gated(module_id) {
2791            return None;
2792        }
2793        self.spawn_nonces
2794            .lock()
2795            .unwrap_or_else(|poisoned| poisoned.into_inner())
2796            .get(module_id)
2797            .cloned()
2798    }
2799
2800    #[cfg(all(test, unix))]
2801    pub(crate) fn legacy_nonce_present_for_test(&self, module: &str) -> bool {
2802        self.spawn_nonces.lock().unwrap().contains_key(module)
2803    }
2804
2805    pub(crate) fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2806        // Prefix ownership follows staged configuration, not the running
2807        // child's launch settings. A respawn may still carry an older spec.
2808        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2809        let protocol = self
2810            .get(&spec.module_id)
2811            .and_then(|module| {
2812                let snapshot = module.inner.snapshot.lock().ok()?;
2813                snapshot
2814                    .process_alive
2815                    .then_some(snapshot.spawned_protocol.unwrap_or(spec.protocol))
2816            })
2817            .unwrap_or(spec.protocol);
2818        self.apply_identity_policy(spec, protocol);
2819    }
2820
2821    #[cfg(all(test, unix))]
2822    pub(crate) fn pause_plain_identity_policy_for_test(
2823        &self,
2824        module: &str,
2825    ) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
2826        let (reached, waiting) = std::sync::mpsc::channel();
2827        let (release, paused) = std::sync::mpsc::channel();
2828        *self.plain_identity_policy_pause.lock().unwrap() = Some((
2829            module.to_owned(),
2830            IdentitySamplePause {
2831                reached,
2832                release: paused,
2833            },
2834        ));
2835        (waiting, release)
2836    }
2837
2838    fn apply_identity_policy(&self, spec: &ModuleSpec, protocol: ModuleProtocol) {
2839        let channel_only = self.module_is_gated(&spec.module_id);
2840        let mut current_nonces = self
2841            .spawn_nonces
2842            .lock()
2843            .unwrap_or_else(|poisoned| poisoned.into_inner());
2844        if protocol == ModuleProtocol::None || channel_only {
2845            current_nonces.remove(&spec.module_id);
2846        }
2847        let spawn_nonce = current_nonces.get(&spec.module_id).cloned();
2848        #[cfg(test)]
2849        if let Some(pause) = self.identity_sample_pause.lock().unwrap().take() {
2850            pause.reached.send(()).unwrap();
2851            pause.release.recv().unwrap();
2852        }
2853        let mut reserved_nonces = self
2854            .reserved_nonces
2855            .lock()
2856            .unwrap_or_else(|poisoned| poisoned.into_inner());
2857        let mut implicit = self
2858            .implicit_protocol_none
2859            .lock()
2860            .unwrap_or_else(|p| p.into_inner());
2861        if spec.reserved {
2862            implicit.remove(&spec.module_id);
2863        } else if protocol == ModuleProtocol::None {
2864            if !reserved_nonces.contains_key(&spec.module_id) {
2865                implicit.insert(spec.module_id.clone());
2866            }
2867        } else if implicit.remove(&spec.module_id) {
2868            reserved_nonces.remove(&spec.module_id);
2869        }
2870        if spec.reserved || protocol == ModuleProtocol::None {
2871            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2872            // reserved name whose module has never spawned has no legitimate
2873            // holder, and the entry's absence is what used to leave the name
2874            // open to the first claimant.
2875            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2876        }
2877        drop(implicit);
2878        drop(reserved_nonces);
2879        drop(current_nonces);
2880        self.refresh_program_binding_nonces(&spec.module_id);
2881        // A later unreserved declaration must not silently unreserve an id that
2882        // was retained after its reserved configuration was removed. The explicit
2883        // release ceremony is the only operation that retires that gate.
2884        self.removal_tombstones
2885            .lock()
2886            .unwrap_or_else(|poisoned| poisoned.into_inner())
2887            .remove(&spec.module_id);
2888    }
2889
2890    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2891    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2892    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2893    /// with no matching prefix are always authorized.
2894    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2895        self.reserved_hello_verification(module_id, presented)
2896            .is_ok()
2897    }
2898
2899    /// Whether HELLO passed a nonce check that was actually required by reserved
2900    /// module policy, or the refusal explaining why a required check failed.
2901    pub(crate) fn reserved_hello_verification(
2902        &self,
2903        module_id: &str,
2904        presented: Option<&str>,
2905    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2906        if self.module_is_gated(module_id) {
2907            return Err(ReservedHelloRejection::Exact {
2908                module_id: module_id.into(),
2909            });
2910        }
2911        self.hello_verification(module_id, presented, false)
2912    }
2913
2914    pub(crate) fn channel_hello_verification(
2915        &self,
2916        module_id: &str,
2917    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2918        if self
2919            .get(module_id)
2920            .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
2921        {
2922            return Err(ReservedHelloRejection::Exact {
2923                module_id: module_id.into(),
2924            });
2925        }
2926        let mut verification = self.hello_verification(module_id, None, true)?;
2927        verification.nonce_verified = true;
2928        Ok(verification)
2929    }
2930
2931    pub(crate) fn prefix_claim_authorized(&self, owner: &str, module_id: &str) -> bool {
2932        self.prefix_claim_publication(owner, module_id).is_some()
2933    }
2934
2935    #[cfg(all(test, unix))]
2936    pub(crate) fn prefix_publication_locked_for_test(&self) -> bool {
2937        matches!(
2938            self.reserved_prefix_owners.try_lock(),
2939            Err(std::sync::TryLockError::WouldBlock)
2940        )
2941    }
2942
2943    // Keep this guard through registry/forwarding publication. Rescans replace
2944    // prefix owners without stopping their launches, so the spawn guard alone
2945    // cannot fence a reservation transfer.
2946    pub(crate) fn prefix_claim_publication(
2947        &self,
2948        owner: &str,
2949        module_id: &str,
2950    ) -> Option<std::sync::MutexGuard<'_, HashMap<String, String>>> {
2951        if subc_protocol::manifest::validate_module_id(module_id).is_err()
2952            || self.module_is_gated(module_id)
2953            || self
2954                .reserved_nonces
2955                .lock()
2956                .unwrap_or_else(|p| p.into_inner())
2957                .contains_key(module_id)
2958        {
2959            return None;
2960        }
2961        let owners = self
2962            .reserved_prefix_owners
2963            .lock()
2964            .unwrap_or_else(|p| p.into_inner());
2965        let authorized = owners
2966            .iter()
2967            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2968            .max_by_key(|(prefix, _)| prefix.len())
2969            .is_some_and(|(prefix, holder)| holder == owner && module_id.len() > prefix.len());
2970        authorized.then_some(owners)
2971    }
2972
2973    fn hello_verification(
2974        &self,
2975        module_id: &str,
2976        presented: Option<&str>,
2977        channel: bool,
2978    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2979        let nonces = self
2980            .reserved_nonces
2981            .lock()
2982            .unwrap_or_else(|poisoned| poisoned.into_inner());
2983        if let Some(expected) = nonces.get(module_id) {
2984            // `None` = reserved with no legitimate holder: refuse every
2985            // presentation, because no process can hold a nonce that was never
2986            // minted. Only a real minted nonce admits, in constant time.
2987            let authorized = channel
2988                || match expected {
2989                    Some(expected) => presented.is_some_and(|p| {
2990                        !p.is_empty() && constant_time_eq(expected.as_bytes(), p.as_bytes())
2991                    }),
2992                    None => false,
2993                };
2994            if authorized {
2995                return Ok(ReservedHelloVerification {
2996                    nonce_verified: true,
2997                    prefix_owner: None,
2998                });
2999            }
3000            return Err(ReservedHelloRejection::Exact {
3001                module_id: module_id.to_string(),
3002            });
3003        }
3004        drop(nonces);
3005
3006        let matched_prefix = self
3007            .reserved_prefix_owners
3008            .lock()
3009            .unwrap_or_else(|poisoned| poisoned.into_inner())
3010            .iter()
3011            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3012            .max_by_key(|(prefix, _)| prefix.len())
3013            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
3014        let Some((prefix, owner_module_id)) = matched_prefix else {
3015            return Ok(ReservedHelloVerification::default());
3016        };
3017
3018        if module_id.len() == prefix.len()
3019            || (!channel
3020                && (self.module_is_gated(&owner_module_id) || self.protocol_none(&owner_module_id)))
3021        {
3022            return Err(ReservedHelloRejection::Prefix {
3023                prefix,
3024                owner_module_id,
3025            });
3026        }
3027
3028        let authorized = if channel {
3029            owner_module_id == module_id
3030        } else {
3031            presented.is_some_and(|presented| {
3032                self.spawn_nonces
3033                .lock()
3034                .unwrap_or_else(|poisoned| poisoned.into_inner())
3035                .get(&owner_module_id)
3036                .is_some_and(|expected| !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes()))
3037                // While the owner is being swapped, children started by
3038                // either of its two processes hold that process's nonce.
3039                || self.swap_nonce_matches(&owner_module_id, presented)
3040            })
3041        };
3042        if authorized {
3043            Ok(ReservedHelloVerification {
3044                nonce_verified: true,
3045                prefix_owner: (owner_module_id != module_id).then_some(owner_module_id),
3046            })
3047        } else {
3048            Err(ReservedHelloRejection::Prefix {
3049                prefix,
3050                owner_module_id,
3051            })
3052        }
3053    }
3054
3055    /// Whether a consumer connection proved it came from a daemon-spawned module.
3056    ///
3057    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
3058    /// accepted only for module ids the supervisor has spawned.
3059    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
3060        if self.module_is_gated(module_id) || self.protocol_none(module_id) {
3061            return false;
3062        }
3063        let nonces = self
3064            .spawn_nonces
3065            .lock()
3066            .unwrap_or_else(|poisoned| poisoned.into_inner());
3067        let current = nonces.get(module_id).is_some_and(|expected| {
3068            !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes())
3069        });
3070        drop(nonces);
3071        // Before cutover both launches may attest. Promotion removes the
3072        // incumbent's token even if its process still needs time to drain.
3073        current || self.swap_nonce_matches(module_id, presented)
3074    }
3075
3076    /// Whether `presented` is either nonce of an open swap for `module_id`.
3077    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
3078        if self.module_is_gated(module_id) || self.protocol_none(module_id) || presented.is_empty()
3079        {
3080            return false;
3081        }
3082        let swaps = self
3083            .swaps
3084            .lock()
3085            .unwrap_or_else(|poisoned| poisoned.into_inner());
3086        swaps.get(module_id).is_some_and(|swap| {
3087            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
3088                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
3089                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
3090                })
3091        })
3092    }
3093
3094    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
3095    /// Called before the candidate process exists.
3096    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
3097        let incumbent_nonce = self
3098            .spawn_nonces
3099            .lock()
3100            .unwrap_or_else(|poisoned| poisoned.into_inner())
3101            .get(module_id)
3102            .cloned();
3103        self.swaps
3104            .lock()
3105            .unwrap_or_else(|poisoned| poisoned.into_inner())
3106            .insert(
3107                module_id.to_string(),
3108                OpenSwap {
3109                    candidate_nonce,
3110                    incumbent_nonce,
3111                    candidate_admitted: false,
3112                },
3113            );
3114        self.refresh_program_binding_nonces(module_id);
3115    }
3116
3117    /// Close the swap for `module_id`, releasing whichever nonce is no longer
3118    /// the module's recorded one.
3119    pub(crate) fn close_swap(&self, module_id: &str) {
3120        self.swaps
3121            .lock()
3122            .unwrap_or_else(|poisoned| poisoned.into_inner())
3123            .remove(module_id);
3124        self.refresh_program_binding_nonces(module_id);
3125    }
3126
3127    /// Install the observer told about swap promotions, replacing any earlier
3128    /// one.
3129    pub(crate) fn set_swap_promotion_observer(
3130        &self,
3131        observer: std::sync::Weak<dyn SwapPromotionObserver>,
3132    ) {
3133        *self
3134            .promotion_observer
3135            .0
3136            .lock()
3137            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
3138    }
3139
3140    /// Tell the installed observer, if it is still alive, that a swap promoted
3141    /// `registration`.
3142    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
3143        let observer = self
3144            .promotion_observer
3145            .0
3146            .lock()
3147            .unwrap_or_else(|poisoned| poisoned.into_inner())
3148            .as_ref()
3149            .and_then(std::sync::Weak::upgrade);
3150        if let Some(observer) = observer {
3151            observer.swap_promoted(registration);
3152        }
3153    }
3154
3155    /// Whether a swap is open for `module_id`.
3156    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
3157        self.swaps
3158            .lock()
3159            .unwrap_or_else(|poisoned| poisoned.into_inner())
3160            .contains_key(module_id)
3161    }
3162
3163    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
3164    /// respawn would, once cutover has made the candidate the module's process.
3165    /// Promotion revokes the incumbent's tokens; the swap record stays open
3166    /// only for lifecycle cleanup while the old process drains.
3167    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
3168        let candidate_nonce = self
3169            .swaps
3170            .lock()
3171            .unwrap_or_else(|poisoned| poisoned.into_inner())
3172            .get_mut(module_id)
3173            .map(|swap| {
3174                swap.incumbent_nonce = None;
3175                swap.candidate_nonce.clone()
3176            });
3177        self.spawn_nonces
3178            .lock()
3179            .unwrap_or_else(|p| p.into_inner())
3180            .remove(module_id);
3181        {
3182            let mut nonces = self
3183                .reserved_nonces
3184                .lock()
3185                .unwrap_or_else(|p| p.into_inner());
3186            if reserved || nonces.contains_key(module_id) {
3187                nonces.insert(module_id.into(), None);
3188            }
3189        }
3190        if self.module_is_gated(module_id) {
3191            return;
3192        }
3193        let Some(nonce) = candidate_nonce.filter(|nonce| !nonce.is_empty()) else {
3194            return;
3195        };
3196        self.set_spawn_nonce(module_id, nonce.clone());
3197        if reserved {
3198            self.set_reserved_nonce(module_id, nonce);
3199        }
3200    }
3201
3202    /// The swap gate for a HELLO claiming `module_id`.
3203    ///
3204    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
3205    /// presents the candidate nonce, which the reserved gate (holding the
3206    /// incumbent's nonce) would refuse as `reserved_module` before swap
3207    /// admission was ever reached. And it applies to unreserved ids too: for an
3208    /// unreserved id the only thing that ever stopped a second process claiming
3209    /// a live id was the `duplicate_module_id` refusal, which is exactly the
3210    /// refusal a swap lifts for its candidate.
3211    ///
3212    /// The incumbent's own nonce falls through to the ordinary gates, which
3213    /// treat it as they always have (a live incumbent is refused as a
3214    /// duplicate). Anything else while a swap is open is refused, including an
3215    /// absent nonce.
3216    pub(crate) fn swap_hello_admission(
3217        &self,
3218        module_id: &str,
3219        presented: Option<&str>,
3220    ) -> SwapHelloAdmission {
3221        if self.protocol_none(module_id) {
3222            return SwapHelloAdmission::ProtocolNone;
3223        }
3224        if self.module_is_gated(module_id) {
3225            return SwapHelloAdmission::ChannelRequired;
3226        }
3227        let swaps = self
3228            .swaps
3229            .lock()
3230            .unwrap_or_else(|poisoned| poisoned.into_inner());
3231        let Some(swap) = swaps.get(module_id) else {
3232            return SwapHelloAdmission::NotSwapping;
3233        };
3234        let Some(presented) = presented.filter(|nonce| !nonce.is_empty()) else {
3235            return SwapHelloAdmission::Refused;
3236        };
3237        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
3238            return if swap.candidate_admitted {
3239                SwapHelloAdmission::Refused
3240            } else {
3241                SwapHelloAdmission::Candidate
3242            };
3243        }
3244        if swap
3245            .incumbent_nonce
3246            .as_deref()
3247            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
3248        {
3249            return SwapHelloAdmission::NotSwapping;
3250        }
3251        SwapHelloAdmission::Refused
3252    }
3253
3254    fn protocol_none(&self, module: &str) -> bool {
3255        self.get(module)
3256            .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3257    }
3258
3259    /// Exclude swap closure through the short, synchronous candidate publication.
3260    pub(crate) fn with_swap_candidate_admission<T>(
3261        &self,
3262        module_id: &str,
3263        publish: impl FnOnce() -> (T, bool),
3264    ) -> Option<T> {
3265        let mut swaps = self.swaps.lock().unwrap_or_else(|p| p.into_inner());
3266        let swap = swaps
3267            .get_mut(module_id)
3268            .filter(|swap| !swap.candidate_admitted)?;
3269        let (result, admitted) = publish();
3270        swap.candidate_admitted = admitted;
3271        Some(result)
3272    }
3273
3274    /// Test/support lookup for the current launch nonce of a supervised spawn.
3275    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3276        self.spawn_nonce(module_id)
3277    }
3278
3279    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
3280    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3281        if self.module_is_gated(module_id) {
3282            return None;
3283        }
3284        self.reserved_nonces
3285            .lock()
3286            .unwrap_or_else(|poisoned| poisoned.into_inner())
3287            .get(module_id)
3288            .cloned()
3289            .flatten()
3290    }
3291
3292    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
3293        // Normally already marked before the process was spawned; marking here
3294        // too keeps `configured_ids` a superset of the roster for any caller
3295        // that inserts a module directly.
3296        self.mark_configured(module.module_id());
3297        let mut modules = self
3298            .modules
3299            .lock()
3300            .unwrap_or_else(|poisoned| poisoned.into_inner());
3301        modules.insert(module.module_id().to_string(), module)
3302    }
3303
3304    /// Record that the supervisor has taken on `module_id`. Called before the
3305    /// module's first process is spawned, so that by the time that process can
3306    /// register, [`Self::is_configured`] already answers true.
3307    fn mark_configured(&self, module_id: &str) {
3308        self.update_configured_ids(|ids| {
3309            ids.insert(module_id.to_string());
3310        });
3311    }
3312
3313    fn publish_configured_ids(&self, ids: Option<&HashSet<String>>) {
3314        let owner = self
3315            .gate_runtime
3316            .lock()
3317            .unwrap_or_else(|p| p.into_inner())
3318            .clone()
3319            .and_then(|owner| owner.upgrade());
3320        if let Some(owner) = owner {
3321            owner.observe_supervisor_configured_ids(ids, Arc::downgrade(&self.configured_ids));
3322        }
3323    }
3324
3325    fn update_configured_ids(&self, update: impl FnOnce(&mut HashSet<String>)) {
3326        match self.configured_ids.lock() {
3327            Ok(mut ids) => {
3328                update(&mut ids);
3329                // Publish while this roster mutation is serialized, so an
3330                // older snapshot can never overwrite a later configured id.
3331                self.publish_configured_ids(Some(&ids));
3332            }
3333            Err(poisoned) => {
3334                update(&mut poisoned.into_inner());
3335                self.publish_configured_ids(None);
3336            }
3337        }
3338    }
3339
3340    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
3341    /// before it was ever put on the roster. A module already on the roster
3342    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
3343    fn unmark_configured_unless_rostered(&self, module_id: &str) {
3344        let modules = self
3345            .modules
3346            .lock()
3347            .unwrap_or_else(|poisoned| poisoned.into_inner());
3348        if !modules.contains_key(module_id) {
3349            self.update_configured_ids(|ids| {
3350                ids.remove(module_id);
3351            });
3352        }
3353    }
3354
3355    /// Whether `module_id` is a module this daemon supervises: on the roster,
3356    /// or about to be (its process is being spawned right now).
3357    ///
3358    /// This, not `get(..).is_some()`, is what "the owner is configured" means
3359    /// for scopes: a supervised module's process can register and sync before
3360    /// [`Self::get`] can return it, and in that window it is still a module
3361    /// that will sync, not one that never will.
3362    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
3363        self.configured_ids
3364            .lock()
3365            .unwrap_or_else(|poisoned| poisoned.into_inner())
3366            .contains(module_id)
3367    }
3368
3369    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
3370        let modules = self
3371            .modules
3372            .lock()
3373            .unwrap_or_else(|poisoned| poisoned.into_inner());
3374        modules.get(module_id).cloned()
3375    }
3376
3377    pub(crate) fn record_late_health_answer(
3378        &self,
3379        module_id: &str,
3380        latency_ms: u64,
3381    ) -> Result<bool, SuperviseError> {
3382        let Some(module) = self.get(module_id) else {
3383            return Ok(false);
3384        };
3385        self.host_health.record_late_module(module_id);
3386        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
3387            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
3388            state.health.last_late_answer_latency_ms = Some(latency_ms);
3389            // A late answer is an answer: the module served the probe, just past
3390            // the deadline. Leaving the miss streak in place while logging
3391            // "proves the module is alive" is how a CPU-starved module that
3392            // answers every probe a few seconds late still marches to the
3393            // threshold and gets killed — the exact kill class `NoAnswer` is
3394            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
3395            // is degradation, and degradation reports; it does not restart.
3396            state.health.consecutive_failures = 0;
3397        })?;
3398        Ok(true)
3399    }
3400
3401    pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
3402        self.host_health.spawn_monitor()
3403    }
3404
3405    /// Arm the one-shot marker for the module process that this caller
3406    /// deliberately initiated severance against. Generic connection teardown
3407    /// must not call this:
3408    /// a surviving process would otherwise retain an exemption for a later
3409    /// genuine crash.
3410    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
3411        let Some(module) = self.get(module_id) else {
3412            return Ok(false);
3413        };
3414        let snapshot = lock_snapshot(&module.inner.snapshot)?;
3415        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
3416            return Ok(false);
3417        };
3418        drop(snapshot);
3419        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
3420    }
3421
3422    pub fn list(&self) -> Vec<SupervisedModule> {
3423        let modules = self
3424            .modules
3425            .lock()
3426            .unwrap_or_else(|poisoned| poisoned.into_inner());
3427        let mut modules = modules.values().cloned().collect::<Vec<_>>();
3428        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
3429        modules
3430    }
3431
3432    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
3433        self.activity_images
3434            .lock()
3435            .unwrap_or_else(|p| p.into_inner())
3436            .remove(module_id);
3437        self.spawn_nonces
3438            .lock()
3439            .unwrap_or_else(|poisoned| poisoned.into_inner())
3440            .remove(module_id);
3441        self.close_swap(module_id);
3442        let mut reserved_nonces = self
3443            .reserved_nonces
3444            .lock()
3445            .unwrap_or_else(|poisoned| poisoned.into_inner());
3446        if reserved_nonces.contains_key(module_id) {
3447            // The old nonce must die with the removed process, but the exact-id
3448            // gate remains until an operator explicitly releases it.
3449            reserved_nonces.insert(module_id.to_string(), None);
3450        }
3451        drop(reserved_nonces);
3452        self.reserved_prefix_owners
3453            .lock()
3454            .unwrap_or_else(|poisoned| poisoned.into_inner())
3455            .retain(|_, owner| owner != module_id);
3456        let removed = self
3457            .modules
3458            .lock()
3459            .unwrap_or_else(|poisoned| poisoned.into_inner())
3460            .remove(module_id);
3461        self.update_configured_ids(|ids| {
3462            ids.remove(module_id);
3463        });
3464        self.remove_program_binding_records(module_id);
3465        removed
3466    }
3467
3468    /// Remember a module removed by a non-preview rescan so route.open can
3469    /// distinguish that intentional removal from an unknown id.
3470    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
3471        self.removal_tombstones
3472            .lock()
3473            .unwrap_or_else(|poisoned| poisoned.into_inner())
3474            .insert(module_id.to_string(), unix_ms_now());
3475    }
3476
3477    /// Return how long ago a rescan removed this module in milliseconds.
3478    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
3479        self.removal_tombstones
3480            .lock()
3481            .unwrap_or_else(|poisoned| poisoned.into_inner())
3482            .get(module_id)
3483            .copied()
3484            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
3485    }
3486
3487    /// Retire a reserved-id gate only after its module has left supervision.
3488    ///
3489    /// A retained gate has no live nonce (`None`), so releasing any other entry
3490    /// would weaken a currently configured or otherwise active reservation.
3491    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
3492        if self.get(module_id).is_some() {
3493            return false;
3494        }
3495        let mut reserved_nonces = self
3496            .reserved_nonces
3497            .lock()
3498            .unwrap_or_else(|poisoned| poisoned.into_inner());
3499        if !matches!(reserved_nonces.get(module_id), Some(None)) {
3500            return false;
3501        }
3502        reserved_nonces.remove(module_id);
3503        self.implicit_protocol_none
3504            .lock()
3505            .unwrap_or_else(|p| p.into_inner())
3506            .remove(module_id);
3507        true
3508    }
3509
3510    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
3511        Arc::clone(&self.operation_lock)
3512    }
3513}
3514
3515/// Process supervisor for subc-owned singleton modules.
3516#[derive(Debug, Clone)]
3517pub struct Supervisor {
3518    registry: Arc<Registry>,
3519    config_delivery: Option<config_delivery::ConfigDelivery>,
3520    restart_policy: RestartPolicy,
3521    drain_timeout: Duration,
3522    connection_file_path: Option<PathBuf>,
3523    capture_logs_dir: Option<PathBuf>,
3524    forwarding: Option<Arc<ForwardingTable>>,
3525    process_liveness: Arc<SupervisorProcessLiveness>,
3526    supervisor_handle: Option<SupervisorHandle>,
3527    health: HealthConfig,
3528    host_health: HostHealth,
3529    daemon_start_clock: crate::clock::StartClock,
3530    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
3531    spawn_events: SpawnEventFeed,
3532    provenance_probe: ExecutableIdentityProbe,
3533    /// Every process spawned through this supervisor (and its clones) and not
3534    /// yet reaped, so daemon shutdown can end them.
3535    child_roster: ChildRoster,
3536    #[cfg(target_os = "linux")]
3537    cgroup_placement: Option<subc_cgroup::Placement>,
3538    #[cfg(test)]
3539    test_after_first_spawn: AfterFirstSpawnHook,
3540}
3541
3542/// A roster entry whose monitor and first process have not started yet.
3543/// Keeping the command receiver here lets discovery serve status immediately
3544/// while start/stop requests wait until bootstrap has published the connection file.
3545pub(crate) struct RegisteredConfiguredModule {
3546    spec: ModuleSpec,
3547    runtime: SupervisorRuntimeConfig,
3548    module: SupervisedModule,
3549    commands: mpsc::Receiver<SupervisorCommand>,
3550}
3551
3552/// Test-only hook run on the path that takes on a new module, right after its
3553/// first `spawn_child` returns (the process exists and could already be
3554/// registering) and before that process is handed to the module's supervise
3555/// loop and put on the roster. Lets a test observe what a fast child would see
3556/// in that window without racing a real one.
3557#[cfg(test)]
3558#[derive(Clone, Default)]
3559struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
3560
3561#[cfg(test)]
3562type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
3563
3564#[cfg(test)]
3565impl fmt::Debug for AfterFirstSpawnHook {
3566    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3567        f.write_str("AfterFirstSpawnHook")
3568    }
3569}
3570
3571#[cfg(test)]
3572impl AfterFirstSpawnHook {
3573    fn run(&self, module_id: &str) {
3574        if let Some(hook) = &self.0 {
3575            hook(module_id);
3576        }
3577    }
3578}
3579
3580impl Supervisor {
3581    #[cfg(all(test, target_os = "linux"))]
3582    pub(crate) fn set_admission_hook_for_test(
3583        &self,
3584        hook: impl Fn(&str, &'static str, Option<u32>) -> bool + Send + Sync + 'static,
3585    ) {
3586        self.child_roster.set_admission_hook_for_test(hook);
3587    }
3588
3589    #[cfg(test)]
3590    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
3591        let supervisor = Self::new(registry, policy);
3592        #[cfg(target_os = "macos")]
3593        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
3594        supervisor
3595    }
3596    /// Verify the trampoline once when configured. Missing private OS support
3597    /// refuses every macOS launch by name but does not stop the daemon's control
3598    /// server. Embedders must explicitly provide a binary with the subc-os hidden
3599    /// entry point; the library must not exec an arbitrary hosting program.
3600    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
3601        let path = path.into();
3602        #[cfg(target_os = "macos")]
3603        {
3604            let result = probe_privacy_trampoline(&path).map(|()| path);
3605            if let Err(cause) = &result {
3606                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
3607            }
3608            self.child_roster.set_privacy_trampoline(result);
3609        }
3610        #[cfg(not(target_os = "macos"))]
3611        let _ = path;
3612        self
3613    }
3614    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
3615    /// capability-requirement log to read.
3616    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
3617        self.child_roster.shutdown_flag()
3618    }
3619
3620    /// The first step of an announced daemon shutdown, before the notice and
3621    /// before any connection is closed.
3622    ///
3623    /// Sets the daemon-shutdown flag first: from here on no module is
3624    /// respawned (crash restart, operator restart, or swap), and every child
3625    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
3626    /// the module exits on the EOF this shutdown gives it or is signalled by a
3627    /// service manager that kills the whole cgroup. Then writes the journal's
3628    /// shutdown marker, which records the instant and closes this daemon
3629    /// incarnation's stretch of the journal.
3630    #[cfg(unix)]
3631    pub(crate) fn begin_daemon_shutdown(&self) {
3632        self.child_roster.close();
3633        if let Some(journal) = &self.terminal_journal {
3634            journal.stamp_shutdown();
3635        }
3636    }
3637
3638    /// Announce a cut while established connections can still carry replies.
3639    /// These budgets bound the attempt to deliver notices and the wait for
3640    /// routes to drain; neither guarantees delivery or child completion, and
3641    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
3642    #[cfg(unix)]
3643    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
3644        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
3645        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
3646        let Some(forwarding) = &self.forwarding else {
3647            return Ok(());
3648        };
3649        let notice_started = Instant::now();
3650        let modules = forwarding
3651            .begin_daemon_drain()
3652            .map_err(SuperviseError::Forwarding)?;
3653        let deadline_ms =
3654            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
3655        let shutdown_reason = self
3656            .supervisor_handle
3657            .as_ref()
3658            .map(SupervisorHandle::take_daemon_shutdown_reason)
3659            .unwrap_or(RouteCloseReason::Disable);
3660        let mut modules_told = 0;
3661        let mut notices = tokio::task::JoinSet::new();
3662        let mut drains = Vec::new();
3663        for connection in forwarding
3664            .module_connections()
3665            .map_err(SuperviseError::Forwarding)?
3666        {
3667            let target = ModuleDrainTarget {
3668                endpoint: connection.endpoint,
3669                sink: connection.sink,
3670                negotiated_ver: connection.negotiated_ver,
3671                abandoned_bindings: Vec::new(),
3672                excluded_subscriptions: 0,
3673            };
3674            notices.spawn(async move {
3675                (
3676                    true,
3677                    send_module_draining(
3678                        &connection.module_id,
3679                        shutdown_reason,
3680                        deadline_ms,
3681                        &target,
3682                        true,
3683                    )
3684                    .await,
3685                )
3686            });
3687        }
3688        for (module_id, endpoint) in modules {
3689            let Some(target) = forwarding
3690                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
3691                .map_err(SuperviseError::Forwarding)?
3692            else {
3693                continue;
3694            };
3695            let routes = forwarding
3696                .endpoint_routes(target.endpoint)
3697                .map_err(SuperviseError::Forwarding)?;
3698            // Only an announced restart promises that modules are coming back.
3699            // Client route notices retain their existing reconnect semantics.
3700            let mut recipients = Vec::new();
3701            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3702            for route in routes {
3703                let client = route.goodbye_target;
3704                if let Some((_, channels)) = clients
3705                    .iter_mut()
3706                    .find(|(existing, _)| existing.connection_id == client.connection_id)
3707                {
3708                    channels.push(client.channel);
3709                } else {
3710                    let channel = client.channel;
3711                    clients.push((client, vec![channel]));
3712                }
3713            }
3714            for (client, mut channels) in clients {
3715                channels.sort_unstable();
3716                channels.dedup();
3717                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3718                    module_id: module_id.clone(),
3719                    channels,
3720                    reason: RouteCloseReason::Restart,
3721                })
3722                .expect("route closing serializes");
3723                recipients.push((client.sink, client.negotiated_ver, closing));
3724            }
3725            for (sink, version, body) in recipients {
3726                notices.spawn(async move {
3727                    let frame = Frame::build_with_version(
3728                        version,
3729                        FrameType::Push,
3730                        control_flags(),
3731                        0,
3732                        0,
3733                        0,
3734                        body,
3735                    )
3736                    .expect("bounded lifecycle notice frame builds");
3737                    (false, sink.send_flushed(frame).await.is_ok())
3738                });
3739            }
3740            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3741            drains.push((module_id, target.endpoint, gauges));
3742        }
3743        // A quiet forwarding table is not proof that queued notices reached the
3744        // socket. Wait for writer flush acknowledgements before testing quiescence.
3745        let notice_deadline = Instant::now() + NOTICE_BUDGET;
3746        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3747            match result {
3748                Ok((true, true)) => modules_told += 1,
3749                Ok((false, true)) => {}
3750                result => warn!(?result, "daemon shutdown notice delivery failed"),
3751            }
3752        }
3753        notices.abort_all();
3754        info!(
3755            ?shutdown_reason,
3756            modules_told, "daemon shutdown module notices delivered"
3757        );
3758        let notice_elapsed = notice_started.elapsed();
3759        let drain_started = Instant::now();
3760        let deadline = drain_started + DRAIN_BUDGET;
3761        let mut waits = tokio::task::JoinSet::new();
3762        for (module_id, endpoint, gauges) in drains {
3763            let forwarding = Arc::clone(forwarding);
3764            let mut runtime = self.runtime_config();
3765            runtime.health.cadence = Duration::from_millis(100);
3766            waits.spawn(async move {
3767                let wait_result = wait_for_forwarding_quiescence(
3768                    &forwarding,
3769                    &module_id,
3770                    &runtime,
3771                    endpoint,
3772                    deadline,
3773                    &gauges,
3774                    DrainScope::Endpoint(endpoint),
3775                    &mut DrainGaugeTrace::new(&gauges),
3776                )
3777                .await;
3778                // Store the measurement before the task returns: quiescence
3779                // may be reached just as the deadline stops the join loop from
3780                // collecting results. Unfinished or failed waits report false.
3781                forwarding
3782                    .record_daemon_drain_result(
3783                        endpoint,
3784                        drained_after_quiescence_wait(&wait_result),
3785                    )
3786                    .map_err(SuperviseError::Forwarding)?;
3787                wait_result
3788            });
3789        }
3790        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3791            if !matches!(result, Ok(Ok(true))) {
3792                warn!(?result, "daemon shutdown drain did not reach quiescence");
3793            }
3794        }
3795        // Measure only the forwarding drain, not child teardown or the caller's
3796        // signal delivery and process-reaping delays.
3797        info!(
3798            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3799            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3800            "daemon shutdown forwarding drain complete"
3801        );
3802        Ok(())
3803    }
3804
3805    /// The last step of a graceful daemon shutdown, after the notice and the
3806    /// drain: send every registered module a module GOODBYE, the same planned
3807    /// stop signal `ck module stop` gives, then close every connection so each
3808    /// subc module sees EOF and starts its own teardown, then end every
3809    /// supervised child that has not exited
3810    /// by its own deadline (its drain budget, capped). Modules lead their own
3811    /// process groups, so a
3812    /// service manager's group kill no longer reaches them; without this a
3813    /// child that does not stop on EOF (every `protocol: "none"` child, which
3814    /// has no connection) would outlive the daemon. Every wait is bounded (see
3815    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
3816    #[cfg(unix)]
3817    pub(crate) async fn end_children_for_daemon_shutdown(
3818        &self,
3819        already_escalated: bool,
3820        escalate: impl std::future::Future<Output = ()>,
3821    ) {
3822        tokio::pin!(escalate);
3823        let mut escalated = already_escalated;
3824        if let Some(forwarding) = &self.forwarding {
3825            let reason = CloseReason::new(
3826                "daemon_shutdown",
3827                "the daemon is exiting after its shutdown notice and drain",
3828            );
3829            if escalated {
3830                // The operator asked to stop waiting: queue the GOODBYEs but
3831                // do not wait for them to be written.
3832                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3833            } else {
3834                tokio::select! {
3835                    biased;
3836                    _ = escalate.as_mut() => {
3837                        info!("second SIGTERM: abandoning module GOODBYE delivery");
3838                        escalated = true;
3839                    }
3840                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3841                }
3842            }
3843            let closed = forwarding.close_all_connections(&reason);
3844            debug!(closed, "closed established connections for daemon shutdown");
3845        }
3846        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3847        // already completed and must not be polled again; the child shutdown
3848        // wait is told it is escalated and gets a future that never fires.
3849        let escalated_here = escalated && !already_escalated;
3850        let remaining_escalate = async move {
3851            if escalated_here {
3852                std::future::pending::<()>().await;
3853            } else {
3854                escalate.await;
3855            }
3856        };
3857        crate::child_roster::end_children_for_daemon_shutdown(
3858            &self.child_roster,
3859            escalated,
3860            remaining_escalate,
3861        )
3862        .await;
3863    }
3864
3865    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3866        Self {
3867            registry,
3868            config_delivery: None,
3869            restart_policy,
3870            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3871            connection_file_path: None,
3872            capture_logs_dir: None,
3873            forwarding: None,
3874            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3875            supervisor_handle: None,
3876            health: HealthConfig::default(),
3877            host_health: HostHealth::default(),
3878            daemon_start_clock: crate::clock::StartClock::capture(),
3879            terminal_journal: None,
3880            spawn_events: SpawnEventFeed::default(),
3881            provenance_probe: ExecutableIdentityProbe::default(),
3882            child_roster: ChildRoster::default(),
3883            #[cfg(target_os = "linux")]
3884            cgroup_placement: None,
3885            #[cfg(test)]
3886            test_after_first_spawn: AfterFirstSpawnHook::default(),
3887        }
3888    }
3889
3890    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3891        self.drain_timeout = drain_timeout;
3892        self
3893    }
3894
3895    /// The drain budget a module gets when its config entry sets none.
3896    pub(crate) fn default_drain_timeout(&self) -> Duration {
3897        self.drain_timeout
3898    }
3899
3900    pub fn with_process_liveness(
3901        mut self,
3902        process_liveness: Arc<SupervisorProcessLiveness>,
3903    ) -> Self {
3904        self.process_liveness = process_liveness;
3905        self
3906    }
3907
3908    /// Connect configuration delivery to observed process lifecycle events.
3909    pub fn with_config_runtime(
3910        mut self,
3911        owner: Arc<crate::config_gate::runtime::RuntimeOwner>,
3912    ) -> Self {
3913        if let Some(handle) = &self.supervisor_handle {
3914            handle.install_binding_retirement(&owner);
3915        }
3916        self.config_delivery = Some(config_delivery::ConfigDelivery(owner));
3917        self
3918    }
3919
3920    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3921        self.connection_file_path = Some(connection_file_path.into());
3922        self
3923    }
3924
3925    /// Enables daemon-owned capture files for supervised stdout and stderr.
3926    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3927        self.capture_logs_dir = Some(logs_dir.into());
3928        self
3929    }
3930
3931    /// Names this daemon lifetime in spawn events, independently of whether a
3932    /// terminal journal is configured.
3933    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3934        // A millisecond start stamp can repeat after clock rollback or a rapid
3935        // restart. Use the connection file's random daemon_id instead: it already
3936        // identifies this daemon lifetime independently of the wall clock.
3937        self.spawn_events.configure_incarnation(daemon_incarnation);
3938        self
3939    }
3940
3941    /// Enables best-effort history shared by every supervised module. Without
3942    /// it, terminal history is kept only in each module's in-memory ring.
3943    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3944        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3945        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3946            path,
3947            daemon_incarnation,
3948        )));
3949        this
3950    }
3951
3952    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3953        self.child_roster.counters = forwarding.counters();
3954        self.forwarding = Some(forwarding);
3955        self
3956    }
3957
3958    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3959        if let Some(delivery) = &self.config_delivery {
3960            supervisor_handle.install_binding_retirement(&delivery.0);
3961        }
3962        self.host_health = supervisor_handle.host_health.clone();
3963        self.spawn_events = supervisor_handle.spawn_events.clone();
3964        self.supervisor_handle = Some(supervisor_handle);
3965        self
3966    }
3967
3968    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3969        self.health = health;
3970        self
3971    }
3972
3973    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3974    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3975    /// record is kept.
3976    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3977        self.child_roster.record_to(path.into());
3978        self
3979    }
3980
3981    #[cfg(target_os = "linux")]
3982    pub fn with_cgroup_placement(
3983        mut self,
3984        cgroup_placement: Option<subc_cgroup::Placement>,
3985    ) -> Self {
3986        self.child_roster
3987            .set_cgroup_placement(cgroup_placement.clone());
3988        self.cgroup_placement = cgroup_placement;
3989        self
3990    }
3991
3992    /// Spawn `spec.program` and start monitoring it.
3993    ///
3994    /// The child is expected to parse `--subc <connection-file-path>`, read the
3995    /// TCP+key connection file, authenticate to the already-running listener, and
3996    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3997    ///
3998    /// This synchronous embedder API blocks on launch admission and store I/O.
3999    /// Do not call it on an async runtime thread; use [`Self::spawn_async`].
4000    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4001        validate_spec(&spec)?;
4002        self.establish_identity(&spec);
4003
4004        let runtime = self.runtime_config();
4005        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4006        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4007        let registration_watch = watch_registration(&self.registry, &spec);
4008        let spawned = spawn_child(
4009            &spec,
4010            runtime.connection_file_path.as_deref(),
4011            self.supervisor_handle.as_ref(),
4012            &runtime.stderr_ring,
4013            runtime.capture_logs_dir.as_deref(),
4014            &runtime.child_roster,
4015            runtime.spawn_delivery(),
4016            #[cfg(target_os = "linux")]
4017            runtime.cgroup_placement.as_ref(),
4018            runtime.register_timeout,
4019        );
4020        #[cfg(test)]
4021        self.test_after_first_spawn.run(&spec.module_id);
4022        let mut child = match spawned {
4023            Ok(child) => child,
4024            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4025                defer_config_spawn(
4026                    &spec,
4027                    &runtime,
4028                    &snapshot,
4029                    RespawnKind::Spawn,
4030                    &runtime.failure_context(runtime.recovery_is_daemon),
4031                )?;
4032                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4033            }
4034            Err(SuperviseError::ConfigHeld { .. }) => {
4035                hold_config_spawn(
4036                    &spec,
4037                    &runtime,
4038                    &snapshot,
4039                    &runtime.failure_context(runtime.recovery_is_daemon),
4040                )?;
4041                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4042            }
4043            Err(err) => {
4044                // Unlike the configured paths, a failed `spawn` leaves nothing
4045                // on the roster, so the module must not stay marked configured.
4046                self.abandon_unrostered(&spec.module_id);
4047                return Err(err);
4048            }
4049        };
4050        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4051        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4052        if let Some(spawn) = activity_spawn {
4053            spawn.complete(&snapshot);
4054        }
4055
4056        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4057    }
4058
4059    /// Spawn without running configuration integrity or keychain checks on the
4060    /// runtime thread. Cancellation during preparation cannot create a child.
4061    pub async fn spawn_async(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4062        validate_spec(&spec)?;
4063        self.establish_identity(&spec);
4064        let mut identity = AsyncInitialIdentity {
4065            supervisor: self,
4066            module: spec.module_id.clone(),
4067            committed: false,
4068        };
4069        let runtime = self.runtime_config();
4070        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4071        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4072        let registration_watch = watch_registration(&self.registry, &spec);
4073        let spawned = spawn_child_async(&spec, &runtime).await;
4074        #[cfg(test)]
4075        self.test_after_first_spawn.run(&spec.module_id);
4076        let mut child = match spawned {
4077            Ok(child) => child,
4078            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4079                defer_config_spawn(
4080                    &spec,
4081                    &runtime,
4082                    &snapshot,
4083                    RespawnKind::Spawn,
4084                    &runtime.failure_context(runtime.recovery_is_daemon),
4085                )?;
4086                identity.committed = true;
4087                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4088            }
4089            Err(SuperviseError::ConfigHeld { .. }) => {
4090                hold_config_spawn(
4091                    &spec,
4092                    &runtime,
4093                    &snapshot,
4094                    &runtime.failure_context(runtime.recovery_is_daemon),
4095                )?;
4096                identity.committed = true;
4097                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4098            }
4099            Err(err) => {
4100                self.abandon_unrostered(&spec.module_id);
4101                return Err(err);
4102            }
4103        };
4104        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4105        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4106        if let Some(spawn) = activity_spawn {
4107            spawn.complete(&snapshot);
4108        }
4109        identity.committed = true;
4110        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4111    }
4112
4113    /// Make `spec`'s module count as configured, with its identity gates
4114    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
4115    /// exists.
4116    ///
4117    /// Every path that takes on a new module calls this before `spawn_child`.
4118    /// The order is the point: the child can connect, register, sync its
4119    /// scopes and ask about them as soon as it is spawned, and the module is
4120    /// only put on the roster after `spawn_child` returns. Were the mark set
4121    /// with the roster entry, a fast child would see its own owner reported
4122    /// as not configured, and a scoped `route.open` in that window would be
4123    /// refused as terminal `scope_not_live` ("will never sync") instead of
4124    /// retryable `scope_not_synced`.
4125    fn establish_identity(&self, spec: &ModuleSpec) {
4126        if let Some(supervisor_handle) = &self.supervisor_handle {
4127            supervisor_handle.apply_identity_configuration(spec);
4128            supervisor_handle.mark_configured(&spec.module_id);
4129        }
4130    }
4131
4132    /// Take back [`Self::establish_identity`]'s configured mark when the
4133    /// module will not be put on the roster after all.
4134    fn abandon_unrostered(&self, module_id: &str) {
4135        if let Some(supervisor_handle) = &self.supervisor_handle {
4136            supervisor_handle.unmark_configured_unless_rostered(module_id);
4137        }
4138    }
4139
4140    /// Record a freshly spawned first process as running. On failure the
4141    /// module never reaches the roster, so its configured mark is taken back.
4142    fn mark_first_process_running(
4143        &self,
4144        spec: &ModuleSpec,
4145        runtime: &SupervisorRuntimeConfig,
4146        snapshot: &SharedSnapshot,
4147        child: &mut SupervisedChild,
4148    ) -> Result<(), SuperviseError> {
4149        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
4150            containment::terminate_blocking(&runtime.child_roster.containment);
4151            self.abandon_unrostered(&spec.module_id);
4152            return Err(err);
4153        }
4154        self.process_liveness
4155            .track(spec.module_id.clone(), Arc::clone(snapshot));
4156        Ok(())
4157    }
4158
4159    /// Start supervising a module declared in daemon configuration.
4160    ///
4161    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
4162    /// failures in the supervisor handle so operator-facing `supervisor.list`
4163    /// reflects every configured module while daemon startup continues.
4164    pub fn supervise_configured(
4165        &self,
4166        spec: ModuleSpec,
4167        enabled: bool,
4168    ) -> Result<SupervisedModule, SuperviseError> {
4169        self.supervise_configured_requested(
4170            spec,
4171            enabled,
4172            self.health.clone(),
4173            None,
4174            self.restart_policy,
4175            DEFAULT_REGISTER_TIMEOUT,
4176            None,
4177        )
4178    }
4179
4180    /// Supervise a configured module with its own health, drain, and crash
4181    /// budget. The restart policy is per-module because the config file is:
4182    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
4183    /// module that is expensive to restart should not be forced onto the same
4184    /// budget as one that is cheap.
4185    pub fn supervise_configured_with_health(
4186        &self,
4187        spec: ModuleSpec,
4188        enabled: bool,
4189        health: HealthConfig,
4190        drain_timeout_ms: Option<u64>,
4191        restart_policy: RestartPolicy,
4192    ) -> Result<SupervisedModule, SuperviseError> {
4193        self.supervise_configured_requested(
4194            spec,
4195            enabled,
4196            health,
4197            drain_timeout_ms,
4198            restart_policy,
4199            DEFAULT_REGISTER_TIMEOUT,
4200            None,
4201        )
4202    }
4203
4204    #[allow(clippy::too_many_arguments)]
4205    pub(crate) fn supervise_configured_requested(
4206        &self,
4207        spec: ModuleSpec,
4208        enabled: bool,
4209        health: HealthConfig,
4210        drain_timeout_ms: Option<u64>,
4211        restart_policy: RestartPolicy,
4212        register_timeout: Duration,
4213        requester: Option<Principal>,
4214    ) -> Result<SupervisedModule, SuperviseError> {
4215        let registered = self.register_configured_with_health(
4216            spec,
4217            enabled,
4218            health,
4219            drain_timeout_ms,
4220            restart_policy,
4221            register_timeout,
4222        )?;
4223        let daemon_initiated = requester.is_none();
4224        self.start_registered_configured(registered, requester, daemon_initiated)
4225    }
4226
4227    #[allow(clippy::too_many_arguments)]
4228    pub(crate) async fn supervise_configured_requested_async(
4229        &self,
4230        spec: ModuleSpec,
4231        enabled: bool,
4232        health: HealthConfig,
4233        drain_timeout_ms: Option<u64>,
4234        restart_policy: RestartPolicy,
4235        register_timeout: Duration,
4236        requester: Option<Principal>,
4237    ) -> Result<SupervisedModule, SuperviseError> {
4238        let registered = self.register_configured_with_health(
4239            spec,
4240            enabled,
4241            health,
4242            drain_timeout_ms,
4243            restart_policy,
4244            register_timeout,
4245        )?;
4246        let daemon_initiated = requester.is_none();
4247        self.start_registered_configured_async(registered, requester, daemon_initiated)
4248            .await
4249    }
4250
4251    /// Record a configured module as starting or disabled, without launching a
4252    /// process or running its monitor. Bootstrap registers the entire roster
4253    /// before publishing discovery, then activates it once children can read
4254    /// the connection file passed in their `--subc` argument.
4255    pub(crate) fn register_configured_with_health(
4256        &self,
4257        spec: ModuleSpec,
4258        enabled: bool,
4259        health: HealthConfig,
4260        drain_timeout_ms: Option<u64>,
4261        restart_policy: RestartPolicy,
4262        register_timeout: Duration,
4263    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
4264        validate_spec(&spec)?;
4265        self.establish_identity(&spec);
4266
4267        let mut runtime = self.runtime_config();
4268        runtime.health = health;
4269        runtime.restart_policy = restart_policy;
4270        runtime.register_timeout = register_timeout;
4271        if let Some(ms) = drain_timeout_ms {
4272            runtime.drain_timeout = Duration::from_millis(ms);
4273            *runtime
4274                .effective_drain_timeout
4275                .lock()
4276                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
4277        }
4278        let snapshot = if enabled {
4279            SupervisorSnapshot::starting()
4280        } else {
4281            SupervisorSnapshot::disabled()
4282        };
4283        let snapshot = {
4284            let mut snapshot = snapshot;
4285            if enabled {
4286                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
4287                {
4288                    snapshot.waiting_for_previous_pid = Some(pid);
4289                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
4290                }
4291            }
4292            snapshot
4293        };
4294        let (module, commands) =
4295            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
4296        Ok(RegisteredConfiguredModule {
4297            spec,
4298            runtime,
4299            module,
4300            commands,
4301        })
4302    }
4303
4304    /// Activate a previously registered module. Queued control requests are
4305    /// served by the monitor after the initial configured spawn has settled.
4306    pub(crate) fn start_registered_configured(
4307        &self,
4308        registered: RegisteredConfiguredModule,
4309        requester: Option<Principal>,
4310        daemon_initiated: bool,
4311    ) -> Result<SupervisedModule, SuperviseError> {
4312        let RegisteredConfiguredModule {
4313            spec,
4314            mut runtime,
4315            module,
4316            commands,
4317        } = registered;
4318        runtime.start_requester = requester;
4319        runtime.recovery_is_daemon = daemon_initiated;
4320        let snapshot = &module.inner.snapshot;
4321        let should_spawn = {
4322            let state = lock_snapshot(snapshot)?;
4323            state.enabled && state.waiting_for_previous_pid.is_none()
4324        };
4325        let child = if should_spawn {
4326            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4327            let registration_watch = watch_registration(&self.registry, &spec);
4328            let spawned = spawn_child(
4329                &spec,
4330                runtime.connection_file_path.as_deref(),
4331                self.supervisor_handle.as_ref(),
4332                &runtime.stderr_ring,
4333                runtime.capture_logs_dir.as_deref(),
4334                &runtime.child_roster,
4335                runtime.spawn_delivery(),
4336                #[cfg(target_os = "linux")]
4337                runtime.cgroup_placement.as_ref(),
4338                runtime.register_timeout,
4339            );
4340            #[cfg(test)]
4341            self.test_after_first_spawn.run(&spec.module_id);
4342            match spawned {
4343                Ok(mut child) => {
4344                    arm_registration_deadline(
4345                        &mut child,
4346                        runtime.register_timeout,
4347                        registration_watch,
4348                    );
4349                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4350                    if let Some(spawn) = activity_spawn {
4351                        spawn.complete(snapshot);
4352                    }
4353                    Some(child)
4354                }
4355                Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4356                    defer_config_spawn(
4357                        &spec,
4358                        &runtime,
4359                        snapshot,
4360                        RespawnKind::Spawn,
4361                        &runtime.failure_context(daemon_initiated),
4362                    )?;
4363                    None
4364                }
4365                Err(mut err) => {
4366                    if matches!(err, SuperviseError::ConfigHeld { .. }) {
4367                        match hold_config_spawn(
4368                            &spec,
4369                            &runtime,
4370                            snapshot,
4371                            &runtime.failure_context(daemon_initiated),
4372                        ) {
4373                            Ok(()) => {
4374                                self.start_monitor(spec, runtime, &module, None, commands);
4375                                return Ok(module);
4376                            }
4377                            Err(parking_error) => err = parking_error,
4378                        }
4379                    }
4380                    if runtime.health.critical {
4381                        error!(
4382                            module_id = %spec.module_id,
4383                            program = %spec.program.display(),
4384                            error = %err,
4385                            "critical configured module failed to spawn; marking failed and alerting"
4386                        );
4387                    } else {
4388                        error!(
4389                            module_id = %spec.module_id,
4390                            program = %spec.program.display(),
4391                            error = %err,
4392                            "configured module failed to spawn; marking failed and continuing"
4393                        );
4394                    }
4395                    fail_snapshot(
4396                        snapshot,
4397                        Some(&spec.module_id),
4398                        None,
4399                        &runtime.failure_context(daemon_initiated),
4400                    );
4401                    None
4402                }
4403            }
4404        } else {
4405            None
4406        };
4407        self.start_monitor(spec, runtime, &module, child, commands);
4408        Ok(module)
4409    }
4410
4411    pub(crate) async fn start_registered_configured_async(
4412        &self,
4413        registered: RegisteredConfiguredModule,
4414        requester: Option<Principal>,
4415        daemon_initiated: bool,
4416    ) -> Result<SupervisedModule, SuperviseError> {
4417        self.start_registered_with_failure_policy(registered, requester, daemon_initiated, None)
4418            .await
4419    }
4420
4421    pub(crate) async fn start_registered_approved_async(
4422        &self,
4423        registered: RegisteredConfiguredModule,
4424        requester: Principal,
4425        generation: crate::config_gate::runtime::ApprovalIdentity,
4426    ) -> Result<SupervisedModule, SuperviseError> {
4427        self.start_registered_with_failure_policy(
4428            registered,
4429            Some(requester),
4430            false,
4431            Some(generation),
4432        )
4433        .await
4434    }
4435
4436    async fn start_registered_with_failure_policy(
4437        &self,
4438        registered: RegisteredConfiguredModule,
4439        requester: Option<Principal>,
4440        daemon_initiated: bool,
4441        approved_generation: Option<crate::config_gate::runtime::ApprovalIdentity>,
4442    ) -> Result<SupervisedModule, SuperviseError> {
4443        let RegisteredConfiguredModule {
4444            spec,
4445            mut runtime,
4446            module,
4447            commands,
4448        } = registered;
4449        runtime.start_requester = requester;
4450        runtime.recovery_is_daemon = daemon_initiated;
4451        let snapshot = &module.inner.snapshot;
4452        let mut startup_error = None;
4453        let failure = FailureContext {
4454            approval: approved_generation.clone(),
4455            ..runtime.failure_context(daemon_initiated)
4456        };
4457        let should_spawn = {
4458            let state = lock_snapshot(snapshot)?;
4459            state.enabled && state.waiting_for_previous_pid.is_none()
4460        };
4461        let child = if should_spawn {
4462            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4463            let registration_watch = watch_registration(&self.registry, &spec);
4464            let spawned = spawn_child_async(&spec, &runtime).await;
4465            #[cfg(test)]
4466            self.test_after_first_spawn.run(&spec.module_id);
4467            match spawned {
4468                Ok(mut child) => {
4469                    arm_registration_deadline(
4470                        &mut child,
4471                        runtime.register_timeout,
4472                        registration_watch,
4473                    );
4474                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4475                    if let Some(spawn) = activity_spawn {
4476                        spawn.complete(snapshot);
4477                    }
4478                    Some(child)
4479                }
4480                Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4481                    defer_config_spawn(&spec, &runtime, snapshot, RespawnKind::Spawn, &failure)?;
4482                    None
4483                }
4484                Err(mut err) => {
4485                    if matches!(err, SuperviseError::ConfigHeld { .. }) {
4486                        match hold_config_spawn(&spec, &runtime, snapshot, &failure) {
4487                            Ok(()) => {
4488                                self.start_monitor(spec, runtime, &module, None, commands);
4489                                return Ok(module);
4490                            }
4491                            Err(parking_error) => err = parking_error,
4492                        }
4493                    }
4494                    if runtime.health.critical {
4495                        error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4496                            "critical configured module failed to spawn; marking failed and alerting");
4497                    } else {
4498                        error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4499                            "configured module failed to spawn; marking failed and continuing");
4500                    }
4501                    fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
4502                    if let Some(generation) = approved_generation {
4503                        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4504                            state.approved_start_failure = Some((generation, err.to_string()));
4505                        });
4506                        startup_error = Some(err);
4507                    }
4508                    None
4509                }
4510            }
4511        } else {
4512            None
4513        };
4514        self.start_monitor(spec, runtime, &module, child, commands);
4515        // Applying an operator-approved configuration must report a spawn error
4516        // without removing the failed module or its monitor. Bootstrap instead
4517        // tolerates failed starts so other configured modules can still run.
4518        startup_error.map_or(Ok(module), Err)
4519    }
4520
4521    fn runtime_config(&self) -> SupervisorRuntimeConfig {
4522        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
4523        SupervisorRuntimeConfig {
4524            registry: self.registry.clone(),
4525            config_delivery: self.config_delivery.clone(),
4526            recovery_is_daemon: true,
4527            start_requester: None,
4528            pending_activity_restart: false,
4529            provenance_probe: self.provenance_probe.clone(),
4530            scheduled_respawn: Arc::default(),
4531            deferred_reload_reply: Arc::default(),
4532            restart_policy: self.restart_policy,
4533            register_timeout: DEFAULT_REGISTER_TIMEOUT,
4534            drain_timeout: self.drain_timeout,
4535            // Shared with this module's roster copy: daemon shutdown waits on
4536            // each child for the module's own drain budget, as resolved now.
4537            child_roster: self
4538                .child_roster
4539                .for_module(Arc::clone(&effective_drain_timeout)),
4540            effective_drain_timeout,
4541            default_drain_timeout: self.drain_timeout,
4542            health: self.health.clone(),
4543            host_health: self.host_health.clone(),
4544            connection_file_path: self.connection_file_path.clone(),
4545            capture_logs_dir: self.capture_logs_dir.clone(),
4546            forwarding: self.forwarding.clone(),
4547            supervisor_handle: self.supervisor_handle.clone(),
4548            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
4549            terminal_ring: Arc::new(Mutex::new(
4550                TerminalRing::new(
4551                    TerminalRingConfig::default(),
4552                    self.daemon_start_clock.started_at_ms(),
4553                )
4554                .with_start_clock(self.daemon_start_clock)
4555                .with_journal(self.terminal_journal.clone())
4556                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
4557            )),
4558            spawn_events: self.spawn_events.clone(),
4559            #[cfg(target_os = "linux")]
4560            cgroup_placement: self.cgroup_placement.clone(),
4561            #[cfg(test)]
4562            test_seed_stale_facts_before_enable_spawn: false,
4563            #[cfg(test)]
4564            test_reload_exit_record_gate: None,
4565        }
4566    }
4567
4568    fn supervised_module(
4569        &self,
4570        spec: ModuleSpec,
4571        runtime: SupervisorRuntimeConfig,
4572        snapshot: SharedSnapshot,
4573        child: Option<SupervisedChild>,
4574    ) -> SupervisedModule {
4575        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
4576        self.start_monitor(spec, runtime, &module, child, commands);
4577        module
4578    }
4579
4580    fn register_module(
4581        &self,
4582        spec: &ModuleSpec,
4583        runtime: &SupervisorRuntimeConfig,
4584        snapshot: SharedSnapshot,
4585    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
4586        runtime.host_health.track(spec, &snapshot);
4587        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
4588            spec: spec.clone(),
4589            enabled: snapshot.lock().unwrap_or_else(|p| p.into_inner()).enabled,
4590            health: runtime.health.clone(),
4591            register_timeout: runtime.register_timeout,
4592            restart_policy: runtime.restart_policy,
4593        }));
4594        let stderr_ring = Arc::clone(&runtime.stderr_ring);
4595        let terminal_ring = Arc::clone(&runtime.terminal_ring);
4596        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
4597        let (tx, rx) = mpsc::channel(4);
4598        let module_id = spec.module_id.clone();
4599        let module = SupervisedModule {
4600            inner: Arc::new(SupervisedModuleInner {
4601                module_id: module_id.clone(),
4602                registry: Arc::clone(&self.registry),
4603                snapshot,
4604                configuration,
4605                stderr_ring,
4606                terminal_ring,
4607                commands: tx,
4608                monitor: Mutex::new(None),
4609                effective_drain_timeout,
4610                provenance_probe: self.provenance_probe.clone(),
4611                retained: runtime.child_roster.retained_view(),
4612            }),
4613        };
4614        // Identity gates and the configured mark must precede the roster entry,
4615        // so a client never sees a module without its admission policy.
4616        if let Some(supervisor_handle) = &self.supervisor_handle {
4617            supervisor_handle.insert(module.clone());
4618        }
4619        (module, rx)
4620    }
4621
4622    fn start_monitor(
4623        &self,
4624        spec: ModuleSpec,
4625        runtime: SupervisorRuntimeConfig,
4626        module: &SupervisedModule,
4627        child: Option<SupervisedChild>,
4628        commands: mpsc::Receiver<SupervisorCommand>,
4629    ) {
4630        let monitor = tokio::spawn(supervise_loop(
4631            spec,
4632            runtime,
4633            Arc::clone(&self.registry),
4634            Arc::clone(&self.process_liveness),
4635            Arc::clone(&module.inner.snapshot),
4636            child,
4637            commands,
4638        ));
4639        *module
4640            .inner
4641            .monitor
4642            .lock()
4643            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
4644    }
4645}
4646
4647impl Default for Supervisor {
4648    fn default() -> Self {
4649        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
4650    }
4651}
4652
4653/// Handle to one supervised child process.
4654#[derive(Clone)]
4655pub struct SupervisedModule {
4656    inner: Arc<SupervisedModuleInner>,
4657}
4658
4659struct SupervisedModuleInner {
4660    retained: crate::child_roster::RetainedView,
4661    module_id: String,
4662    registry: Arc<Registry>,
4663    snapshot: SharedSnapshot,
4664    configuration: Arc<Mutex<SupervisedConfiguration>>,
4665    stderr_ring: Arc<Mutex<StderrRing>>,
4666    terminal_ring: Arc<Mutex<TerminalRing>>,
4667    commands: mpsc::Sender<SupervisorCommand>,
4668    monitor: Mutex<Option<JoinHandle<()>>>,
4669    effective_drain_timeout: Arc<Mutex<Duration>>,
4670    provenance_probe: ExecutableIdentityProbe,
4671}
4672
4673impl fmt::Debug for SupervisedModule {
4674    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4675        f.debug_struct("SupervisedModule")
4676            .field("module_id", &self.inner.module_id)
4677            .field("status", &self.status())
4678            .finish_non_exhaustive()
4679    }
4680}
4681
4682impl SupervisedModule {
4683    pub(crate) fn check_approved_start_failure(
4684        &self,
4685        generation: &crate::config_gate::runtime::ApprovalIdentity,
4686    ) -> Result<bool, SuperviseError> {
4687        let state = lock_snapshot(&self.inner.snapshot)?;
4688        if let Some((failed_generation, detail)) = &state.approved_start_failure {
4689            if failed_generation == generation {
4690                return Err(SuperviseError::ReloadFailed {
4691                    module_id: self.inner.module_id.clone(),
4692                    reason: detail.clone(),
4693                });
4694            }
4695        }
4696        Ok(state.approved_start_failure.is_some())
4697    }
4698
4699    pub(crate) fn has_approved_start_failure(
4700        &self,
4701        identity: &crate::config_gate::runtime::ApprovalIdentity,
4702    ) -> bool {
4703        lock_snapshot(&self.inner.snapshot).is_ok_and(|state| {
4704            state
4705                .approved_start_failure
4706                .as_ref()
4707                .is_some_and(|(failed, _)| failed == identity)
4708        })
4709    }
4710
4711    pub(crate) fn needs_approved_gate_restart(
4712        &self,
4713        approval: &crate::config_gate::runtime::ApprovalIdentity,
4714    ) -> bool {
4715        lock_snapshot(&self.inner.snapshot)
4716            .is_ok_and(|state| state.needs_approved_gate_restart(approval))
4717    }
4718
4719    #[cfg(all(test, unix))]
4720    pub(crate) fn config_hold_origin_for_test(
4721        &self,
4722    ) -> (bool, Option<crate::config_gate::runtime::ApprovalIdentity>) {
4723        let state = lock_snapshot(&self.inner.snapshot).unwrap();
4724        (
4725            state.config_hold_daemon_initiated,
4726            state.config_hold_approval.clone(),
4727        )
4728    }
4729
4730    pub(crate) async fn config_policy_probe(
4731        &self,
4732        connection: crate::ConnectionId,
4733        generation: u64,
4734        attempt: u64,
4735    ) -> Result<(), SuperviseError> {
4736        self.inner
4737            .commands
4738            .send(SupervisorCommand::ConfigPolicyProbe {
4739                connection,
4740                generation,
4741                attempt,
4742            })
4743            .await
4744            .map_err(|_| SuperviseError::CommandClosed {
4745                module_id: self.inner.module_id.clone(),
4746            })
4747    }
4748    pub(crate) async fn config_exit_hold(&self) -> Result<(), SuperviseError> {
4749        let (reply, result) = oneshot::channel();
4750        self.inner
4751            .commands
4752            .send(SupervisorCommand::ConfigExitHold { reply })
4753            .await
4754            .map_err(|_| SuperviseError::CommandClosed {
4755                module_id: self.module_id().to_string(),
4756            })?;
4757        result.await.map_err(|_| SuperviseError::CommandClosed {
4758            module_id: self.module_id().to_string(),
4759        })?
4760    }
4761    pub(crate) async fn config_delivery_failed(
4762        &self,
4763        connection: crate::ConnectionId,
4764        hold: bool,
4765    ) -> Result<(), SuperviseError> {
4766        self.inner
4767            .commands
4768            .send(SupervisorCommand::ConfigDeliveryFailed { connection, hold })
4769            .await
4770            .map_err(|_| SuperviseError::CommandClosed {
4771                module_id: self.module_id().to_string(),
4772            })
4773    }
4774
4775    pub(crate) async fn config_apply_restart(&self, generation: u64) -> Result<(), SuperviseError> {
4776        self.inner
4777            .commands
4778            .send(SupervisorCommand::ConfigApplyRestart {
4779                generation,
4780                supersedes_swap: false,
4781            })
4782            .await
4783            .map_err(|_| SuperviseError::CommandClosed {
4784                module_id: self.inner.module_id.clone(),
4785            })
4786    }
4787    pub fn module_id(&self) -> &str {
4788        &self.inner.module_id
4789    }
4790
4791    /// Test-only: put one probe miss on the streak, the way
4792    /// `handle_health_probe_failure` does, so tests can assert what a later
4793    /// event does to the streak without driving the whole probe loop.
4794    #[cfg(test)]
4795    pub(crate) fn record_health_probe_failure_for_test(
4796        &self,
4797        detail: &str,
4798    ) -> Result<(), SuperviseError> {
4799        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
4800            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
4801            state.health.detail = Some(detail.to_string());
4802        })
4803    }
4804
4805    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
4806        Ok(lock_snapshot(&self.inner.snapshot)?.state)
4807    }
4808
4809    pub(crate) fn registration_end_reason(
4810        &self,
4811    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
4812        let snapshot = lock_snapshot(&self.inner.snapshot)?;
4813        Ok(match snapshot.state {
4814            ModuleState::Draining if snapshot.draining_to_replace => {
4815                Some(RegistrationEndReason::SupervisorRestart)
4816            }
4817            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
4818            ModuleState::Restarting if snapshot.draining_to_replace => {
4819                Some(RegistrationEndReason::SupervisorRestart)
4820            }
4821            _ => None,
4822        })
4823    }
4824
4825    /// The module's retained stderr, newest lines last.
4826    ///
4827    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
4828    /// module, `supervisor.list` renders every module, and putting it in the
4829    /// shared snapshot would make each status read carry a payload almost nobody
4830    /// asked for. Callers that want the text ask for it.
4831    pub fn stderr_tail(
4832        &self,
4833        max_lines: Option<usize>,
4834        max_bytes: Option<usize>,
4835    ) -> StderrTailSnapshot {
4836        self.inner
4837            .stderr_ring
4838            .lock()
4839            .unwrap_or_else(|poisoned| poisoned.into_inner())
4840            .snapshot(max_lines, max_bytes)
4841    }
4842
4843    /// The module's bounded terminal history, oldest retained exit first.
4844    ///
4845    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
4846    /// daemon whose in-memory history was necessarily reset.
4847    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
4848        self.inner
4849            .terminal_ring
4850            .lock()
4851            .unwrap_or_else(|poisoned| poisoned.into_inner())
4852            .snapshot()
4853    }
4854
4855    /// Retained observations from the current ring and all journal generations.
4856    ///
4857    /// Blocking: this reads the journal files. Async callers use
4858    /// `Self::read_durable_terminal_history`.
4859    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
4860        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
4861    }
4862
4863    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
4864    /// read (up to every retained generation) never occupies a runtime worker.
4865    /// Fails only if the blocking task could not finish (runtime shutdown or a
4866    /// panic in the read).
4867    pub(crate) async fn read_durable_terminal_history(
4868        &self,
4869    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
4870        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
4871        let module_id = self.inner.module_id.clone();
4872        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
4873            .await
4874    }
4875
4876    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
4877        self.status_with_snapshot_lock(&self.inner.snapshot, None)
4878            .map(|(status, _)| status)
4879    }
4880
4881    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
4882        let state = lock_snapshot(&self.inner.snapshot)?;
4883        Ok(state
4884            .failed_retry
4885            .pending
4886            .filter(|_| state.state == ModuleState::Failed && state.enabled)
4887            .map(|retry| {
4888                unix_ms_now().saturating_add(
4889                    u64::try_from(
4890                        retry
4891                            .deadline
4892                            .saturating_duration_since(Instant::now())
4893                            .as_millis(),
4894                    )
4895                    .unwrap_or(u64::MAX),
4896                )
4897            }))
4898    }
4899
4900    #[cfg(all(test, unix))]
4901    pub(crate) fn retry_deadline_for_test(&self) -> Option<Instant> {
4902        lock_snapshot(&self.inner.snapshot)
4903            .unwrap()
4904            .failed_retry
4905            .pending
4906            .map(|retry| retry.deadline)
4907    }
4908
4909    pub(crate) fn record_deliberate_severance(
4910        &self,
4911        identity: ProcessIdentity,
4912    ) -> Result<bool, SuperviseError> {
4913        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4914        if snapshot.pid != Some(identity.pid)
4915            || snapshot.process_start_time != Some(identity.start_time)
4916        {
4917            return Ok(false);
4918        }
4919        snapshot.deliberate_severance = Some(identity);
4920        Ok(true)
4921    }
4922
4923    /// Read status for a channel-0 renderer and report a contended snapshot lock.
4924    ///
4925    /// Internal supervision callers use [`Self::status`] so writer-side machinery
4926    /// does not produce reader-observability logs.
4927    pub(crate) fn status_for_control(
4928        &self,
4929        caller: &'static str,
4930    ) -> Result<ModuleStatus, SuperviseError> {
4931        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
4932            .map(|(status, _)| status)
4933    }
4934
4935    fn status_with_snapshot_lock(
4936        &self,
4937        snapshot: &SharedSnapshot,
4938        caller: Option<&'static str>,
4939    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
4940        let restart_policy = self
4941            .inner
4942            .configuration
4943            .lock()
4944            .map_err(|_| SuperviseError::StatePoisoned {
4945                module_id: Some(self.inner.module_id.clone()),
4946            })?
4947            .restart_policy;
4948        let mut guard = match caller {
4949            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
4950            None => lock_snapshot(snapshot)?,
4951        };
4952        // Read the budget through the pruning path so a reader sees the same
4953        // in-window count the restart decision would use, not a stale total.
4954        let restart_count = guard.crash_restarts_in_window(restart_policy.window, Instant::now());
4955        let snapshot = guard.clone();
4956        drop(guard);
4957        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
4958            SuperviseError::StatePoisoned {
4959                module_id: Some(self.inner.module_id.clone()),
4960            }
4961        })?;
4962        let registration_active = self
4963            .inner
4964            .registry
4965            .get_module(&self.inner.module_id)
4966            .map_err(SuperviseError::Registry)?
4967            .is_some();
4968        let protocol = snapshot
4969            .spawned_protocol
4970            .unwrap_or(self.declared_protocol()?);
4971        let running_process =
4972            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
4973        // Registration is the difference between the two protocols and the only
4974        // one: a subc module that has not registered cannot serve a request even
4975        // though its process is up, and a `none` module never registers at all,
4976        // so requiring it there would pin `live` to false for the whole life of
4977        // a perfectly healthy process.
4978        let live = match protocol {
4979            ModuleProtocol::Subc => running_process && registration_active,
4980            ModuleProtocol::None => running_process,
4981        };
4982        // Read only the retained cache: a status read must not resend signals,
4983        // rewrite the record, or keep the containment resource alive.
4984        let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
4985        let carry_configured = self
4986            .inner
4987            .configuration
4988            .lock()
4989            .map_err(|_| SuperviseError::StatePoisoned {
4990                module_id: Some(self.inner.module_id.clone()),
4991            })?
4992            .spec
4993            .carry_containment;
4994        let recovery_block = carries_containment(
4995            carry_configured,
4996            self.inner
4997                .retained
4998                .has_carried_container(&self.inner.module_id),
4999        )
5000        .then(|| self.inner.retained.recovery_block())
5001        .flatten();
5002        let launch_nonce_source = observed_launch_nonce_source(&snapshot);
5003        let pid = snapshot.reported_pid();
5004        let mut health = snapshot.health;
5005        if let Some(detail) = recovery_block
5006            .as_ref()
5007            .or_else(|| retained.as_ref().map(|(_, detail)| detail))
5008        {
5009            health.detail = Some(match health.detail {
5010                Some(previous) => format!("{previous}; {detail}"),
5011                None => detail.clone(),
5012            });
5013        }
5014
5015        Ok((
5016            ModuleStatus {
5017                admission_refused: snapshot.admission_refused,
5018                legacy_registration_spawn: snapshot.legacy_registration_spawn,
5019                registration_declaration: snapshot.registration_declaration.clone(),
5020                stop_reason: snapshot.stop_reason,
5021                waiting_for_previous_pid: retained
5022                    .as_ref()
5023                    .map(|(pid, _)| *pid)
5024                    .filter(|pid| *pid > 0)
5025                    .or(snapshot.waiting_for_previous_pid),
5026                waiting_for_previous_detail: recovery_block
5027                    .or_else(|| retained.map(|(_, detail)| detail))
5028                    .or(snapshot.waiting_for_previous_detail),
5029                module_id: self.inner.module_id.clone(),
5030                launch_nonce_source,
5031                state: snapshot.state,
5032                enabled: snapshot.enabled,
5033                process_alive: snapshot.process_alive,
5034                registration_active,
5035                protocol,
5036                live,
5037                restart_count,
5038                lifetime_restarts: snapshot.lifetime_restarts,
5039                spawn_generation: snapshot.spawn_generation,
5040                swap_outcome: snapshot.swap_outcome,
5041                max_restarts: restart_policy.max_restarts,
5042                restart_window: restart_policy.window,
5043                drain_timeout,
5044                restart_backoff: restart_policy.backoff,
5045                restart_max_backoff: restart_policy.max_backoff,
5046                pid,
5047                spawned_at_ms: snapshot.spawned_at_ms,
5048                spawned_from: snapshot.spawned_from,
5049                launch_spec_digest: snapshot.launch_spec_digest,
5050                process_start_time: snapshot.process_start_time,
5051                last_exit: snapshot.last_exit,
5052                health,
5053            },
5054            snapshot.spawned_file_identity.clone(),
5055        ))
5056    }
5057
5058    #[cfg(test)]
5059    pub(crate) fn hold_snapshot_for_test(
5060        &self,
5061        acquired: std::sync::mpsc::Sender<()>,
5062        hold: Duration,
5063    ) -> std::thread::JoinHandle<()> {
5064        let snapshot = Arc::clone(&self.inner.snapshot);
5065        std::thread::spawn(move || {
5066            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
5067            acquired
5068                .send(())
5069                .expect("test receiver waits for snapshot lock");
5070            std::thread::sleep(hold);
5071        })
5072    }
5073
5074    /// The status and the running-image check for `supervisor.provenance`,
5075    /// taken from one status read. The exec acknowledgement can land between
5076    /// two separate reads, and the reply would then pair "no pid yet" with an
5077    /// image observed after the module started, which describes no single
5078    /// moment.
5079    pub(crate) async fn status_and_running_image_agreement(
5080        &self,
5081    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
5082        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
5083        let image = self
5084            .inner
5085            .provenance_probe
5086            .observe(
5087                status.pid,
5088                status.spawned_from.as_deref(),
5089                identity,
5090                status.process_start_time,
5091            )
5092            .await;
5093        Ok((status, image))
5094    }
5095
5096    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
5097        let snapshot = match lock_snapshot(&self.inner.snapshot) {
5098            Ok(snapshot) => snapshot.clone(),
5099            Err(_) => {
5100                return subc_control::RunningImageAgreement::Unavailable {
5101                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
5102                };
5103            }
5104        };
5105        self.inner
5106            .provenance_probe
5107            .observe(
5108                snapshot.reported_pid(),
5109                snapshot.spawned_from.as_deref(),
5110                snapshot.spawned_file_identity.clone(),
5111                snapshot.process_start_time,
5112            )
5113            .await
5114    }
5115
5116    /// Memory and CPU time of the module's current process, read now. Only the
5117    /// process the supervisor spawned is read, not processes it has started.
5118    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
5119        #[cfg(windows)]
5120        {
5121            let image = lock_snapshot(&self.inner.snapshot)
5122                .ok()
5123                .and_then(|snapshot| {
5124                    snapshot
5125                        .spawned_file_identity
5126                        .as_ref()
5127                        .and_then(|identity| identity.windows_image.clone())
5128                });
5129            match image {
5130                Some(image) => {
5131                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
5132                }
5133                None => subc_control::ChildResourceUsage::Unavailable {
5134                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
5135                },
5136            }
5137        }
5138        #[cfg(not(windows))]
5139        {
5140            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
5141                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
5142                Err(_) => {
5143                    return subc_control::ChildResourceUsage::Unavailable {
5144                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
5145                    }
5146                }
5147            };
5148            crate::child_resources::read(pid, start_time)
5149        }
5150    }
5151
5152    #[cfg(all(test, any(target_os = "linux", windows)))]
5153    pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
5154        self.inner
5155            .snapshot
5156            .try_lock()
5157            .map(|snapshot| snapshot.subtree_target.clone())
5158            .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
5159    }
5160
5161    #[cfg(test)]
5162    pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
5163        self.inner.snapshot.lock().unwrap().subtree_target = target;
5164    }
5165
5166    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
5167        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
5168        Ok(match snapshot.state {
5169            ModuleState::Restarting => true,
5170            ModuleState::Failed | ModuleState::Disabled => false,
5171            _ if snapshot.enabled && snapshot.respawn_pending => true,
5172            _ => daemon_will_restart(
5173                &mut snapshot,
5174                &self
5175                    .inner
5176                    .configuration
5177                    .lock()
5178                    .map_err(|_| SuperviseError::StatePoisoned {
5179                        module_id: Some(self.inner.module_id.clone()),
5180                    })?
5181                    .restart_policy,
5182                Instant::now(),
5183            ),
5184        })
5185    }
5186
5187    #[cfg(test)]
5188    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
5189        self.is_warming_with_snapshot_lock(None)
5190    }
5191
5192    pub(crate) fn is_warming_for_control(
5193        &self,
5194        caller: &'static str,
5195    ) -> Result<bool, SuperviseError> {
5196        self.is_warming_with_snapshot_lock(Some(caller))
5197    }
5198
5199    fn is_warming_with_snapshot_lock(
5200        &self,
5201        caller: Option<&'static str>,
5202    ) -> Result<bool, SuperviseError> {
5203        let snapshot = match caller {
5204            Some(caller) => {
5205                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
5206            }
5207            None => lock_snapshot(&self.inner.snapshot)?,
5208        }
5209        .clone();
5210        Ok(matches!(
5211            snapshot.state,
5212            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
5213        ))
5214    }
5215
5216    /// Drain the module and stop monitoring it.
5217    pub async fn drain(&self) -> Result<(), SuperviseError> {
5218        self.stop().await
5219    }
5220
5221    pub(crate) async fn retire_requested(
5222        &self,
5223        requester: Option<Principal>,
5224    ) -> Result<(), SuperviseError> {
5225        match self.state()? {
5226            ModuleState::Stopped => return Ok(()),
5227            ModuleState::Failed
5228                if lock_snapshot(&self.inner.snapshot)?
5229                    .failed_retry
5230                    .pending
5231                    .is_none() =>
5232            {
5233                return Ok(())
5234            }
5235            ModuleState::Failed
5236            | ModuleState::Starting
5237            | ModuleState::Running
5238            | ModuleState::Unresponsive
5239            | ModuleState::Restarting
5240            | ModuleState::Draining
5241            | ModuleState::Disabled => {}
5242        }
5243
5244        let (reply_tx, reply_rx) = oneshot::channel();
5245        self.inner
5246            .commands
5247            .send(SupervisorCommand::Retire {
5248                requester,
5249                reply: reply_tx,
5250            })
5251            .await
5252            .map_err(|_| SuperviseError::CommandClosed {
5253                module_id: self.inner.module_id.clone(),
5254            })?;
5255        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5256            module_id: self.inner.module_id.clone(),
5257        })?
5258    }
5259
5260    pub async fn stop(&self) -> Result<(), SuperviseError> {
5261        self.stop_requested(None).await
5262    }
5263
5264    pub(crate) async fn stop_requested(
5265        &self,
5266        requester: Option<Principal>,
5267    ) -> Result<(), SuperviseError> {
5268        match self.state()? {
5269            ModuleState::Stopped => return Ok(()),
5270            ModuleState::Failed
5271                if lock_snapshot(&self.inner.snapshot)?
5272                    .failed_retry
5273                    .pending
5274                    .is_none() =>
5275            {
5276                return Ok(())
5277            }
5278            ModuleState::Failed
5279            | ModuleState::Starting
5280            | ModuleState::Running
5281            | ModuleState::Unresponsive
5282            | ModuleState::Restarting
5283            | ModuleState::Draining
5284            | ModuleState::Disabled => {}
5285        }
5286
5287        let (reply_tx, reply_rx) = oneshot::channel();
5288        self.inner
5289            .commands
5290            .send(SupervisorCommand::Drain {
5291                requester,
5292                reply: reply_tx,
5293            })
5294            .await
5295            .map_err(|_| SuperviseError::CommandClosed {
5296                module_id: self.inner.module_id.clone(),
5297            })?;
5298        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5299            module_id: self.inner.module_id.clone(),
5300        })?
5301    }
5302
5303    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
5304        self.restart_requested(drain_timeout_ms, None).await
5305    }
5306
5307    pub(crate) async fn restart_requested(
5308        &self,
5309        drain_timeout_ms: Option<u64>,
5310        requester: Option<Principal>,
5311    ) -> Result<(), SuperviseError> {
5312        let queued_at = Instant::now();
5313        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
5314        let (reply_tx, reply_rx) = oneshot::channel();
5315        self.inner
5316            .commands
5317            .send(SupervisorCommand::Restart {
5318                requester,
5319                drain_timeout_ms,
5320                received_at_generation,
5321                queued_at,
5322                reply: reply_tx,
5323            })
5324            .await
5325            .map_err(|_| SuperviseError::CommandClosed {
5326                module_id: self.inner.module_id.clone(),
5327            })?;
5328        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5329            module_id: self.inner.module_id.clone(),
5330        })?
5331    }
5332
5333    /// Blue/green restart: see `SupervisorCommand::Swap` and the
5334    /// `supervisor_swap` module. Returns once the swap has cut over (the old
5335    /// process then drains in the background of the supervise loop), has been
5336    /// superseded by an approved configuration restart, or has failed. Both
5337    /// supersession and failure leave the old process serving until replacement.
5338    pub async fn swap(
5339        &self,
5340        ready_timeout: Option<Duration>,
5341    ) -> Result<SwapOutcome, SuperviseError> {
5342        self.swap_requested(ready_timeout, None).await
5343    }
5344
5345    pub(crate) async fn swap_requested(
5346        &self,
5347        ready_timeout: Option<Duration>,
5348        requester: Option<Principal>,
5349    ) -> Result<SwapOutcome, SuperviseError> {
5350        let (reply_tx, reply_rx) = oneshot::channel();
5351        self.inner
5352            .commands
5353            .send(SupervisorCommand::Swap {
5354                requester,
5355                ready_timeout,
5356                reply: reply_tx,
5357            })
5358            .await
5359            .map_err(|_| SuperviseError::CommandClosed {
5360                module_id: self.inner.module_id.clone(),
5361            })?;
5362        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5363            module_id: self.inner.module_id.clone(),
5364        })?
5365    }
5366
5367    pub async fn reload(&self) -> Result<(), SuperviseError> {
5368        self.reload_requested(None).await
5369    }
5370
5371    pub(crate) async fn reload_requested(
5372        &self,
5373        requester: Option<Principal>,
5374    ) -> Result<(), SuperviseError> {
5375        let (reply_tx, reply_rx) = oneshot::channel();
5376        self.inner
5377            .commands
5378            .send(SupervisorCommand::Reload {
5379                requester,
5380                reply: reply_tx,
5381            })
5382            .await
5383            .map_err(|_| SuperviseError::CommandClosed {
5384                module_id: self.inner.module_id.clone(),
5385            })?;
5386        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5387            module_id: self.inner.module_id.clone(),
5388        })?
5389    }
5390
5391    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
5392        self.set_enabled_requested(enabled, None).await
5393    }
5394
5395    pub(crate) async fn set_enabled_requested(
5396        &self,
5397        enabled: bool,
5398        requester: Option<Principal>,
5399    ) -> Result<bool, SuperviseError> {
5400        self.set_enabled_with_budget_policy(enabled, requester, true)
5401            .await
5402    }
5403
5404    pub(crate) async fn release_config_hold(&self) -> Result<bool, SuperviseError> {
5405        self.set_enabled_with_budget_policy(true, None, false).await
5406    }
5407
5408    #[cfg(all(test, unix))]
5409    pub(crate) fn seed_crash_budget_for_test(&self, count: u32) {
5410        let policy = self.inner.configuration.lock().unwrap().restart_policy;
5411        let mut state = self.inner.snapshot.lock().unwrap();
5412        for _ in 0..count {
5413            state.record_crash_restart(&policy, Instant::now());
5414        }
5415    }
5416
5417    async fn set_enabled_with_budget_policy(
5418        &self,
5419        enabled: bool,
5420        requester: Option<Principal>,
5421        reset_budget: bool,
5422    ) -> Result<bool, SuperviseError> {
5423        let (reply_tx, reply_rx) = oneshot::channel();
5424        self.inner
5425            .commands
5426            .send(SupervisorCommand::SetEnabled {
5427                requester,
5428                enabled,
5429                reset_budget,
5430                reply: reply_tx,
5431            })
5432            .await
5433            .map_err(|_| SuperviseError::CommandClosed {
5434                module_id: self.inner.module_id.clone(),
5435            })?;
5436        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5437            module_id: self.inner.module_id.clone(),
5438        })?
5439    }
5440
5441    /// The current process's protocol, or the configured protocol when down.
5442    /// A rescan stores the next launch spec without changing how an existing
5443    /// process registers, serves routes, is probed, or exits.
5444    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
5445        let configured = self
5446            .inner
5447            .configuration
5448            .lock()
5449            .map_err(|_| SuperviseError::StatePoisoned {
5450                module_id: Some(self.inner.module_id.clone()),
5451            })?
5452            .spec
5453            .protocol;
5454        let state = lock_snapshot(&self.inner.snapshot)?;
5455        Ok(state.spawned_protocol.unwrap_or(configured))
5456    }
5457
5458    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
5459        let configuration =
5460            self.inner
5461                .configuration
5462                .lock()
5463                .map_err(|_| SuperviseError::StatePoisoned {
5464                    module_id: Some(self.inner.module_id.clone()),
5465                })?;
5466        Ok((configuration.spec.clone(), configuration.health.clone()))
5467    }
5468
5469    pub(crate) fn configured_register_timeout(&self) -> Result<Duration, SuperviseError> {
5470        self.inner
5471            .configuration
5472            .lock()
5473            .map(|configuration| configuration.register_timeout)
5474            .map_err(|_| SuperviseError::StatePoisoned {
5475                module_id: Some(self.inner.module_id.clone()),
5476            })
5477    }
5478
5479    pub(crate) fn configured_enabled(&self) -> Result<bool, SuperviseError> {
5480        self.inner
5481            .configuration
5482            .lock()
5483            .map(|configuration| configuration.enabled)
5484            .map_err(|_| SuperviseError::StatePoisoned {
5485                module_id: Some(self.inner.module_id.clone()),
5486            })
5487    }
5488
5489    /// Replace this module's launch spec, keeping its health and drain policy,
5490    /// the way a rescan does for a changed config entry. The running process is
5491    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
5492    #[cfg(any(test, feature = "test-support"))]
5493    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
5494        let (_, health) = self.configuration()?;
5495        let drain_timeout_ms = u64::try_from(
5496            self.inner
5497                .effective_drain_timeout
5498                .lock()
5499                .unwrap_or_else(|poisoned| poisoned.into_inner())
5500                .as_millis(),
5501        )
5502        .ok();
5503        self.update_configuration(spec, health, drain_timeout_ms)
5504            .await
5505    }
5506
5507    pub(crate) async fn reconcile_approved_configuration(
5508        &self,
5509        approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5510    ) -> Result<(), SuperviseError> {
5511        let entry = approved
5512            .module_entry(&self.inner.module_id)
5513            .map_err(|error| SuperviseError::InvalidSpec {
5514                reason: error.to_string(),
5515            })?;
5516        validate_spec(&entry.module_spec())?;
5517        let (reply, received) = oneshot::channel();
5518        self.inner
5519            .commands
5520            .send(SupervisorCommand::ApprovedConfiguration { approved, reply })
5521            .await
5522            .map_err(|_| SuperviseError::CommandClosed {
5523                module_id: self.inner.module_id.clone(),
5524            })?;
5525        received.await.map_err(|_| SuperviseError::CommandClosed {
5526            module_id: self.inner.module_id.clone(),
5527        })??;
5528        let mut configuration =
5529            self.inner
5530                .configuration
5531                .lock()
5532                .map_err(|_| SuperviseError::StatePoisoned {
5533                    module_id: Some(self.inner.module_id.clone()),
5534                })?;
5535        configuration.spec = entry.module_spec();
5536        configuration.enabled = entry.enabled;
5537        configuration.health = entry.health;
5538        configuration.register_timeout = entry.register_timeout;
5539        configuration.restart_policy = entry.restart;
5540        Ok(())
5541    }
5542
5543    pub(crate) async fn update_configuration(
5544        &self,
5545        spec: ModuleSpec,
5546        health: HealthConfig,
5547        drain_timeout_ms: Option<u64>,
5548    ) -> Result<(), SuperviseError> {
5549        if spec.module_id != self.inner.module_id {
5550            return Err(SuperviseError::InvalidSpec {
5551                reason: "a supervised module's module_id cannot be changed".to_string(),
5552            });
5553        }
5554        validate_spec(&spec)?;
5555        let (reply_tx, reply_rx) = oneshot::channel();
5556        self.inner
5557            .commands
5558            .send(SupervisorCommand::UpdateConfiguration {
5559                spec: spec.clone(),
5560                health: health.clone(),
5561                drain_timeout_ms,
5562                reply: reply_tx,
5563            })
5564            .await
5565            .map_err(|_| SuperviseError::CommandClosed {
5566                module_id: self.inner.module_id.clone(),
5567            })?;
5568        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5569            module_id: self.inner.module_id.clone(),
5570        })??;
5571        let mut configuration =
5572            self.inner
5573                .configuration
5574                .lock()
5575                .map_err(|_| SuperviseError::StatePoisoned {
5576                    module_id: Some(self.inner.module_id.clone()),
5577                })?;
5578        configuration.spec = spec;
5579        configuration.health = health;
5580        Ok(())
5581    }
5582}
5583
5584impl Drop for SupervisedModuleInner {
5585    fn drop(&mut self) {
5586        let Ok(mut monitor) = self.monitor.lock() else {
5587            return;
5588        };
5589        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
5590            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
5591                state.state = ModuleState::Stopped;
5592                clear_current_process_facts(state);
5593            });
5594            monitor.abort();
5595        }
5596        let _ = monitor.take();
5597    }
5598}
5599
5600#[derive(Debug)]
5601enum SupervisorCommand {
5602    ApprovedConfiguration {
5603        approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5604        reply: oneshot::Sender<Result<(), SuperviseError>>,
5605    },
5606    ConfigPolicyProbe {
5607        connection: crate::ConnectionId,
5608        generation: u64,
5609        attempt: u64,
5610    },
5611    ConfigApplyRestart {
5612        generation: u64,
5613        supersedes_swap: bool,
5614    },
5615    ConfigExitHold {
5616        reply: oneshot::Sender<Result<(), SuperviseError>>,
5617    },
5618    ConfigDeliveryFailed {
5619        connection: crate::ConnectionId,
5620        hold: bool,
5621    },
5622    Drain {
5623        requester: Option<Principal>,
5624        reply: oneshot::Sender<Result<(), SuperviseError>>,
5625    },
5626    Retire {
5627        requester: Option<Principal>,
5628        reply: oneshot::Sender<Result<(), SuperviseError>>,
5629    },
5630    Restart {
5631        requester: Option<Principal>,
5632        /// Operator override for this one restart's drain budget, in ms. `None`
5633        /// uses the module's configured/default budget; `Some(0)` cuts
5634        /// immediately (wedge bounce: a stuck request never settles, so
5635        /// waiting only delays recovery).
5636        drain_timeout_ms: Option<u64>,
5637        /// The module's `spawn_generation` when the request was received, before
5638        /// it waited in the command queue. A queued restart whose module has
5639        /// since spawned a newer process is already satisfied (see the handler).
5640        received_at_generation: u64,
5641        /// When the request entered the command queue, so the handler can log
5642        /// how long it waited behind the loop's other work.
5643        queued_at: Instant,
5644        reply: oneshot::Sender<Result<(), SuperviseError>>,
5645    },
5646    Reload {
5647        requester: Option<Principal>,
5648        reply: oneshot::Sender<Result<(), SuperviseError>>,
5649    },
5650    SetEnabled {
5651        reset_budget: bool,
5652        requester: Option<Principal>,
5653        enabled: bool,
5654        reply: oneshot::Sender<Result<bool, SuperviseError>>,
5655    },
5656    UpdateConfiguration {
5657        spec: ModuleSpec,
5658        health: HealthConfig,
5659        /// Per-module drain override from the new config; `None` re-resolves to
5660        /// the supervisor-wide default.
5661        drain_timeout_ms: Option<u64>,
5662        reply: oneshot::Sender<Result<(), SuperviseError>>,
5663    },
5664    Swap {
5665        requester: Option<Principal>,
5666        /// How long the candidate may take to register and declare itself
5667        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
5668        ready_timeout: Option<Duration>,
5669        /// Answered at cutover or failure; the incumbent's drain follows.
5670        reply: oneshot::Sender<Result<SwapOutcome, SuperviseError>>,
5671    },
5672}
5673
5674/// Successful daemon-local swap disposition. Supersession leaves the incumbent
5675/// serving until an approved replacement starts with the latest configured
5676/// command; the legacy control acknowledgment does not carry this disposition.
5677#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5678pub enum SwapOutcome {
5679    Cutover,
5680    SupersededByConfigRestart,
5681}
5682
5683impl SwapOutcome {
5684    pub fn as_str(self) -> &'static str {
5685        match self {
5686            Self::Cutover => "cutover",
5687            Self::SupersededByConfigRestart => "superseded_by_config_restart",
5688        }
5689    }
5690}
5691
5692#[derive(Debug)]
5693pub enum SuperviseError {
5694    ConfigHeld {
5695        module_id: String,
5696        code: String,
5697        detail: String,
5698    },
5699    ConfigDeliveryDeferred {
5700        module_id: String,
5701    },
5702    InvalidSpec {
5703        reason: String,
5704    },
5705    Spawn {
5706        program: PathBuf,
5707        source: io::Error,
5708        cgroup_path: Option<PathBuf>,
5709    },
5710    Cgroup {
5711        module_id: String,
5712        source: io::Error,
5713    },
5714    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
5715    /// than spawn a reserved module without its identity binding.
5716    LaunchNonce {
5717        reason: String,
5718    },
5719    Wait {
5720        module_id: String,
5721        source: io::Error,
5722    },
5723    Kill {
5724        module_id: String,
5725        source: io::Error,
5726    },
5727    Forwarding(ForwardingError),
5728    Registry(RegistryError),
5729    ReloadUnavailable {
5730        module_id: String,
5731        reason: String,
5732    },
5733    /// An operator restart/reload was requested for a module that is currently
5734    /// disabled. Restart/reload cycle a *running* module; a disabled module must
5735    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
5736    /// by a restart, so these commands are rejected instead of re-enabling it.
5737    Disabled {
5738        module_id: String,
5739    },
5740    ReloadFailed {
5741        module_id: String,
5742        reason: String,
5743    },
5744    RegistrationStillActive {
5745        module_id: String,
5746        waited: Duration,
5747    },
5748    StatePoisoned {
5749        module_id: Option<String>,
5750    },
5751    CommandClosed {
5752        module_id: String,
5753    },
5754    /// A restart or reload arrived while a swap's candidate was warming. The
5755    /// swap owns the module until it cuts over or fails; a stop or disable
5756    /// would have aborted it instead.
5757    SwapInProgress {
5758        module_id: String,
5759    },
5760    /// A swap was refused before anything was spawned.
5761    SwapRefused {
5762        module_id: String,
5763        reason: SwapRefusal,
5764    },
5765    /// A swap spawned a candidate and gave up on it. The candidate has been
5766    /// killed and its slot freed; the incumbent was left serving and was never
5767    /// drained, except in the one `CutoverLost` case described on that arm.
5768    SwapFailed {
5769        module_id: String,
5770        arm: SwapFailureArm,
5771        detail: String,
5772        /// How the candidate exited, when it exited on its own before the
5773        /// supervisor gave up on it.
5774        candidate_exit: Option<ExitReport>,
5775    },
5776}
5777
5778/// Why a swap was refused before a candidate was spawned.
5779#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5780pub enum SwapRefusal {
5781    /// The module's config does not declare `overlap: "safe"`.
5782    OverlapExclusive,
5783    /// The module is not registered, so there is no incumbent to keep serving
5784    /// and nothing a swap would improve on; a plain restart is the tool.
5785    NotRegistered,
5786    /// The module does not speak the subc wire, so a candidate could never
5787    /// register or declare itself ready.
5788    ProtocolNone,
5789    /// The supervisor lacks the forwarding table (to cut routes over) or the
5790    /// shared handle (to admit the candidate's HELLO) that a swap needs.
5791    NotConfigured,
5792    /// A swap is already open for this module.
5793    AlreadySwapping,
5794}
5795
5796impl SwapRefusal {
5797    pub fn as_str(self) -> &'static str {
5798        match self {
5799            Self::OverlapExclusive => "overlap_exclusive",
5800            Self::NotRegistered => "not_registered",
5801            Self::ProtocolNone => "protocol_none",
5802            Self::NotConfigured => "not_configured",
5803            Self::AlreadySwapping => "already_swapping",
5804        }
5805    }
5806}
5807
5808/// Which failure arm ended a swap. Every arm but one leaves the incumbent
5809/// serving and undrained; see `CutoverLost`.
5810#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5811pub enum SwapFailureArm {
5812    /// The candidate process could not be started.
5813    SpawnFailed,
5814    /// The candidate did not register within the readiness budget.
5815    NeverRegistered,
5816    /// The candidate registered but did not declare itself ready in time.
5817    NeverReady,
5818    /// The candidate exited before cutover.
5819    CandidateExited,
5820    /// The candidate declared itself ready but failed its health probe.
5821    CandidateUnhealthy,
5822    /// An operator stop, disable or retire arrived while the candidate warmed.
5823    /// The candidate was killed and the operator's command then carried out on
5824    /// the incumbent.
5825    Interrupted,
5826    /// The candidate's connection closed at the moment of cutover. If it
5827    /// closed before forwarding moved, the incumbent is untouched. If it closed
5828    /// between the forwarding and registry halves of cutover, forwarding can no
5829    /// longer route to the incumbent, so the module is restarted plainly.
5830    CutoverLost,
5831}
5832
5833impl SwapFailureArm {
5834    pub fn as_str(self) -> &'static str {
5835        match self {
5836            Self::SpawnFailed => "spawn_failed",
5837            Self::NeverRegistered => "never_registered",
5838            Self::NeverReady => "never_ready",
5839            Self::CandidateExited => "candidate_exited",
5840            Self::CandidateUnhealthy => "candidate_unhealthy",
5841            Self::Interrupted => "interrupted",
5842            Self::CutoverLost => "cutover_lost",
5843        }
5844    }
5845}
5846
5847impl SuperviseError {
5848    pub(crate) fn is_config_gate_error(&self) -> bool {
5849        match self {
5850            Self::ConfigHeld { .. } => true,
5851            Self::Spawn { source, .. } => source
5852                .get_ref()
5853                .is_some_and(|cause| cause.is::<crate::config_gate::GateError>()),
5854            _ => false,
5855        }
5856    }
5857}
5858
5859impl fmt::Display for SuperviseError {
5860    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5861        match self {
5862            Self::ConfigHeld { module_id, code, detail } => write!(f, "{code}: module '{module_id}' held: {detail}"),
5863            Self::ConfigDeliveryDeferred { module_id } => write!(f, "config_gated: module '{module_id}' is waiting for its exited delivery target's failure record"),
5864            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
5865            Self::Spawn {
5866                program,
5867                source,
5868                cgroup_path: Some(cgroup_path),
5869            } => write!(
5870                f,
5871                "failed to place module in cgroup '{}' while spawning '{}': {source}",
5872                cgroup_path.display(),
5873                program.display()
5874            ),
5875            Self::Spawn {
5876                program,
5877                source,
5878                cgroup_path: None,
5879            } => write!(
5880                f,
5881                "failed to spawn module '{}': {source}",
5882                program.display()
5883            ),
5884            Self::Cgroup { module_id, source } => {
5885                write!(
5886                    f,
5887                    "failed to prepare cgroup for module '{module_id}': {source}"
5888                )
5889            }
5890            Self::LaunchNonce { reason } => {
5891                write!(
5892                    f,
5893                    "failed to generate reserved-module launch nonce: {reason}"
5894                )
5895            }
5896            Self::Wait { module_id, source } => {
5897                write!(f, "failed to wait for module '{module_id}': {source}")
5898            }
5899            Self::Kill { module_id, source } => {
5900                write!(f, "failed to kill module '{module_id}': {source}")
5901            }
5902            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
5903            Self::Registry(err) => write!(f, "registry error: {err}"),
5904            Self::ReloadUnavailable { module_id, reason } => {
5905                write!(f, "reload unavailable for module '{module_id}': {reason}")
5906            }
5907            Self::Disabled { module_id } => {
5908                write!(
5909                    f,
5910                    "module '{module_id}' is disabled; enable it before restart or reload"
5911                )
5912            }
5913            Self::ReloadFailed { module_id, reason } => {
5914                write!(f, "reload failed for module '{module_id}': {reason}")
5915            }
5916            Self::RegistrationStillActive { module_id, waited } => write!(
5917                f,
5918                "module '{module_id}' registration remained active after waiting {waited:?}"
5919            ),
5920            Self::StatePoisoned { module_id } => match module_id {
5921                Some(module_id) => {
5922                    write!(f, "supervisor state for module '{module_id}' was poisoned")
5923                }
5924                None => write!(f, "supervisor state was poisoned"),
5925            },
5926            Self::CommandClosed { module_id } => {
5927                write!(
5928                    f,
5929                    "supervisor command channel for module '{module_id}' is closed"
5930                )
5931            }
5932            Self::SwapInProgress { module_id } => write!(
5933                f,
5934                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
5935            ),
5936            Self::SwapRefused { module_id, reason } => match reason {
5937                SwapRefusal::OverlapExclusive => write!(
5938                    f,
5939                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
5940                ),
5941                SwapRefusal::NotRegistered => write!(
5942                    f,
5943                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
5944                ),
5945                SwapRefusal::ProtocolNone => write!(
5946                    f,
5947                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
5948                ),
5949                SwapRefusal::NotConfigured => write!(
5950                    f,
5951                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
5952                ),
5953                SwapRefusal::AlreadySwapping => {
5954                    write!(f, "module '{module_id}' is already being swapped")
5955                }
5956            },
5957            Self::SwapFailed {
5958                module_id,
5959                arm,
5960                detail,
5961                ..
5962            } => write!(
5963                f,
5964                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
5965                arm.as_str()
5966            ),
5967        }
5968    }
5969}
5970
5971impl Error for SuperviseError {
5972    fn source(&self) -> Option<&(dyn Error + 'static)> {
5973        match self {
5974            Self::Spawn { source, .. }
5975            | Self::Cgroup { source, .. }
5976            | Self::Wait { source, .. }
5977            | Self::Kill { source, .. } => Some(source),
5978            Self::Forwarding(err) => Some(err),
5979            Self::Registry(err) => Some(err),
5980            Self::LaunchNonce { .. }
5981            | Self::ConfigDeliveryDeferred { .. }
5982            | Self::ConfigHeld { .. }
5983            | Self::InvalidSpec { .. }
5984            | Self::ReloadUnavailable { .. }
5985            | Self::Disabled { .. }
5986            | Self::ReloadFailed { .. }
5987            | Self::RegistrationStillActive { .. }
5988            | Self::StatePoisoned { .. }
5989            | Self::CommandClosed { .. }
5990            | Self::SwapInProgress { .. }
5991            | Self::SwapRefused { .. }
5992            | Self::SwapFailed { .. } => None,
5993        }
5994    }
5995}
5996
5997pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
5998    if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
5999        return Err(SuperviseError::InvalidSpec {
6000            reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
6001        });
6002    }
6003    if spec.module_id.trim().is_empty() {
6004        return Err(SuperviseError::InvalidSpec {
6005            reason: "module_id must not be empty".to_string(),
6006        });
6007    }
6008
6009    Ok(())
6010}
6011
6012#[derive(Debug, Default)]
6013struct HealthProbeRuntime {
6014    spawn_generation: Option<u64>,
6015    configured_health: Option<HealthConfig>,
6016    registered_connection: Option<crate::ConnectionId>,
6017    advertised: bool,
6018    next_probe_at: Option<Instant>,
6019    probe_index: u64,
6020}
6021
6022fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
6023    lock_snapshot(snapshot)
6024        .ok()
6025        .and_then(|state| state.spawned_protocol)
6026        .unwrap_or(spec.protocol)
6027}
6028
6029impl HealthProbeRuntime {
6030    fn refresh_registration(
6031        &mut self,
6032        spec: &ModuleSpec,
6033        runtime: &SupervisorRuntimeConfig,
6034        registry: &Registry,
6035        snapshot: &SharedSnapshot,
6036    ) {
6037        let generation = lock_snapshot(snapshot)
6038            .ok()
6039            .map(|state| state.spawn_generation);
6040        if self.spawn_generation != generation {
6041            // A replacement cannot reuse a deadline that is already due for its
6042            // predecessor. HTTP children get a full cadence to start; wire
6043            // children still wait for their own health-advertising HELLO.
6044            *self = Self {
6045                spawn_generation: generation,
6046                ..Self::default()
6047            };
6048        }
6049        if self.configured_health.as_ref() != Some(&runtime.health) {
6050            self.configured_health = Some(runtime.health.clone());
6051            self.next_probe_at = None;
6052            self.registered_connection = None;
6053            self.probe_index = 0;
6054        }
6055        // A non-wire process never registers. Only an explicitly configured
6056        // HTTP endpoint can arm its health probe; an absent HELLO is not a
6057        // health failure for that kind of process.
6058        if running_protocol(spec, snapshot) == ModuleProtocol::None {
6059            self.registered_connection = None;
6060            self.advertised = runtime.health.http.is_some();
6061            if !self.advertised {
6062                self.next_probe_at = None;
6063                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6064                    let unknown = ModuleHealthStatus::default();
6065                    if state.health != unknown {
6066                        state.health = unknown;
6067                        state.last_probe_evidence = None;
6068                    }
6069                });
6070            } else if self.next_probe_at.is_none() {
6071                self.next_probe_at = Some(
6072                    Instant::now()
6073                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6074                );
6075            }
6076            return;
6077        }
6078
6079        let registration = match registry.get_module(&spec.module_id) {
6080            Ok(registration) => registration,
6081            Err(err) => {
6082                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
6083                self.advertised = false;
6084                self.next_probe_at = None;
6085                return;
6086            }
6087        };
6088
6089        let Some(registration) = registration else {
6090            self.registered_connection = None;
6091            self.advertised = false;
6092            self.next_probe_at = None;
6093            return;
6094        };
6095
6096        let advertised = registration
6097            .control_ops
6098            .iter()
6099            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
6100        if !advertised {
6101            self.registered_connection = Some(registration.connection_id);
6102            self.advertised = false;
6103            self.next_probe_at = None;
6104            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6105                if state.health.status != SupervisorHealthStatus::Unknown
6106                    || state.health.consecutive_failures != 0
6107                    || state.health.last_probe_ms.is_some()
6108                    || state.health.detail.is_some()
6109                    || state.health.metrics.is_some()
6110                {
6111                    state.health.status = SupervisorHealthStatus::Unknown;
6112                    state.last_probe_evidence = None;
6113                    state.health.consecutive_failures = 0;
6114                    state.health.last_probe_ms = None;
6115                    state.health.detail = None;
6116                    state.health.metrics = None;
6117                }
6118            });
6119            return;
6120        }
6121
6122        let reregistered = self.registered_connection != Some(registration.connection_id);
6123        self.registered_connection = Some(registration.connection_id);
6124        self.advertised = true;
6125        if reregistered || self.next_probe_at.is_none() {
6126            self.probe_index = 0;
6127            self.next_probe_at = Some(
6128                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6129            );
6130            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6131                state.health.status = SupervisorHealthStatus::Unknown;
6132                state.last_probe_evidence = None;
6133                state.health.consecutive_failures = 0;
6134                state.health.detail = None;
6135                state.health.metrics = None;
6136            });
6137        }
6138    }
6139
6140    fn wake_at(&self) -> Option<Instant> {
6141        if !self.advertised {
6142            return None;
6143        }
6144        self.next_probe_at
6145    }
6146
6147    fn due(&self) -> bool {
6148        self.advertised
6149            && self
6150                .next_probe_at
6151                .is_some_and(|next| Instant::now() >= next)
6152    }
6153
6154    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
6155        self.probe_index = self.probe_index.wrapping_add(1);
6156        self.next_probe_at = Some(
6157            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
6158        );
6159    }
6160}
6161
6162/// Evidence is separate from diagnostic prose so host starvation can hold only
6163/// silence-based restarts. A missing lane is still actionable during overload;
6164/// a timeout can instead mean the host has not scheduled a healthy module.
6165#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6166enum HealthProbeEvidence {
6167    /// The module's control lane is gone. Proof of death.
6168    LaneDead,
6169    /// No reply within the deadline. Proves nothing about the module's state.
6170    NoAnswer,
6171    /// The module replied, but not with a usable health report. Proves it is alive.
6172    BadAnswer,
6173    /// The daemon could not ask. Says nothing about the module.
6174    Misconfigured,
6175}
6176
6177#[derive(Debug)]
6178struct HealthProbeError {
6179    evidence: HealthProbeEvidence,
6180    message: String,
6181}
6182
6183impl HealthProbeError {
6184    fn lane_dead(message: impl Into<String>) -> Self {
6185        Self::with(HealthProbeEvidence::LaneDead, message)
6186    }
6187
6188    fn no_answer(message: impl Into<String>) -> Self {
6189        Self::with(HealthProbeEvidence::NoAnswer, message)
6190    }
6191
6192    fn bad_answer(message: impl Into<String>) -> Self {
6193        Self::with(HealthProbeEvidence::BadAnswer, message)
6194    }
6195
6196    fn misconfigured(message: impl Into<String>) -> Self {
6197        Self::with(HealthProbeEvidence::Misconfigured, message)
6198    }
6199
6200    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
6201        Self {
6202            evidence,
6203            message: message.into(),
6204        }
6205    }
6206
6207    /// Whether this observation is proof the module cannot serve.
6208    ///
6209    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
6210    /// variant that fires under CPU starvation, and treating it as proof is the
6211    /// defect this enum exists to make impossible to reintroduce silently.
6212    ///
6213    /// Outside a host-overload hold, silence still reaches the normal failure
6214    /// threshold so an isolated, genuinely wedged module can recover.
6215    #[allow(dead_code)]
6216    fn is_proof_of_death(&self) -> bool {
6217        matches!(self.evidence, HealthProbeEvidence::LaneDead)
6218    }
6219
6220    /// Short stable label for logs and the health snapshot.
6221    ///
6222    /// An operator reading `ck health` currently cannot tell "the module is gone"
6223    /// from "the module did not answer in five seconds", because both render as
6224    /// prose in the same field. These labels are what make the two
6225    /// distinguishable at a glance, and they are what a later restart-policy
6226    /// change will be argued from.
6227    fn label(&self) -> &'static str {
6228        match self.evidence {
6229            HealthProbeEvidence::LaneDead => "lane-dead",
6230            HealthProbeEvidence::NoAnswer => "no-answer",
6231            HealthProbeEvidence::BadAnswer => "bad-answer",
6232            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
6233        }
6234    }
6235}
6236
6237impl fmt::Display for HealthProbeError {
6238    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
6239        f.write_str(&self.message)
6240    }
6241}
6242
6243async fn run_health_probe_cycle(
6244    spec: &ModuleSpec,
6245    runtime: &SupervisorRuntimeConfig,
6246    registry: &Registry,
6247    process_liveness: &SupervisorProcessLiveness,
6248    snapshot: &SharedSnapshot,
6249    child: &mut Option<SupervisedChild>,
6250) {
6251    let now_ms = unix_ms_now();
6252    let delivery_probe = match &runtime.config_delivery {
6253        Some(delivery) => {
6254            delivery
6255                .health_probe(&spec.module_id, runtime.forwarding.as_deref(), registry)
6256                .await
6257        }
6258        None => None,
6259    };
6260    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
6261        .then_some(runtime.health.http.as_deref())
6262        .flatten();
6263    let result = match http {
6264        Some(url) => probe_http_health(url, runtime.health.deadline).await,
6265        None => probe_module_health(&spec.module_id, runtime, None).await,
6266    };
6267    match result {
6268        Ok(report) => {
6269            handle_health_report(
6270                spec,
6271                runtime,
6272                registry,
6273                process_liveness,
6274                snapshot,
6275                child,
6276                report,
6277                now_ms,
6278                delivery_probe,
6279            )
6280            .await;
6281        }
6282        Err(err) => {
6283            if http.is_some() {
6284                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6285                    state.health.status = SupervisorHealthStatus::Failing;
6286                });
6287            }
6288            handle_health_probe_failure(
6289                spec,
6290                runtime,
6291                registry,
6292                process_liveness,
6293                snapshot,
6294                child,
6295                err,
6296                now_ms,
6297                delivery_probe,
6298            )
6299            .await;
6300        }
6301    }
6302}
6303
6304pub(crate) struct HttpProbeTarget<'a> {
6305    address: std::net::SocketAddr,
6306    localhost: bool,
6307    authority: &'a str,
6308    path: String,
6309}
6310
6311/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
6312/// or TLS. A URL cannot turn a local health check into an outbound connection.
6313pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
6314    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
6315        return Err("must not contain whitespace, controls, or a fragment".into());
6316    }
6317    let rest = url
6318        .strip_prefix("http://")
6319        .ok_or("must use plain http://")?;
6320    let split = rest.find(['/', '?']).unwrap_or(rest.len());
6321    let (authority, suffix) = rest.split_at(split);
6322    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
6323        ("::1", rest)
6324    } else {
6325        let split = authority.find(':').unwrap_or(authority.len());
6326        authority.split_at(split)
6327    };
6328    let ip: std::net::IpAddr = match host {
6329        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
6330        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
6331        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
6332    };
6333    let port = if port.is_empty() {
6334        80
6335    } else {
6336        port.strip_prefix(':')
6337            .and_then(|p| p.parse::<u16>().ok())
6338            .filter(|p| *p > 0)
6339            .ok_or("must have a valid nonzero TCP port")?
6340    };
6341    let path = if suffix.is_empty() {
6342        "/".into()
6343    } else if suffix.starts_with('?') {
6344        format!("/{suffix}")
6345    } else {
6346        suffix.into()
6347    };
6348    Ok(HttpProbeTarget {
6349        address: std::net::SocketAddr::new(ip, port),
6350        localhost: host == "localhost",
6351        authority,
6352        path,
6353    })
6354}
6355
6356async fn probe_http_health(
6357    url: &str,
6358    deadline: Duration,
6359) -> Result<HealthReport, HealthProbeError> {
6360    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
6361    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
6362    // Keep partial diagnostics outside the timed future so cancellation does
6363    // not discard a status line or body bytes already received.
6364    let mut response_status = String::new();
6365    let mut body = Vec::new();
6366    let probe = async {
6367        // Resolve localhost ourselves so a hosts-file override cannot turn
6368        // this into an outbound request, while IPv6-only local servers work.
6369        let connection = match tokio::net::TcpStream::connect(target.address).await {
6370            Err(_) if target.localhost => {
6371                tokio::net::TcpStream::connect((
6372                    std::net::Ipv6Addr::LOCALHOST,
6373                    target.address.port(),
6374                ))
6375                .await
6376            }
6377            result => result,
6378        };
6379        let mut stream = connection.map_err(|error| {
6380            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
6381        })?;
6382        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
6383            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
6384        let mut reader = BufReader::new(stream);
6385        let mut budget = 16 * 1024;
6386        let status = http_line(&mut reader, &mut budget).await?;
6387        let mut words = status.split_ascii_whitespace();
6388        let version = words.next();
6389        let code = words
6390            .next()
6391            .filter(|word| word.len() == 3)
6392            .and_then(|word| word.parse::<u16>().ok());
6393        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
6394            || !code.is_some_and(|code| (100..600).contains(&code))
6395        {
6396            return Err(HealthProbeError::bad_answer(format!(
6397                "invalid HTTP status: {status}"
6398            )));
6399        }
6400        let code = code.expect("validated status code");
6401        response_status = status.clone();
6402        let mut length = None;
6403        let mut chunked = false;
6404        loop {
6405            let line = http_line(&mut reader, &mut budget).await?;
6406            if line.is_empty() {
6407                break;
6408            }
6409            if let Some((name, value)) = line.split_once(':') {
6410                if name.eq_ignore_ascii_case("content-length") {
6411                    length = Some(value.trim().parse::<u64>().map_err(|_| {
6412                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
6413                    })?);
6414                } else if name.eq_ignore_ascii_case("transfer-encoding") {
6415                    chunked = value.trim().eq_ignore_ascii_case("chunked");
6416                }
6417            }
6418        }
6419        if chunked {
6420            while body.len() < 200 {
6421                let line = http_line(&mut reader, &mut budget).await?;
6422                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
6423                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
6424                if size == 0 {
6425                    break;
6426                }
6427                let count = size.min((200 - body.len()) as u64) as usize;
6428                let start = body.len();
6429                (&mut reader)
6430                    .take(count as u64)
6431                    .read_to_end(&mut body)
6432                    .await
6433                    .map_err(|error| {
6434                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6435                    })?;
6436                if body.len() - start != count {
6437                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
6438                }
6439                if size > count as u64 || body.len() == 200 {
6440                    break;
6441                }
6442                if !http_line(&mut reader, &mut budget).await?.is_empty() {
6443                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
6444                }
6445            }
6446        } else {
6447            reader
6448                .take(length.unwrap_or(200).min(200))
6449                .read_to_end(&mut body)
6450                .await
6451                .map_err(|error| {
6452                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6453                })?;
6454        }
6455        if (200..300).contains(&code) {
6456            Ok(HealthReport::ok())
6457        } else {
6458            Err(HealthProbeError::bad_answer(
6459                "HTTP health endpoint returned non-2xx",
6460            ))
6461        }
6462    };
6463    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
6464        Err(HealthProbeError::no_answer(format!(
6465            "HTTP probe timed out after {deadline:?}"
6466        )))
6467    });
6468    if let Err(error) = &mut result {
6469        if !response_status.is_empty() {
6470            error.message = format!(
6471                "{}; {response_status}: {}",
6472                error.message,
6473                String::from_utf8_lossy(&body)
6474            );
6475        }
6476    }
6477    result
6478}
6479
6480async fn http_line(
6481    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
6482    remaining: &mut usize,
6483) -> Result<String, HealthProbeError> {
6484    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
6485    let mut line = Vec::new();
6486    (&mut *reader)
6487        .take(*remaining as u64)
6488        .read_until(b'\n', &mut line)
6489        .await
6490        .map_err(|error| {
6491            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
6492        })?;
6493    *remaining -= line.len();
6494    if !line.ends_with(b"\r\n") {
6495        return Err(HealthProbeError::bad_answer(
6496            "HTTP headers are incomplete or exceed 16 KiB",
6497        ));
6498    }
6499    line.truncate(line.len() - 2);
6500    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
6501}
6502
6503async fn probe_module_health(
6504    module_id: &str,
6505    runtime: &SupervisorRuntimeConfig,
6506    drain_deadline: Option<Instant>,
6507) -> Result<HealthReport, HealthProbeError> {
6508    let Some(forwarding) = runtime.forwarding.as_ref() else {
6509        return Err(HealthProbeError::misconfigured(
6510            "supervisor was not configured with a forwarding table",
6511        ));
6512    };
6513    let probe_started_at = Instant::now();
6514    let mut deadline = probe_started_at + runtime.health.deadline;
6515    if let Some(drain_deadline) = drain_deadline {
6516        deadline = deadline.min(drain_deadline);
6517    }
6518    let pending = if drain_deadline.is_some() {
6519        forwarding.begin_drain_health_probe_rpc_for(
6520            module_id,
6521            MODULE_CONTROL_OP_HEALTH_CHECK,
6522            probe_started_at,
6523            deadline,
6524        )
6525    } else {
6526        forwarding.begin_health_probe_rpc_for(
6527            module_id,
6528            MODULE_CONTROL_OP_HEALTH_CHECK,
6529            probe_started_at,
6530            deadline,
6531        )
6532    }
6533    .map_err(|err| {
6534        // The endpoint is not registered, so there is no live control lane to
6535        // ask. That is the module being absent, not slow.
6536        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6537    })?;
6538    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
6539}
6540
6541/// [`probe_module_health`] for one endpoint rather than the id's active one.
6542///
6543/// A swap probes two processes that no by-id lookup reaches: its candidate
6544/// before cutover, and its superseded incumbent (for busy gauges) while the
6545/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
6546/// bounds the by-id drain probe.
6547async fn probe_endpoint_health(
6548    endpoint: crate::ModuleEndpointId,
6549    runtime: &SupervisorRuntimeConfig,
6550    deadline_cap: Option<Instant>,
6551) -> Result<HealthReport, HealthProbeError> {
6552    let Some(forwarding) = runtime.forwarding.as_ref() else {
6553        return Err(HealthProbeError::misconfigured(
6554            "supervisor was not configured with a forwarding table",
6555        ));
6556    };
6557    let probe_started_at = Instant::now();
6558    let mut deadline = probe_started_at + runtime.health.deadline;
6559    if let Some(cap) = deadline_cap {
6560        deadline = deadline.min(cap);
6561    }
6562    let pending = forwarding
6563        .begin_endpoint_health_probe_rpc_for(
6564            endpoint,
6565            MODULE_CONTROL_OP_HEALTH_CHECK,
6566            probe_started_at,
6567            deadline,
6568        )
6569        .map_err(|err| {
6570            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6571        })?;
6572    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
6573}
6574
6575/// Send a begun health probe and classify its answer.
6576async fn await_health_probe(
6577    forwarding: &ForwardingTable,
6578    pending: PendingModuleControlRpc,
6579    deadline: Instant,
6580    probe_budget: Duration,
6581) -> Result<HealthReport, HealthProbeError> {
6582    let PendingModuleControlRpc {
6583        endpoint,
6584        module_sink,
6585        negotiated_ver,
6586        corr,
6587        receiver,
6588    } = pending;
6589    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6590        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
6591    })?;
6592    let frame = Frame::build_with_version(
6593        negotiated_ver,
6594        FrameType::Request,
6595        control_flags(),
6596        0,
6597        0,
6598        corr,
6599        body,
6600    )
6601    .map_err(|err| {
6602        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
6603    })?;
6604
6605    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
6606    // blocks waiting for capacity when the module's egress queue is full, and an
6607    // unbounded await here freezes the whole supervision actor (it stops polling
6608    // Child::wait and supervisor commands), making the module unrecoverable
6609    // in-band. On timeout the probe fails like any transport failure.
6610    match timeout_at(deadline, module_sink.send(frame)).await {
6611        Ok(Ok(())) => {}
6612        Ok(Err(err)) => {
6613            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6614            // A closed sink means the module's egress channel is gone -- the
6615            // receiving half is dropped when its connection tears down. Proof.
6616            return Err(HealthProbeError::lane_dead(format!(
6617                "failed to send health.check: {err}"
6618            )));
6619        }
6620        Err(_elapsed) => {
6621            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6622            // A full egress queue means the module is not draining its socket, which
6623            // is consistent with a wedged module AND with one whose reader is merely
6624            // starved. Silence, not proof.
6625            return Err(HealthProbeError::no_answer(
6626                "health.check send timed out before enqueue (module egress full)",
6627            ));
6628        }
6629    }
6630
6631    match timeout_at(deadline, receiver).await {
6632        Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => Err(HealthProbeError::bad_answer(
6633            "health.check returned an unexpected config response",
6634        )),
6635        // Each arm records WHAT WAS OBSERVED. Four of them are the module
6636        // demonstrably answering -- rejected, non-health, malformed, wrong op --
6637        // and those prove it is alive even though the probe failed.
6638        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6639            response.health_report().ok_or_else(|| {
6640                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
6641            })
6642        }
6643        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
6644            format!("health.check rejected: {}", body.message),
6645        )),
6646        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6647            Err(HealthProbeError::lane_dead(message))
6648        }
6649        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6650            Err(HealthProbeError::bad_answer(message))
6651        }
6652        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6653            Err(HealthProbeError::bad_answer(format!(
6654                "expected module-control op '{expected}', got '{actual}'"
6655            )))
6656        }
6657        // A reply that crosses the deadline before this waiter observes it is
6658        // still proof of life. The forwarding path records its end-to-end latency
6659        // before delivering this classification.
6660        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
6661            "module answered health.check after its daemon deadline",
6662        )),
6663        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
6664            "health.check waiter was canceled before the module responded",
6665        )),
6666        Err(_) => {
6667            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
6668            Err(HealthProbeError::no_answer(format!(
6669                "module did not answer health.check within {probe_budget:?}"
6670            )))
6671        }
6672    }
6673}
6674
6675#[allow(clippy::too_many_arguments)]
6676async fn handle_health_report(
6677    spec: &ModuleSpec,
6678    runtime: &SupervisorRuntimeConfig,
6679    registry: &Registry,
6680    process_liveness: &SupervisorProcessLiveness,
6681    snapshot: &SharedSnapshot,
6682    child: &mut Option<SupervisedChild>,
6683    report: HealthReport,
6684    now_ms: u64,
6685    delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6686) {
6687    let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6688        (Some(delivery), Some(probe)) => {
6689            delivery
6690                .health_result(
6691                    &spec.module_id,
6692                    runtime.forwarding.as_deref(),
6693                    registry,
6694                    probe,
6695                    report.status == HealthStatus::Ok,
6696                )
6697                .await
6698        }
6699        _ => false,
6700    };
6701    let status = supervisor_health_status(report.status);
6702    let detail = report.detail.clone();
6703    let metrics = truncate_health_metrics(report.metrics);
6704    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6705        state.health.status = status;
6706        state.last_probe_evidence = None;
6707        state.health.last_probe_ms = Some(now_ms);
6708        state.health.detail = detail.clone();
6709        state.health.metrics = metrics.clone();
6710        state.health.consecutive_failures = 0;
6711        state.failed_retry.observe_health(
6712            report.status == HealthStatus::Ok,
6713            runtime.restart_policy.window,
6714        );
6715    });
6716
6717    if delivery_failed {
6718        return;
6719    }
6720    let action = match report.status {
6721        HealthStatus::Ok => return,
6722        HealthStatus::Degraded => runtime.health.on_degraded,
6723        HealthStatus::Failing => runtime.health.on_failing,
6724    };
6725    apply_l3_health_action(
6726        spec,
6727        runtime,
6728        registry,
6729        process_liveness,
6730        snapshot,
6731        child,
6732        status,
6733        detail.as_deref(),
6734        action,
6735        now_ms,
6736    )
6737    .await;
6738}
6739
6740#[allow(clippy::too_many_arguments)]
6741async fn handle_health_probe_failure(
6742    spec: &ModuleSpec,
6743    runtime: &SupervisorRuntimeConfig,
6744    registry: &Registry,
6745    process_liveness: &SupervisorProcessLiveness,
6746    snapshot: &SharedSnapshot,
6747    child: &mut Option<SupervisedChild>,
6748    err: HealthProbeError,
6749    now_ms: u64,
6750    delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6751) {
6752    let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6753        (Some(delivery), Some(probe)) => {
6754            delivery
6755                .health_result(
6756                    &spec.module_id,
6757                    runtime.forwarding.as_deref(),
6758                    registry,
6759                    probe,
6760                    false,
6761                )
6762                .await
6763        }
6764        _ => false,
6765    };
6766    let threshold = runtime.health.failure_threshold.max(1);
6767    let held = if err.evidence == HealthProbeEvidence::NoAnswer {
6768        runtime.host_health.record_late_module(&spec.module_id)
6769    } else {
6770        false
6771    };
6772    let mut failures = 0;
6773    // Carry the evidence class into the operator-visible detail. Without it,
6774    // "module did not answer within 5s" and "the control lane is gone" are two
6775    // prose strings in the same field, and the reader has to know the codebase to
6776    // tell which one is proof of anything.
6777    let detail = if held {
6778        format!("[{}] {err}; host overloaded", err.label())
6779    } else {
6780        format!("[{}] {err}", err.label())
6781    };
6782    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6783        state.last_probe_evidence = Some(err.evidence);
6784        state
6785            .failed_retry
6786            .observe_health(false, runtime.restart_policy.window);
6787        // A failed wire probe invalidates the last report, even before the
6788        // restart threshold. HTTP probes already mark failures as Failing.
6789        if held {
6790            state.health.status = SupervisorHealthStatus::Degraded;
6791        } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
6792            state.health.status = SupervisorHealthStatus::Unknown;
6793        }
6794        state.health.last_probe_ms = Some(now_ms);
6795        // Held silence breaks the streak: after the host recovers, only new
6796        // failures count. Replaying pre-overload misses would restart immediately.
6797        state.health.consecutive_failures = if held {
6798            0
6799        } else {
6800            state.health.consecutive_failures.saturating_add(1)
6801        };
6802        state.health.detail = Some(detail.clone());
6803        state.health.metrics = None;
6804        failures = state.health.consecutive_failures;
6805    });
6806
6807    if delivery_failed {
6808        return;
6809    }
6810    if held || failures < threshold {
6811        warn!(
6812            module_id = %spec.module_id,
6813            consecutive_failures = failures,
6814            threshold,
6815            evidence = err.label(),
6816            detail = %detail,
6817            "health.check probe failed"
6818        );
6819        return;
6820    }
6821
6822    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6823        state.state = ModuleState::Unresponsive;
6824        state.health.status = SupervisorHealthStatus::Unresponsive;
6825    });
6826    // The evidence class is logged at the kill site because this is the line an
6827    // operator reads after an unexplained restart. A streak of `no-answer` under
6828    // machine load is the known false-positive shape; a `lane-dead` is not.
6829    if runtime.health.critical {
6830        error!(
6831            module_id = %spec.module_id,
6832            status = "unresponsive",
6833            evidence = err.label(),
6834            detail = %detail,
6835            "critical module health alert"
6836        );
6837    } else {
6838        warn!(
6839            module_id = %spec.module_id,
6840            status = "unresponsive",
6841            evidence = err.label(),
6842            detail = %detail,
6843            "module health threshold breached"
6844        );
6845    }
6846    if let Err(err) = health_restart_child(
6847        spec,
6848        runtime,
6849        registry,
6850        process_liveness,
6851        snapshot,
6852        child,
6853        SupervisorHealthStatus::Unresponsive,
6854        Some(&detail),
6855        now_ms,
6856    )
6857    .await
6858    {
6859        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6860    }
6861}
6862
6863#[allow(clippy::too_many_arguments)]
6864async fn apply_l3_health_action(
6865    spec: &ModuleSpec,
6866    runtime: &SupervisorRuntimeConfig,
6867    registry: &Registry,
6868    process_liveness: &SupervisorProcessLiveness,
6869    snapshot: &SharedSnapshot,
6870    child: &mut Option<SupervisedChild>,
6871    status: SupervisorHealthStatus,
6872    detail: Option<&str>,
6873    action: HealthAction,
6874    now_ms: u64,
6875) {
6876    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
6877    match action {
6878        HealthAction::Report => {
6879            info!(
6880                module_id = %spec.module_id,
6881                status = ?status,
6882                detail,
6883                "module reported non-ok health"
6884            );
6885        }
6886        HealthAction::Alert => {
6887            error!(
6888                module_id = %spec.module_id,
6889                status = ?status,
6890                detail,
6891                "module health alert"
6892            );
6893        }
6894        HealthAction::Restart => {
6895            if let Err(err) = health_restart_child(
6896                spec,
6897                runtime,
6898                registry,
6899                process_liveness,
6900                snapshot,
6901                child,
6902                status,
6903                detail,
6904                now_ms,
6905            )
6906            .await
6907            {
6908                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6909            }
6910        }
6911    }
6912}
6913
6914#[allow(clippy::too_many_arguments)]
6915async fn health_restart_child(
6916    spec: &ModuleSpec,
6917    runtime: &SupervisorRuntimeConfig,
6918    registry: &Registry,
6919    process_liveness: &SupervisorProcessLiveness,
6920    snapshot: &SharedSnapshot,
6921    child: &mut Option<SupervisedChild>,
6922    status: SupervisorHealthStatus,
6923    detail: Option<&str>,
6924    now_ms: u64,
6925) -> Result<(), SuperviseError> {
6926    budgeted_restart_child(
6927        spec,
6928        runtime,
6929        registry,
6930        process_liveness,
6931        snapshot,
6932        child,
6933        status,
6934        detail,
6935        now_ms,
6936        true,
6937    )
6938    .await
6939}
6940
6941#[allow(clippy::too_many_arguments)]
6942async fn budgeted_restart_child(
6943    spec: &ModuleSpec,
6944    runtime: &SupervisorRuntimeConfig,
6945    registry: &Registry,
6946    process_liveness: &SupervisorProcessLiveness,
6947    snapshot: &SharedSnapshot,
6948    child: &mut Option<SupervisedChild>,
6949    status: SupervisorHealthStatus,
6950    detail: Option<&str>,
6951    now_ms: u64,
6952    health_failure: bool,
6953) -> Result<(), SuperviseError> {
6954    if let Some(error) = runtime
6955        .config_delivery
6956        .as_ref()
6957        .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
6958        .filter(|error| error.code != "config_delivery_deferred")
6959    {
6960        return Err(config_delivery::gate_hold_error(
6961            spec,
6962            error,
6963            runtime.config_delivery.as_ref(),
6964        ));
6965    }
6966    let recovery_reason = if health_failure {
6967        "health_check"
6968    } else {
6969        "configuration_delivery"
6970    };
6971    let (enabled, schedule) = {
6972        let mut state = lock_snapshot(snapshot)?;
6973        let enabled = state.enabled;
6974        let schedule = if enabled {
6975            state.next_crash_restart(&runtime.restart_policy, Instant::now())
6976        } else {
6977            None
6978        };
6979        (enabled, schedule)
6980    };
6981
6982    if !enabled {
6983        return Err(SuperviseError::Disabled {
6984            module_id: spec.module_id.clone(),
6985        });
6986    }
6987
6988    if schedule.is_none() {
6989        if health_failure {
6990            record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
6991        }
6992        error!(
6993            module_id = %spec.module_id,
6994            status = ?status,
6995            detail,
6996            max_restarts = runtime.restart_policy.max_restarts,
6997            window_secs = runtime.restart_policy.window.as_secs(),
6998            reason = %runtime.restart_policy.budget_exhausted_detail(),
6999            recovery_reason,
7000            "automatic restart budget exhausted; marking module failed"
7001        );
7002        let stop_notice = begin_forwarding_drain_if_configured(
7003            spec,
7004            runtime,
7005            registry,
7006            snapshot,
7007            Some(true),
7008            RouteCloseReason::Disable,
7009        )
7010        .await?;
7011        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7012            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
7013        })?;
7014        let result = drain_optional_child(
7015            &spec.module_id,
7016            spec.protocol,
7017            stop_notice,
7018            registry,
7019            runtime.forwarding.as_deref(),
7020            snapshot,
7021            &runtime.terminal_ring,
7022            &runtime.spawn_events,
7023            child,
7024            runtime.drain_timeout,
7025            ModuleState::Failed,
7026            Some(true),
7027            &runtime.failure_context(true),
7028        )
7029        .await;
7030        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7031            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
7032                state.failed_retry.schedule(&spec.module_id);
7033            }
7034        })?;
7035        result?;
7036        process_liveness.untrack_if_current(&spec.module_id, snapshot);
7037        return Ok(());
7038    }
7039
7040    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
7041    let mut restart_count = 0;
7042    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7043        restart_count = state.crash_restarts.len();
7044        state.state = ModuleState::Unresponsive;
7045        if health_failure {
7046            state.health.status = status;
7047            state.health.last_action = Some(HealthAction::Restart.to_string());
7048            state.health.last_action_ms = Some(now_ms);
7049        }
7050    })?;
7051    warn!(
7052        module_id = %spec.module_id,
7053        status = ?status,
7054        detail,
7055        restart_count,
7056        restart_in_window = schedule.restart_in_window,
7057        delay_ms = schedule.delay.as_millis() as u64,
7058        recovery_reason,
7059        "automatic module restart"
7060    );
7061
7062    let stop_notice = begin_forwarding_drain_if_configured(
7063        spec,
7064        runtime,
7065        registry,
7066        snapshot,
7067        Some(true),
7068        RouteCloseReason::Restart,
7069    )
7070    .await?;
7071    drain_optional_child(
7072        &spec.module_id,
7073        spec.protocol,
7074        stop_notice,
7075        registry,
7076        runtime.forwarding.as_deref(),
7077        snapshot,
7078        &runtime.terminal_ring,
7079        &runtime.spawn_events,
7080        child,
7081        runtime.drain_timeout,
7082        ModuleState::Restarting,
7083        Some(true),
7084        &runtime.failure_context(true),
7085    )
7086    .await?;
7087    schedule_respawn(
7088        runtime,
7089        snapshot,
7090        &spec.module_id,
7091        schedule.delay,
7092        RespawnKind::Spawn,
7093        true,
7094    )
7095}
7096
7097fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
7098    if let Some(reply) = runtime
7099        .deferred_reload_reply
7100        .lock()
7101        .unwrap_or_else(|p| p.into_inner())
7102        .take()
7103    {
7104        let _ = reply.send(Err(SuperviseError::ReloadFailed {
7105            module_id: module_id.to_string(),
7106            reason: reason.to_string(),
7107        }));
7108    }
7109}
7110
7111fn park_config_spawn(
7112    spec: &ModuleSpec,
7113    runtime: &SupervisorRuntimeConfig,
7114    snapshot: &SharedSnapshot,
7115    failure: &FailureContext,
7116    admission_refused: Option<String>,
7117) -> Result<(), SuperviseError> {
7118    *runtime
7119        .scheduled_respawn
7120        .lock()
7121        .unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
7122    if let Some(reply) = runtime
7123        .deferred_reload_reply
7124        .lock()
7125        .unwrap_or_else(|poisoned| poisoned.into_inner())
7126        .take()
7127    {
7128        let _ = reply.send(Err(SuperviseError::ConfigHeld {
7129            module_id: spec.module_id.clone(),
7130            code: "config_gated".into(),
7131            detail: "module is held by configuration admission".into(),
7132        }));
7133    }
7134    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7135        state.admission_refused = admission_refused;
7136        state.config_hold_daemon_initiated = failure.daemon_initiated;
7137        state.config_hold_approval = failure.approval.clone();
7138        state.config_hold_parked = true;
7139        state.state = ModuleState::Stopped;
7140        state.respawn_pending = false;
7141        state.failed_retry.pending = None;
7142        // Parking ends this attempt and discards its failure timestamp. If a
7143        // later release fails, its retry delay starts at that new failure.
7144        state.failed_retry.failed_at = None;
7145        clear_current_process_facts(state);
7146    })
7147}
7148
7149fn try_park_config_attempt(
7150    spec: &ModuleSpec,
7151    runtime: &SupervisorRuntimeConfig,
7152    snapshot: &SharedSnapshot,
7153    failure: &FailureContext,
7154    program_base: Option<&subc_config::Digest>,
7155) -> Result<bool, SuperviseError> {
7156    let result = (|| {
7157        if !lock_snapshot(snapshot)?.enabled {
7158            return Ok(false);
7159        }
7160        #[cfg(test)]
7161        if let Some(delivery) = &runtime.config_delivery {
7162            delivery.0.pause_before_config_park_for_test();
7163        }
7164        let Some(delivery) = &runtime.config_delivery else {
7165            return Ok(false);
7166        };
7167        let parked = delivery
7168            .0
7169            .with_recorded_admission_hold(&spec.module_id, program_base, |refusal| {
7170                park_config_spawn(spec, runtime, snapshot, failure, refusal)
7171            })
7172            .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(delivery)))?;
7173        if let Some(result) = parked {
7174            result?;
7175            Ok(true)
7176        } else {
7177            Ok(false)
7178        }
7179    })();
7180    if let Err(error) = &result {
7181        // A failed park still belongs to the attempt that tried to park. Mark it
7182        // Failed here, before the monitor can use a daemon/operator origin saved
7183        // for a different attempt to decide whether to schedule a slow retry.
7184        fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7185        record_approved_attempt_failure(spec, snapshot, failure, error);
7186    }
7187    result
7188}
7189
7190fn record_approved_attempt_failure(
7191    spec: &ModuleSpec,
7192    snapshot: &SharedSnapshot,
7193    failure: &FailureContext,
7194    error: &SuperviseError,
7195) {
7196    let Some(approval) = &failure.approval else {
7197        return;
7198    };
7199    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7200        if state.enabled
7201            && state.state == ModuleState::Failed
7202            && state
7203                .approved_start_failure
7204                .as_ref()
7205                .is_none_or(|(identity, _)| identity != approval)
7206        {
7207            state.approved_start_failure = Some((approval.clone(), error.to_string()));
7208        }
7209    });
7210}
7211
7212fn hold_config_spawn(
7213    spec: &ModuleSpec,
7214    runtime: &SupervisorRuntimeConfig,
7215    snapshot: &SharedSnapshot,
7216    failure: &FailureContext,
7217) -> Result<(), SuperviseError> {
7218    if !lock_snapshot(snapshot)?.enabled {
7219        return Ok(());
7220    }
7221    if !try_park_config_attempt(spec, runtime, snapshot, failure, None)? {
7222        // This caller consumed an actual start attempt. If its hold ended before
7223        // parking, retain that attempt instead of waiting for a lost release.
7224        defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
7225    }
7226    Ok(())
7227}
7228
7229fn schedule_respawn(
7230    runtime: &SupervisorRuntimeConfig,
7231    snapshot: &SharedSnapshot,
7232    module_id: &str,
7233    delay: Duration,
7234    kind: RespawnKind,
7235    daemon_initiated: bool,
7236) -> Result<(), SuperviseError> {
7237    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
7238    update_snapshot(snapshot, Some(module_id), |state| {
7239        state.respawn_pending = true
7240    })?;
7241    *runtime
7242        .scheduled_respawn
7243        .lock()
7244        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7245        deadline: Instant::now() + delay,
7246        kind,
7247        daemon_initiated,
7248        approval: None,
7249    });
7250    Ok(())
7251}
7252
7253const CONFIG_ADMISSION_RETRY_DELAY: Duration = Duration::from_millis(100);
7254
7255fn defer_config_spawn(
7256    spec: &ModuleSpec,
7257    runtime: &SupervisorRuntimeConfig,
7258    snapshot: &SharedSnapshot,
7259    kind: RespawnKind,
7260    failure: &FailureContext,
7261) -> Result<(), SuperviseError> {
7262    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7263        state.state = ModuleState::Restarting;
7264        state.respawn_pending = true;
7265        clear_current_process_facts(state);
7266    })?;
7267    *runtime
7268        .scheduled_respawn
7269        .lock()
7270        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7271        deadline: Instant::now() + CONFIG_ADMISSION_RETRY_DELAY,
7272        kind,
7273        daemon_initiated: failure.daemon_initiated,
7274        approval: failure.approval.clone(),
7275    });
7276    Ok(())
7277}
7278
7279fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
7280    let _ = update_snapshot(snapshot, Some(module_id), |state| {
7281        state.health.last_action = Some(action);
7282        state.health.last_action_ms = Some(now_ms);
7283    });
7284}
7285
7286fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
7287    match status {
7288        HealthStatus::Ok => SupervisorHealthStatus::Ok,
7289        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
7290        HealthStatus::Failing => SupervisorHealthStatus::Failing,
7291    }
7292}
7293
7294/// Caps the metrics blob stored in the cached supervisor snapshot, which is
7295/// returned to every `supervisor.list` and `supervisor.health` caller.
7296///
7297/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
7298/// path: that request exists to return a module's complete metrics object, and
7299/// `ck health <module-id>` documents it as the way to see what the cached view
7300/// truncates. The asymmetry is the feature.
7301///
7302/// So a new caller must decide which side it is on rather than assume the cap is
7303/// universal. Reaching for it on a fresh-probe path would silently reintroduce
7304/// the truncation that path exists to avoid.
7305fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
7306    let metrics = metrics?;
7307    match serde_json::to_vec(&metrics) {
7308        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
7309            "truncated": true,
7310            "original_bytes": encoded.len(),
7311        })),
7312        Ok(_) | Err(_) => Some(metrics),
7313    }
7314}
7315
7316/// Spread health probes so a fleet-wide restart does not converge them.
7317///
7318/// The delay is derived from the module id and probe index rather than a random
7319/// source, so it is deterministic per module: a module keeps its own offset
7320/// across daemon restarts instead of re-rolling into a collision.
7321fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
7322    if cadence.is_zero() {
7323        return Duration::ZERO;
7324    }
7325    let cadence_ms = cadence.as_millis() as u64;
7326    // This early return is REDUNDANT, deliberately, and a mutation run will show
7327    // it surviving removal. Recording why here so the next person to notice does
7328    // not have to re-derive it:
7329    //
7330    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
7331    //   a zero cadence and builds the Duration from whole milliseconds, so a
7332    //   sub-millisecond cadence cannot come from config.
7333    // - Even if reached it changes no answer. The `.max(1)` below makes the span
7334    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
7335    //   -- exactly what this returns.
7336    //
7337    // Kept as a guard against a future widening of the config parser (accepting
7338    // microseconds, say), which would make the sub-millisecond case reachable.
7339    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
7340    // divides by zero. Remove this and nothing changes.
7341    if cadence_ms == 0 {
7342        return cadence;
7343    }
7344    // Note that this never returns less than one cadence, including for the FIRST
7345    // probe. So a freshly registered module reports health `unknown` for a full
7346    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
7347    // ready to answer.
7348    //
7349    // That is a property of the supervisor's schedule, not of any module: an
7350    // operator watching a restart sees `unknown` and cannot tell it from a module
7351    // that is slow to warm. Measured on two unrelated modules, both flipping to
7352    // `ok` between 22s and 32s after restart.
7353    //
7354    // Left as-is because spreading the first probe is what keeps a fleet-wide
7355    // restart from firing fourteen simultaneous probes into a cold machine. The
7356    // alternative -- probe at t+0 and jitter only from the second onward -- trades
7357    // that thundering herd for a faster first reading.
7358    let jitter_span = (cadence_ms / 10).max(1);
7359    let hash = module_id.as_bytes().iter().fold(
7360        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
7361        |acc, byte| {
7362            acc.wrapping_mul(1099511628211)
7363                .wrapping_add(u64::from(*byte))
7364        },
7365    );
7366    cadence + Duration::from_millis(hash % jitter_span)
7367}
7368
7369#[cfg(test)]
7370mod tests {
7371    use super::*;
7372
7373    #[test]
7374    fn registration_retirement_self_prefixed_owner_is_not_a_prefix_submodule() {
7375        let handle = SupervisorHandle::new();
7376        handle.set_spawn_nonce("prefix:owner", "owner-nonce".into());
7377        handle.set_reserved_prefixes("prefix:owner", &["prefix:".into()]);
7378        let owner = handle
7379            .reserved_hello_verification("prefix:owner", Some("owner-nonce"))
7380            .unwrap();
7381        assert!(owner.nonce_verified);
7382        assert!(
7383            owner.prefix_owner.is_none(),
7384            "an unreserved programmatic owner is not its own prefix child"
7385        );
7386        let child = handle
7387            .reserved_hello_verification("prefix:child", Some("owner-nonce"))
7388            .unwrap();
7389        assert!(child.nonce_verified);
7390        assert_eq!(child.prefix_owner.as_deref(), Some("prefix:owner"));
7391    }
7392
7393    #[test]
7394    fn readding_a_module_clears_its_rescan_removal_tombstone() {
7395        let handle = SupervisorHandle::new();
7396        let module_id = "readded-tombstone";
7397        handle.record_rescan_removal(module_id);
7398        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
7399
7400        handle.apply_identity_configuration(&ModuleSpec {
7401            module_id: module_id.to_string(),
7402            program: PathBuf::from("/test/module"),
7403            args: Vec::new(),
7404            env: Vec::new(),
7405            reserved: false,
7406            reserved_prefixes: Vec::new(),
7407            protocol: ModuleProtocol::Subc,
7408            overlap: Default::default(),
7409            carry_containment: false,
7410        });
7411
7412        assert!(
7413            handle.removal_tombstone_age_ms(module_id).is_none(),
7414            "a re-added module must not retain a stale removal tombstone"
7415        );
7416    }
7417
7418    /// What one module's owner looked like from the control plane at the
7419    /// instant after its first process was spawned.
7420    #[derive(Debug, PartialEq, Eq)]
7421    struct OwnerInSpawnWindow {
7422        module_id: String,
7423        configured: bool,
7424        on_roster: bool,
7425        admission_refusal: Option<&'static str>,
7426    }
7427
7428    /// A supervised module's process can connect, register, sync its scopes
7429    /// and describe them as soon as it is spawned. A plain `spawn` does not put
7430    /// the module on the roster until afterward; configured supervision does
7431    /// so before spawning. In both cases the owner must
7432    /// already read as configured, so a scoped `route.open` against it is
7433    /// refused as retryable `scope_not_synced` and not as terminal
7434    /// `scope_not_live` ("will never sync").
7435    ///
7436    /// The hook runs just after each initial spawn attempt, before returning to
7437    /// the caller, so no race with a real child's registration is needed.
7438    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7439    async fn a_new_module_is_configured_before_its_first_process_can_register() {
7440        use crate::scopes::ScopeTable;
7441        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
7442
7443        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
7444        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
7445            module_id: module_id.to_string(),
7446            program,
7447            args: Vec::new(),
7448            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7449                .into_iter()
7450                .map(|key| (key.to_string(), dir.path().display().to_string()))
7451                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7452                .collect(),
7453            reserved: false,
7454            reserved_prefixes: Vec::new(),
7455            protocol: ModuleProtocol::Subc,
7456            overlap: Default::default(),
7457            carry_containment: false,
7458        };
7459        let live = super::terminal_history_tests::fake_aft_stub_path();
7460        let missing = dir.path().join("definitely-missing-module");
7461
7462        let handle = SupervisorHandle::new();
7463        let mut supervisor =
7464            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7465                .with_handle(handle.clone());
7466        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
7467        let hook_handle = handle.clone();
7468        let hook_observed = Arc::clone(&observed);
7469        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7470            // Exactly what the control plane computes for a scoped route.open
7471            // naming this module as the owner of a scope it has not synced.
7472            let configured = hook_handle.is_configured(module_id);
7473            let selector = ScopeSelector::new(
7474                Principal::Reserved {
7475                    module_id: module_id.to_string(),
7476                },
7477                "s".to_string(),
7478                Some(1),
7479            );
7480            let carrier = Principal::Reserved {
7481                module_id: "carrier".to_string(),
7482            };
7483            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
7484                .admit(&carrier, module_id, &selector, configured)
7485            {
7486                Ok(_) => None,
7487                Err(refusal) => Some(refusal.code),
7488            };
7489            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
7490                module_id: module_id.to_string(),
7491                configured,
7492                on_roster: hook_handle.get(module_id).is_some(),
7493                admission_refusal,
7494            });
7495        })));
7496
7497        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
7498        let configured = supervisor
7499            .supervise_configured(stub("configured", live.clone()), true)
7500            .unwrap();
7501        let with_health = supervisor
7502            .supervise_configured_with_health(
7503                stub("with-health", live.clone()),
7504                true,
7505                HealthConfig::default(),
7506                None,
7507                RestartPolicy::default(),
7508            )
7509            .unwrap();
7510        // The failed-spawn path still puts the module on the roster (as
7511        // failed), so it is configured throughout.
7512        let failed = supervisor
7513            .supervise_configured_with_health(
7514                stub("failed-spawn", missing.clone()),
7515                true,
7516                HealthConfig::default(),
7517                None,
7518                RestartPolicy::default(),
7519            )
7520            .unwrap();
7521        // A failed plain `spawn` puts nothing on the roster, so its mark is
7522        // taken back once the spawn has failed.
7523        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
7524
7525        let expected = [
7526            "plain",
7527            "configured",
7528            "with-health",
7529            "failed-spawn",
7530            "spawn-error",
7531        ]
7532        .into_iter()
7533        .map(|module_id| OwnerInSpawnWindow {
7534            module_id: module_id.to_string(),
7535            configured: true,
7536            on_roster: !matches!(module_id, "plain" | "spawn-error"),
7537            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
7538        })
7539        .collect::<Vec<_>>();
7540        assert_eq!(*observed.lock().unwrap(), expected);
7541
7542        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
7543            assert!(
7544                handle.get(module_id).is_some(),
7545                "{module_id} is on the roster"
7546            );
7547            assert!(
7548                handle.is_configured(module_id),
7549                "{module_id} stays configured"
7550            );
7551        }
7552        assert!(handle.get("spawn-error").is_none());
7553        assert!(
7554            !handle.is_configured("spawn-error"),
7555            "a plain spawn that failed must not leave its module marked configured"
7556        );
7557
7558        // Leaving the roster clears the mark with it.
7559        handle.retire("failed-spawn");
7560        assert!(!handle.is_configured("failed-spawn"));
7561
7562        for module in [plain, configured, with_health] {
7563            module.stop().await.unwrap();
7564        }
7565        drop(failed);
7566    }
7567
7568    /// A configured module reads as starting until the supervisor has admitted
7569    /// its first process (exec confirmed, identity durably recorded), even
7570    /// though that process is already running by then and can register over
7571    /// HELLO, which is what lists it in catalog.list. So a client that needs
7572    /// the supervisor state must wait for that state; seeing the module in
7573    /// the catalog does not mean supervisor.list already says running.
7574    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7575    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
7576        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
7577        let spec = ModuleSpec {
7578            module_id: "spawned-not-admitted".to_string(),
7579            program: super::terminal_history_tests::fake_aft_stub_path(),
7580            args: Vec::new(),
7581            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7582                .into_iter()
7583                .map(|key| (key.to_string(), dir.path().display().to_string()))
7584                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7585                .collect(),
7586            reserved: false,
7587            reserved_prefixes: Vec::new(),
7588            protocol: ModuleProtocol::Subc,
7589            overlap: Default::default(),
7590            carry_containment: false,
7591        };
7592        let handle = SupervisorHandle::new();
7593        let mut supervisor =
7594            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7595                .with_handle(handle.clone());
7596        let observed = Arc::new(Mutex::new(None));
7597        let hook_handle = handle.clone();
7598        let hook_observed = Arc::clone(&observed);
7599        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7600            let state = hook_handle
7601                .get(module_id)
7602                .expect("a configured module is on the roster before it is spawned")
7603                .state()
7604                .unwrap();
7605            *hook_observed.lock().unwrap() = Some(state);
7606        })));
7607
7608        let module = supervisor.supervise_configured(spec, true).unwrap();
7609        assert_eq!(
7610            *observed.lock().unwrap(),
7611            Some(ModuleState::Starting),
7612            "between the spawn and its admission the supervisor still reports starting"
7613        );
7614        assert_eq!(module.state().unwrap(), ModuleState::Running);
7615        module.stop().await.unwrap();
7616    }
7617
7618    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
7619        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
7620        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
7621            snapshot.process_alive = true;
7622            snapshot.pid = Some(41);
7623            snapshot.spawned_at_ms = Some(42);
7624            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
7625            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
7626            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
7627                device: 43,
7628                inode: 44,
7629                #[cfg(windows)]
7630                windows_image: None,
7631            });
7632        })
7633        .unwrap();
7634        snapshot
7635    }
7636
7637    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
7638        let snapshot = lock_snapshot(snapshot).unwrap();
7639        assert!(!snapshot.process_alive);
7640        assert_eq!(snapshot.pid, None);
7641        assert_eq!(snapshot.spawned_at_ms, None);
7642        assert_eq!(snapshot.spawned_from, None);
7643        assert_eq!(snapshot.launch_spec_digest, None);
7644        assert_eq!(snapshot.spawned_file_identity, None);
7645    }
7646
7647    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7648    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
7649        let supervisor =
7650            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7651        let mut runtime = supervisor.runtime_config();
7652        runtime.test_seed_stale_facts_before_enable_spawn = true;
7653        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
7654        let mut child = None;
7655        let spec = ModuleSpec {
7656            module_id: "failed-enable-clears-facts".to_string(),
7657            program: PathBuf::from("/definitely/missing/failed-enable-module"),
7658            args: Vec::new(),
7659            env: Vec::new(),
7660            reserved: false,
7661            reserved_prefixes: Vec::new(),
7662            protocol: ModuleProtocol::Subc,
7663            overlap: Default::default(),
7664            carry_containment: false,
7665        };
7666
7667        let result = set_child_enabled(
7668            &spec,
7669            &runtime,
7670            &supervisor.registry,
7671            &supervisor.process_liveness,
7672            &snapshot,
7673            &mut child,
7674            true,
7675            &runtime.failure_context(false),
7676        )
7677        .await;
7678
7679        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
7680        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7681        assert_snapshot_process_facts_cleared(&snapshot);
7682    }
7683
7684    #[tokio::test]
7685    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
7686        let supervisor =
7687            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7688        let runtime = supervisor.runtime_config();
7689        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
7690            ModuleState::Restarting,
7691            true,
7692        )));
7693        let spec = ModuleSpec {
7694            module_id: "start-stranded-restarting".to_string(),
7695            program: super::terminal_history_tests::fake_aft_stub_path(),
7696            args: Vec::new(),
7697            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
7698            reserved: false,
7699            reserved_prefixes: Vec::new(),
7700            protocol: ModuleProtocol::None,
7701            overlap: Default::default(),
7702            carry_containment: false,
7703        };
7704        let mut child = None;
7705        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
7706        assert!(!super::set_child_enabled(
7707            &spec,
7708            &runtime,
7709            &Registry::default(),
7710            &supervisor.process_liveness,
7711            &snapshot,
7712            &mut child,
7713            true,
7714            &runtime.failure_context(false),
7715        )
7716        .await
7717        .unwrap());
7718        assert!(child.is_none());
7719        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
7720        assert!(super::set_child_enabled(
7721            &spec,
7722            &runtime,
7723            &Registry::default(),
7724            &supervisor.process_liveness,
7725            &snapshot,
7726            &mut child,
7727            true,
7728            &runtime.failure_context(false),
7729        )
7730        .await
7731        .unwrap());
7732        assert_eq!(
7733            lock_snapshot(&snapshot).unwrap().state,
7734            ModuleState::Running
7735        );
7736        let mut child = child.unwrap();
7737        child.start_kill().unwrap();
7738        child.wait().await.unwrap();
7739    }
7740
7741    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7742    async fn failed_reload_spawn_clears_current_process_facts() {
7743        let supervisor =
7744            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7745        let mut runtime = supervisor.runtime_config();
7746        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
7747        let snapshot = stale_process_snapshot(ModuleState::Running, true);
7748        let mut child = None;
7749        let spec = ModuleSpec {
7750            module_id: "failed-reload-clears-facts".to_string(),
7751            program: PathBuf::from("/unused/failed-reload-module"),
7752            args: Vec::new(),
7753            env: Vec::new(),
7754            reserved: false,
7755            reserved_prefixes: Vec::new(),
7756            protocol: ModuleProtocol::Subc,
7757            overlap: Default::default(),
7758            carry_containment: false,
7759        };
7760
7761        let result = handle_reload_spawn_failure(
7762            &spec,
7763            &runtime,
7764            &supervisor.process_liveness,
7765            &snapshot,
7766            &mut child,
7767            "forced reload spawn failure".to_string(),
7768        )
7769        .await;
7770
7771        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
7772        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7773        assert_snapshot_process_facts_cleared(&snapshot);
7774    }
7775
7776    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7777    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
7778        let supervisor =
7779            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7780        let snapshot = stale_process_snapshot(ModuleState::Running, true);
7781        let module = supervisor.supervised_module(
7782            ModuleSpec {
7783                module_id: "drop-clears-facts".to_string(),
7784                program: PathBuf::from("/unused/drop-module"),
7785                args: Vec::new(),
7786                env: Vec::new(),
7787                reserved: false,
7788                reserved_prefixes: Vec::new(),
7789                protocol: ModuleProtocol::Subc,
7790                overlap: Default::default(),
7791                carry_containment: false,
7792            },
7793            supervisor.runtime_config(),
7794            Arc::clone(&snapshot),
7795            None,
7796        );
7797        assert!(!module
7798            .inner
7799            .monitor
7800            .lock()
7801            .unwrap()
7802            .as_ref()
7803            .unwrap()
7804            .is_finished());
7805
7806        drop(module);
7807
7808        assert_eq!(
7809            lock_snapshot(&snapshot).unwrap().state,
7810            ModuleState::Stopped
7811        );
7812        assert_snapshot_process_facts_cleared(&snapshot);
7813    }
7814
7815    #[cfg(unix)]
7816    #[tokio::test]
7817    async fn rescan_preserves_running_protocol_until_respawn() {
7818        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
7819        let initial = ModuleSpec {
7820            module_id: "rescan-protocol".into(),
7821            program: PathBuf::from("/bin/sleep"),
7822            args: vec!["60".into()],
7823            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7824                .into_iter()
7825                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
7826                .collect(),
7827            reserved: false,
7828            reserved_prefixes: vec![],
7829            protocol: ModuleProtocol::None,
7830            overlap: Default::default(),
7831            carry_containment: false,
7832        };
7833        let supervisor =
7834            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7835        let module = supervisor.spawn(initial.clone()).unwrap();
7836        assert!(module.status().unwrap().live);
7837        let mut next = initial;
7838        next.protocol = ModuleProtocol::Subc;
7839        module
7840            .update_configuration(next.clone(), HealthConfig::default(), None)
7841            .await
7842            .unwrap();
7843        assert!(
7844            module.status().unwrap().live,
7845            "rescan must not require HELLO from the old non-wire process"
7846        );
7847        let runtime = supervisor.runtime_config();
7848        let action = on_child_exit(
7849            &next,
7850            RestartPolicy::default(),
7851            &supervisor.registry,
7852            &module.inner.snapshot,
7853            &runtime.terminal_ring,
7854            &runtime.spawn_events,
7855            &runtime.child_roster,
7856            ExitReport {
7857                kind: ExitKind::Clean,
7858                code: Some(0),
7859                signal: None,
7860                at_ms: unix_ms_now(),
7861            },
7862            true,
7863        )
7864        .await;
7865        assert!(
7866            matches!(action, NextAction::Restart { .. }),
7867            "the old non-wire process's clean exit must restart"
7868        );
7869        module.drain().await.unwrap();
7870    }
7871
7872    #[cfg(unix)]
7873    #[tokio::test]
7874    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
7875        use std::os::unix::fs::PermissionsExt;
7876        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
7877        let script = dir.join("module.sh");
7878        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
7879        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
7880        let record_path = dir.join("live-children.json");
7881        let supervisor =
7882            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7883                .with_live_children_record(&record_path);
7884        for (program, args) in [
7885            (PathBuf::from("sleep"), vec!["60".into()]),
7886            (script, vec![]),
7887        ] {
7888            let spec = ModuleSpec {
7889                module_id: "image-identity".into(),
7890                program: program.clone(),
7891                args,
7892                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7893                    .into_iter()
7894                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
7895                    .collect(),
7896                reserved: false,
7897                reserved_prefixes: vec![],
7898                protocol: ModuleProtocol::None,
7899                overlap: Default::default(),
7900                carry_containment: false,
7901            };
7902            let module = supervisor.spawn(spec).unwrap();
7903            #[cfg(target_os = "macos")]
7904            {
7905                // SETEXEC confirmation is asynchronous; the orphan record must
7906                // identify the final image, never the intermediate trampoline.
7907                let deadline = Instant::now() + Duration::from_secs(5);
7908                while crate::live_children::read_record(&record_path)
7909                    .unwrap()
7910                    .iter()
7911                    .all(|entry| entry.executable.is_none())
7912                {
7913                    assert!(Instant::now() < deadline, "module image was not confirmed");
7914                    tokio::time::sleep(Duration::from_millis(5)).await;
7915                }
7916            }
7917            let entry = crate::live_children::read_record(&record_path)
7918                .unwrap()
7919                .pop()
7920                .unwrap();
7921            let observed = subc_os::Process::open(entry.pid)
7922                .unwrap()
7923                .unwrap()
7924                .observe()
7925                .unwrap();
7926            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
7927            module.drain().await.unwrap();
7928            assert_eq!(
7929                verdict,
7930                crate::live_children::IdentityVerdict::Matches,
7931                "program {program:?}: recorded {entry:?}, observed {observed:?}"
7932            );
7933        }
7934    }
7935
7936    #[cfg(unix)]
7937    fn http_fixture(
7938        dir: &std::path::Path,
7939        url: &str,
7940        threshold: u32,
7941    ) -> crate::daemon_config::ConfiguredModule {
7942        let path = dir.join("subc.jsonc");
7943        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
7944            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
7945            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
7946            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
7947            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
7948        }}}).to_string()).unwrap();
7949        crate::daemon_config::load(&path)
7950            .unwrap()
7951            .unwrap()
7952            .modules
7953            .pop()
7954            .unwrap()
7955    }
7956
7957    #[cfg(unix)]
7958    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
7959        timeout(Duration::from_secs(5), async {
7960            loop {
7961                if module.status().unwrap().health.status == status {
7962                    break;
7963                }
7964                sleep(Duration::from_millis(5)).await;
7965            }
7966        })
7967        .await
7968        .unwrap_or_else(|_| {
7969            panic!(
7970                "expected {status:?}, got {:?}",
7971                module.status().unwrap().health
7972            )
7973        });
7974    }
7975
7976    #[cfg(unix)]
7977    #[tokio::test]
7978    async fn http_health_status_flips_ok_failing_ok() {
7979        use tokio::io::{AsyncReadExt, AsyncWriteExt};
7980        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
7981        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
7982        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
7983        let serving_status = status.clone();
7984        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
7985        let server = tokio::spawn(async move {
7986            loop {
7987                let (mut stream, _) = listener.accept().await.unwrap();
7988                let mut request = [0u8; 2048];
7989                let count = stream.read(&mut request).await.unwrap();
7990                assert!(count > 0, "a probe must send an HTTP request");
7991                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
7992                let body = if code == 200 {
7993                    "ready"
7994                } else {
7995                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
7996                };
7997                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
7998                let _ = stream.write_all(response.as_bytes()).await;
7999            }
8000        });
8001        let configured = http_fixture(&dir, &url, 1000);
8002        let module =
8003            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8004                .supervise_configured_with_health(
8005                    configured.module_spec(),
8006                    true,
8007                    configured.health,
8008                    configured.drain_timeout_ms,
8009                    configured.restart,
8010                )
8011                .unwrap();
8012        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8013        status.store(503, std::sync::atomic::Ordering::SeqCst);
8014        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
8015        assert!(module
8016            .status()
8017            .unwrap()
8018            .health
8019            .detail
8020            .unwrap()
8021            .contains("scratch failure"));
8022        status.store(200, std::sync::atomic::Ordering::SeqCst);
8023        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8024        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
8025        let before = module.status().unwrap();
8026        let (spec, mut health) = module.configuration().unwrap();
8027        health.http = None;
8028        module
8029            .update_configuration(spec.clone(), health.clone(), Some(10))
8030            .await
8031            .unwrap();
8032        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
8033        health.http = Some(url);
8034        module
8035            .update_configuration(spec, health, Some(10))
8036            .await
8037            .unwrap();
8038        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8039        assert_eq!(
8040            module.status().unwrap().pid,
8041            before.pid,
8042            "changing a probe must apply live, not restart its process"
8043        );
8044        let (spec, mut health) = module.configuration().unwrap();
8045        health.failure_threshold = 2;
8046        module
8047            .update_configuration(spec, health, Some(10))
8048            .await
8049            .unwrap();
8050        status.store(503, std::sync::atomic::Ordering::SeqCst);
8051        timeout(Duration::from_secs(5), async {
8052            while module.status().unwrap().spawn_generation == before.spawn_generation {
8053                sleep(Duration::from_millis(5)).await;
8054            }
8055        })
8056        .await
8057        .expect("sustained HTTP 503 responses must trigger the health restart policy");
8058        module.drain().await.unwrap();
8059        server.abort();
8060    }
8061
8062    #[cfg(unix)]
8063    #[tokio::test]
8064    async fn http_health_replacement_recovers_after_one_restart() {
8065        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8066        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
8067        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8068        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8069        let mut configured = http_fixture(&dir, &url, 2);
8070        configured.restart.max_restarts = 2;
8071        let module =
8072            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8073                .supervise_configured_with_health(
8074                    configured.module_spec(),
8075                    true,
8076                    configured.health,
8077                    configured.drain_timeout_ms,
8078                    configured.restart,
8079                )
8080                .unwrap();
8081        let first_generation = module.status().unwrap().spawn_generation;
8082        let observed = module.clone();
8083        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
8084        let server = tokio::spawn(async move {
8085            let mut generation = first_generation;
8086            let mut probes = 0;
8087            loop {
8088                let (mut stream, _) = listener.accept().await.unwrap();
8089                let mut request = [0; 2048];
8090                assert!(stream.read(&mut request).await.unwrap() > 0);
8091                let current = observed.status().unwrap().spawn_generation;
8092                if current != generation {
8093                    generation = current;
8094                    probes = 0;
8095                }
8096                probes += 1;
8097                // The original process never recovers. Each replacement has one
8098                // transient miss, then answers successfully for its remaining life.
8099                let healthy = generation != first_generation && probes > 1;
8100                let code = if healthy { 200 } else { 503 };
8101                let response = format!(
8102                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
8103                );
8104                let _ = stream.write_all(response.as_bytes()).await;
8105                if healthy {
8106                    healthy_tx.send(()).await.unwrap();
8107                }
8108            }
8109        });
8110        timeout(Duration::from_secs(5), async {
8111            let mut healthy_probes = 0;
8112            loop {
8113                if module.state().unwrap() == ModuleState::Failed {
8114                    break;
8115                }
8116                while healthy_rx.try_recv().is_ok() {
8117                    healthy_probes += 1;
8118                }
8119                if healthy_probes >= 3
8120                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
8121                {
8122                    break;
8123                }
8124                sleep(Duration::from_millis(5)).await;
8125            }
8126        })
8127        .await
8128        .expect("the replacement must settle healthy without exhausting its restart budget");
8129        let status = module.status().unwrap();
8130        module.drain().await.unwrap();
8131        server.abort();
8132        assert_eq!(status.state, ModuleState::Running);
8133        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
8134        assert_eq!(status.health.consecutive_failures, 0);
8135        assert_eq!(status.lifetime_restarts, 1);
8136        assert_eq!(status.restart_count, 1);
8137    }
8138
8139    #[cfg(unix)]
8140    #[tokio::test]
8141    async fn http_health_sustained_failure_exhausts_restart_budget() {
8142        use std::collections::BTreeMap;
8143        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8144        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
8145        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8146        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8147        let mut configured = http_fixture(&dir, &url, 2);
8148        configured.restart.max_restarts = 2;
8149        let module =
8150            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8151                .supervise_configured_with_health(
8152                    configured.module_spec(),
8153                    true,
8154                    configured.health,
8155                    configured.drain_timeout_ms,
8156                    configured.restart,
8157                )
8158                .unwrap();
8159        let observed = module.clone();
8160        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
8161        let served = probes.clone();
8162        let server = tokio::spawn(async move {
8163            loop {
8164                let (mut stream, _) = listener.accept().await.unwrap();
8165                let mut request = [0; 2048];
8166                assert!(stream.read(&mut request).await.unwrap() > 0);
8167                *served
8168                    .lock()
8169                    .unwrap()
8170                    .entry(observed.status().unwrap().spawn_generation)
8171                    .or_default() += 1;
8172                let _ = stream
8173                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
8174                    .await;
8175            }
8176        });
8177        timeout(Duration::from_secs(5), async {
8178            while module.state().unwrap() != ModuleState::Failed {
8179                sleep(Duration::from_millis(5)).await;
8180            }
8181        })
8182        .await
8183        .expect("sustained failures must still exhaust the health restart budget");
8184        let status = module.status().unwrap();
8185        server.abort();
8186        assert_eq!(status.restart_count, 2);
8187        assert_eq!(status.lifetime_restarts, 2);
8188        assert!(!status.process_alive);
8189        assert_eq!(
8190            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
8191            vec![2, 2, 2]
8192        );
8193    }
8194
8195    #[cfg(unix)]
8196    #[tokio::test]
8197    async fn spawn_clears_the_previous_process_health() {
8198        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
8199        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
8200        configured.health.cadence = Duration::from_secs(3600);
8201        let module =
8202            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8203                .supervise_configured_with_health(
8204                    configured.module_spec(),
8205                    false,
8206                    configured.health,
8207                    configured.drain_timeout_ms,
8208                    configured.restart,
8209                )
8210                .unwrap();
8211        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
8212            state.health.status = SupervisorHealthStatus::Unresponsive;
8213            state.health.consecutive_failures = 3;
8214            state.health.detail = Some("old process refused connections".into());
8215            state.health.metrics = Some(serde_json::json!({"old": true}));
8216            state.health.last_probe_ms = Some(123);
8217            state.health.last_action = Some("restart".into());
8218            state.health.last_action_ms = Some(124);
8219        })
8220        .unwrap();
8221        module.set_enabled(true).await.unwrap();
8222        let health = module.status().unwrap().health;
8223        module.drain().await.unwrap();
8224        assert_eq!(health.consecutive_failures, 0);
8225        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
8226        assert_eq!(health.detail, None);
8227        assert_eq!(health.metrics, None);
8228        assert_eq!(health.last_probe_ms, None);
8229        assert_eq!(health.last_action.as_deref(), Some("restart"));
8230        assert_eq!(health.last_action_ms, Some(124));
8231    }
8232
8233    #[cfg(unix)]
8234    #[tokio::test]
8235    async fn http_health_no_listener_restarts_after_consecutive_failures() {
8236        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
8237        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8238        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
8239        drop(unused);
8240        let configured = http_fixture(&dir, &url, 2);
8241        let module =
8242            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8243                .supervise_configured_with_health(
8244                    configured.module_spec(),
8245                    true,
8246                    configured.health,
8247                    configured.drain_timeout_ms,
8248                    configured.restart,
8249                )
8250                .unwrap();
8251        let before = module.status().unwrap().spawn_generation;
8252        timeout(Duration::from_secs(5), async {
8253            loop {
8254                let status = module.status().unwrap();
8255                if status.spawn_generation > before {
8256                    assert!(status.lifetime_restarts > 0);
8257                    break;
8258                }
8259                sleep(Duration::from_millis(5)).await;
8260            }
8261        })
8262        .await
8263        .expect("sustained HTTP refusal must trigger the health restart policy");
8264        module.drain().await.unwrap();
8265    }
8266
8267    #[cfg(unix)]
8268    #[tokio::test]
8269    async fn http_health_timeout_honours_deadline() {
8270        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8271        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8272        let server = tokio::spawn(async move {
8273            let _held = listener.accept().await.unwrap();
8274            std::future::pending::<()>().await;
8275        });
8276        let error = timeout(
8277            Duration::from_secs(1),
8278            probe_http_health(&url, Duration::from_millis(10)),
8279        )
8280        .await
8281        .expect("the probe must enforce its own deadline")
8282        .unwrap_err();
8283        server.abort();
8284        assert!(error.to_string().contains("timed out"));
8285    }
8286
8287    #[cfg(unix)]
8288    #[tokio::test]
8289    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
8290        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8291        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
8292        let url = format!(
8293            "http://localhost:{}/healthz",
8294            listener.local_addr().unwrap().port()
8295        );
8296        let server = tokio::spawn(async move {
8297            let (mut stream, _) = listener.accept().await.unwrap();
8298            let mut request = [0u8; 2048];
8299            assert!(stream.read(&mut request).await.unwrap() > 0);
8300            stream
8301                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
8302                .await
8303                .unwrap();
8304        });
8305        // The deadline only bounds a hang. A probe that never tried the IPv6
8306        // address would be refused on 127.0.0.1 and fail at once, so a longer
8307        // deadline does not weaken the assertion; one second timed out under a
8308        // loaded parallel test run.
8309        assert_eq!(
8310            probe_http_health(&url, Duration::from_secs(10))
8311                .await
8312                .unwrap()
8313                .status,
8314            HealthStatus::Ok
8315        );
8316        server.await.unwrap();
8317    }
8318
8319    #[cfg(unix)]
8320    #[tokio::test]
8321    async fn http_health_timeout_keeps_partial_status_and_body() {
8322        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8323        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8324        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8325        let server = tokio::spawn(async move {
8326            let (mut stream, _) = listener.accept().await.unwrap();
8327            let mut request = [0u8; 2048];
8328            assert!(stream.read(&mut request).await.unwrap() > 0);
8329            stream
8330                .write_all(
8331                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
8332                )
8333                .await
8334                .unwrap();
8335            std::future::pending::<()>().await;
8336        });
8337        let error = probe_http_health(&url, Duration::from_secs(1))
8338            .await
8339            .unwrap_err()
8340            .to_string();
8341        server.abort();
8342        assert!(
8343            error.contains("timed out")
8344                && error.contains("503 Unavailable")
8345                && error.contains("partial diagnostic"),
8346            "{error}"
8347        );
8348    }
8349
8350    #[cfg(unix)]
8351    #[tokio::test]
8352    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
8353        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8354        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8355        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8356        let server = tokio::spawn(async move {
8357            let (mut stream, _) = listener.accept().await.unwrap();
8358            let mut request = [0u8; 2048];
8359            assert!(stream.read(&mut request).await.unwrap() > 0);
8360            let body = format!("{}not-in-diagnostic", "x".repeat(400));
8361            let response = format!(
8362                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
8363                body.len()
8364            );
8365            stream.write_all(response.as_bytes()).await.unwrap();
8366        });
8367        let error = probe_http_health(&url, Duration::from_secs(1))
8368            .await
8369            .unwrap_err()
8370            .to_string();
8371        server.await.unwrap();
8372        assert!(error.ends_with(&"x".repeat(200)), "{error}");
8373        assert_eq!(error.split(": ").last().unwrap().len(), 200);
8374        assert!(!error.contains("not-in-diagnostic"));
8375    }
8376
8377    #[cfg(unix)]
8378    #[tokio::test]
8379    async fn http_health_real_nats_server_monitoring() {
8380        if std::process::Command::new("nats-server")
8381            .arg("--version")
8382            .env("XDG_DATA_HOME", std::env::temp_dir())
8383            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
8384            .env("XDG_CONFIG_HOME", std::env::temp_dir())
8385            .output()
8386            .is_err()
8387        {
8388            eprintln!(
8389                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
8390            );
8391            return;
8392        }
8393        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
8394        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8395        let port = monitor.local_addr().unwrap().port();
8396        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8397        let client_port = client.local_addr().unwrap().port();
8398        let config = dir.join("server.conf");
8399        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
8400        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
8401        configured.program = PathBuf::from("nats-server");
8402        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
8403        drop(monitor);
8404        drop(client);
8405        let module =
8406            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8407                .supervise_configured_with_health(
8408                    configured.module_spec(),
8409                    true,
8410                    configured.health,
8411                    configured.drain_timeout_ms,
8412                    configured.restart,
8413                )
8414                .unwrap();
8415        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8416        module.drain().await.unwrap();
8417    }
8418
8419    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8420    async fn configuration_update_does_not_replace_captured_running_process_facts() {
8421        let supervisor =
8422            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8423        let snapshot = stale_process_snapshot(ModuleState::Running, true);
8424        let initial = ModuleSpec {
8425            module_id: "rescan-preserves-spawn-facts".to_string(),
8426            program: PathBuf::from("/spawned/module"),
8427            args: Vec::new(),
8428            env: Vec::new(),
8429            reserved: false,
8430            reserved_prefixes: Vec::new(),
8431            protocol: ModuleProtocol::Subc,
8432            overlap: Default::default(),
8433            carry_containment: false,
8434        };
8435        let module = supervisor.supervised_module(
8436            initial.clone(),
8437            supervisor.runtime_config(),
8438            snapshot,
8439            None,
8440        );
8441        let before = module.status().unwrap();
8442        let mut replacement = initial;
8443        replacement.program = PathBuf::from("/rescanned/replacement-module");
8444
8445        module
8446            .update_configuration(replacement, HealthConfig::default(), None)
8447            .await
8448            .unwrap();
8449
8450        let after = module.status().unwrap();
8451        assert_eq!(after.pid, before.pid);
8452        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
8453        assert_eq!(after.spawned_from, before.spawned_from);
8454        drop(module);
8455    }
8456}
8457
8458fn unix_ms_now() -> u64 {
8459    SystemTime::now()
8460        .duration_since(UNIX_EPOCH)
8461        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
8462        .unwrap_or(0)
8463}
8464
8465async fn supervise_loop(
8466    mut spec: ModuleSpec,
8467    mut runtime: SupervisorRuntimeConfig,
8468    registry: Arc<Registry>,
8469    process_liveness: Arc<SupervisorProcessLiveness>,
8470    snapshot: SharedSnapshot,
8471    mut child: Option<SupervisedChild>,
8472    mut commands: mpsc::Receiver<SupervisorCommand>,
8473) {
8474    let mut health_probe = HealthProbeRuntime::default();
8475    // Registry writes (including embedded callers) notify this module only.
8476    // Subscribe before the first refresh; watch retains changes that arrive
8477    // while commands or probes are running. No polling fallback is needed.
8478    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
8479        Ok(changes) => Some(changes),
8480        Err(err) => {
8481            // A poisoned registry cannot accept further writes, so it cannot
8482            // recover via a timer. Keep exit and command handling alive.
8483            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
8484            None
8485        }
8486    };
8487    // All restart backoffs run here, including health and operator requests.
8488    // While one is pending the loop serves commands, so disable or drain can
8489    // cancel the replacement without spawning a process just to stop it.
8490    let mut pending_respawn: Option<PendingRespawn> = None;
8491    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
8492    // before anything else so a stop that interrupted a swap runs at once.
8493    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
8494    let mut previous_instance_recheck = tokio::time::interval_at(
8495        Instant::now() + Duration::from_secs(5),
8496        Duration::from_secs(5),
8497    );
8498    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
8499    'supervision: loop {
8500        #[cfg(test)]
8501        {
8502            lock_snapshot(&snapshot).unwrap().actor_select = None;
8503        }
8504        #[cfg(target_os = "macos")]
8505        if let Some(active) = child.as_mut() {
8506            active.confirm_privacy_exec().await;
8507        }
8508        if let Some(scheduled) = runtime
8509            .scheduled_respawn
8510            .lock()
8511            .unwrap_or_else(|p| p.into_inner())
8512            .take()
8513        {
8514            runtime.recovery_is_daemon = scheduled.daemon_initiated;
8515            pending_respawn = Some(scheduled);
8516        }
8517        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
8518            pending_respawn = None;
8519            cancel_deferred_reload(
8520                &runtime,
8521                &spec.module_id,
8522                "respawn cancelled by a supervisor command",
8523            );
8524        }
8525        if child.is_none()
8526            && pending_respawn.is_none()
8527            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
8528        {
8529            cancel_deferred_reload(
8530                &runtime,
8531                &spec.module_id,
8532                "respawn cancelled before a replacement was spawned",
8533            );
8534            let mut stranded = false;
8535            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8536                state.respawn_pending = false;
8537                state.coalesced_restart_pending = false;
8538                if matches!(
8539                    state.state,
8540                    ModuleState::Restarting
8541                        | ModuleState::Starting
8542                        | ModuleState::Draining
8543                        | ModuleState::Unresponsive
8544                ) {
8545                    error!(module_id = %spec.module_id, state = ?state.state,
8546                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
8547                    stranded = true;
8548                }
8549            });
8550            if stranded {
8551                fail_snapshot(
8552                    &snapshot,
8553                    Some(&spec.module_id),
8554                    None,
8555                    &runtime.failure_context(runtime.recovery_is_daemon),
8556                );
8557            }
8558        }
8559        if child.is_none()
8560            && (runtime.child_roster.is_closed()
8561                || pending_respawn.is_none()
8562                || !lock_snapshot(&snapshot)
8563                    .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
8564        {
8565            containment::terminate(&runtime.child_roster.containment).await;
8566        }
8567        if let Some(command) = requeued.pop_front() {
8568            if !handle_supervisor_command(
8569                command,
8570                &mut spec,
8571                &mut runtime,
8572                &registry,
8573                &process_liveness,
8574                &snapshot,
8575                &mut child,
8576                &mut commands,
8577                &mut requeued,
8578            )
8579            .await
8580            {
8581                containment::terminate(&runtime.child_roster.containment).await;
8582                return;
8583            }
8584            if child.is_some() || !respawn_still_pending(&snapshot) {
8585                pending_respawn = None;
8586                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8587                    state.respawn_pending = false
8588                });
8589            }
8590            continue;
8591        }
8592        if child.is_some() {
8593            runtime.recovery_is_daemon = true;
8594            #[cfg(test)]
8595            {
8596                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
8597            }
8598            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
8599            let active = child.as_mut().expect("child checked above");
8600            if active.registered_since_spawn() {
8601                if let Some(delivery) = &runtime.config_delivery {
8602                    delivery.queue_registered(
8603                        &spec.module_id,
8604                        runtime.forwarding.as_ref(),
8605                        &registry,
8606                        active,
8607                    );
8608                }
8609            }
8610            if runtime.child_roster.is_closed() || !active.awaiting_registration() {
8611                active.registration_deadline = None;
8612            }
8613            let registration_deadline = active.registration_deadline;
8614            let no_checks = runtime.health.http.is_none()
8615                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
8616                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
8617            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
8618                state
8619                    .failed_retry
8620                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
8621            });
8622            let wake_at = match (health_probe.wake_at(), uptime_reset) {
8623                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
8624                (probe, None) => probe,
8625            };
8626            let wake_at = match (wake_at, registration_deadline) {
8627                (Some(wake), Some(register)) => Some(wake.min(register)),
8628                (wake, register) => wake.or(register),
8629            };
8630            #[cfg(test)]
8631            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
8632            let probe_sleep = supervisor_sleep(wake_at);
8633            tokio::pin!(probe_sleep);
8634            // A `protocol: "none"` child (a plain process that never registers
8635            // over the subc wire, such as nats-server) with no HTTP health
8636            // check has nothing to probe, so it waits only for its exit or a
8637            // supervisor command. A rescan that adds an HTTP check is a command.
8638            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
8639            #[cfg(test)]
8640            {
8641                let registration_pending = wire_child
8642                    && registration_changes
8643                        .as_ref()
8644                        .is_some_and(|changes| changes.has_changed().unwrap());
8645                let mut state = lock_snapshot(&snapshot).unwrap();
8646                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
8647                    generation: state.spawn_generation,
8648                    turn: state.actor_turns,
8649                    registered_connection: health_probe.registered_connection,
8650                    next_probe_at: health_probe.next_probe_at,
8651                    wake_after,
8652                });
8653            }
8654            let registration = active
8655                .awaiting_registration()
8656                .then(|| active.registration_spawn.clone())
8657                .flatten();
8658            #[cfg(all(test, target_os = "linux"))]
8659            let registration_before_exit = !runtime.child_roster.admission_event_for_test(
8660                &spec.module_id,
8661                "observe_registration_before_exit",
8662                Some(active.pid),
8663            ) && registration.is_some();
8664            #[cfg(not(all(test, target_os = "linux")))]
8665            let registration_before_exit = false;
8666            #[cfg(all(test, target_os = "linux"))]
8667            let registration_child_is_running = |child: &mut SupervisedChild| {
8668                // Model the interval after SIGKILL is accepted but before the
8669                // child becomes waitable, without delaying the actual kill.
8670                registration_before_exit || registration_child_is_running(child)
8671            };
8672            let registration_failed = async {
8673                match &registration {
8674                    Some(spawn) => spawn.wait_registration_failure().await,
8675                    None => std::future::pending().await,
8676                }
8677            };
8678            let active_child = child.as_mut().expect("child checked above");
8679            tokio::select! {
8680                _ = registration_failed => {
8681                    if !active_child.registered_since_spawn() && registration_child_is_running(active_child) {
8682                        active_child.registration_deadline = None;
8683                        active_child.spawn_failure = Some("registration_channel_closed".into());
8684                        if let Err(error) = stop_unregistered_child(active_child).await {
8685                            fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8686                                &runtime.spawn_events, format!("registration_channel_closed: {error}"),
8687                                &runtime.failure_context(true));
8688                            child = None;
8689                        }
8690                    }
8691                    continue;
8692                }
8693                wait_result = active_child.wait(), if !registration_before_exit => {
8694                    if let Ok(mut state) = lock_snapshot(&snapshot) {
8695                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
8696                    }
8697                    #[cfg(test)]
8698                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8699                    // Every arm below that gives up on the CHILD must keep the
8700                    // supervision task itself alive (child = None, loop
8701                    // continues into command-serving mode). Returning here
8702                    // closes the command channel, which makes the module
8703                    // permanently unrestartable in-band: a clean child exit
8704                    // of an enabled module once wedged the fleet this way
8705                    // ('supervisor command channel is closed') and required a
8706                    // full daemon restart to recover.
8707                    let exit_report = match wait_result {
8708                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
8709                        Err(err) => {
8710                            active_child.drain_stderr(&spec.module_id).await;
8711                            fail_after_wait_error(
8712                                &spec.module_id,
8713                                &snapshot,
8714                                &runtime.terminal_ring,
8715                                &runtime.spawn_events,
8716                                format!("failed to wait for supervised module: {err}"),
8717                            &runtime.failure_context(true),
8718                            );
8719                            untrack_if_registration_released(
8720                                &process_liveness,
8721                                &registry,
8722                                &spec.module_id,
8723                                &snapshot,
8724                            );
8725                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
8726                            child = None;
8727                            continue;
8728                        }
8729                    };
8730                    active_child.drain_stderr(&spec.module_id).await;
8731
8732                    let delivery=runtime.config_delivery.clone();
8733                    let exited_spawn=active_child.spawned_at;
8734                    let spawn=active_child.config_delivery.clone();
8735                    let refusal=active_child.spawn_failure.as_deref().filter(|detail|detail.starts_with("program_refused:") && !active_child.live_program_binding.as_ref().is_some_and(|binding|binding.is_refused())).zip(active_child.admission_base.clone()).map(|(detail,base)|(base,subc_config::raw_digest(detail.as_bytes())));
8736                    let module_id=spec.module_id.clone();
8737                    // Publication may own the store mutex for local durable I/O.
8738                    // Keep commands selectable until the child's exit is recorded;
8739                    // no respawn disposition is chosen before that observation.
8740                    let observed=async move {
8741                        if let Some(delivery)=delivery {
8742                            delivery.0.on_blocking(move |owner| {
8743                                if let Some(spawn)=spawn { owner.delivery_child_exited(&spawn)?; }
8744                                if let Some((base,identity))=refusal {owner.record_program_refusal(&module_id,&base,identity)?;}
8745                                Ok(())
8746                            }).await
8747                        } else {Ok(())}
8748                    };
8749                    tokio::pin!(observed);
8750                    loop {
8751                        tokio::select! {
8752                            result=&mut observed=>{
8753                                if let Err(error)=result {warn!(module_id=%spec.module_id,%error,"configuration exit observation failed");}
8754                                break;
8755                            }
8756                            command=commands.recv()=>{
8757                                let Some(command)=command else {containment::terminate(&runtime.child_roster.containment).await;return;};
8758                                let deferred=||SuperviseError::ConfigDeliveryDeferred {module_id:spec.module_id.clone()};
8759                                let command=match command {
8760                                    SupervisorCommand::Restart {reply,..}|SupervisorCommand::Reload {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8761                                    SupervisorCommand::Swap {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8762                                    command @ (SupervisorCommand::ConfigApplyRestart {..}|SupervisorCommand::ConfigPolicyProbe {..}|SupervisorCommand::ConfigDeliveryFailed {..})=>{requeued.push_back(command);continue;}
8763                                    command=>command,
8764                                };
8765                                if !handle_supervisor_command(command,&mut spec,&mut runtime,&registry,&process_liveness,&snapshot,&mut child,&mut commands,&mut requeued).await {
8766                                    containment::terminate(&runtime.child_roster.containment).await;return;
8767                                }
8768                                if child.as_ref().is_none_or(|current|current.spawned_at!=exited_spawn) {continue 'supervision;}
8769                            }
8770                        }
8771                    }
8772                    let active_child=child.as_mut().expect("exit observation retained its child");
8773                    // An unavailable image can race the program's own exit.
8774                    // Only the admission kill is budget-free; another exit must
8775                    // retain its ordinary crash/clean-exit accounting.
8776                    let admission_kill = admission_exit::is_admission_kill(active_child.admission_kill_sent, &exit_report);
8777                    let deferred_spawn=admission_kill && active_child.spawn_failure.as_deref().is_some_and(|detail|detail.starts_with("config_delivery_deferred:"));
8778                    let refused_program = active_child.live_program_binding.as_ref().is_some_and(|binding| binding.is_refused());
8779                    let exit_held = !deferred_spawn && (refused_program || runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.exit_is_held(&spec.module_id)));
8780                    let parked = exit_held && (!refused_program || active_child.admission_base.is_some())
8781                        && try_park_config_attempt(&spec, &runtime, &snapshot, &runtime.failure_context(true),
8782                            if refused_program { active_child.admission_base.as_ref() } else { None })
8783                            .unwrap_or_else(|error| { warn!(module_id=%spec.module_id, %error, "exit hold could not be recorded; using ordinary exit accounting"); false });
8784                    let next = if deferred_spawn {
8785                        record_terminal_with_detail(&spec.module_id,&runtime.terminal_ring,&runtime.spawn_events,&exit_report,TerminalDisposition::Restarting,active_child.spawn_failure.clone());
8786                        let _=update_snapshot(&snapshot,Some(&spec.module_id),|state| {
8787                            state.state=ModuleState::Restarting;
8788                            clear_current_process_facts(state);
8789                            state.last_exit=Some(exit_report.clone());
8790                        });
8791                        NextAction::Restart {schedule:None}
8792                    } else if parked {
8793                        record_terminal_with_detail(&spec.module_id, &runtime.terminal_ring, &runtime.spawn_events, &exit_report, TerminalDisposition::Stopped, active_child.spawn_failure.clone().or_else(|| Some("config_gated".into())));
8794                        NextAction::Stop { registration_released: registry.get_module(&spec.module_id).ok().flatten().is_none() }
8795                    } else { on_child_exit(
8796                        &spec,
8797                        runtime.restart_policy,
8798                        &registry,
8799                        &snapshot,
8800                        &runtime.terminal_ring,
8801                        &runtime.spawn_events,
8802                        &runtime.child_roster,
8803                        exit_report,
8804                        true,
8805                    ).await };
8806                    runtime.start_requester = None;
8807                    runtime.pending_activity_restart = false;
8808                    // The exit is recorded, so a daemon shutdown may stop
8809                    // waiting for this child (see `SupervisedChild::wait`).
8810                    let carry = matches!(next, NextAction::Restart { .. })
8811                        && active_child.spawn_failure.is_none()
8812                        && !runtime.child_roster.is_closed();
8813                    if let Err(error) = active_child.finish_exit(carry).await {
8814                        containment::terminate(&runtime.child_roster.containment).await;
8815                        fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8816                            &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
8817                            &runtime.failure_context(true));
8818                        child = None;
8819                        continue;
8820                    }
8821                    match next {
8822                        NextAction::Stop { registration_released } => {
8823                            if registration_released {
8824                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
8825                            }
8826                            child = None;
8827                        }
8828                        NextAction::Restart { schedule } => {
8829                            let delay = if deferred_spawn {CONFIG_ADMISSION_RETRY_DELAY} else {schedule.map_or(
8830                                runtime.restart_policy.delay_for_restart(0),
8831                                |schedule| schedule.delay,
8832                            )};
8833                            if let Some(schedule) = schedule {
8834                                log_crash_respawn(&spec.module_id, schedule);
8835                            }
8836                            // The exited child is fully recorded at this point,
8837                            // so release it and count the backoff down in the
8838                            // command-serving branch below rather than sleeping
8839                            // here: commands cannot be received from inside this
8840                            // select arm, and an operator disable or drain that
8841                            // arrives during the backoff must cancel the pending
8842                            // respawn instead of waiting for it to spawn first.
8843                            child = None;
8844                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true, approval: None });
8845                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
8846                        }
8847                    }
8848                }
8849                command = commands.recv() => {
8850                    #[cfg(test)]
8851                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8852                    let Some(command) = command else {
8853                        containment::terminate(&runtime.child_roster.containment).await;
8854                        return;
8855                    };
8856                    if !handle_supervisor_command(
8857                        command,
8858                        &mut spec,
8859                        &mut runtime,
8860                        &registry,
8861                        &process_liveness,
8862                        &snapshot,
8863                        &mut child,
8864                        &mut commands,
8865                        &mut requeued,
8866                    ).await {
8867                        containment::terminate(&runtime.child_roster.containment).await;
8868                        return;
8869                    }
8870                }
8871                _ = async {
8872                    match registration_changes.as_mut() {
8873                        Some(changes) => { let _ = changes.changed().await; }
8874                        None => std::future::pending().await,
8875                    }
8876                }, if wire_child => {
8877                    #[cfg(test)]
8878                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8879                }
8880                _ = &mut probe_sleep => {
8881                    #[cfg(test)]
8882                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8883                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
8884                        // Recheck after the wake: HELLO, exit or shutdown may
8885                        // have become ready in the same executor tick.
8886                        active_child.registration_deadline = None;
8887                        if !runtime.child_roster.is_closed()
8888                            && !active_child.registered_since_spawn()
8889                            && registration_child_is_running(active_child)
8890                        {
8891                            warn!(module_id = %spec.module_id, pid = active_child.pid,
8892                                wait_secs = runtime.register_timeout.as_secs(),
8893                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
8894                            active_child.spawn_failure = Some("registration_timeout".into());
8895                            if let Err(error) = stop_unregistered_child(active_child).await {
8896                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
8897                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8898                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
8899                                    &runtime.failure_context(true));
8900                                child = None;
8901                            }
8902                            // The ordinary exit handler records the child's exit,
8903                            // spends crash budget, and schedules recovery.
8904                            continue;
8905                        }
8906                    }
8907                    if health_probe.due() {
8908                        run_health_probe_cycle(
8909                            &spec,
8910                            &runtime,
8911                            &registry,
8912                            &process_liveness,
8913                            &snapshot,
8914                            &mut child,
8915                        ).await;
8916                        if child.is_none() {
8917                            runtime.start_requester = None;
8918                            runtime.pending_activity_restart = false;
8919                        }
8920                        if child.is_some() {
8921                            health_probe.schedule_next(&spec, runtime.health.cadence);
8922                        }
8923                    }
8924                }
8925            }
8926        } else if let Some(pending) = pending_respawn.clone() {
8927            tokio::select! {
8928                _ = runtime.child_roster.closed() => {
8929                    pending_respawn = None;
8930                    let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8931                        state.state = ModuleState::Stopped;
8932                        state.respawn_pending = false;
8933                    });
8934                    containment::terminate(&runtime.child_roster.containment).await;
8935                }
8936                _ = async {
8937                    sleep_until(pending.deadline).await;
8938                    if let Some(delivery) = &runtime.config_delivery {
8939                        delivery.unfenced(&spec.module_id).await;
8940                    }
8941                } => {
8942                    pending_respawn = None;
8943                    let failure = pending.failure_context(&runtime);
8944                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
8945                    // A command handled below while the backoff elapsed may
8946                    // have stopped the module; never respawn past an operator's
8947                    // disable or drain.
8948                    if !respawn_still_pending(&snapshot) {
8949                        continue;
8950                    }
8951                    // The daemon began shutting down during the backoff: the
8952                    // spawn would be refused anyway, and refusing it here
8953                    // leaves the module stopped instead of reporting a
8954                    // failed restart.
8955                    if runtime.child_roster.is_closed() {
8956                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8957                            state.state = ModuleState::Stopped;
8958                        });
8959                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
8960                        continue;
8961                    }
8962                    if let Err(err) = release_dead_registration(
8963                        &registry,
8964                        runtime.forwarding.as_deref(),
8965                        &snapshot,
8966                        &spec.module_id,
8967                        &failure,
8968                    ).await {
8969                        record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
8970                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
8971                        continue;
8972                    }
8973
8974                    if matches!(pending.kind, RespawnKind::Reload) {
8975                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
8976                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
8977                        if matches!(result, Err(SuperviseError::ConfigDeliveryDeferred { .. })) {
8978                            *runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()) = reply;
8979                            continue;
8980                        }
8981                        if result.is_ok() && runtime.pending_activity_restart {
8982                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
8983                        }
8984                        runtime.pending_activity_restart = false;
8985                        if let Some(reply) = reply { let _ = reply.send(result); }
8986                        continue;
8987                    }
8988                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
8989                    match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
8990                        Ok(next_child) => {
8991                            child = Some(next_child);
8992                            if runtime.pending_activity_restart {
8993                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
8994                                runtime.pending_activity_restart = false;
8995                            }
8996                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
8997                        }
8998                        Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
8999                            pending_respawn = Some(PendingRespawn { deadline:Instant::now()+CONFIG_ADMISSION_RETRY_DELAY, ..pending });
9000                            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
9001                        }
9002                        Err(SuperviseError::ConfigHeld { .. }) => {
9003                            let _ = hold_config_spawn(&spec, &runtime, &snapshot, &failure);
9004                            runtime.pending_activity_restart = false;
9005                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9006                        }
9007                        Err(err) => {
9008                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &failure);
9009                            record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9010                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9011                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
9012                        }
9013                    }
9014                }
9015                command = commands.recv() => {
9016                    let Some(command) = command else {
9017                        containment::terminate(&runtime.child_roster.containment).await;
9018                        return;
9019                    };
9020                    if !handle_supervisor_command(
9021                        command,
9022                        &mut spec,
9023                        &mut runtime,
9024                        &registry,
9025                        &process_liveness,
9026                        &snapshot,
9027                        &mut child,
9028                        &mut commands,
9029                        &mut requeued,
9030                    ).await {
9031                        containment::terminate(&runtime.child_roster.containment).await;
9032                        return;
9033                    }
9034                    // Reconcile the pending respawn with what the command did:
9035                    // a start may already have spawned a fresh child,
9036                    // while a disable or drain moved the snapshot out of the
9037                    // state the respawn was counting down from.
9038                    if child.is_some() || !respawn_still_pending(&snapshot) {
9039                        pending_respawn = None;
9040                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9041                    }
9042                }
9043            }
9044        } else {
9045            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
9046                (state.enabled && state.state == ModuleState::Failed)
9047                    .then_some(state.failed_retry.pending)
9048                    .flatten()
9049            });
9050            let command = if lock_snapshot(&snapshot)
9051                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
9052            {
9053                tokio::select! {
9054                    command = commands.recv() => command,
9055                    _ = previous_instance_recheck.tick() => {
9056                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
9057                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
9058                            if enabled && !runtime.child_roster.is_closed() {
9059                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9060                                match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9061                                    Ok(next_child) => child = Some(next_child),
9062                                    Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9063                                        let _ = defer_config_spawn(&spec, &runtime, &snapshot, RespawnKind::Spawn, &runtime.failure_context(true));
9064                                    }
9065                                    Err(SuperviseError::ConfigHeld { .. }) => {
9066                                        let _ = hold_config_spawn(&spec, &runtime, &snapshot, &runtime.failure_context(true));
9067                                    }
9068                                    Err(error) => {
9069                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
9070                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
9071                                    }
9072                                }
9073                            } else {
9074                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9075                                    state.waiting_for_previous_pid = None;
9076                                    state.waiting_for_previous_detail = None;
9077                                });
9078                            }
9079                        }
9080                        continue;
9081                    }
9082                }
9083            } else {
9084                tokio::select! {
9085                    biased;
9086                    command = commands.recv() => command,
9087                    _ = async {
9088                        match slow_retry {
9089                            Some(retry) => sleep_until(retry.deadline).await,
9090                            None => std::future::pending().await,
9091                        }
9092                    } => {
9093                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
9094                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9095                            state.failed_retry.pending = None;
9096                        });
9097                        if runtime.child_roster.is_closed() {
9098                            continue;
9099                        }
9100                        info!(module_id = %spec.module_id, attempt = retry.attempt,
9101                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
9102                        if let Err(error) = set_child_enabled(
9103                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
9104                            &runtime.failure_context(true),
9105                        ).await {
9106                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
9107                        }
9108                        continue;
9109                    }
9110                }
9111            };
9112            let Some(command) = command else {
9113                containment::terminate(&runtime.child_roster.containment).await;
9114                return;
9115            };
9116            if !handle_supervisor_command(
9117                command,
9118                &mut spec,
9119                &mut runtime,
9120                &registry,
9121                &process_liveness,
9122                &snapshot,
9123                &mut child,
9124                &mut commands,
9125                &mut requeued,
9126            )
9127            .await
9128            {
9129                containment::terminate(&runtime.child_roster.containment).await;
9130                return;
9131            }
9132        }
9133    }
9134}
9135
9136// Preserve the deadline until the future is polled. Starting a relative sleep
9137// after the executor yields would restart part of the countdown.
9138async fn supervisor_sleep(wake_at: Option<Instant>) {
9139    match wake_at {
9140        Some(deadline) => sleep_until(deadline).await,
9141        None => std::future::pending().await,
9142    }
9143}
9144
9145fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
9146    info!(
9147        module_id,
9148        restart_in_window = schedule.restart_in_window,
9149        delay_ms = schedule.delay.as_millis() as u64,
9150        "respawning after crash"
9151    );
9152}
9153
9154/// Whether the respawn a backoff was counting down to is still wanted. A
9155/// disable or drain handled while the backoff elapsed moves the snapshot out
9156/// of `Restarting`, and the operator's stop must win over the pending respawn,
9157/// so every sleep-then-spawn path re-validates against the live snapshot
9158/// instead of assuming the state it left behind still holds.
9159fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
9160    matches!(
9161        lock_snapshot(snapshot),
9162        Ok(state) if state.enabled && state.state == ModuleState::Restarting
9163    )
9164}
9165
9166enum NextAction {
9167    Stop {
9168        registration_released: bool,
9169    },
9170    Restart {
9171        schedule: Option<CrashRestartSchedule>,
9172    },
9173}
9174
9175#[allow(clippy::too_many_arguments)]
9176async fn handle_supervisor_command(
9177    command: SupervisorCommand,
9178    spec: &mut ModuleSpec,
9179    runtime: &mut SupervisorRuntimeConfig,
9180    registry: &Arc<Registry>,
9181    process_liveness: &SupervisorProcessLiveness,
9182    snapshot: &SharedSnapshot,
9183    child: &mut Option<SupervisedChild>,
9184    commands: &mut mpsc::Receiver<SupervisorCommand>,
9185    requeued: &mut VecDeque<SupervisorCommand>,
9186) -> bool {
9187    if let SupervisorCommand::SetEnabled {
9188        enabled: true,
9189        reset_budget: false,
9190        reply,
9191        ..
9192    } = command
9193    {
9194        let parked = lock_snapshot(snapshot).ok().and_then(|state| {
9195            (state.config_hold_parked
9196                && state.enabled
9197                && !state.process_alive
9198                && child.is_none()
9199                && state.state == ModuleState::Stopped)
9200                .then(|| {
9201                    (
9202                        state.config_hold_daemon_initiated,
9203                        state.config_hold_approval.clone(),
9204                    )
9205                })
9206        });
9207        // Clearing a configuration-delivery block must not restart a child that
9208        // exited on its own or exhausted its crash budget. Resume only an actor
9209        // stopped by a configuration hold; otherwise leave its slow retry alone.
9210        let result = if let Some((daemon_initiated, approval)) = parked {
9211            runtime.recovery_is_daemon = daemon_initiated;
9212            let failure = FailureContext {
9213                approval,
9214                ..runtime.failure_context(daemon_initiated)
9215            };
9216            let result = set_child_enabled_with_budget_policy(
9217                spec,
9218                runtime,
9219                registry,
9220                process_liveness,
9221                snapshot,
9222                child,
9223                true,
9224                &failure,
9225                false,
9226            )
9227            .await;
9228            if let Err(error) = &result {
9229                record_approved_attempt_failure(spec, snapshot, &failure, error);
9230            }
9231            result
9232        } else {
9233            Ok(false)
9234        };
9235        let _ = reply.send(result);
9236        return true;
9237    }
9238    let blocked = runtime
9239        .config_delivery
9240        .as_ref()
9241        .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
9242        .filter(|error| error.code != "config_delivery_deferred")
9243        .map(|error| {
9244            config_delivery::gate_hold_error(spec, error, runtime.config_delivery.as_ref())
9245        })
9246        .or_else(|| recovery_block_error(spec, &runtime.child_roster))
9247        .or_else(|| {
9248            runtime
9249                .child_roster
9250                .previous_instance(&spec.module_id)
9251                .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
9252        });
9253    let command = if let Some(error) = blocked {
9254        match command {
9255            SupervisorCommand::Restart { reply, .. } | SupervisorCommand::Reload { reply, .. } => {
9256                let _ = reply.send(Err(error));
9257                return true;
9258            }
9259            SupervisorCommand::Swap { reply, .. } => {
9260                let _ = reply.send(Err(error));
9261                return true;
9262            }
9263            SupervisorCommand::ConfigApplyRestart { .. } => return true,
9264            SupervisorCommand::SetEnabled {
9265                enabled: true,
9266                requester,
9267                reply,
9268                ..
9269            } => {
9270                if matches!(error, SuperviseError::ConfigHeld { .. })
9271                    && lock_snapshot(snapshot).is_ok_and(|state| !state.enabled)
9272                    && runtime
9273                        .config_delivery
9274                        .as_ref()
9275                        .is_some_and(|delivery| delivery.0.can_stage_unstamp_enable())
9276                {
9277                    // Stage only the pending unstamp's desired state. A stamped
9278                    // unapproved actor stays refused; admission release starts
9279                    // an unstamped actor only after reconciliation completes.
9280                    let mut changed = false;
9281                    let result = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9282                        changed = !state.enabled;
9283                        state.enabled = true;
9284                        if child.is_none() && state.waiting_for_previous_pid.is_none() {
9285                            state.state = ModuleState::Starting;
9286                            state.failed_retry.cancel();
9287                        }
9288                    })
9289                    .and_then(|_| {
9290                        if child.is_none()
9291                            && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none()
9292                        {
9293                            hold_config_spawn(
9294                                spec,
9295                                runtime,
9296                                snapshot,
9297                                &runtime.failure_context(false),
9298                            )?;
9299                        }
9300                        Ok(changed)
9301                    });
9302                    if result.is_ok() && changed {
9303                        runtime.start_requester = requester;
9304                    }
9305                    let _ = reply.send(result);
9306                    return true;
9307                }
9308                let _ = reply.send(Err(error));
9309                return true;
9310            }
9311            other => other,
9312        }
9313    } else {
9314        command
9315    };
9316    // A refused start or restart leaves the Failed actor's existing recovery
9317    // deadline intact. Cancel only after the command passes admission.
9318    if matches!(
9319        command,
9320        SupervisorCommand::Drain { .. }
9321            | SupervisorCommand::Retire { .. }
9322            | SupervisorCommand::SetEnabled { .. }
9323            | SupervisorCommand::Restart { .. }
9324    ) {
9325        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9326            state.failed_retry.cancel();
9327        });
9328    }
9329    match command {
9330        SupervisorCommand::ApprovedConfiguration { approved, reply } => {
9331            let result = async {
9332                let next = approved.module_entry(&spec.module_id).map_err(|error| {
9333                    SuperviseError::InvalidSpec { reason: error.to_string() }
9334                })?;
9335                let retry_approved_failure = {
9336                    let mut state = lock_snapshot(snapshot)?;
9337                    if let Some((generation, detail)) = &state.approved_start_failure {
9338                        if *generation == approved.identity() && next.enabled {
9339                            return Err(SuperviseError::Spawn {
9340                                program: spec.program.clone(),
9341                                source: std::io::Error::other(detail.clone()),
9342                                cgroup_path: None,
9343                            });
9344                        }
9345                    }
9346                    let retry = state.approved_start_failure.is_some();
9347                    state.approved_start_failure = None;
9348                    retry
9349                };
9350                stage_supervisor_configuration(
9351                    spec, runtime, snapshot, next.module_spec(), next.health,
9352                    next.drain_timeout_ms,
9353                );
9354                runtime.restart_policy = next.restart;
9355                runtime.register_timeout = next.register_timeout;
9356                if let Some(active) = child.as_mut()
9357                    .filter(|child| child.registration_deadline.is_some())
9358                {
9359                    active.registration_deadline = Some(active.spawned_at + next.register_timeout);
9360                }
9361                let enabled = lock_snapshot(snapshot)?.enabled;
9362                // An approval retries a changed failed entry or its own earlier
9363                // failed start, not an unrelated crash-budget failure. For an
9364                // unchanged entry, reconciled means the approved configuration
9365                // is effective; it does not claim the process has recovered.
9366                // A terminal gate stop is different: a new approval owns its
9367                // recovery even when the daemon entry itself did not change.
9368                let retry_gate_stopped = lock_snapshot(snapshot)?
9369                    .needs_approved_gate_restart(&approved.identity());
9370                let retry_failed = enabled && child.is_none()
9371                    && ((lock_snapshot(snapshot)?.state == ModuleState::Failed
9372                        && (retry_approved_failure || approved.changes_entry(&spec.module_id)))
9373                        || retry_gate_stopped);
9374                if retry_failed {
9375                    // This approval owns a new start, so the failed attempt's
9376                    // daemon retry must not survive either result of that start.
9377                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
9378                        state.failed_retry.cancel();
9379                        if retry_gate_stopped {
9380                            state.stop_reason = None;
9381                            state.config_exit_failed_approval = None;
9382                        }
9383                    })?;
9384                }
9385                if enabled != next.enabled || retry_failed {
9386                    let failure = FailureContext {
9387                        approval: next.enabled.then(|| approved.identity()),
9388                        ..runtime.failure_context(false)
9389                    };
9390                    let held = next.enabled && runtime.config_delivery.as_ref()
9391                        .is_some_and(|delivery| delivery.0.has_recorded_admission_hold(&spec.module_id)
9392                            || delivery.0.check_module_hold(&spec.module_id).is_err_and(|error| error.code == "config_delivery_deferred"));
9393                    if held {
9394                        // Preserve module-delivery failure/recovery holds. The
9395                        // admission release starts the actor when its own hold ends.
9396                        reset_restart_count(snapshot, &spec.module_id)?;
9397                        update_snapshot(snapshot, Some(&spec.module_id), |state| {
9398                            state.enabled = true;
9399                            if child.is_none() && state.waiting_for_previous_pid.is_none() {
9400                                state.state = ModuleState::Starting;
9401                            }
9402                        })?;
9403                        if child.is_none() && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none() {
9404                            hold_config_spawn(spec, runtime, snapshot, &failure)?;
9405                        }
9406                    } else {
9407                        match set_child_enabled_with_budget_policy(
9408                            spec, runtime, registry, process_liveness, snapshot, child,
9409                            next.enabled, &failure, true,
9410                        ).await {
9411                            Ok(_) => {},
9412                            Err(SuperviseError::ConfigHeld { .. }) if next.enabled => {},
9413                            Err(error) => return Err(error),
9414                        }
9415                        if !next.enabled {
9416                            containment::terminate(&runtime.child_roster.containment).await;
9417                        }
9418                    }
9419                }
9420                if (child.as_ref().is_some_and(|child| child.registration_spawn.is_some() && child.admission_base.is_none())
9421                        || (child.is_none() && lock_snapshot(snapshot)?.state == ModuleState::Restarting
9422                            && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.stamp_channel_cutover_pending())))
9423                    && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.module_is_gated(&spec.module_id))
9424                {
9425                    let failure = FailureContext {
9426                        approval: Some(approved.identity()),
9427                        ..runtime.failure_context(false)
9428                    };
9429                    // A stamp cannot adopt an unverified legacy process, even
9430                    // when its approved replacement no longer speaks subc.
9431                    // Stopping that process spends no crash restart budget.
9432                    if child.is_some() {
9433                        restart_child_with_budget_policy(spec, runtime, registry,
9434                            process_liveness, snapshot, child, runtime.drain_timeout, false).await?;
9435                    }
9436                    // The stamp is already committed. A deferred or held launch
9437                    // belongs to the normal retry/hold loop, not a failed apply.
9438                    update_snapshot(snapshot, Some(&spec.module_id), |state| state.respawn_pending = false)?;
9439                    match spawn_and_mark_running_async(spec, runtime, snapshot).await {
9440                        Ok(replacement) => *child = Some(replacement),
9441                        Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9442                            defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, &failure)?;
9443                        }
9444                        Err(SuperviseError::ConfigHeld { .. }) => hold_config_spawn(spec, runtime, snapshot, &failure)?,
9445                        Err(error) => {
9446                            fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
9447                            return Err(error);
9448                        }
9449                    }
9450                }
9451                info!(module_id=%spec.module_id, config_generation=approved.generation(), "approved daemon entry reconciled");
9452                Ok(())
9453            }
9454            .await;
9455            if let Err(error) = &result {
9456                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9457                    if state.enabled
9458                        && state.state == ModuleState::Failed
9459                        && state
9460                            .approved_start_failure
9461                            .as_ref()
9462                            .is_none_or(|(generation, _)| *generation != approved.identity())
9463                    {
9464                        state.approved_start_failure =
9465                            Some((approved.identity(), error.to_string()));
9466                    }
9467                });
9468            }
9469            let _ = reply.send(result);
9470            true
9471        }
9472        SupervisorCommand::ConfigPolicyProbe {
9473            connection,
9474            generation,
9475            attempt,
9476        } => {
9477            let Some(delivery) = &runtime.config_delivery else {
9478                return true;
9479            };
9480            let Ok(gate) = delivery.0.module_gate(&spec.module_id) else {
9481                return true;
9482            };
9483            let guard = gate.lock().await;
9484            let pending = delivery
9485                .0
9486                .on_blocking(move |owner| {
9487                    owner.policy_probe_pending(connection, generation, attempt)
9488                })
9489                .await
9490                .unwrap_or(false)
9491                && child.is_some()
9492                && registry
9493                    .get_module(&spec.module_id)
9494                    .ok()
9495                    .flatten()
9496                    .is_some_and(|registration| registration.connection_id == connection);
9497            drop(guard);
9498            if pending {
9499                run_health_probe_cycle(spec, runtime, registry, process_liveness, snapshot, child)
9500                    .await;
9501            }
9502            true
9503        }
9504        SupervisorCommand::ConfigApplyRestart {
9505            generation,
9506            supersedes_swap,
9507        } => {
9508            let Some(owner) = runtime
9509                .config_delivery
9510                .as_ref()
9511                .map(|delivery| delivery.0.clone())
9512            else {
9513                return true;
9514            };
9515            let Ok(gate) = owner.module_gate(&spec.module_id) else {
9516                return true;
9517            };
9518            let _guard = gate.lock().await;
9519            let spawn = child
9520                .as_ref()
9521                .and_then(|child| child.config_delivery.clone());
9522            let module = spec.module_id.clone();
9523            let current = owner
9524                .on_blocking(move |owner| {
9525                    Ok((supersedes_swap
9526                        || owner
9527                            .pending_config_restarts()?
9528                            .contains(&(module.clone(), generation)))
9529                        && owner.requires_fresher_spawn(&module, spawn.as_ref())?
9530                            == Some(generation))
9531                })
9532                .await
9533                .unwrap_or(false);
9534            if !current
9535                || !lock_snapshot(snapshot).is_ok_and(|state| {
9536                    state.enabled && !state.respawn_pending && state.process_alive
9537                })
9538            {
9539                return true;
9540            }
9541            drop(_guard);
9542            info!(module_id=%spec.module_id, config_generation=generation, "approved configuration restart preserves the crash budget");
9543            if let Err(error) = restart_child_with_budget_policy(
9544                spec,
9545                runtime,
9546                registry,
9547                process_liveness,
9548                snapshot,
9549                child,
9550                runtime.drain_timeout,
9551                false,
9552            )
9553            .await
9554            {
9555                warn!(module_id=%spec.module_id, %error, "approved configuration restart could not be scheduled");
9556            }
9557            true
9558        }
9559        SupervisorCommand::ConfigExitHold { reply } => {
9560            // A notification owns no start, backoff, reload reply or previous-
9561            // instance wait. Only an idle, registered initial start has no work
9562            // to preserve; its initialized startup context can be parked here.
9563            let idle_start = child.is_none()
9564                && lock_snapshot(snapshot).is_ok_and(|state| {
9565                    state.enabled
9566                        && state.state == ModuleState::Starting
9567                        && !state.process_alive
9568                        && !state.respawn_pending
9569                        && state.waiting_for_previous_pid.is_none()
9570                        && state.waiting_for_previous_detail.is_none()
9571                        && state.failed_retry.pending.is_none()
9572                })
9573                && runtime
9574                    .scheduled_respawn
9575                    .lock()
9576                    .unwrap_or_else(|p| p.into_inner())
9577                    .is_none()
9578                && runtime
9579                    .deferred_reload_reply
9580                    .lock()
9581                    .unwrap_or_else(|p| p.into_inner())
9582                    .is_none();
9583            let result = if idle_start {
9584                try_park_config_attempt(
9585                    spec,
9586                    runtime,
9587                    snapshot,
9588                    &runtime.failure_context(runtime.recovery_is_daemon),
9589                    None,
9590                )
9591                .map(|_| ())
9592            } else {
9593                Ok(())
9594            };
9595            let _ = reply.send(result);
9596            true
9597        }
9598        SupervisorCommand::ConfigDeliveryFailed { connection, hold } => {
9599            if registry
9600                .get_module(&spec.module_id)
9601                .ok()
9602                .flatten()
9603                .is_none_or(|registration| registration.connection_id != connection)
9604            {
9605                return true;
9606            }
9607            runtime.recovery_is_daemon = true;
9608            // With a recoverable generation, delivery-failure recovery spends
9609            // one crash-budget entry: the module rejected its configuration.
9610            // Approved configuration restarts spend none.
9611            if hold {
9612                let notice = begin_forwarding_drain_if_configured(
9613                    spec,
9614                    runtime,
9615                    registry,
9616                    snapshot,
9617                    Some(true),
9618                    RouteCloseReason::Disable,
9619                )
9620                .await;
9621                if let Ok(notice) = notice {
9622                    let drained = drain_optional_child(
9623                        &spec.module_id,
9624                        spec.protocol,
9625                        notice,
9626                        registry,
9627                        runtime.forwarding.as_deref(),
9628                        snapshot,
9629                        &runtime.terminal_ring,
9630                        &runtime.spawn_events,
9631                        child,
9632                        runtime.drain_timeout,
9633                        ModuleState::Failed,
9634                        Some(true),
9635                        &runtime.failure_context(true),
9636                    )
9637                    .await;
9638                    if drained.is_ok() {
9639                        match try_park_config_attempt(
9640                            spec,
9641                            runtime,
9642                            snapshot,
9643                            &runtime.failure_context(true),
9644                            None,
9645                        ) {
9646                            Ok(true) => {}
9647                            Ok(false) => {
9648                                if let Err(error) = budgeted_restart_child(
9649                                    spec,
9650                                    runtime,
9651                                    registry,
9652                                    process_liveness,
9653                                    snapshot,
9654                                    child,
9655                                    SupervisorHealthStatus::Unknown,
9656                                    Some("configuration delivery failed after its hold ended"),
9657                                    unix_ms_now(),
9658                                    false,
9659                                )
9660                                .await
9661                                {
9662                                    warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9663                                }
9664                            }
9665                            Err(error) => {
9666                                warn!(module_id=%spec.module_id, %error, "configuration failure could not be parked")
9667                            }
9668                        }
9669                    }
9670                }
9671            } else if let Err(error) = budgeted_restart_child(
9672                spec,
9673                runtime,
9674                registry,
9675                process_liveness,
9676                snapshot,
9677                child,
9678                SupervisorHealthStatus::Unknown,
9679                Some("configuration delivery failed"),
9680                unix_ms_now(),
9681                false,
9682            )
9683            .await
9684            {
9685                warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9686            }
9687            true
9688        }
9689        SupervisorCommand::Drain { requester, reply } => {
9690            // A plain stop runs no forwarding drain, so nothing reaches the
9691            // module over its connection before the wait: ask by signal.
9692            let result = drain_optional_child(
9693                &spec.module_id,
9694                spec.protocol,
9695                StopNotice::NotSent,
9696                registry,
9697                runtime.forwarding.as_deref(),
9698                snapshot,
9699                &runtime.terminal_ring,
9700                &runtime.spawn_events,
9701                child,
9702                runtime.drain_timeout,
9703                ModuleState::Stopped,
9704                None,
9705                &FailureContext::operator(),
9706            )
9707            .await;
9708            containment::terminate(&runtime.child_roster.containment).await;
9709            let registration_released = result.is_ok();
9710            if registration_released {
9711                runtime.record_activity_transition(
9712                    "subc.module_stopped",
9713                    &spec.module_id,
9714                    requester,
9715                );
9716            }
9717            let _ = reply.send(result);
9718            if registration_released {
9719                process_liveness.untrack_if_current(&spec.module_id, snapshot);
9720            }
9721            false
9722        }
9723        SupervisorCommand::Retire { requester, reply } => {
9724            let result = async {
9725                let stop_notice = begin_forwarding_drain_if_configured(
9726                    spec,
9727                    runtime,
9728                    registry,
9729                    snapshot,
9730                    None,
9731                    RouteCloseReason::Disable,
9732                )
9733                .await?;
9734                drain_optional_child(
9735                    &spec.module_id,
9736                    spec.protocol,
9737                    stop_notice,
9738                    registry,
9739                    runtime.forwarding.as_deref(),
9740                    snapshot,
9741                    &runtime.terminal_ring,
9742                    &runtime.spawn_events,
9743                    child,
9744                    runtime.drain_timeout,
9745                    ModuleState::Stopped,
9746                    None,
9747                    &FailureContext::operator(),
9748                )
9749                .await
9750            }
9751            .await;
9752            containment::terminate(&runtime.child_roster.containment).await;
9753            let registration_released = result.is_ok();
9754            if registration_released {
9755                runtime.record_activity_transition(
9756                    "subc.module_stopped",
9757                    &spec.module_id,
9758                    requester,
9759                );
9760            }
9761            let _ = reply.send(result);
9762            if registration_released {
9763                process_liveness.untrack_if_current(&spec.module_id, snapshot);
9764            }
9765            false
9766        }
9767        SupervisorCommand::Restart {
9768            requester,
9769            drain_timeout_ms,
9770            received_at_generation,
9771            queued_at,
9772            reply,
9773        } => {
9774            // Without this line a restart that waited in the queue (behind a
9775            // health probe cycle or another command) was invisible: the log
9776            // showed only the drain timing out, minutes after the operator's call.
9777            info!(
9778                module_id = %spec.module_id,
9779                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
9780                "restart command dequeued"
9781            );
9782            // ACK AT INITIATION, not completion. The blocking form deadlocked any
9783            // caller whose own request lane rides the module being restarted: the
9784            // caller's in-flight request keeps the drain from quiescing, the drain
9785            // keeps the restart from completing, and the completion keeps the reply
9786            // from releasing the caller — so the drain always timed out and cut the
9787            // initiator with a GOODBYE, even on a healthy module. Replying once the
9788            // restart is validated lets a self-lane caller settle, which is exactly
9789            // what makes the drain succeed. Completion is observable via
9790            // supervisor.list / module status; a post-ack failure lands the module
9791            // in a visible terminal state below rather than in a reply nobody can
9792            // receive.
9793            let validation = match lock_snapshot(snapshot) {
9794                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
9795                    module_id: spec.module_id.clone(),
9796                }),
9797                Ok(_) => Ok(()),
9798                Err(err) => Err(err),
9799            };
9800            let initiated = validation.is_ok();
9801            let _ = reply.send(validation);
9802            // A restart asks for a fresh process. Commands run one at a time,
9803            // so a restart queued behind another restart (two operator calls
9804            // in quick succession) is dequeued the moment the first one has
9805            // spawned its replacement -- before that process has sent HELLO.
9806            // Running it would drain and kill the process the first restart
9807            // just produced, which is the opposite of what both callers asked
9808            // for. If a process spawned after this request was received is
9809            // still supervised, the request is already satisfied. Not when the
9810            // configuration changed since that spawn: then the newer process
9811            // predates the spec this restart may exist to apply.
9812            // Generation publication can lag a child that is already serving
9813            // while its identity is recorded. Only process creation after request
9814            // receipt satisfies a restart, not publication after receipt.
9815            let satisfied_by_generation = if initiated
9816                && child
9817                    .as_ref()
9818                    .is_some_and(|child| child.spawned_at > queued_at)
9819            {
9820                lock_snapshot(snapshot).ok().and_then(|state| {
9821                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
9822                })
9823            } else {
9824                None
9825            };
9826            let satisfied_by_pending = initiated
9827                && child.is_none()
9828                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
9829                    // No child exists yet, so this replacement will necessarily
9830                    // be created after the queued request, unlike a published
9831                    // generation whose process may already have been serving.
9832                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
9833                    if pending {
9834                        state.coalesced_restart_pending = true;
9835                    }
9836                    pending
9837                });
9838            if satisfied_by_pending {
9839                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
9840            } else if let Some(generation) = satisfied_by_generation {
9841                info!(
9842                    module_id = %spec.module_id,
9843                    received_at_generation,
9844                    "restart already satisfied by generation {generation}; not restarting again"
9845                );
9846            } else if initiated {
9847                // Precedence: this restart's operator override, else the module's
9848                // configured budget (already resolved into the runtime).
9849                let drain_timeout = drain_timeout_ms
9850                    .map(Duration::from_millis)
9851                    .unwrap_or(runtime.drain_timeout);
9852                if let Err(err) = restart_child(
9853                    spec,
9854                    runtime,
9855                    registry,
9856                    process_liveness,
9857                    snapshot,
9858                    child,
9859                    drain_timeout,
9860                )
9861                .await
9862                {
9863                    warn!(
9864                        module_id = %spec.module_id,
9865                        error = %err,
9866                        "operator restart failed after initiation ack; module state carries the outcome"
9867                    );
9868                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9869                        state.state = ModuleState::Failed;
9870                        clear_current_process_facts(state);
9871                    });
9872                } else {
9873                    runtime.start_requester = requester;
9874                    runtime.pending_activity_restart = true;
9875                }
9876            }
9877            true
9878        }
9879        SupervisorCommand::Reload { requester, reply } => {
9880            let result =
9881                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
9882            if result.is_ok() {
9883                runtime.start_requester = requester;
9884                runtime.pending_activity_restart = true;
9885            }
9886            if result.is_ok()
9887                && runtime
9888                    .scheduled_respawn
9889                    .lock()
9890                    .unwrap_or_else(|p| p.into_inner())
9891                    .as_ref()
9892                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
9893            {
9894                *runtime
9895                    .deferred_reload_reply
9896                    .lock()
9897                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
9898            } else {
9899                let _ = reply.send(result);
9900            }
9901            true
9902        }
9903        SupervisorCommand::SetEnabled {
9904            enabled,
9905            requester,
9906            reply,
9907            reset_budget,
9908        } => {
9909            let previous_requester = runtime.start_requester.clone();
9910            if enabled {
9911                runtime.start_requester = requester.clone();
9912            }
9913            let result = set_child_enabled_with_budget_policy(
9914                spec,
9915                runtime,
9916                registry,
9917                process_liveness,
9918                snapshot,
9919                child,
9920                enabled,
9921                &runtime.failure_context(false),
9922                reset_budget,
9923            )
9924            .await;
9925            if !enabled {
9926                containment::terminate(&runtime.child_roster.containment).await;
9927            }
9928            if matches!(result, Ok(true)) {
9929                runtime.pending_activity_restart = false;
9930                if enabled {
9931                    runtime.start_requester = requester;
9932                } else {
9933                    runtime.start_requester = None;
9934                    runtime.record_activity_transition(
9935                        "subc.module_disabled",
9936                        &spec.module_id,
9937                        requester,
9938                    );
9939                }
9940            } else {
9941                runtime.start_requester = previous_requester;
9942            }
9943            let _ = reply.send(result);
9944            true
9945        }
9946        SupervisorCommand::UpdateConfiguration {
9947            spec: next_spec,
9948            health,
9949            drain_timeout_ms,
9950            reply,
9951        } => {
9952            if runtime
9953                .config_delivery
9954                .as_ref()
9955                .is_some_and(|delivery| delivery.0.file_configuration_is_gated(&spec.module_id))
9956            {
9957                let _ = reply.send(Err(SuperviseError::ConfigHeld {
9958                    module_id: spec.module_id.clone(),
9959                    code: "config_gated".into(),
9960                    detail: "live file reconfiguration cannot replace stamped authority".into(),
9961                }));
9962                return true;
9963            }
9964            stage_supervisor_configuration(
9965                spec,
9966                runtime,
9967                snapshot,
9968                next_spec,
9969                health,
9970                drain_timeout_ms,
9971            );
9972            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9973                state.configuration_updated_since_spawn = true;
9974            });
9975            let _ = reply.send(Ok(()));
9976            true
9977        }
9978        SupervisorCommand::Swap {
9979            requester,
9980            ready_timeout,
9981            reply,
9982        } => {
9983            if spec.carry_containment
9984                || containment::current(&runtime.child_roster.containment).is_some()
9985            {
9986                let _ = reply.send(Err(SuperviseError::InvalidSpec {
9987                    reason: "a carried container cannot overlap a swap candidate".to_owned(),
9988                }));
9989                return true;
9990            }
9991            let mut swap_runtime = runtime.clone();
9992            swap_runtime.start_requester = requester;
9993            let before = lock_snapshot(snapshot)
9994                .ok()
9995                .map(|state| state.spawn_generation);
9996            let end = swap::run_swap(
9997                spec,
9998                &swap_runtime,
9999                registry,
10000                process_liveness,
10001                snapshot,
10002                child,
10003                commands,
10004                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
10005                reply,
10006            )
10007            .await;
10008            if before
10009                != lock_snapshot(snapshot)
10010                    .ok()
10011                    .map(|state| state.spawn_generation)
10012            {
10013                runtime.start_requester = swap_runtime.start_requester;
10014            }
10015            requeued.extend(end.requeue);
10016            true
10017        }
10018    }
10019}
10020
10021fn stage_supervisor_configuration(
10022    spec: &mut ModuleSpec,
10023    runtime: &mut SupervisorRuntimeConfig,
10024    snapshot: &SharedSnapshot,
10025    next_spec: ModuleSpec,
10026    health: HealthConfig,
10027    drain_timeout_ms: Option<u64>,
10028) {
10029    if let Some(handle) = &runtime.supervisor_handle {
10030        handle.apply_identity_configuration(&next_spec);
10031    }
10032    let launch_changed = *spec != next_spec;
10033    *spec = next_spec;
10034    if launch_changed {
10035        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10036            state.configuration_updated_since_spawn = true
10037        });
10038    }
10039    let health_changed = runtime.health != health;
10040    runtime.health = health;
10041    // Reset the cadence and old endpoint's failure streak on a live
10042    // health-policy change rather than waiting for its old deadline.
10043    if health_changed {
10044        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10045            state.health = ModuleHealthStatus::default();
10046            state.last_probe_evidence = None;
10047        });
10048    }
10049    runtime.drain_timeout = drain_timeout_ms
10050        .map(Duration::from_millis)
10051        .unwrap_or(runtime.default_drain_timeout);
10052    *runtime
10053        .effective_drain_timeout
10054        .lock()
10055        .unwrap_or_else(|p| p.into_inner()) = runtime.drain_timeout;
10056}
10057
10058async fn restart_child(
10059    spec: &ModuleSpec,
10060    runtime: &SupervisorRuntimeConfig,
10061    registry: &Registry,
10062    process_liveness: &SupervisorProcessLiveness,
10063    snapshot: &SharedSnapshot,
10064    child: &mut Option<SupervisedChild>,
10065    drain_timeout: Duration,
10066) -> Result<(), SuperviseError> {
10067    restart_child_with_budget_policy(
10068        spec,
10069        runtime,
10070        registry,
10071        process_liveness,
10072        snapshot,
10073        child,
10074        drain_timeout,
10075        true,
10076    )
10077    .await
10078}
10079
10080#[allow(clippy::too_many_arguments)]
10081async fn restart_child_with_budget_policy(
10082    spec: &ModuleSpec,
10083    runtime: &SupervisorRuntimeConfig,
10084    registry: &Registry,
10085    process_liveness: &SupervisorProcessLiveness,
10086    snapshot: &SharedSnapshot,
10087    child: &mut Option<SupervisedChild>,
10088    drain_timeout: Duration,
10089    reset_budget: bool,
10090) -> Result<(), SuperviseError> {
10091    // Restart cycles a running module; it must not silently start a disabled one.
10092    if !lock_snapshot(snapshot)?.enabled {
10093        return Err(SuperviseError::Disabled {
10094            module_id: spec.module_id.clone(),
10095        });
10096    }
10097    let stop_notice = begin_forwarding_drain_with_timeout(
10098        spec,
10099        runtime,
10100        registry,
10101        snapshot,
10102        None,
10103        RouteCloseReason::Restart,
10104        drain_timeout,
10105    )
10106    .await?;
10107
10108    if child.is_some() {
10109        drain_optional_child(
10110            &spec.module_id,
10111            spec.protocol,
10112            stop_notice,
10113            registry,
10114            runtime.forwarding.as_deref(),
10115            snapshot,
10116            &runtime.terminal_ring,
10117            &runtime.spawn_events,
10118            child,
10119            drain_timeout,
10120            ModuleState::Restarting,
10121            Some(true),
10122            &FailureContext::operator(),
10123        )
10124        .await?;
10125    } else {
10126        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10127            state.enabled = true;
10128            state.state = ModuleState::Restarting;
10129            clear_current_process_facts(state);
10130        })?;
10131        release_dead_registration(
10132            registry,
10133            runtime.forwarding.as_deref(),
10134            snapshot,
10135            &spec.module_id,
10136            &runtime.failure_context(false),
10137        )
10138        .await?;
10139    }
10140
10141    if reset_budget {
10142        reset_restart_count(snapshot, &spec.module_id)?;
10143    }
10144    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10145    schedule_respawn(
10146        runtime,
10147        snapshot,
10148        &spec.module_id,
10149        runtime.restart_policy.backoff,
10150        RespawnKind::Spawn,
10151        false,
10152    )
10153}
10154
10155async fn reload_child(
10156    spec: &ModuleSpec,
10157    runtime: &SupervisorRuntimeConfig,
10158    registry: &Registry,
10159    process_liveness: &SupervisorProcessLiveness,
10160    snapshot: &SharedSnapshot,
10161    child: &mut Option<SupervisedChild>,
10162) -> Result<(), SuperviseError> {
10163    // Reload cycles a running module; it must not silently start a disabled one.
10164    if !lock_snapshot(snapshot)?.enabled {
10165        return Err(SuperviseError::Disabled {
10166            module_id: spec.module_id.clone(),
10167        });
10168    }
10169    let stop_notice = begin_forwarding_drain(
10170        spec,
10171        runtime,
10172        registry,
10173        snapshot,
10174        Some(true),
10175        RouteCloseReason::Reload,
10176    )
10177    .await?;
10178
10179    if child.is_some() {
10180        drain_optional_child(
10181            &spec.module_id,
10182            spec.protocol,
10183            stop_notice,
10184            registry,
10185            runtime.forwarding.as_deref(),
10186            snapshot,
10187            &runtime.terminal_ring,
10188            &runtime.spawn_events,
10189            child,
10190            runtime.drain_timeout,
10191            ModuleState::Restarting,
10192            Some(true),
10193            &FailureContext::operator(),
10194        )
10195        .await?;
10196    } else {
10197        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10198            state.enabled = true;
10199            state.state = ModuleState::Restarting;
10200            clear_current_process_facts(state);
10201        })?;
10202        release_dead_registration(
10203            registry,
10204            runtime.forwarding.as_deref(),
10205            snapshot,
10206            &spec.module_id,
10207            &runtime.failure_context(false),
10208        )
10209        .await?;
10210    }
10211
10212    reset_restart_count(snapshot, &spec.module_id)?;
10213    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10214    schedule_respawn(
10215        runtime,
10216        snapshot,
10217        &spec.module_id,
10218        runtime.restart_policy.backoff,
10219        RespawnKind::Reload,
10220        false,
10221    )
10222}
10223
10224async fn finish_reload_child(
10225    spec: &ModuleSpec,
10226    runtime: &SupervisorRuntimeConfig,
10227    registry: &Registry,
10228    process_liveness: &SupervisorProcessLiveness,
10229    snapshot: &SharedSnapshot,
10230    child: &mut Option<SupervisedChild>,
10231) -> Result<(), SuperviseError> {
10232    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10233    let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10234        Ok(next_child) => next_child,
10235        Err(error @ SuperviseError::ConfigDeliveryDeferred { .. }) => {
10236            defer_config_spawn(
10237                spec,
10238                runtime,
10239                snapshot,
10240                RespawnKind::Reload,
10241                &runtime.failure_context(runtime.recovery_is_daemon),
10242            )?;
10243            return Err(error);
10244        }
10245        Err(error @ SuperviseError::ConfigHeld { .. }) => {
10246            let failure = runtime.failure_context(runtime.recovery_is_daemon);
10247            if try_park_config_attempt(spec, runtime, snapshot, &failure, None)? {
10248                return Err(error);
10249            }
10250            defer_config_spawn(spec, runtime, snapshot, RespawnKind::Reload, &failure)?;
10251            return Err(SuperviseError::ConfigDeliveryDeferred {
10252                module_id: spec.module_id.clone(),
10253            });
10254        }
10255        Err(err) => {
10256            return handle_reload_spawn_failure(
10257                spec,
10258                runtime,
10259                process_liveness,
10260                snapshot,
10261                child,
10262                format!("new child failed to spawn: {err}"),
10263            )
10264            .await;
10265        }
10266    };
10267    *child = Some(next_child);
10268
10269    let wait_outcome = {
10270        let active_child = child.as_mut().expect("new reload child was just stored");
10271        wait_for_registration_after_reload(
10272            registry,
10273            &spec.module_id,
10274            snapshot,
10275            active_child,
10276            REGISTRY_RELEASE_TIMEOUT,
10277        )
10278        .await?
10279    };
10280
10281    match wait_outcome {
10282        RegistrationWaitOutcome::Registered => {
10283            child
10284                .as_mut()
10285                .expect("registered reload child")
10286                .registration_deadline = None;
10287            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
10288            Ok(())
10289        }
10290        RegistrationWaitOutcome::Exited(exit_report) => {
10291            if let Some(active_child) = child.as_mut() {
10292                active_child.drain_stderr(&spec.module_id).await;
10293            }
10294            // Keep the exited child's roster guard until its terminal is written.
10295            // Shutdown waits on that guard, not on the child Option used for respawn.
10296            let mut exited_child = child.take().expect("exited reload child is still stored");
10297            #[cfg(test)]
10298            if let Some(gate) = &runtime.test_reload_exit_record_gate {
10299                gate.reached.notify_one();
10300                gate.resume.notified().await;
10301            }
10302            let result = handle_reload_child_registration_failure(
10303                spec,
10304                runtime,
10305                registry,
10306                process_liveness,
10307                snapshot,
10308                child,
10309                ReloadRegistrationFailure {
10310                    exit_report: registration_failure_exit_report(exit_report),
10311                    reason: exited_child
10312                        .spawn_failure
10313                        .clone()
10314                        .unwrap_or_else(|| "new child exited before registering".to_string()),
10315                },
10316            )
10317            .await;
10318            exited_child
10319                .finish_exit(false)
10320                .await
10321                .map_err(|source| SuperviseError::Wait {
10322                    module_id: spec.module_id.clone(),
10323                    source,
10324                })?;
10325            result
10326        }
10327        RegistrationWaitOutcome::TimedOut => {
10328            let mut timed_out_child = child
10329                .take()
10330                .expect("timed-out reload child is still running");
10331            timed_out_child
10332                .start_kill()
10333                .map_err(|source| SuperviseError::Kill {
10334                    module_id: spec.module_id.clone(),
10335                    source,
10336                })?;
10337            let status = timed_out_child
10338                .wait()
10339                .await
10340                .map_err(|source| SuperviseError::Wait {
10341                    module_id: spec.module_id.clone(),
10342                    source,
10343                })?;
10344            timed_out_child.drain_stderr(&spec.module_id).await;
10345            let result = handle_reload_child_registration_failure(
10346                spec,
10347                runtime,
10348                registry,
10349                process_liveness,
10350                snapshot,
10351                child,
10352                ReloadRegistrationFailure {
10353                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
10354                        snapshot,
10355                        &timed_out_child,
10356                        &status,
10357                    )),
10358                    reason: format!(
10359                        "new child did not register within {:?}",
10360                        REGISTRY_RELEASE_TIMEOUT
10361                    ),
10362                },
10363            )
10364            .await;
10365            timed_out_child
10366                .finish_exit(false)
10367                .await
10368                .map_err(|source| SuperviseError::Wait {
10369                    module_id: spec.module_id.clone(),
10370                    source,
10371                })?;
10372            result
10373        }
10374    }
10375}
10376
10377#[allow(clippy::too_many_arguments)]
10378async fn set_child_enabled(
10379    spec: &ModuleSpec,
10380    runtime: &SupervisorRuntimeConfig,
10381    registry: &Registry,
10382    process_liveness: &SupervisorProcessLiveness,
10383    snapshot: &SharedSnapshot,
10384    child: &mut Option<SupervisedChild>,
10385    enabled: bool,
10386    failure: &FailureContext,
10387) -> Result<bool, SuperviseError> {
10388    set_child_enabled_with_budget_policy(
10389        spec,
10390        runtime,
10391        registry,
10392        process_liveness,
10393        snapshot,
10394        child,
10395        enabled,
10396        failure,
10397        true,
10398    )
10399    .await
10400}
10401
10402#[allow(clippy::too_many_arguments)]
10403async fn set_child_enabled_with_budget_policy(
10404    spec: &ModuleSpec,
10405    runtime: &SupervisorRuntimeConfig,
10406    registry: &Registry,
10407    process_liveness: &SupervisorProcessLiveness,
10408    snapshot: &SharedSnapshot,
10409    child: &mut Option<SupervisedChild>,
10410    enabled: bool,
10411    failure: &FailureContext,
10412    reset_budget: bool,
10413) -> Result<bool, SuperviseError> {
10414    let (current_enabled, current_state, respawn_pending) = {
10415        let state = lock_snapshot(snapshot)?;
10416        (state.enabled, state.state, state.respawn_pending)
10417    };
10418    // `start` (enable on an already-enabled module) heals TERMINAL states instead
10419    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
10420    // clean (Stopped) has no live process and no other in-band recovery — the
10421    // operator's start is the explicit recovery act and resets the budget. Without
10422    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
10423    // which the 2026-07-14 aft outage proved is a trap when the failed module is
10424    // the one providing every agent's shell.
10425    let revive_terminal = enabled
10426        && current_enabled
10427        && child.is_none()
10428        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
10429            || (current_state == ModuleState::Restarting && !respawn_pending));
10430    if current_enabled == enabled && !revive_terminal {
10431        return Ok(false);
10432    }
10433
10434    if enabled {
10435        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10436            state.enabled = true;
10437            state.state = ModuleState::Starting;
10438            clear_current_process_facts(state);
10439        })?;
10440        #[cfg(test)]
10441        if runtime.test_seed_stale_facts_before_enable_spawn {
10442            update_snapshot(snapshot, Some(&spec.module_id), |state| {
10443                state.process_alive = true;
10444                state.pid = Some(41);
10445                state.spawned_at_ms = Some(42);
10446                state.spawned_from = Some(PathBuf::from("/spawned/module"));
10447                state.spawned_file_identity = Some(SpawnedFileIdentity {
10448                    device: 43,
10449                    inode: 44,
10450                    #[cfg(windows)]
10451                    windows_image: None,
10452                });
10453            })?;
10454        }
10455        release_dead_registration(
10456            registry,
10457            runtime.forwarding.as_deref(),
10458            snapshot,
10459            &spec.module_id,
10460            failure,
10461        )
10462        .await?;
10463        if reset_budget {
10464            reset_restart_count(snapshot, &spec.module_id)?;
10465        }
10466        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10467        let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10468            Ok(next_child) => next_child,
10469            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
10470                defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
10471                return Ok(true);
10472            }
10473            Err(error @ SuperviseError::ConfigHeld { .. }) => {
10474                if let Err(parking_error) = hold_config_spawn(spec, runtime, snapshot, failure) {
10475                    process_liveness.untrack_if_current(&spec.module_id, snapshot);
10476                    return Err(parking_error);
10477                }
10478                return Err(error);
10479            }
10480            Err(err) => {
10481                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
10482                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10483                return Err(err);
10484            }
10485        };
10486        *child = Some(next_child);
10487        debug!(module_id = %spec.module_id, "supervised module enabled");
10488        Ok(true)
10489    } else {
10490        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10491            state.config_hold_parked = false;
10492            state.config_hold_daemon_initiated = false;
10493            state.config_hold_approval = None;
10494        })?;
10495        let stop_notice = begin_forwarding_drain_if_configured(
10496            spec,
10497            runtime,
10498            registry,
10499            snapshot,
10500            Some(false),
10501            RouteCloseReason::Disable,
10502        )
10503        .await?;
10504        drain_optional_child(
10505            &spec.module_id,
10506            spec.protocol,
10507            stop_notice,
10508            registry,
10509            runtime.forwarding.as_deref(),
10510            snapshot,
10511            &runtime.terminal_ring,
10512            &runtime.spawn_events,
10513            child,
10514            runtime.drain_timeout,
10515            ModuleState::Disabled,
10516            Some(false),
10517            failure,
10518        )
10519        .await?;
10520        debug!(module_id = %spec.module_id, "supervised module disabled");
10521        Ok(true)
10522    }
10523}
10524
10525#[allow(clippy::too_many_arguments)]
10526async fn on_child_exit(
10527    spec: &ModuleSpec,
10528    policy: RestartPolicy,
10529    registry: &Registry,
10530    snapshot: &SharedSnapshot,
10531    terminal_ring: &Arc<Mutex<TerminalRing>>,
10532    spawn_events: &SpawnEventFeed,
10533    roster: &ChildRoster,
10534    exit_report: ExitReport,
10535    daemon_initiated: bool,
10536) -> NextAction {
10537    // Once the daemon has begun shutting down, no exit is a crash to recover
10538    // from: the module is exiting because the daemon is going away (EOF on its
10539    // connection, or a service manager signalling the whole cgroup). Record it
10540    // as such and never schedule a respawn, which would only start a process
10541    // for the shutdown to end again.
10542    if roster.is_closed() {
10543        return on_child_exit_during_daemon_shutdown(
10544            spec,
10545            registry,
10546            snapshot,
10547            terminal_ring,
10548            spawn_events,
10549            exit_report,
10550        )
10551        .await;
10552    }
10553    // Deliberate stops the supervisor asks for (operator stop, disable,
10554    // restart, reload, swap, a health restart, a drain that runs out of budget)
10555    // takes the child out of the supervise loop and reaps it in
10556    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
10557    // that reaches this point is either unrequested or a first-HELLO timeout,
10558    // which is deliberately classified as a crash even after a clean TERM exit.
10559    //
10560    // For a subc-wire module a clean exit is still a stop: those modules are
10561    // written to re-raise SIGTERM, so a stray outside signal already reads as a
10562    // crash, and exiting 0 is a deliberate choice the module made. A
10563    // `protocol: "none"` module is a stock program we cannot change, and many
10564    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
10565    // stop would leave the module down for good after any stray signal, so it
10566    // goes through the crash path instead: it spends restart budget, respawns
10567    // with the crash backoff, and ends `failed` when the budget runs out.
10568    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
10569        && running_protocol(spec, snapshot) == ModuleProtocol::None;
10570    match exit_report.kind {
10571        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
10572            info!(
10573                module_id = %spec.module_id,
10574                exit_code = ?exit_report.code,
10575                exit_signal = ?exit_report.signal,
10576                "supervised module exited cleanly"
10577            );
10578            if let Err(err) = publish_after_terminal_record(
10579                &spec.module_id,
10580                || {
10581                    record_terminal(
10582                        &spec.module_id,
10583                        terminal_ring,
10584                        spawn_events,
10585                        &exit_report,
10586                        TerminalDisposition::Stopped,
10587                    )
10588                },
10589                || {
10590                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10591                        state.state = ModuleState::Stopped;
10592                        clear_current_process_facts(state);
10593                        state.last_exit = Some(exit_report.clone());
10594                    })
10595                },
10596            ) {
10597                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
10598            }
10599            let registration_released = match wait_for_registration_release(
10600                registry,
10601                &spec.module_id,
10602                REGISTRY_RELEASE_TIMEOUT,
10603            )
10604            .await
10605            {
10606                Ok(()) => true,
10607                Err(err) => {
10608                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
10609                    false
10610                }
10611            };
10612            NextAction::Stop {
10613                registration_released,
10614            }
10615        }
10616        ExitKind::Clean | ExitKind::Crash => {
10617            if unrequested_clean_exit_of_protocol_none {
10618                warn!(
10619                    module_id = %spec.module_id,
10620                    exit_code = ?exit_report.code,
10621                    exit_signal = ?exit_report.signal,
10622                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
10623                );
10624            } else {
10625                warn!(
10626                    module_id = %spec.module_id,
10627                    exit_code = ?exit_report.code,
10628                    exit_signal = ?exit_report.signal,
10629                    "supervised module exited abnormally (crash)"
10630                );
10631            }
10632            let mut restart_schedule = None;
10633            let mut disposition = TerminalDisposition::Disabled;
10634            let mut final_state = ModuleState::Disabled;
10635            // Set only when the budget is what stopped the module, so the
10636            // terminal record says which limit was hit rather than leaving
10637            // `failed` to be read as "crashed once, badly".
10638            let mut disposition_detail = lock_snapshot(snapshot)
10639                .ok()
10640                .and_then(|mut state| state.spawn_failure.take());
10641            let now = Instant::now();
10642            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10643                if state.enabled {
10644                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
10645                        // Reserving the last slot exhausts the budget for the
10646                        // replacement, not for this exit. Publish its scheduled
10647                        // recovery atomically with the reservation so connection
10648                        // cleanup sees it while terminal recording still holds
10649                        // the visible state at Running.
10650                        state.respawn_pending = true;
10651                        final_state = ModuleState::Restarting;
10652                        restart_schedule = Some(schedule);
10653                        disposition = TerminalDisposition::Restarting;
10654                    } else {
10655                        final_state = ModuleState::Failed;
10656                        disposition = TerminalDisposition::Failed;
10657                        let budget = policy.budget_exhausted_detail();
10658                        disposition_detail =
10659                            Some(disposition_detail.take().map_or_else(
10660                                || budget.clone(),
10661                                |cause| format!("{cause}; {budget}"),
10662                            ));
10663                    }
10664                } else {
10665                    disposition = TerminalDisposition::Disabled;
10666                }
10667            }) {
10668                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
10669                return NextAction::Stop {
10670                    registration_released: false,
10671                };
10672            }
10673            if disposition == TerminalDisposition::Failed {
10674                // The window is in the message, not only in the fields: this line
10675                // is read in a scrollback where a bare `max_restarts=3` reads as a
10676                // lifetime cap and sends the operator looking for three crashes
10677                // that never happened together.
10678                error!(
10679                    module_id = %spec.module_id,
10680                    max_restarts = policy.max_restarts,
10681                    window_secs = policy.window.as_secs(),
10682                    "module stopped: {}",
10683                    policy.budget_exhausted_detail()
10684                );
10685            }
10686            if let Err(err) = publish_after_terminal_record(
10687                &spec.module_id,
10688                || {
10689                    record_terminal_with_detail(
10690                        &spec.module_id,
10691                        terminal_ring,
10692                        spawn_events,
10693                        &exit_report,
10694                        disposition,
10695                        disposition_detail,
10696                    )
10697                },
10698                || {
10699                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10700                        state.state = final_state;
10701                        clear_current_process_facts(state);
10702                        if final_state == ModuleState::Failed
10703                            && daemon_initiated
10704                            && !roster.is_closed()
10705                        {
10706                            state.failed_retry.schedule(&spec.module_id);
10707                        }
10708                        state.last_exit = Some(exit_report.clone());
10709                    })
10710                },
10711            ) {
10712                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
10713            }
10714
10715            if let Some(schedule) = restart_schedule {
10716                NextAction::Restart {
10717                    schedule: Some(schedule),
10718                }
10719            } else {
10720                let registration_released = match wait_for_registration_release(
10721                    registry,
10722                    &spec.module_id,
10723                    REGISTRY_RELEASE_TIMEOUT,
10724                )
10725                .await
10726                {
10727                    Ok(()) => true,
10728                    Err(err) => {
10729                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
10730                        false
10731                    }
10732                };
10733                NextAction::Stop {
10734                    registration_released,
10735                }
10736            }
10737        }
10738        ExitKind::DeliberateSeverance => {
10739            warn!(
10740                module_id = %spec.module_id,
10741                exit_code = ?exit_report.code,
10742                exit_signal = ?exit_report.signal,
10743                "supervised module exited after deliberate connection severance"
10744            );
10745            let should_restart = match lock_snapshot(snapshot) {
10746                Ok(state) => state.enabled,
10747                Err(err) => {
10748                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
10749                    return NextAction::Stop {
10750                        registration_released: false,
10751                    };
10752                }
10753            };
10754            let final_state = if should_restart {
10755                ModuleState::Restarting
10756            } else {
10757                ModuleState::Disabled
10758            };
10759            if let Err(err) = publish_after_terminal_record(
10760                &spec.module_id,
10761                || {
10762                    record_terminal(
10763                        &spec.module_id,
10764                        terminal_ring,
10765                        spawn_events,
10766                        &exit_report,
10767                        terminal_disposition(final_state),
10768                    )
10769                },
10770                || {
10771                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10772                        state.state = final_state;
10773                        clear_current_process_facts(state);
10774                        state.last_exit = Some(exit_report.clone());
10775                        state.lifetime_restarts += 1;
10776                    })
10777                },
10778            ) {
10779                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
10780                return NextAction::Stop {
10781                    registration_released: false,
10782                };
10783            }
10784
10785            if should_restart {
10786                NextAction::Restart { schedule: None }
10787            } else {
10788                let registration_released = match wait_for_registration_release(
10789                    registry,
10790                    &spec.module_id,
10791                    REGISTRY_RELEASE_TIMEOUT,
10792                )
10793                .await
10794                {
10795                    Ok(()) => true,
10796                    Err(err) => {
10797                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
10798                        false
10799                    }
10800                };
10801                NextAction::Stop {
10802                    registration_released,
10803                }
10804            }
10805        }
10806    }
10807}
10808
10809async fn on_child_exit_during_daemon_shutdown(
10810    spec: &ModuleSpec,
10811    registry: &Registry,
10812    snapshot: &SharedSnapshot,
10813    terminal_ring: &Arc<Mutex<TerminalRing>>,
10814    spawn_events: &SpawnEventFeed,
10815    exit_report: ExitReport,
10816) -> NextAction {
10817    info!(
10818        module_id = %spec.module_id,
10819        exit_code = ?exit_report.code,
10820        exit_signal = ?exit_report.signal,
10821        exit_kind = ?exit_report.kind,
10822        "supervised module exited during daemon shutdown; not restarting it"
10823    );
10824    if let Err(err) = publish_after_terminal_record(
10825        &spec.module_id,
10826        || {
10827            record_terminal(
10828                &spec.module_id,
10829                terminal_ring,
10830                spawn_events,
10831                &exit_report,
10832                TerminalDisposition::DaemonShutdown,
10833            )
10834        },
10835        || {
10836            update_snapshot(snapshot, Some(&spec.module_id), |state| {
10837                state.state = ModuleState::Stopped;
10838                clear_current_process_facts(state);
10839                state.last_exit = Some(exit_report.clone());
10840            })
10841        },
10842    ) {
10843        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
10844    }
10845    let registration_released =
10846        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
10847            .await
10848            .is_ok();
10849    NextAction::Stop {
10850        registration_released,
10851    }
10852}
10853
10854// Keep the status lock out of terminal I/O. Readers may observe the previous
10855// state while recording is in progress, but never a completed exit without its
10856// history. Journal errors are logged by the journal and retained in the ring;
10857// a recording panic still publishes the final state before resuming the unwind.
10858fn publish_after_terminal_record<T>(
10859    module_id: &str,
10860    record: impl FnOnce(),
10861    publish: impl FnOnce() -> T,
10862) -> T {
10863    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
10864    let published = publish();
10865    if let Err(panic) = recorded {
10866        error!(
10867            module_id,
10868            "terminal recording panicked; published final state before resuming unwind"
10869        );
10870        std::panic::resume_unwind(panic);
10871    }
10872    published
10873}
10874
10875fn fail_after_wait_error(
10876    module_id: &str,
10877    snapshot: &SharedSnapshot,
10878    terminal_ring: &Arc<Mutex<TerminalRing>>,
10879    spawn_events: &SpawnEventFeed,
10880    detail: String,
10881    failure: &FailureContext,
10882) {
10883    let report = wait_error_exit_report();
10884    publish_after_terminal_record(
10885        module_id,
10886        || {
10887            record_terminal_with_detail(
10888                module_id,
10889                terminal_ring,
10890                spawn_events,
10891                &report,
10892                TerminalDisposition::Failed,
10893                Some(detail),
10894            )
10895        },
10896        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
10897    );
10898}
10899
10900#[cfg(test)]
10901fn record_wait_error_terminal(
10902    module_id: &str,
10903    terminal_ring: &Arc<Mutex<TerminalRing>>,
10904    spawn_events: &SpawnEventFeed,
10905) {
10906    record_terminal(
10907        module_id,
10908        terminal_ring,
10909        spawn_events,
10910        &wait_error_exit_report(),
10911        TerminalDisposition::Failed,
10912    );
10913}
10914
10915fn record_terminal(
10916    module_id: &str,
10917    terminal_ring: &Arc<Mutex<TerminalRing>>,
10918    spawn_events: &SpawnEventFeed,
10919    exit_report: &ExitReport,
10920    disposition: TerminalDisposition,
10921) {
10922    record_terminal_with_detail(
10923        module_id,
10924        terminal_ring,
10925        spawn_events,
10926        exit_report,
10927        disposition,
10928        None,
10929    );
10930}
10931
10932/// The ring lock is held only to capture the read (see
10933/// `TerminalJournal::capture_read`), so this module's exits keep recording
10934/// while the journal files are read. Blocking: it reads files.
10935fn durable_terminal_history_of(
10936    terminal_ring: &Mutex<TerminalRing>,
10937    module_id: &str,
10938) -> subc_control::TerminalHistory {
10939    let read = terminal_ring
10940        .lock()
10941        .unwrap_or_else(|p| p.into_inner())
10942        .capture_durable_history();
10943    read.read(module_id)
10944}
10945
10946fn record_terminal_with_detail(
10947    module_id: &str,
10948    terminal_ring: &Arc<Mutex<TerminalRing>>,
10949    spawn_events: &SpawnEventFeed,
10950    exit_report: &ExitReport,
10951    disposition: TerminalDisposition,
10952    disposition_detail: Option<String>,
10953) {
10954    #[cfg(test)]
10955    terminal_history_tests::before_terminal_record(spawn_events);
10956    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
10957    let record = TerminalRecord {
10958        exit_code: exit_report.code,
10959        exit_signal: exit_report.signal,
10960        at_ms: exit_report.at_ms,
10961        disposition,
10962        exit_kind: exit_report.kind.into(),
10963        disposition_detail,
10964    };
10965    terminal_ring
10966        .lock()
10967        .unwrap_or_else(|poisoned| poisoned.into_inner())
10968        .record_exit(module_id, record);
10969}
10970
10971fn untrack_if_registration_released(
10972    process_liveness: &SupervisorProcessLiveness,
10973    registry: &Registry,
10974    module_id: &str,
10975    snapshot: &SharedSnapshot,
10976) {
10977    match registry.get_module(module_id) {
10978        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
10979        Ok(Some(_)) => {}
10980        Err(err) => {
10981            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
10982        }
10983    }
10984}
10985
10986/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
10987/// then apply the module's configured entries minus daemon-private capture keys.
10988///
10989/// Separated from `spawn_child` only so it can be asserted without spawning a
10990/// process — a duplicate of this logic in a test would pass while the real one
10991/// drifted, which is the defect class this function exists to avoid.
10992/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
10993/// nonce. A `protocol: "none"` module gets neither, because it cannot use
10994/// either and the argument would stop a stock binary from starting at all.
10995/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
10996///
10997/// The plain-spawn form, kept for the tests that assert its plan; spawns go
10998/// through [`apply_wire_spawn_args_for_admission`].
10999#[cfg(test)]
11000fn apply_wire_spawn_args(
11001    command: &mut Command,
11002    spec: &ModuleSpec,
11003    connection_file_path: Option<&std::path::Path>,
11004    handle: Option<&SupervisorHandle>,
11005) -> Result<Option<NonceHandoff>, SuperviseError> {
11006    apply_wire_spawn_args_for_role(
11007        command,
11008        spec,
11009        connection_file_path,
11010        handle,
11011        SpawnRole::Plain,
11012    )
11013}
11014
11015/// The read end of a spawn's launch-nonce pipe, prepared by
11016/// [`apply_wire_spawn_args_for_admission`] and installed as the child's descriptor 3
11017/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
11018/// server before spawn and authenticates its client after the child exists.
11019#[cfg(unix)]
11020type NonceHandoff = subc_os::LaunchNonceHandoff;
11021#[cfg(windows)]
11022type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
11023#[cfg(not(any(unix, windows)))]
11024type NonceHandoff = std::convert::Infallible;
11025
11026/// Create the launch nonce that a new module process must present when it
11027/// registers or opens a consumer connection, including a replacement process
11028/// started alongside a still-serving module.
11029///
11030/// A plain spawn replaces the module's recorded nonce. A swap candidate records
11031/// a separate candidate token so the still-serving incumbent and its consumers
11032/// keep their nonce. Both records are installed before the process exists, so
11033/// the child's initial HELLO registration cannot arrive ahead of its nonce.
11034///
11035/// On Unix the nonce is delivered only through a pipe. It is written into
11036/// a pipe whose read end the child gets as descriptor 3, named by
11037/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
11038/// process of the same user cannot read it with `ps eww`. That handoff is
11039/// returned rather than installed here. The combined descriptor placement runs
11040/// after the cgroup callback and placement barrier, so replacing slots 3-5
11041/// cannot close a descriptor either callback still needs.
11042/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
11043#[cfg(test)]
11044fn apply_wire_spawn_args_for_role(
11045    command: &mut Command,
11046    spec: &ModuleSpec,
11047    connection_file_path: Option<&std::path::Path>,
11048    handle: Option<&SupervisorHandle>,
11049    role: SpawnRole,
11050) -> Result<Option<NonceHandoff>, SuperviseError> {
11051    apply_wire_spawn_args_for_admission(command, spec, connection_file_path, handle, role, None)
11052}
11053
11054fn apply_wire_spawn_args_for_admission(
11055    command: &mut Command,
11056    spec: &ModuleSpec,
11057    connection_file_path: Option<&std::path::Path>,
11058    handle: Option<&SupervisorHandle>,
11059    role: SpawnRole,
11060    admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11061) -> Result<Option<NonceHandoff>, SuperviseError> {
11062    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
11063    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
11064    // included: a daemon started from a module's process tree inherits it,
11065    // and passing it on would point the child at a descriptor it does not
11066    // have.
11067    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
11068    #[cfg(windows)]
11069    {
11070        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
11071        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
11072    }
11073    // Remove inherited or configured copies too: withholding must mean absent.
11074    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
11075    command.env_remove(subc_os::registration_channel::REGISTRATION_FD_ENV);
11076    command.env_remove(subc_os::registration_channel::REGISTRATION_PIPE_ENV);
11077    command.env_remove(subc_os::registration_channel::DAEMON_PID_ENV);
11078    if role == SpawnRole::Plain {
11079        if let Some(handle) = handle {
11080            #[cfg(all(test, unix))]
11081            {
11082                let pause = {
11083                    let mut slot = handle.plain_identity_policy_pause.lock().unwrap();
11084                    if slot
11085                        .as_ref()
11086                        .is_some_and(|(module, _)| module == &spec.module_id)
11087                    {
11088                        slot.take().map(|(_, pause)| pause)
11089                    } else {
11090                        None
11091                    }
11092                };
11093                if let Some(pause) = pause {
11094                    pause.reached.send(()).unwrap();
11095                    pause.release.recv().unwrap();
11096                }
11097            }
11098            // A rescan preserves the running child's nonce and reservation.
11099            // Replacement spawn preparation applies the new protocol now,
11100            // before protocol:none can return without minting a nonce.
11101            handle.apply_identity_policy(spec, spec.protocol);
11102        }
11103    }
11104    if spec.protocol == ModuleProtocol::None {
11105        return Ok(None);
11106    }
11107    if let Some(connection_file_path) = connection_file_path {
11108        command.arg(SUBC_ARG).arg(connection_file_path);
11109    }
11110
11111    if admission.is_some_and(|admission| admission.launch.is_some()) {
11112        if let Some(handle) = handle {
11113            match role {
11114                SpawnRole::Plain => {
11115                    handle
11116                        .spawn_nonces
11117                        .lock()
11118                        .unwrap_or_else(|p| p.into_inner())
11119                        .remove(&spec.module_id);
11120                    if spec.reserved {
11121                        handle
11122                            .reserved_nonces
11123                            .lock()
11124                            .unwrap_or_else(|p| p.into_inner())
11125                            .insert(spec.module_id.clone(), None);
11126                    }
11127                }
11128                // Swaps still need their lifecycle record, but the empty value
11129                // is not a bearer token and is never accepted by nonce HELLO.
11130                SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, String::new()),
11131            }
11132        }
11133        return Ok(None);
11134    }
11135    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
11136    // route.open attestation. Reserved modules additionally use the same nonce
11137    // for HELLO id-squatting protection. A respawn rotates both records.
11138    let nonce = generate_launch_nonce()?;
11139    if let Some(handle) = handle {
11140        match role {
11141            SpawnRole::Plain => {
11142                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
11143                if spec.reserved {
11144                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
11145                }
11146            }
11147            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
11148        }
11149    }
11150    #[cfg(unix)]
11151    let handoff = {
11152        let handoff =
11153            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
11154                program: spec.program.clone(),
11155                source,
11156                cgroup_path: None,
11157            })?;
11158        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
11159        Some(handoff)
11160    };
11161    #[cfg(windows)]
11162    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
11163    #[cfg(not(any(unix, windows)))]
11164    let handoff = None;
11165    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
11166    // until the daemon's per-spawn delivery records show that every running
11167    // Windows module has read its secret from the named pipe.
11168    #[cfg(not(unix))]
11169    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
11170    Ok(handoff)
11171}
11172
11173fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
11174    command.env_remove(CK_LOG_ENV);
11175    // The spawn role is the supervisor's to set, and only on a swap candidate
11176    // (see `apply_spawn_role`). Removing it here, rather than just not setting
11177    // it, is what makes it absent on a plain spawn: the daemon's own
11178    // environment could carry it, and so could a spec built outside daemon
11179    // config (config refuses it as an `env` key). A module reading it on a
11180    // plain restart would pick the long swap budget and leave callers waiting.
11181    command.env_remove(SUBC_SPAWN_ROLE_ENV);
11182    for (key, value) in &spec.env {
11183        // cortexkit-log currently exposes retention only as a Rust struct, not
11184        // environment names. These values are daemon-private sink metadata and
11185        // must never become a public child-process contract by being inherited.
11186        if matches!(
11187            key.as_str(),
11188            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
11189        ) || key == SUBC_SPAWN_ROLE_ENV
11190        {
11191            continue;
11192        }
11193        command.env(key, value);
11194    }
11195}
11196
11197#[cfg(windows)]
11198fn prepare_windows_registration_pipe(
11199    command: &mut Command,
11200    spec: &ModuleSpec,
11201) -> Result<Option<SpawnRegistrationChannel>, SuperviseError> {
11202    if spec.protocol == ModuleProtocol::None {
11203        return Ok(None);
11204    }
11205    let pipe = subc_os::registration_channel::RegistrationPipe::new().map_err(|source| {
11206        SuperviseError::Spawn {
11207            program: spec.program.clone(),
11208            source,
11209            cgroup_path: None,
11210        }
11211    })?;
11212    command
11213        .env(
11214            subc_os::registration_channel::REGISTRATION_PIPE_ENV,
11215            pipe.name(),
11216        )
11217        .env(
11218            subc_os::registration_channel::DAEMON_PID_ENV,
11219            std::process::id().to_string(),
11220        );
11221    Ok(Some(pipe))
11222}
11223
11224#[cfg(windows)]
11225fn prepare_windows_nonce_pipe(
11226    command: &mut Command,
11227    spec: &ModuleSpec,
11228    nonce: &str,
11229) -> Option<NonceHandoff> {
11230    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
11231        Ok(handoff) => {
11232            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
11233            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
11234            // still register until every module reads the pipe. A reader without
11235            // this variable set to env returns the pipe error instead of using
11236            // an inherited or configured environment copy.
11237            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
11238            Some(handoff)
11239        }
11240        Err(error) => {
11241            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
11242            None
11243        }
11244    }
11245}
11246
11247#[cfg(windows)]
11248fn start_windows_nonce_pipe(
11249    handoff: Option<NonceHandoff>,
11250    pid: u32,
11251    spec: &ModuleSpec,
11252) -> Option<subc_os::LaunchNoncePipeDelivery> {
11253    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
11254    // same maximum wait used for registration during a reload. Older modules
11255    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
11256    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
11257        Ok(delivery) => Some(delivery),
11258        Err(error) => {
11259            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
11260            None
11261        }
11262    })
11263}
11264
11265fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
11266    #[cfg(windows)]
11267    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
11268        return Some(
11269            if snapshot
11270                .pipe_consumed
11271                .as_ref()
11272                .is_some_and(|consumed| consumed.get().is_some())
11273            {
11274                "pipe"
11275            } else {
11276                "env"
11277            }
11278            .to_owned(),
11279        );
11280    }
11281    let _ = snapshot;
11282    None
11283}
11284
11285/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
11286/// of a blue/green swap.
11287#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11288enum SpawnRole {
11289    Plain,
11290    SwapCandidate,
11291}
11292
11293/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
11294/// `apply_child_env` has already removed the variable for every spawn.
11295fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
11296    if role == SpawnRole::SwapCandidate {
11297        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
11298    }
11299}
11300
11301fn previous_instance_error(
11302    spec: &ModuleSpec,
11303    pid: u32,
11304    reason: crate::live_children::RetainedReason,
11305) -> SuperviseError {
11306    SuperviseError::Spawn {
11307        program: spec.program.clone(),
11308        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
11309        cgroup_path: None,
11310    }
11311}
11312
11313fn carries_containment(configured: bool, has_container: bool) -> bool {
11314    configured || has_container
11315}
11316
11317fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
11318    if !carries_containment(
11319        spec.carry_containment,
11320        containment::current(&roster.containment).is_some(),
11321    ) {
11322        return None;
11323    }
11324    Some(SuperviseError::Spawn {
11325        program: spec.program.clone(),
11326        source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
11327        cgroup_path: None,
11328    })
11329}
11330
11331#[allow(clippy::too_many_arguments)]
11332fn spawn_child(
11333    spec: &ModuleSpec,
11334    connection_file_path: Option<&std::path::Path>,
11335    handle: Option<&SupervisorHandle>,
11336    ring: &Arc<Mutex<StderrRing>>,
11337    capture_logs_dir: Option<&std::path::Path>,
11338    roster: &ChildRoster,
11339    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11340    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11341    register_timeout: Duration,
11342) -> Result<SupervisedChild, SuperviseError> {
11343    #[cfg(all(test, unix))]
11344    if let Some(source) = containment_tests::join_failure(spec, roster) {
11345        containment::terminate_blocking(&roster.containment);
11346        return Err(SuperviseError::Spawn {
11347            program: spec.program.clone(),
11348            source,
11349            cgroup_path: None,
11350        });
11351    }
11352    let result = spawn_child_in_slot(
11353        spec,
11354        connection_file_path,
11355        handle,
11356        ring,
11357        capture_logs_dir,
11358        roster,
11359        config_delivery,
11360        #[cfg(target_os = "linux")]
11361        cgroup_placement,
11362        SpawnRole::Plain,
11363        false,
11364        register_timeout,
11365    );
11366    if result.is_err() {
11367        if roster.admission_pending(&spec.module_id) {
11368            containment::terminate_deferred(&roster.containment);
11369        } else {
11370            containment::terminate_blocking(&roster.containment);
11371        }
11372    }
11373    result
11374}
11375
11376/// Spawn one process of `spec` into a slot.
11377///
11378/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
11379/// A swap candidate needs a different cgroup from the process it is replacing,
11380/// which is still alive: in the same cgroup the two would be one kill domain,
11381/// and killing a failed candidate could take the incumbent with it.
11382///
11383/// The stderr capture file is `<module_id>.stderr.log` for every process of
11384/// the module, whichever slot it is in, because that is the one file
11385/// `ck module logs` reads. During a swap's overlap both processes append to it;
11386/// the daemon writes whole lines, so the two interleave by line, which is also
11387/// the merged view an operator wants while a swap runs.
11388#[allow(clippy::too_many_arguments)]
11389fn spawn_child_in_slot(
11390    spec: &ModuleSpec,
11391    connection_file_path: Option<&std::path::Path>,
11392    handle: Option<&SupervisorHandle>,
11393    ring: &Arc<Mutex<StderrRing>>,
11394    capture_logs_dir: Option<&std::path::Path>,
11395    roster: &ChildRoster,
11396    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11397    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11398    role: SpawnRole,
11399    alternate_slot: bool,
11400    register_timeout: Duration,
11401) -> Result<SupervisedChild, SuperviseError> {
11402    let prepared = prepare_child_in_slot(
11403        spec,
11404        connection_file_path,
11405        handle,
11406        capture_logs_dir,
11407        roster,
11408        config_delivery,
11409        #[cfg(target_os = "linux")]
11410        cgroup_placement,
11411        role,
11412        alternate_slot,
11413        false,
11414    )?;
11415    let gate = config_delivery
11416        .map(|delivery| delivery.module_gate(spec))
11417        .transpose()?;
11418    let _gate = gate
11419        .as_ref()
11420        .map(|gate| {
11421            gate.try_lock()
11422                .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11423                    module_id: spec.module_id.clone(),
11424                })
11425        })
11426        .transpose()?;
11427    if let (Some(delivery), Some(admission)) = (config_delivery, &prepared.spawn_admission) {
11428        delivery.check(spec, admission)?;
11429    }
11430    let checked_base = prepared
11431        .spawn_admission
11432        .as_ref()
11433        .and_then(|admission| admission.approved_base())
11434        .cloned();
11435    let mut result = launch_child_in_slot(spec, ring, roster, prepared, register_timeout);
11436    drop(_gate);
11437    if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11438        (config_delivery, &result)
11439    {
11440        if let Some(base) = checked_base.as_ref().filter(|_| code == "program_refused") {
11441            if !delivery.record_refusal(spec, base, detail)? {
11442                result = Err(spawn_plan_stale(spec));
11443            }
11444        }
11445    }
11446    result
11447}
11448
11449struct AsyncInitialIdentity<'a> {
11450    supervisor: &'a Supervisor,
11451    module: String,
11452    committed: bool,
11453}
11454impl Drop for AsyncInitialIdentity<'_> {
11455    fn drop(&mut self) {
11456        if !self.committed {
11457            self.supervisor.abandon_unrostered(&self.module);
11458        }
11459    }
11460}
11461
11462/// Child-free preparation can be abandoned without orphaning a process. After
11463/// this value returns, the async caller validates its launch witness and creates
11464/// and publishes the child without another await.
11465struct PreparedChild {
11466    registration_spawn: Option<Arc<RegistrationSpawn>>,
11467    registration_handle: Option<SupervisorHandle>,
11468    #[cfg(any(unix, windows))]
11469    registration_channel: Option<SpawnRegistrationChannel>,
11470    config_context: Option<config_delivery::ConfigDelivery>,
11471    binding_publication: Option<ProgramBindingPublication>,
11472    launch_spec: ModuleSpec,
11473    spawn_admission: Option<crate::config_gate::runtime::SpawnAdmission>,
11474    command: Command,
11475    carrying: bool,
11476    carried: Option<Arc<containment::CarriedContainment>>,
11477    #[cfg(unix)]
11478    launch_identity: Option<containment::LaunchIdentity>,
11479    #[cfg(unix)]
11480    group_target: containment::GroupTarget,
11481    #[cfg(target_os = "linux")]
11482    cgroup_placement: Option<subc_cgroup::Placement>,
11483    #[cfg(target_os = "linux")]
11484    cgroup_name: String,
11485    #[cfg(target_os = "linux")]
11486    cgroup_cleanup: PreparedCgroup,
11487    cgroup_path: Option<PathBuf>,
11488    #[cfg(unix)]
11489    admission_gate: Option<admission::Gate>,
11490    output_sink: ChildOutputSink,
11491    #[cfg(target_os = "macos")]
11492    privacy_exec: Option<PrivacyExec>,
11493    #[cfg(windows)]
11494    image_capture: subc_os::ExecutableCapture,
11495    #[cfg(windows)]
11496    nonce_handoff: Option<NonceHandoff>,
11497    spawned_at: Instant,
11498    config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
11499    delivery_registration: Option<config_delivery::RegistrationGuard>,
11500    program_binding: Option<SpawnProgramBinding>,
11501}
11502
11503#[cfg(target_os = "linux")]
11504struct PreparedCgroup(Option<(subc_cgroup::Placement, String)>);
11505#[cfg(target_os = "linux")]
11506impl Drop for PreparedCgroup {
11507    fn drop(&mut self) {
11508        if let Some((placement, name)) = &self.0 {
11509            remove_module_cgroup(placement, name);
11510        }
11511    }
11512}
11513
11514pub(crate) struct SpawnProgramBinding {
11515    pub(crate) path: PathBuf,
11516    // Keep the inode alive if the configured name is replaced while preparation
11517    // is suspended, so replacement cannot pass through inode reuse.
11518    _file: std::fs::File,
11519    pub(crate) identity: subc_os::RetainedFileIdentity,
11520}
11521
11522impl SpawnProgramBinding {
11523    pub(crate) fn capture(spec: &ModuleSpec) -> Result<Self, SuperviseError> {
11524        Self::capture_after_resolution(spec, |_| {})
11525    }
11526
11527    fn capture_after_resolution(
11528        spec: &ModuleSpec,
11529        before_open: impl FnOnce(&std::path::Path),
11530    ) -> Result<Self, SuperviseError> {
11531        require_program_search_path(spec)?;
11532        #[cfg(windows)]
11533        let windows_capture =
11534            windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
11535                program: spec.program.clone(),
11536                source,
11537                cgroup_path: None,
11538            })?;
11539        #[cfg(windows)]
11540        let path = windows_capture.path().to_path_buf();
11541        #[cfg(not(windows))]
11542        let path = if spec.program.is_absolute() || spec.program.components().count() > 1 {
11543            spec.program.clone()
11544        } else {
11545            let path = spec
11546                .env
11547                .iter()
11548                .rev()
11549                .find(|(key, _)| key == "PATH")
11550                .map(|(_, path)| std::ffi::OsString::from(path))
11551                .or_else(|| std::env::var_os("PATH"))
11552                .unwrap_or_else(|| {
11553                    if cfg!(target_os = "macos") {
11554                        // Match Darwin's spawn search when PATH is absent, not
11555                        // the current-directory search of an explicitly empty PATH.
11556                        "/usr/bin:/bin".into()
11557                    } else {
11558                        Default::default()
11559                    }
11560                });
11561            std::env::split_paths(&path)
11562                .map(|dir| dir.join(&spec.program))
11563                .find(|path| {
11564                    if !path.is_file() {
11565                        return false;
11566                    }
11567                    #[cfg(unix)]
11568                    {
11569                        rustix::fs::accessat(
11570                            rustix::fs::CWD,
11571                            path,
11572                            rustix::fs::Access::EXEC_OK,
11573                            rustix::fs::AtFlags::EACCESS,
11574                        )
11575                        .is_ok()
11576                    }
11577                    #[cfg(not(unix))]
11578                    {
11579                        true
11580                    }
11581                })
11582                .ok_or_else(|| SuperviseError::Spawn {
11583                    program: spec.program.clone(),
11584                    source: io::Error::new(io::ErrorKind::NotFound, "executable not found in PATH"),
11585                    cgroup_path: None,
11586                })?
11587        };
11588        let resolved = std::fs::canonicalize(&path).map_err(|source| SuperviseError::Spawn {
11589            program: spec.program.clone(),
11590            source,
11591            cgroup_path: None,
11592        })?;
11593        before_open(&resolved);
11594        // Resolving links does not open their target. The descriptor open must
11595        // remain nonblocking and no-follow if that resolved name is substituted.
11596        let file =
11597            subc_os::open_regular_nofollow(&resolved).map_err(|source| SuperviseError::Spawn {
11598                program: spec.program.clone(),
11599                source: io::Error::other(source),
11600                cgroup_path: None,
11601            })?;
11602        let identity =
11603            subc_os::retained_file_identity(&file).map_err(|source| SuperviseError::Spawn {
11604                program: spec.program.clone(),
11605                source,
11606                cgroup_path: None,
11607            })?;
11608        Ok(Self {
11609            path,
11610            _file: file,
11611            identity,
11612        })
11613    }
11614    fn current(&self) -> bool {
11615        #[cfg(unix)]
11616        {
11617            subc_os::file_identity(&self.path) == Some(self.identity)
11618        }
11619        #[cfg(windows)]
11620        {
11621            std::fs::canonicalize(&self.path)
11622                .ok()
11623                .and_then(|path| subc_os::open_regular_nofollow(&path).ok())
11624                .and_then(|file| subc_os::retained_file_identity(&file).ok())
11625                == Some(self.identity)
11626        }
11627    }
11628}
11629
11630fn spawn_plan_stale(spec: &ModuleSpec) -> SuperviseError {
11631    SuperviseError::ConfigDeliveryDeferred {
11632        module_id: spec.module_id.clone(),
11633    }
11634}
11635
11636fn require_program_search_path(spec: &ModuleSpec) -> Result<(), SuperviseError> {
11637    if !cfg!(target_os = "macos")
11638        && !spec.program.is_absolute()
11639        && spec.program.components().count() == 1
11640        && !spec.env.iter().any(|(key, _)| {
11641            if cfg!(windows) {
11642                key.eq_ignore_ascii_case("PATH")
11643            } else {
11644                key == "PATH"
11645            }
11646        })
11647        && std::env::var_os("PATH").is_none()
11648    {
11649        // The OS default search is not a current-directory search. Refusing an
11650        // unspecified search path prevents approving a decoy in the current directory.
11651        return Err(SuperviseError::Spawn {
11652            program: spec.program.clone(),
11653            source: io::Error::new(io::ErrorKind::InvalidInput, "bare program requires PATH"),
11654            cgroup_path: None,
11655        });
11656    }
11657    Ok(())
11658}
11659
11660async fn spawn_child_async(
11661    spec: &ModuleSpec,
11662    runtime: &SupervisorRuntimeConfig,
11663) -> Result<SupervisedChild, SuperviseError> {
11664    #[cfg(all(test, unix))]
11665    if let Some(source) = containment_tests::join_failure(spec, &runtime.child_roster) {
11666        containment::terminate_blocking(&runtime.child_roster.containment);
11667        return Err(SuperviseError::Spawn {
11668            program: spec.program.clone(),
11669            source,
11670            cgroup_path: None,
11671        });
11672    }
11673    let result = spawn_child_in_slot_async(spec, runtime, SpawnRole::Plain, false).await;
11674    if result.is_err() {
11675        if runtime.child_roster.admission_pending(&spec.module_id) {
11676            containment::terminate_deferred(&runtime.child_roster.containment);
11677        } else {
11678            containment::terminate_blocking(&runtime.child_roster.containment);
11679        }
11680    }
11681    result
11682}
11683
11684async fn spawn_child_in_slot_async(
11685    spec: &ModuleSpec,
11686    runtime: &SupervisorRuntimeConfig,
11687    role: SpawnRole,
11688    alternate_slot: bool,
11689) -> Result<SupervisedChild, SuperviseError> {
11690    let gate = runtime
11691        .config_delivery
11692        .as_ref()
11693        .map(|delivery| delivery.0.module_gate(&spec.module_id))
11694        .transpose()
11695        .map_err(|error| SuperviseError::InvalidSpec {
11696            reason: error.to_string(),
11697        })?;
11698    if gate.as_ref().is_some_and(|gate| gate.try_lock().is_err()) {
11699        return Err(SuperviseError::ConfigDeliveryDeferred {
11700            module_id: spec.module_id.clone(),
11701        });
11702    }
11703    let prepared_spec = spec.clone();
11704    let prepared_runtime = runtime.clone();
11705    let prepared = tokio::task::spawn_blocking(move || {
11706        let prepared = prepare_child_in_slot(
11707            &prepared_spec,
11708            prepared_runtime.connection_file_path.as_deref(),
11709            prepared_runtime.supervisor_handle.as_ref(),
11710            prepared_runtime.capture_logs_dir.as_deref(),
11711            &prepared_runtime.child_roster,
11712            prepared_runtime.spawn_delivery(),
11713            #[cfg(target_os = "linux")]
11714            prepared_runtime.cgroup_placement.as_ref(),
11715            role,
11716            alternate_slot,
11717            true,
11718        );
11719        prepared
11720    })
11721    .await
11722    .map_err(|_| SuperviseError::InvalidSpec {
11723        reason: "configuration launch preparation stopped".into(),
11724    })??;
11725    if let Some(registration) = &prepared.delivery_registration {
11726        if !registration.launch_current()? {
11727            return Err(spawn_plan_stale(spec));
11728        }
11729    }
11730    let _gate = match gate {
11731        Some(gate) => {
11732            Some(
11733                gate.try_lock_owned()
11734                    .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11735                        module_id: spec.module_id.clone(),
11736                    })?,
11737            )
11738        }
11739        None => None,
11740    };
11741    if let (Some(delivery), Some(admission)) = (runtime.spawn_delivery(), &prepared.spawn_admission)
11742    {
11743        delivery.check(spec, admission)?;
11744    }
11745    if prepared
11746        .spawn_admission
11747        .as_ref()
11748        .is_none_or(|admission| admission.launch.is_none())
11749        && prepared
11750            .program_binding
11751            .as_ref()
11752            .is_some_and(|binding| !binding.current())
11753    {
11754        return Err(spawn_plan_stale(spec));
11755    }
11756    // No await may separate this last identity check from ownership publication.
11757    let checked_base = prepared
11758        .spawn_admission
11759        .as_ref()
11760        .and_then(|admission| admission.approved_base())
11761        .cloned();
11762    let mut result = launch_child_in_slot(
11763        spec,
11764        &runtime.stderr_ring,
11765        &runtime.child_roster,
11766        prepared,
11767        runtime.register_timeout,
11768    );
11769    drop(_gate);
11770    if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11771        (runtime.config_delivery.as_ref(), &result)
11772    {
11773        if let Some(base) = checked_base.filter(|_| code == "program_refused") {
11774            let module = spec.module_id.clone();
11775            let identity = subc_config::raw_digest(detail.as_bytes());
11776            let recorded = delivery
11777                .0
11778                .on_blocking(move |owner| owner.record_program_refusal(&module, &base, identity))
11779                .await;
11780            match recorded {
11781                Ok(true) => {}
11782                Ok(false) => result = Err(spawn_plan_stale(spec)),
11783                Err(error) => {
11784                    result = Err(config_delivery::gate_hold_error(
11785                        spec,
11786                        error,
11787                        Some(delivery),
11788                    ))
11789                }
11790            }
11791        }
11792    }
11793    result
11794}
11795
11796fn validate_spawn_program(
11797    spec: &ModuleSpec,
11798    admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11799    delivery: Option<config_delivery::SpawnDelivery<'_>>,
11800) -> Result<(), SuperviseError> {
11801    if let Some(admission) = admission {
11802        if let Some(program) = &admission.launch {
11803            if let Err(error) = program.validate_for_spawn() {
11804                // A captured unsigned image is a definite refusal. Uncertain
11805                // reads are deferred by validate_for_spawn, not attributed to
11806                // the operator as a changed or unapprovable program.
11807                if let (Some(context), Some(base)) = (delivery, admission.approved_base()) {
11808                    if error.code == "program_refused"
11809                        && !context.record_refusal(spec, base, &error.detail)?
11810                    {
11811                        return Err(spawn_plan_stale(spec));
11812                    }
11813                }
11814                return Err(config_delivery::gate_hold_error(
11815                    spec,
11816                    error,
11817                    delivery.map(|context| context.runtime()).as_ref(),
11818                ));
11819            }
11820        }
11821    }
11822    Ok(())
11823}
11824
11825#[allow(clippy::too_many_arguments)]
11826fn prepare_child_in_slot(
11827    spec: &ModuleSpec,
11828    connection_file_path: Option<&std::path::Path>,
11829    handle: Option<&SupervisorHandle>,
11830    capture_logs_dir: Option<&std::path::Path>,
11831    roster: &ChildRoster,
11832    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11833    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11834    role: SpawnRole,
11835    alternate_slot: bool,
11836    asynchronous: bool,
11837) -> Result<PreparedChild, SuperviseError> {
11838    require_program_search_path(spec)?;
11839    let spawn_admission = config_delivery
11840        .map(|delivery| delivery.prepare(spec))
11841        .transpose()?;
11842    #[cfg(target_os = "macos")]
11843    let gated = spawn_admission
11844        .as_ref()
11845        .is_some_and(|admission| admission.approved_base().is_some());
11846    let launch_spec = spawn_admission
11847        .as_ref()
11848        .and_then(|admission| admission.launch.as_ref())
11849        .map(|launch| launch.spec.clone())
11850        .unwrap_or_else(|| spec.clone());
11851    let spec = &launch_spec;
11852    if let Some(error) = recovery_block_error(spec, roster) {
11853        return Err(error);
11854    }
11855    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
11856        return Err(previous_instance_error(spec, pid, reason));
11857    }
11858    if roster.is_closed() {
11859        return Err(SuperviseError::Spawn {
11860            program: spec.program.clone(),
11861            source: io::Error::other("the daemon is shutting down; not starting a new process"),
11862            cgroup_path: None,
11863        });
11864    }
11865    let carried = containment::current(&roster.containment);
11866    let carrying = carries_containment(spec.carry_containment, carried.is_some());
11867    if role != SpawnRole::Plain && carrying {
11868        return Err(SuperviseError::InvalidSpec {
11869            reason: "a carried container cannot overlap a swap candidate".to_owned(),
11870        });
11871    }
11872    #[cfg(unix)]
11873    let launch_identity = if carrying && carried.is_none() {
11874        Some(
11875            containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
11876                program: spec.program.clone(),
11877                source,
11878                cgroup_path: None,
11879            })?,
11880        )
11881    } else {
11882        None
11883    };
11884    #[cfg(target_os = "linux")]
11885    let cgroup_placement = match &carried {
11886        Some(container) => container.cgroup().map(|(placement, _)| placement),
11887        None => cgroup_placement,
11888    };
11889    #[cfg(unix)]
11890    let group_target = if let Some(container) = carried
11891        .as_ref()
11892        .filter(|container| container.group().is_some())
11893    {
11894        containment::GroupTarget::Pinned(Arc::clone(container))
11895    } else {
11896        #[cfg(target_os = "linux")]
11897        let cgroup_container = carrying && cgroup_placement.is_some();
11898        #[cfg(not(target_os = "linux"))]
11899        let cgroup_container = false;
11900        if cgroup_container {
11901            containment::GroupTarget::NoGroup
11902        } else {
11903            containment::GroupTarget::OwnLeader
11904        }
11905    };
11906    #[cfg(target_os = "linux")]
11907    let cgroup_name = {
11908        // Slot names alone are not kill domains: a retired incumbent may still
11909        // be draining when a later enable/restart spawns into the same slot.
11910        // Decimal entropy keeps the suffix unambiguous; Placement performs
11911        // the module-id escaping and constructs the filesystem path.
11912        if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
11913            name.to_owned()
11914        } else if cgroup_placement.is_none() {
11915            swap::cgroup_name(&spec.module_id, alternate_slot)
11916        } else {
11917            let nonce = generate_launch_nonce()?;
11918            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
11919            // Leave room for byte escaping and the suffix under NAME_MAX. The
11920            // label is only for humans; the nonce identifies the kill domain.
11921            let mut end = spec.module_id.len().min(64);
11922            while !spec.module_id.is_char_boundary(end) {
11923                end -= 1;
11924            }
11925            format!(
11926                "{}_{suffix}",
11927                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
11928            )
11929        }
11930    };
11931    #[cfg(not(target_os = "linux"))]
11932    let _ = alternate_slot;
11933    #[cfg(windows)]
11934    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
11935        program: spec.program.clone(),
11936        source,
11937        cgroup_path: None,
11938    })?;
11939    #[cfg(windows)]
11940    let program_binding = if asynchronous {
11941        let mut resolved = spec.clone();
11942        resolved.program = image_capture.path().to_path_buf();
11943        Some(SpawnProgramBinding::capture(&resolved)?)
11944    } else {
11945        None
11946    };
11947    #[cfg(not(windows))]
11948    let program_binding = (asynchronous || cfg!(target_os = "macos"))
11949        .then(|| SpawnProgramBinding::capture(spec))
11950        .transpose()?;
11951    #[cfg(target_os = "macos")]
11952    validate_spawn_program(spec, spawn_admission.as_ref(), config_delivery)?;
11953    #[cfg(target_os = "macos")]
11954    let (mut command, privacy_exec, exec_ack) = privacy_command(
11955        spec,
11956        program_binding
11957            .as_ref()
11958            .expect("macOS captures every launch"),
11959        roster,
11960        // Ungated launches retain legacy unsuspended exec, including scripts. Suspended
11961        // verification admits approved programs only. prepare() above fails
11962        // closed if the gate cannot decide; an image sample is not authority.
11963        gated,
11964    )?;
11965    #[cfg(windows)]
11966    let mut command = Command::new(image_capture.path());
11967    #[cfg(not(any(target_os = "macos", windows)))]
11968    let mut command = Command::new(
11969        program_binding
11970            .as_ref()
11971            .map_or(spec.program.as_path(), |binding| binding.path.as_path()),
11972    );
11973    command.args(&spec.args);
11974    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
11975    // that is the whole of the intent, so remove that one key rather than the
11976    // environment.
11977    //
11978    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
11979    // and took the POSIX environment with it. Modules spawned that way had no
11980    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
11981    // logging:
11982    //
11983    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
11984    //     both unset it fell back to the temp dir alone and `ck` could not find
11985    //     a daemon running on the same machine from inside any module's process
11986    //     tree — reporting a path the file has never lived at, which reads as
11987    //     "the daemon did not write its file".
11988    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
11989    //     the RELATIVE `.local/share`, so a module deriving its own store path
11990    //     resolved it against its own CWD. That is the store-fragmentation
11991    //     defect the daemon already refuses in config (`parse_doc` rejects a
11992    //     relative `storage.data_home`) arriving by derivation instead.
11993    //   * anything a module spawns inherited it: git without ~/.gitconfig,
11994    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
11995    //     quietly rather than erroring.
11996    //
11997    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
11998    // offered one candidate under /tmp while the file sat in /run/user/1000.
11999    //
12000    // A configured module is unaffected either way: `module_spec()` puts the
12001    // resolved CK_LOG into `spec.env`, which is applied below and therefore
12002    // wins over anything ambient.
12003    apply_child_env(&mut command, spec);
12004    apply_spawn_role(&mut command, role);
12005    let nonce_handoff = apply_wire_spawn_args_for_admission(
12006        &mut command,
12007        spec,
12008        connection_file_path,
12009        handle,
12010        role,
12011        spawn_admission.as_ref(),
12012    )?;
12013    #[cfg(unix)]
12014    let mut slots = subc_os::registration_channel::ChildDescriptorSlots::default();
12015    #[cfg(unix)]
12016    let registration_channel = if spec.protocol == ModuleProtocol::Subc {
12017        let (parent, child, metadata) = subc_os::registration_channel::registration_socketpair()
12018            .map_err(|source| SuperviseError::Spawn {
12019                program: spec.program.clone(),
12020                source,
12021                cgroup_path: None,
12022            })?;
12023        slots
12024            .channel(child)
12025            .map_err(|source| SuperviseError::Spawn {
12026                program: spec.program.clone(),
12027                source,
12028                cgroup_path: None,
12029            })?;
12030        command
12031            .env(subc_os::registration_channel::REGISTRATION_FD_ENV, metadata)
12032            .env(
12033                subc_os::registration_channel::DAEMON_PID_ENV,
12034                std::process::id().to_string(),
12035            );
12036        Some(parent)
12037    } else {
12038        None
12039    };
12040    #[cfg(windows)]
12041    let registration_channel = prepare_windows_registration_pipe(&mut command, spec)?;
12042    let binding_publication = if spawn_admission
12043        .as_ref()
12044        .is_some_and(|admission| admission.launch.is_some())
12045    {
12046        handle.and_then(|handle| handle.begin_program_binding(&spec.module_id, role))
12047    } else {
12048        None
12049    };
12050    let registration_spawn = (spec.protocol == ModuleProtocol::Subc)
12051        .then(|| {
12052            handle.map(|handle| {
12053                RegistrationSpawn::new_with_binding_requirement(
12054                    handle,
12055                    &spec.module_id,
12056                    role,
12057                    binding_publication.clone(),
12058                    spawn_admission
12059                        .as_ref()
12060                        .is_some_and(|admission| admission.launch.is_some()),
12061                )
12062            })
12063        })
12064        .flatten();
12065
12066    #[cfg(target_os = "linux")]
12067    let cgroup_path = cgroup_placement
12068        .map(|placement| {
12069            if let Some(container) = &carried {
12070                let path = container
12071                    .cgroup_path()
12072                    .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
12073                placement.existing_module_path(path)?.ok_or_else(|| {
12074                    io::Error::new(
12075                        io::ErrorKind::NotFound,
12076                        "carried cgroup disappeared before replacement",
12077                    )
12078                })
12079            } else {
12080                if carrying {
12081                    containment::prepare_cgroup_path(placement, &cgroup_name)
12082                } else {
12083                    placement.module_path(&cgroup_name)
12084                }
12085            }
12086        })
12087        .transpose()
12088        .map_err(|source| SuperviseError::Cgroup {
12089            module_id: spec.module_id.clone(),
12090            source,
12091        })?;
12092    #[cfg(not(target_os = "linux"))]
12093    let cgroup_path: Option<PathBuf> = None;
12094    // Own a newly created directory before any fallible admission or command
12095    // preparation. Existing carried containment must never be removed here.
12096    #[cfg(target_os = "linux")]
12097    let cgroup_cleanup = PreparedCgroup(
12098        cgroup_placement
12099            .filter(|_| carried.is_none())
12100            .map(|placement| (placement.clone(), cgroup_name.clone())),
12101    );
12102    #[cfg(unix)]
12103    let mut admission_gate = if carrying {
12104        Some(
12105            admission::Gate::install(&mut command, spec, roster).map_err(|source| {
12106                SuperviseError::Spawn {
12107                    program: spec.program.clone(),
12108                    source,
12109                    cgroup_path: cgroup_path.clone(),
12110                }
12111            })?,
12112        )
12113    } else {
12114        None
12115    };
12116    #[cfg(target_os = "linux")]
12117    if let Some(path) = &cgroup_path {
12118        apply_cgroup_placement(&mut command, spec, path)?;
12119    }
12120
12121    let output_sink = if let Some(logs_dir) = capture_logs_dir {
12122        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
12123        match ChildOutputSink::open(&path, capture_retention(spec)) {
12124            Ok(sink) => sink,
12125            Err(error) => {
12126                warn!(
12127                    module_id = %spec.module_id,
12128                    path = %path.display(),
12129                    error = %error,
12130                    "could not open child output capture file; forwarding to stderr"
12131                );
12132                ChildOutputSink::Stderr
12133            }
12134        }
12135    } else {
12136        ChildOutputSink::Stderr
12137    };
12138
12139    command.stdout(Stdio::piped());
12140    command.stderr(Stdio::piped());
12141    command.kill_on_drop(!carrying);
12142    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
12143    // before exec). In the daemon's group, a service manager that kills the
12144    // job's process group when the daemon exits (launchd's default) killed
12145    // every module at the same moment its control connection closed, so no
12146    // module ever ran its EOF teardown on a daemon stop. Outside that group a
12147    // module is reached only by the daemon: the EOF it sees when its
12148    // connection closes, and the bounded stop in `child_roster` for anything
12149    // still running after that. On Linux this composes with the cgroup
12150    // placement above: that is a pre_exec write to cgroup.procs, std performs
12151    // setpgid in the child before running pre_exec callbacks, and the two
12152    // change independent process attributes.
12153    //
12154    // stdin is /dev/null because a process outside the terminal's foreground
12155    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
12156    // by hand would otherwise hand down. Under a service manager stdin is
12157    // already /dev/null.
12158    #[cfg(unix)]
12159    command.process_group(
12160        carried
12161            .as_ref()
12162            .map(|container| container.verified_join_group())
12163            .transpose()
12164            .map_err(|source| SuperviseError::Spawn {
12165                program: spec.program.clone(),
12166                source,
12167                cgroup_path: cgroup_path.clone(),
12168            })?
12169            .flatten()
12170            .unwrap_or(0) as i32,
12171    );
12172    command.stdin(Stdio::null());
12173    #[cfg(unix)]
12174    if let Some(gate) = &mut admission_gate {
12175        gate.install_placement(&mut command);
12176    }
12177    // Fill all reserved slots in one allocation-free step, after placement has
12178    // stopped using descriptors that the fixed slots may replace.
12179    #[cfg(unix)]
12180    {
12181        if let Some(handoff) = nonce_handoff {
12182            slots.nonce(handoff);
12183        }
12184        #[cfg(target_os = "macos")]
12185        slots
12186            .acknowledgement(exec_ack.into_descriptor())
12187            .map_err(|source| SuperviseError::Spawn {
12188                program: spec.program.clone(),
12189                source,
12190                cgroup_path: cgroup_path.clone(),
12191            })?;
12192        slots.install(command.as_std_mut());
12193    }
12194    #[cfg(not(any(unix, windows)))]
12195    let _ = nonce_handoff;
12196
12197    // Containment, step 1 of 3 (issue #109): create the child suspended so it
12198    // cannot run a single instruction -- and therefore cannot spawn a
12199    // grandchild -- before it is in the job. See `contain_spawned_child` for the
12200    // other two steps and why the window matters.
12201    #[cfg(windows)]
12202    {
12203        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
12204            program: spec.program.clone(),
12205            source,
12206            cgroup_path: None,
12207        })?;
12208        subc_jobobject::suspend_on_create_async(&mut command);
12209    }
12210    // Timestamp before the spawn call: on Unix the child can serve before that
12211    // call even returns. This conservative bound never labels an older child
12212    // as a replacement for a request received during process creation.
12213    let spawned_at = Instant::now();
12214    let delivery_context = config_delivery;
12215    let config_delivery = config_delivery
12216        .map(|delivery| {
12217            delivery.spawned(
12218                spec,
12219                spawn_admission
12220                    .as_ref()
12221                    .expect("delivery prepares spawn admission"),
12222                spawned_at,
12223                role,
12224            )
12225        })
12226        .transpose()?
12227        .flatten();
12228    let mut delivery_registration =
12229        delivery_context.map(|context| context.registration_guard(config_delivery.as_ref(), role));
12230    if let Some(registration) = &mut delivery_registration {
12231        registration.defer_cleanup();
12232    }
12233    if asynchronous {
12234        #[cfg(test)]
12235        if let Some(context) = delivery_context {
12236            context.pause_preflight_for_test();
12237        }
12238    }
12239    validate_spawn_program(spec, spawn_admission.as_ref(), delivery_context)?;
12240    Ok(PreparedChild {
12241        registration_spawn,
12242        registration_handle: handle.cloned(),
12243        #[cfg(any(unix, windows))]
12244        registration_channel,
12245        config_context: delivery_context.map(|delivery| delivery.runtime()),
12246        binding_publication,
12247        launch_spec,
12248        spawn_admission,
12249        command,
12250        carrying,
12251        carried: carried.clone(),
12252        #[cfg(unix)]
12253        launch_identity,
12254        #[cfg(unix)]
12255        group_target,
12256        #[cfg(target_os = "linux")]
12257        cgroup_placement: cgroup_placement.cloned(),
12258        #[cfg(target_os = "linux")]
12259        cgroup_cleanup,
12260        #[cfg(target_os = "linux")]
12261        cgroup_name,
12262        cgroup_path,
12263        #[cfg(unix)]
12264        admission_gate,
12265        output_sink,
12266        #[cfg(target_os = "macos")]
12267        privacy_exec,
12268        #[cfg(windows)]
12269        image_capture,
12270        #[cfg(windows)]
12271        nonce_handoff,
12272        spawned_at,
12273        config_delivery,
12274        delivery_registration,
12275        program_binding,
12276    })
12277}
12278
12279fn launch_child_in_slot(
12280    _spec: &ModuleSpec,
12281    ring: &Arc<Mutex<StderrRing>>,
12282    roster: &ChildRoster,
12283    prepared: PreparedChild,
12284    register_timeout: Duration,
12285) -> Result<SupervisedChild, SuperviseError> {
12286    let launch_spec = prepared.launch_spec.clone();
12287    let registration_declaration = prepared
12288        .spawn_admission
12289        .as_ref()
12290        .and_then(|admission| admission.registration_declaration(&launch_spec.module_id));
12291    let admission_base = prepared
12292        .spawn_admission
12293        .as_ref()
12294        .and_then(|admission| admission.approved_base())
12295        .cloned();
12296    let spec = &launch_spec;
12297    let expected_image = prepared
12298        .spawn_admission
12299        .as_ref()
12300        .and_then(|admission| admission.launch.as_ref())
12301        .map(|launch| launch.image);
12302    let program_launch = prepared
12303        .spawn_admission
12304        .and_then(|admission| admission.launch);
12305    let config_owner = prepared
12306        .config_context
12307        .as_ref()
12308        .map(|context| context.0.clone());
12309    let binding_publication = prepared.binding_publication;
12310    #[cfg(test)]
12311    if let Some(context) = &prepared.config_context {
12312        context.0.before_spawn_exec_for_test();
12313    }
12314    #[cfg(feature = "test-support")]
12315    if let Some(context) = &prepared.config_context {
12316        context
12317            .0
12318            .pause_spawn_before_exec()
12319            .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(context)))?;
12320    }
12321    let mut command = prepared.command;
12322    let carrying = prepared.carrying;
12323    let carried = prepared.carried;
12324    #[cfg(unix)]
12325    let launch_identity = prepared.launch_identity;
12326    #[cfg(unix)]
12327    let group_target = prepared.group_target;
12328    #[cfg(target_os = "linux")]
12329    let cgroup_placement = prepared.cgroup_placement.as_ref();
12330    #[cfg(target_os = "linux")]
12331    let cgroup_name = prepared.cgroup_name;
12332    #[cfg(target_os = "linux")]
12333    let mut cgroup_cleanup = prepared.cgroup_cleanup;
12334    #[cfg(target_os = "linux")]
12335    let _ = cgroup_cleanup.0.take();
12336    let cgroup_path = prepared.cgroup_path;
12337    #[cfg(unix)]
12338    let admission_gate = prepared.admission_gate;
12339    let output_sink = prepared.output_sink;
12340    #[cfg(target_os = "macos")]
12341    let mut privacy_exec = prepared.privacy_exec;
12342    #[cfg(target_os = "macos")]
12343    if let Some(pending) = &mut privacy_exec {
12344        pending.admission_binding = binding_publication.clone();
12345        if expected_image.is_some() {
12346            pending.expected = expected_image;
12347        }
12348    }
12349    #[cfg(any(unix, windows))]
12350    let registration_channel = prepared.registration_channel;
12351    let registration_spawn = prepared.registration_spawn;
12352    let registration_handle = prepared.registration_handle;
12353    #[cfg(windows)]
12354    let image_capture = prepared.image_capture;
12355    #[cfg(windows)]
12356    let nonce_handoff = prepared.nonce_handoff;
12357    let spawned_at = prepared.spawned_at;
12358    let config_delivery = prepared.config_delivery;
12359    let delivery_registration = prepared.delivery_registration;
12360    let _program_binding = prepared.program_binding;
12361    #[cfg(all(test, unix))]
12362    containment_tests::record_spawn_attempt(spec, &mut command);
12363    #[cfg(unix)]
12364    let spawned = match admission_gate {
12365        Some(gate) => gate
12366            .spawn(
12367                command,
12368                spec,
12369                roster,
12370                launch_identity,
12371                #[cfg(target_os = "linux")]
12372                cgroup_placement
12373                    .zip(cgroup_path.as_ref())
12374                    .map(|(placement, path)| {
12375                        (placement.clone(), cgroup_name.clone(), path.clone())
12376                    }),
12377            )
12378            .map(|(child, admitted)| (child, Some(admitted))),
12379        None => {
12380            let spawned =
12381                subc_os::child_spawn::tokio_spawn(&mut command).map(|child| (child, None));
12382            // The callback owns all child-only sources. Closing them here is
12383            // necessary for channel EOF and macOS exec acknowledgement EOF.
12384            drop(command);
12385            spawned
12386        }
12387    };
12388    #[cfg(not(unix))]
12389    let spawned = subc_os::child_spawn::tokio_spawn(&mut command);
12390    let spawned = match spawned {
12391        Ok(child) => child,
12392        Err(source) => {
12393            if carried
12394                .as_ref()
12395                .and_then(|container| container.group())
12396                .is_some()
12397            {
12398                containment::terminate_blocking(&roster.containment);
12399            }
12400            #[cfg(target_os = "linux")]
12401            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12402                remove_module_cgroup(placement, &cgroup_name);
12403            }
12404            return Err(SuperviseError::Spawn {
12405                program: spec.program.clone(),
12406                source,
12407                cgroup_path,
12408            });
12409        }
12410    };
12411    #[cfg(unix)]
12412    let (mut child, mut admitted) = spawned;
12413    #[cfg(unix)]
12414    let group_target = admitted
12415        .as_ref()
12416        .map(|attempt| attempt.group_target())
12417        .unwrap_or(group_target);
12418    #[cfg(not(unix))]
12419    let mut child = spawned;
12420    #[cfg(test)]
12421    spawn_record_tests::after_create(&child, spec);
12422
12423    // Bind while the child is still suspended, before job assignment resumes it.
12424    // Keeping the executable handle open across CreateProcessW denies rename
12425    // and deletion, so the file cannot be exchanged during spawn. Bind that
12426    // file ID to the process handle returned by spawn, never a reopened PID.
12427    #[cfg(windows)]
12428    let windows_image = match image_capture.bind_tokio(&child) {
12429        Ok(image) => Arc::new(image),
12430        Err(source) => {
12431            kill_and_reap_rejected_child(&mut child);
12432            return Err(SuperviseError::Spawn {
12433                program: spec.program.clone(),
12434                source,
12435                cgroup_path: None,
12436            });
12437        }
12438    };
12439    // Assign the suspended child to its module job; resume only after its
12440    // process identity has been durably recorded for orphan recovery.
12441    #[cfg(windows)]
12442    let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
12443    let spawned_at_ms = unix_ms_now();
12444    #[cfg(not(windows))]
12445    let spawned_from = spec.program.clone();
12446    #[cfg(windows)]
12447    let spawned_from = windows_image.path().to_path_buf();
12448    #[cfg(not(windows))]
12449    let spawned_file_identity = spawned_file_identity(&spawned_from);
12450    #[cfg(windows)]
12451    let spawned_file_identity = Some(SpawnedFileIdentity {
12452        device: 0,
12453        inode: 0,
12454        windows_image: Some(Arc::clone(&windows_image)),
12455    });
12456    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
12457        program: spec.program.clone(),
12458        source: io::Error::other("spawned child exposed no live pid"),
12459        cgroup_path: cgroup_path.clone(),
12460    })?;
12461    #[cfg(windows)]
12462    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
12463    #[cfg(not(windows))]
12464    let process_start_time = admitted
12465        .as_ref()
12466        .map(|admitted| admitted.birth)
12467        .or_else(|| crate::provenance::process_start_time(pid));
12468    #[cfg(windows)]
12469    let process_start_time = Some(windows_image.start_time());
12470    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
12471    #[cfg(all(test, target_os = "macos"))]
12472    privacy_exec_boundary_tests::before_image_sample(spec, pid);
12473    #[cfg(all(test, target_os = "linux"))]
12474    assert!(roster.admission_event_for_test(&spec.module_id, "before_image_binding", Some(pid)));
12475    // Record the committed kernel image, not the configured pathname: PATH
12476    // lookup and shebang interpretation can select a different executable.
12477    // Linux's posix_spawn can return before that image is installed, so the
12478    // observer also waits for the kernel's first-exec flag to clear.
12479    #[cfg(not(windows))]
12480    let recorded_image = observe_spawned_image(pid);
12481    #[cfg(all(test, target_os = "linux"))]
12482    let recorded_image = if roster.admission_event_for_test(
12483        &spec.module_id,
12484        "after_spawn_image_sample",
12485        Some(pid),
12486    ) {
12487        recorded_image
12488    } else {
12489        None
12490    };
12491    #[cfg(target_os = "linux")]
12492    // An exited child cannot receive HELLO service and has no readable /proc/exe.
12493    // Keep its real exit for the supervisor instead of turning a crash into a hold.
12494    let exited_before_binding = expected_image.is_some()
12495        && recorded_image.and_then(|image| image.executable) != expected_image
12496        && child.try_wait().ok().flatten().is_some();
12497    #[cfg(target_os = "linux")]
12498    let unavailable_initial_image = expected_image.is_some()
12499        && recorded_image.and_then(|image| image.executable).is_none()
12500        && !exited_before_binding;
12501    #[cfg(not(target_os = "linux"))]
12502    let unavailable_initial_image = false;
12503    #[cfg(target_os = "linux")]
12504    if expected_image
12505        .is_some_and(|expected| recorded_image.and_then(|image| image.executable) != Some(expected))
12506        && !exited_before_binding
12507        && !unavailable_initial_image
12508    {
12509        kill_module_cgroup(cgroup_placement, &cgroup_name);
12510        kill_and_reap_rejected_child(&mut child, group_target.clone());
12511        return Err(SuperviseError::ConfigHeld {
12512            module_id: spec.module_id.clone(),
12513            code: "program_refused".into(),
12514            detail: format!(
12515                "spawned executable {:?} does not match hashed program {:?}",
12516                recorded_image.and_then(|image| image.executable),
12517                expected_image
12518            ),
12519        });
12520    }
12521    #[cfg(not(target_os = "linux"))]
12522    let _ = expected_image;
12523    #[cfg(target_os = "macos")]
12524    let binding_ready = privacy_exec.is_none();
12525    #[cfg(target_os = "linux")]
12526    let binding_ready = !exited_before_binding;
12527    #[cfg(not(any(target_os = "macos", target_os = "linux")))]
12528    let binding_ready = cfg!(windows);
12529    // spawn() confirms only the first exec, into the trampoline. Never persist
12530    // the trampoline image; the asynchronous acknowledgement publishes the
12531    // module image once the trampoline has replaced itself with the module.
12532    #[cfg(target_os = "macos")]
12533    let recorded_image = if privacy_exec.is_some() {
12534        None
12535    } else {
12536        recorded_image
12537    };
12538    #[cfg(target_os = "linux")]
12539    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
12540    #[cfg(not(target_os = "linux"))]
12541    let recorded_cgroup_name = None;
12542    #[cfg(unix)]
12543    let preadmitted = admitted
12544        .as_mut()
12545        .and_then(|admitted| admitted.leader.take());
12546    #[cfg(not(unix))]
12547    let preadmitted: Option<crate::child_roster::RosterGuard> = None;
12548    let roster_guard = if let Some(guard) = preadmitted {
12549        Ok(guard)
12550    } else {
12551        roster.admit(
12552            spec.module_id.clone(),
12553            pid,
12554            spec.protocol,
12555            process_start_time,
12556            crate::child_roster::RecordedIdentity {
12557                container: None,
12558                #[cfg(unix)]
12559                carried: None,
12560                #[cfg(not(windows))]
12561                start_time: recorded_image.map(|image| image.start_time),
12562                #[cfg(windows)]
12563                start_time: process_start_time,
12564                #[cfg(not(windows))]
12565                executable: recorded_image
12566                    .and_then(|image| image.executable)
12567                    .map(crate::live_children::ExecutableIdentity::from),
12568                #[cfg(windows)]
12569                executable: Some(crate::live_children::ExecutableIdentity::from(
12570                    windows_image.identity(),
12571                )),
12572                cgroup_name: recorded_cgroup_name,
12573                #[cfg(target_os = "linux")]
12574                cgroup_placement: cgroup_placement.cloned(),
12575            },
12576        )
12577    };
12578    let roster_guard = match roster_guard {
12579        Ok(guard) => guard,
12580        Err(failure) => {
12581            return failure.with_cleanup(|source| {
12582                #[cfg(windows)]
12583                if let Some(job) = &job {
12584                    let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12585                }
12586                #[cfg(target_os = "linux")]
12587                kill_module_cgroup(cgroup_placement, &cgroup_name);
12588                #[cfg(unix)]
12589                let rejected_group = group_target.clone();
12590                kill_and_reap_rejected_child(
12591                    &mut child,
12592                    #[cfg(unix)]
12593                    rejected_group,
12594                );
12595                #[cfg(target_os = "linux")]
12596                if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12597                    remove_module_cgroup(placement, &cgroup_name);
12598                }
12599                Err(SuperviseError::Spawn {
12600                    program: spec.program.clone(),
12601                    source: io::Error::other(format!(
12602                        "could not durably record module child: {source}"
12603                    )),
12604                    cgroup_path,
12605                })
12606            });
12607        }
12608    };
12609    #[cfg(unix)]
12610    if admitted.is_some() {
12611        if let Some(image) = recorded_image {
12612            if let Err(source) = roster_guard.confirm_executable(image) {
12613                kill_and_reap_rejected_child(&mut child, group_target.clone());
12614                return Err(SuperviseError::Spawn {
12615                    program: spec.program.clone(),
12616                    source,
12617                    cgroup_path,
12618                });
12619            }
12620        }
12621    }
12622    // The pre-spawn check can pass just before shutdown begins. Ownership is
12623    // published at the start of admission, even while a record write is queued.
12624    // If shutdown already saw an empty roster before that publication, this
12625    // check sees the roster closed and ends the child before starting a module.
12626    if roster.is_closed() {
12627        if carrying {
12628            #[cfg(unix)]
12629            let rejected_group = admitted
12630                .as_ref()
12631                .map(|admitted| admitted.rejection_target())
12632                .unwrap_or_else(|| group_target.clone());
12633            #[cfg(windows)]
12634            if let Some(job) = &job {
12635                let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12636            }
12637            kill_and_reap_rejected_child(
12638                &mut child,
12639                #[cfg(unix)]
12640                rejected_group,
12641            );
12642            #[cfg(all(test, unix))]
12643            let _ =
12644                roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12645            drop(roster_guard);
12646            return Err(SuperviseError::Spawn {
12647                program: spec.program.clone(),
12648                source: io::Error::other(
12649                    "the daemon began shutting down during carried admission; ended the container",
12650                ),
12651                cgroup_path,
12652            });
12653        }
12654        // This child was never admitted, so there is no module protocol shutdown to wait for.
12655        #[cfg(target_os = "linux")]
12656        kill_module_cgroup(cgroup_placement, &cgroup_name);
12657        if let Err(error) = child.start_kill() {
12658            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
12659        }
12660        #[cfg(target_os = "linux")]
12661        if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12662            // This spawn was never admitted, so shutdown has no roster entry
12663            // to await. Do not detach its cleanup: the runtime could exit
12664            // before that task reaps the rejected child and removes its group.
12665            while matches!(child.try_wait(), Ok(None)) {
12666                std::thread::yield_now();
12667            }
12668            if matches!(
12669                subc_cgroup::kill_module(Some(placement), &cgroup_name),
12670                subc_cgroup::KillOutcome::Killed
12671            ) {
12672                if let Ok(path) = placement.module_path(&cgroup_name) {
12673                    while std::fs::read_to_string(path.join("cgroup.events"))
12674                        .ok()
12675                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
12676                    {
12677                        std::thread::yield_now();
12678                    }
12679                }
12680            }
12681            remove_module_cgroup(placement, &cgroup_name);
12682        }
12683        drop(roster_guard);
12684        return Err(SuperviseError::Spawn {
12685            program: spec.program.clone(),
12686            source: io::Error::other(
12687                "the daemon began shutting down while this process was starting; ended it",
12688            ),
12689            cgroup_path,
12690        });
12691    }
12692
12693    let stdout_pump = match child.stdout.take() {
12694        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
12695        None => {
12696            warn!(
12697                module_id = %spec.module_id,
12698                "spawned child exposed no stdout pipe; file capture will be incomplete"
12699            );
12700            None
12701        }
12702    };
12703    let stderr_pump = match child.stderr.take() {
12704        Some(stderr) => {
12705            let generation = ring
12706                .lock()
12707                .unwrap_or_else(|poisoned| poisoned.into_inner())
12708                .begin_process();
12709            Some(StderrPump {
12710                task: tokio::spawn(pump_stderr_to(
12711                    stderr,
12712                    Arc::clone(ring),
12713                    generation,
12714                    output_sink,
12715                )),
12716                generation,
12717            })
12718        }
12719        None => {
12720            // Spawning succeeded but the pipe did not materialise. Recording it as
12721            // uncaptured keeps the tail honest: the alternative is an empty tail
12722            // that reads as a module which printed nothing.
12723            ring.lock()
12724                .unwrap_or_else(|poisoned| poisoned.into_inner())
12725                .mark_not_captured("stderr pipe was not available on spawn");
12726            warn!(
12727                module_id = %spec.module_id,
12728                "spawned child exposed no stderr pipe; tail will be unavailable"
12729            );
12730            None
12731        }
12732    };
12733
12734    let exec = Arc::new(crate::activity::ExecOutcome::new());
12735    #[cfg(target_os = "macos")]
12736    let pending_exec = privacy_exec.is_some();
12737    #[cfg(not(target_os = "macos"))]
12738    let pending_exec = false;
12739    if !pending_exec {
12740        exec.finish(crate::activity::ExecState::CommittedExec);
12741    }
12742    #[cfg(windows)]
12743    let subtree_target = job
12744        .as_ref()
12745        .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
12746        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
12747    #[cfg(target_os = "linux")]
12748    let subtree_target = cgroup_path
12749        .map(crate::subtree_resources::Target::Cgroup)
12750        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
12751    #[cfg(not(any(target_os = "linux", windows)))]
12752    let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
12753    let mut supervised = SupervisedChild {
12754        registration_spawn,
12755        #[cfg(any(unix, windows))]
12756        registration_channel,
12757        live_program_binding: None,
12758        registered_config_connection: None,
12759        admission_base,
12760        registration_declaration,
12761        _program_binding: binding_publication,
12762        subtree_target,
12763        child: containment::ProcessChild::plain(child, pid),
12764        exec: crate::activity::ExecOwner(exec),
12765        #[cfg(windows)]
12766        nonce_delivery,
12767        protocol: spec.protocol,
12768        #[cfg(target_os = "linux")]
12769        module_id: cgroup_name,
12770        #[cfg(target_os = "linux")]
12771        cgroup_placement: cgroup_placement.cloned(),
12772        #[cfg(windows)]
12773        job,
12774        stdout_pump,
12775        stderr_pump,
12776        stderr_ring: Arc::clone(ring),
12777        spawned_at,
12778        config_delivery,
12779        registration_deadline: None,
12780        registration_watch: None,
12781        spawned_at_ms,
12782        spawned_from,
12783        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
12784        spawned_file_identity,
12785        process_start_time,
12786        process_identity,
12787        pid,
12788        roster_guard: Some(roster_guard),
12789        #[cfg(target_os = "macos")]
12790        privacy_exec,
12791        #[cfg(target_os = "macos")]
12792        report_ready: Arc::new(OnceLock::new()),
12793        spawn_failure: None,
12794        admission_kill_sent: false,
12795    };
12796    if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
12797        supervised.child.reject(
12798            #[cfg(unix)]
12799            group_target,
12800        );
12801        #[cfg(all(test, unix))]
12802        let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12803        return Err(error);
12804    }
12805    if let (Some(launch), Some(owner), Some(base)) = (
12806        program_launch,
12807        config_owner,
12808        supervised.admission_base.clone(),
12809    ) {
12810        supervised.live_program_binding =
12811            content_binding::LiveProgramBinding::new(launch, &supervised, owner, base);
12812        if let (Some(publication), Some(live)) = (
12813            &supervised._program_binding,
12814            &supervised.live_program_binding,
12815        ) {
12816            publication.attach(live);
12817        }
12818    }
12819    #[cfg(unix)]
12820    if let Some(admitted) = admitted.take() {
12821        admitted.disarm();
12822    }
12823    #[cfg(windows)]
12824    if let Err(error) = resume_suspended_child(pid, spec) {
12825        containment::terminate_blocking(&roster.containment);
12826        let _ = supervised.child.start_kill();
12827        return Err(error);
12828    }
12829    if binding_ready {
12830        // An unreadable initial image is not a definite mismatch. Keep the
12831        // owned child and delivery registration until its reaped status decides
12832        // whether it crashed on its own or was stopped by admission's kill.
12833        let checked = if unavailable_initial_image {
12834            Err(crate::config_gate::GateError::new(
12835                "config_delivery_deferred",
12836                "initial process image is unavailable",
12837            ))
12838        } else {
12839            match (
12840                &supervised._program_binding,
12841                &supervised.live_program_binding,
12842            ) {
12843                (Some(binding), _) => binding.verify(),
12844                (None, Some(live)) => live.check(),
12845                _ if supervised.admission_base.is_some() => {
12846                    Err(crate::config_gate::GateError::new(
12847                        "config_delivery_deferred",
12848                        "live program witness is unavailable",
12849                    ))
12850                }
12851                _ => Ok(()),
12852            }
12853        };
12854        #[cfg(all(test, target_os = "linux"))]
12855        let checked = if roster.admission_event_for_test(
12856            &spec.module_id,
12857            "after_content_witness_check",
12858            Some(pid),
12859        ) {
12860            checked
12861        } else {
12862            Err(crate::config_gate::GateError::new(
12863                "config_delivery_deferred",
12864                "current process image is unavailable",
12865            ))
12866        };
12867        if let Err(error) = checked {
12868            if let Some(binding) = &mut supervised._program_binding {
12869                binding.reject();
12870            }
12871            // Reap through the normal child-reaping path, retaining the real exit record and
12872            // its roster ownership. A refusal must also end any helpers.
12873            if error.code == "program_refused" || supervised.try_wait().ok().flatten().is_none() {
12874                supervised.spawn_failure = Some(format!("{}: {}", error.code, error.detail));
12875                #[cfg(all(test, target_os = "linux"))]
12876                assert!(roster.admission_event_for_test(
12877                    &spec.module_id,
12878                    if unavailable_initial_image {
12879                        "before_initial_image_kill"
12880                    } else {
12881                        "before_content_witness_kill"
12882                    },
12883                    Some(pid),
12884                ));
12885                let _ = supervised.start_admission_kill();
12886                #[cfg(all(test, target_os = "linux"))]
12887                if supervised.admission_kill_sent {
12888                    assert!(roster.admission_event_for_test(
12889                        &spec.module_id,
12890                        "after_admission_kill_sent",
12891                        Some(pid),
12892                    ));
12893                }
12894            }
12895        }
12896    }
12897    if let Some(registration) = delivery_registration {
12898        registration.commit();
12899    }
12900    if let Some(spawn) = &supervised.registration_spawn {
12901        if let Some(identity) = supervised.process_identity {
12902            let _ = spawn.identity.set(identity);
12903        }
12904        if supervised.spawn_failure.is_some() {
12905            spawn.refuse();
12906        } else if binding_ready {
12907            spawn.ready();
12908        }
12909        #[cfg(any(unix, windows))]
12910        if let (Some(handle), Some(channel)) =
12911            (registration_handle, supervised.registration_channel.take())
12912        {
12913            handle.serve_registration(channel, spawn.clone(), pid, spawned_at + register_timeout);
12914        }
12915    }
12916    Ok(supervised)
12917}
12918
12919#[cfg(test)]
12920mod spawn_record_tests {
12921    use super::*;
12922
12923    thread_local! {
12924        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
12925    }
12926
12927    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
12928        LAST_PID.set(child.id());
12929        #[cfg(windows)]
12930        if let Some((_, address)) = _spec
12931            .env
12932            .iter()
12933            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
12934        {
12935            use std::io::{Read, Write};
12936            let mut stream = std::net::TcpStream::connect(address).unwrap();
12937            stream
12938                .write_all(&child.id().unwrap().to_ne_bytes())
12939                .unwrap();
12940            let mut release = [0];
12941            stream.read_exact(&mut release).unwrap();
12942        }
12943    }
12944
12945    #[cfg(windows)]
12946    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
12947        if spec
12948            .env
12949            .iter()
12950            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
12951        {
12952            Err(io::Error::other("injected module job assignment failure"))
12953        } else {
12954            Ok(())
12955        }
12956    }
12957
12958    fn spec() -> ModuleSpec {
12959        ModuleSpec {
12960            module_id: "spawn-record-worker".into(),
12961            #[cfg(unix)]
12962            program: "/bin/sleep".into(),
12963            #[cfg(windows)]
12964            program: std::env::current_exe().unwrap(),
12965            #[cfg(unix)]
12966            args: vec!["60".into()],
12967            #[cfg(windows)]
12968            args: vec![
12969                "--exact".into(),
12970                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
12971                "--nocapture".into(),
12972            ],
12973            env: vec![],
12974            reserved: false,
12975            reserved_prefixes: vec![],
12976            protocol: ModuleProtocol::None,
12977            overlap: Default::default(),
12978            carry_containment: false,
12979        }
12980    }
12981
12982    #[tokio::test]
12983    async fn record_sync_failure_fails_spawn_and_reaps_child() {
12984        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
12985        let supervisor =
12986            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12987                .with_live_children_record(home.join("live-children.json"));
12988        crate::live_children::fail_next_file_sync();
12989        let result = supervisor.spawn(spec());
12990        let failed = result.is_err();
12991        if let Ok(module) = result {
12992            module.stop().await.unwrap();
12993        }
12994        assert!(failed, "record failure must fail spawn");
12995        let pid = LAST_PID.get().expect("the real OS child was created");
12996        assert!(
12997            subc_os::Process::open(pid).unwrap().is_none(),
12998            "child {pid} survived rejected admission or was not reaped"
12999        );
13000    }
13001
13002    #[cfg(target_os = "macos")]
13003    #[test]
13004    #[ignore = "manual on-device latency measurement"]
13005    fn durable_record_write_latency_50() {
13006        let home = cortexkit_test_support::ScratchDir::new("record-latency");
13007        let path = home.join("live-children.json");
13008        let children = vec![crate::live_children::LiveChild {
13009            module_id: "latency-worker".into(),
13010            pid: std::process::id(),
13011            protocol: ModuleProtocol::None,
13012            start_time: Some(1),
13013            executable: None,
13014            cgroup_name: None,
13015            container: None,
13016        }];
13017        let mut samples = Vec::new();
13018        for _ in 0..50 {
13019            let start = std::time::Instant::now();
13020            crate::live_children::write_record(&path, &children).unwrap();
13021            samples.push(start.elapsed());
13022        }
13023        samples.sort();
13024        eprintln!(
13025            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
13026            (samples[24] + samples[25]) / 2,
13027            samples[47]
13028        );
13029    }
13030
13031    #[cfg(windows)]
13032    #[test]
13033    fn windows_store_touch_fixture() {
13034        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
13035            std::fs::write(path, b"module ran").unwrap();
13036        }
13037    }
13038
13039    #[cfg(windows)]
13040    #[tokio::test]
13041    async fn windows_daemon_fixture() {
13042        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
13043            return;
13044        };
13045        let mut spec = spec();
13046        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
13047        let supervisor =
13048            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13049        let _module = supervisor.spawn(spec).unwrap();
13050    }
13051
13052    #[cfg(windows)]
13053    #[test]
13054    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
13055        use std::io::Read;
13056        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13057        // An outer job models a scheduler/service host. The daemon's root and
13058        // root job must nest without requiring breakaway from this job.
13059        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13060        command
13061            .args([
13062                "--exact",
13063                "supervise::spawn_record_tests::windows_daemon_fixture",
13064                "--nocapture",
13065            ])
13066            .env(
13067                "SUBC_TEST_DAEMON_BARRIER",
13068                listener.local_addr().unwrap().to_string(),
13069            );
13070        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13071        listener.set_nonblocking(true).unwrap();
13072        let deadline = std::time::Instant::now() + Duration::from_secs(30);
13073        let mut stream = loop {
13074            if let Ok((stream, _)) = listener.accept() {
13075                break stream;
13076            }
13077            assert!(
13078                daemon.child.try_wait().unwrap().is_none(),
13079                "daemon failed before rendezvous"
13080            );
13081            assert!(
13082                std::time::Instant::now() < deadline,
13083                "daemon did not reach CreateProcess failpoint"
13084            );
13085            std::thread::yield_now();
13086        };
13087        stream
13088            .set_read_timeout(Some(Duration::from_secs(30)))
13089            .unwrap();
13090        let mut pid = [0; 4];
13091        stream.read_exact(&mut pid).unwrap();
13092        let pid = u32::from_ne_bytes(pid);
13093        assert!(subc_jobobject::process_exists(pid));
13094        // Terminate only the daemon, not its outer job: otherwise the host job
13095        // would also kill a child whose daemon never joined its root job.
13096        daemon.child.kill().unwrap();
13097        daemon.child.wait().unwrap();
13098        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
13099        if !exited {
13100            daemon.job.terminate().unwrap();
13101        }
13102        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
13103    }
13104
13105    #[cfg(windows)]
13106    #[tokio::test]
13107    async fn windows_containment_failure_never_resumes() {
13108        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
13109        let sentinel = home.join("store-touched");
13110        let mut spec = spec();
13111        spec.env
13112            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
13113        spec.env.push((
13114            "SUBC_TEST_STORE_TOUCH".into(),
13115            sentinel.display().to_string(),
13116        ));
13117        let supervisor =
13118            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13119        let result = supervisor.spawn(spec);
13120        // A version that resumes despite assignment failure must have time to
13121        // execute its sentinel before dropping the returned module kills it.
13122        if let Ok(module) = result {
13123            let _ = tokio::time::timeout(Duration::from_secs(10), async {
13124                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
13125                    tokio::task::yield_now().await;
13126                }
13127            })
13128            .await;
13129            let _ = module.stop().await;
13130            panic!(
13131                "assignment failure resumed a module; store touched={}",
13132                sentinel.exists()
13133            );
13134        }
13135        assert!(
13136            !sentinel.exists(),
13137            "refused module executed store-touch sentinel"
13138        );
13139        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
13140    }
13141
13142    #[cfg(windows)]
13143    #[tokio::test]
13144    async fn windows_nested_daemon_fixture() {
13145        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
13146            return;
13147        };
13148        let mut spec = spec();
13149        spec.env.push((
13150            "SUBC_TEST_STORE_TOUCH".into(),
13151            sentinel.to_str().unwrap().into(),
13152        ));
13153        let supervisor =
13154            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13155        let module = supervisor
13156            .spawn(spec)
13157            .expect("per-module job must nest below daemon and host jobs");
13158        tokio::time::timeout(Duration::from_secs(10), async {
13159            while !PathBuf::from(&sentinel).exists() {
13160                tokio::task::yield_now().await;
13161            }
13162        })
13163        .await
13164        .unwrap();
13165        module.stop().await.unwrap();
13166    }
13167
13168    #[cfg(windows)]
13169    #[test]
13170    fn windows_module_job_nests_under_host_job() {
13171        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
13172        let sentinel = home.join("nested-store-touched");
13173        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13174        command
13175            .args([
13176                "--exact",
13177                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
13178                "--nocapture",
13179            ])
13180            .env("SUBC_TEST_NESTED_STORE", &sentinel);
13181        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13182        assert!(daemon.child.wait().unwrap().success());
13183        assert!(sentinel.exists(), "nested protocol:none module never ran");
13184    }
13185}
13186
13187#[cfg(target_os = "linux")]
13188pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
13189    use subc_cgroup::KillOutcome;
13190    match subc_cgroup::kill_module(placement, module_id) {
13191        KillOutcome::Killed => {}
13192        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
13193            debug!(
13194                module_id,
13195                "cgroup tree kill unavailable; using direct-child kill"
13196            );
13197        }
13198        KillOutcome::IoError { path, error } => {
13199            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
13200        }
13201    }
13202}
13203
13204/// Assign a freshly spawned Windows child without ever resuming on failure.
13205///
13206/// Steps 2 and 3 of the suspended-create contract: the job is created and the
13207/// child assigned **while it is still suspended** (step 1 is
13208/// `suspend_on_create_async` at the spawn site). The caller resumes only after
13209/// the child's roster admission is durable. Failed containment kills and reaps
13210/// the suspended child instead of running an unprotected module.
13211#[cfg(windows)]
13212fn contain_spawned_child(
13213    child: &mut Child,
13214    spec: &ModuleSpec,
13215    carried: Option<&Arc<containment::CarriedContainment>>,
13216) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
13217    let assignment = || {
13218        let job = match carried {
13219            Some(container) => container.job(),
13220            None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
13221        };
13222        #[cfg(test)]
13223        spawn_record_tests::assignment_failpoint(spec)?;
13224        job.lock()
13225            .unwrap_or_else(|p| p.into_inner())
13226            .assign(child)?;
13227        Ok::<_, io::Error>(job)
13228    };
13229    match assignment() {
13230        Ok(job) => Ok(Some(job)),
13231        Err(source) => {
13232            kill_and_reap_rejected_child(child);
13233            Err(SuperviseError::Spawn {
13234                program: spec.program.clone(),
13235                source: io::Error::other(format!(
13236                    "Windows module job containment failed: {source}"
13237                )),
13238                cgroup_path: None,
13239            })
13240        }
13241    }
13242}
13243
13244/// Rejection is synchronous: do not return a spawn error while the child can
13245/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
13246fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
13247    #[cfg(unix)]
13248    group.signal_rejected(child);
13249    let _ = child.start_kill();
13250    while matches!(child.try_wait(), Ok(None)) {
13251        std::thread::yield_now();
13252    }
13253}
13254
13255/// Resume a suspended child, killing it if it cannot be started.
13256///
13257/// A suspended process holds a pid and does nothing, so there is no useful
13258/// state to return: the caller gets an error and the spawn fails.
13259#[cfg(windows)]
13260fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
13261    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
13262        // Kill it here rather than leaving a suspended process for the caller
13263        // to notice; `kill_on_drop` would eventually do this, but the module
13264        // would have been reported as running in between.
13265        let _ = subc_os::child_spawn::status(
13266            std::process::Command::new("taskkill.exe")
13267                .args(["/PID", &pid.to_string(), "/T", "/F"])
13268                .stdin(Stdio::null())
13269                .stdout(Stdio::null())
13270                .stderr(Stdio::null()),
13271        );
13272        return Err(SuperviseError::Spawn {
13273            program: spec.program.clone(),
13274            source,
13275            cgroup_path: None,
13276        });
13277    }
13278    Ok(())
13279}
13280
13281#[cfg(target_os = "linux")]
13282fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13283    match placement.remove_module(module_id) {
13284        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
13285        Err(error) => warn!(
13286            module_id,
13287            error = %error,
13288            "could not remove module cgroup after process exit; continuing teardown"
13289        ),
13290    }
13291}
13292
13293#[cfg(target_os = "linux")]
13294const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
13295#[cfg(target_os = "linux")]
13296const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
13297
13298#[cfg(target_os = "linux")]
13299async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13300    let path = placement.module_path(module_id).ok();
13301    if path.as_ref().is_some_and(|path| {
13302        std::fs::read_to_string(path.join("cgroup.events"))
13303            .ok()
13304            .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
13305    }) {
13306        remove_module_cgroup(placement, module_id);
13307        return;
13308    }
13309
13310    // Freeze before enumerating: a frozen task cannot exit and release its PID
13311    // for reuse by a process outside this module while SIGTERM is being sent.
13312    if terminate_reaped_cgroup(placement, module_id).await {
13313        if let Some(path) = &path {
13314            let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
13315            while cgroup_is_populated(path) && Instant::now() < deadline {
13316                sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
13317            }
13318        }
13319    }
13320
13321    // Reaping the direct child is not proof its descendants exited. Kill any
13322    // survivors; when `cgroup.kill` succeeds, wait for the kernel to report the
13323    // cgroup empty before rmdir.
13324    if matches!(
13325        subc_cgroup::kill_module(Some(placement), module_id),
13326        subc_cgroup::KillOutcome::Killed
13327    ) {
13328        if let Some(path) = &path {
13329            while cgroup_is_populated(path) {
13330                sleep(Duration::from_millis(1)).await;
13331            }
13332        }
13333    }
13334    remove_module_cgroup(placement, module_id);
13335}
13336
13337#[cfg(target_os = "linux")]
13338fn cgroup_is_populated(path: &std::path::Path) -> bool {
13339    std::fs::read_to_string(path.join("cgroup.events"))
13340        .ok()
13341        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
13342}
13343
13344#[cfg(target_os = "linux")]
13345async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
13346    let frozen =
13347        match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
13348            .await
13349        {
13350            subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
13351            outcome => {
13352                tracing::debug!(
13353                    module_id,
13354                    ?outcome,
13355                    "cgroup freeze failed; skipping SIGTERM grace"
13356                );
13357                return false;
13358            }
13359        };
13360    let listed = match frozen.processes() {
13361        subc_cgroup::ProcsOutcome::Listed(pids) => {
13362            for pid in pids.into_iter().filter(|pid| *pid > 1) {
13363                if let Some(pid) = rustix::process::Pid::from_raw(pid) {
13364                    if let Err(error) =
13365                        rustix::process::kill_process(pid, rustix::process::Signal::TERM)
13366                    {
13367                        tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
13368                    }
13369                }
13370            }
13371            true
13372        }
13373        outcome => {
13374            tracing::debug!(
13375                module_id,
13376                ?outcome,
13377                "cgroup process listing failed; skipping SIGTERM grace"
13378            );
13379            false
13380        }
13381    };
13382    match frozen.thaw() {
13383        subc_cgroup::ThawOutcome::Thawed => listed,
13384        outcome => {
13385            tracing::debug!(
13386                module_id,
13387                ?outcome,
13388                "cgroup thaw failed; proceeding to cgroup.kill"
13389            );
13390            false
13391        }
13392    }
13393}
13394
13395#[cfg(target_os = "linux")]
13396fn apply_cgroup_placement(
13397    command: &mut Command,
13398    spec: &ModuleSpec,
13399    path: &std::path::Path,
13400) -> Result<(), SuperviseError> {
13401    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
13402        module_id: spec.module_id.clone(),
13403        source,
13404    })
13405}
13406
13407fn capture_retention(spec: &ModuleSpec) -> Retention {
13408    let defaults = Retention::default();
13409    let value = |name: &str| {
13410        spec.env
13411            .iter()
13412            .rev()
13413            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
13414    };
13415    Retention {
13416        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
13417            .and_then(|value| value.parse().ok())
13418            .unwrap_or(defaults.max_file_mb),
13419        keep: value(CAPTURE_KEEP_ENV)
13420            .and_then(|value| value.parse().ok())
13421            .unwrap_or(defaults.keep),
13422        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
13423            .and_then(|value| value.parse().ok())
13424            .unwrap_or(defaults.max_age_days),
13425    }
13426}
13427
13428/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
13429/// module's registration to the exact process the supervisor spawned.
13430fn generate_launch_nonce() -> Result<String, SuperviseError> {
13431    let mut bytes = [0u8; 32];
13432    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
13433        reason: source.to_string(),
13434    })?;
13435    let mut hex = String::with_capacity(64);
13436    for b in bytes {
13437        use std::fmt::Write;
13438        let _ = write!(hex, "{b:02x}");
13439    }
13440    Ok(hex)
13441}
13442
13443/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
13444/// signal about how many leading bytes matched.
13445fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
13446    if a.len() != b.len() {
13447        return false;
13448    }
13449    let mut diff = 0u8;
13450    for (x, y) in a.iter().zip(b.iter()) {
13451        diff |= x ^ y;
13452    }
13453    diff == 0
13454}
13455
13456/// The kernel's image after an acknowledged exec, shared by ordinary launches
13457/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
13458/// not identities inferred from a configured pathname.
13459#[cfg(not(windows))]
13460fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
13461    let process = subc_os::Process::open(pid).ok().flatten()?;
13462    #[cfg(target_os = "linux")]
13463    return sample_after_linux_exec(
13464        || {
13465            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
13466            let committed = linux_exec_committed(&stat);
13467            #[cfg(test)]
13468            spawn_image_tests::report_exec_state(committed);
13469            committed
13470        },
13471        || process.observe(),
13472    );
13473    #[cfg(not(target_os = "linux"))]
13474    process.observe()
13475}
13476
13477/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
13478/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
13479/// installation. Until it clears, /proc/pid/exe can still name the daemon.
13480#[cfg(target_os = "linux")]
13481fn linux_exec_committed(stat: &str) -> Option<bool> {
13482    const PF_FORKNOEXEC: u64 = 0x40;
13483    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
13484    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
13485}
13486
13487#[cfg(target_os = "linux")]
13488fn sample_after_linux_exec(
13489    mut exec_committed: impl FnMut() -> Option<bool>,
13490    observe: impl FnOnce() -> Option<subc_os::Observation>,
13491) -> Option<subc_os::Observation> {
13492    let deadline = std::time::Instant::now() + Duration::from_secs(1);
13493    while !exec_committed()? {
13494        // A failed or indefinitely stalled exec must not publish the inherited
13495        // image as authority to signal a future orphan. Missing identity keeps
13496        // the existing conservative orphan-sweep behavior.
13497        if std::time::Instant::now() >= deadline {
13498            return None;
13499        }
13500        std::thread::sleep(Duration::from_micros(50));
13501    }
13502    observe()
13503}
13504
13505#[cfg(all(test, target_os = "linux"))]
13506mod spawn_image_tests {
13507    use super::*;
13508    use std::cell::{Cell, RefCell};
13509
13510    enum ExecEvent {
13511        Uncommitted,
13512        Image(Option<subc_os::Observation>),
13513    }
13514
13515    thread_local! {
13516        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
13517    }
13518
13519    pub(super) fn report_exec_state(committed: Option<bool>) {
13520        if committed == Some(false) {
13521            EXEC_EVENTS.with(|events| {
13522                if let Some(events) = events.borrow_mut().take() {
13523                    let _ = events.send(ExecEvent::Uncommitted);
13524                }
13525            });
13526        }
13527    }
13528
13529    #[test]
13530    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
13531        use std::io::{Read, Write};
13532        use std::os::fd::AsRawFd;
13533
13534        let descriptor = std::fs::File::open("/dev/null").unwrap();
13535        let mut command = std::process::Command::new("/bin/sleep");
13536        command.arg("30");
13537        let (mut ready, mut release) =
13538            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
13539                .unwrap();
13540        let spawn = std::thread::spawn(move || command.spawn());
13541        let mut message = [0u8; 8];
13542        ready.read_exact(&mut message).unwrap();
13543        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
13544        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
13545        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
13546        let (events, received) = std::sync::mpsc::channel();
13547        let observer = std::thread::spawn(move || {
13548            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
13549            let image = observe_spawned_image(pid);
13550            let _ = events.send(ExecEvent::Image(image));
13551        });
13552        // Release only once the production observer has seen the uncommitted
13553        // exec, or has incorrectly returned the inherited image. Either way,
13554        // release and reap before asserting so a regression cannot leak a child.
13555        let first = received.recv_timeout(Duration::from_secs(5));
13556        release.write_all(&[1]).unwrap();
13557        let mut child = spawn.join().unwrap().unwrap();
13558        let after = match first {
13559            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
13560                Ok(ExecEvent::Image(image)) => image,
13561                _ => None,
13562            },
13563            Ok(ExecEvent::Image(image)) => image,
13564            Err(_) => None,
13565        };
13566        observer.join().unwrap();
13567        child.kill().unwrap();
13568        child.wait().unwrap();
13569
13570        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
13571        assert_eq!(
13572            inherited.unwrap().executable,
13573            subc_os::file_identity(&std::env::current_exe().unwrap())
13574        );
13575        assert_eq!(
13576            after.unwrap().executable,
13577            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
13578        );
13579    }
13580
13581    #[test]
13582    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
13583        let committed = Cell::new(false);
13584        let inherited = subc_os::Observation {
13585            start_time: 23,
13586            executable: Some(subc_os::FileIdentity {
13587                device: 29,
13588                inode: 31,
13589            }),
13590        };
13591        let module = subc_os::Observation {
13592            start_time: 23,
13593            executable: Some(subc_os::FileIdentity {
13594                device: 29,
13595                inode: 37,
13596            }),
13597        };
13598        let polls = Cell::new(0);
13599        let image = sample_after_linux_exec(
13600            || {
13601                polls.set(polls.get() + 1);
13602                // Hold the inherited image through the first poll, as in the
13603                // posix_spawn parent-wake window, then commit the module image.
13604                committed.set(polls.get() == 2);
13605                Some(committed.get())
13606            },
13607            || Some(if committed.get() { module } else { inherited }),
13608        );
13609        assert_eq!(image, Some(module));
13610        assert_eq!(polls.get(), 2);
13611    }
13612
13613    #[test]
13614    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
13615        assert_eq!(
13616            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
13617            Some(false)
13618        );
13619        assert_eq!(
13620            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
13621            Some(true)
13622        );
13623        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
13624        assert_eq!(linux_exec_committed("17 (a) R"), None);
13625    }
13626
13627    #[test]
13628    fn unreadable_exec_state_never_samples_an_inherited_image() {
13629        assert_eq!(
13630            sample_after_linux_exec(
13631                || None,
13632                || panic!("must not sample before exec confirmation")
13633            ),
13634            None
13635        );
13636    }
13637
13638    #[test]
13639    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
13640        assert_eq!(
13641            sample_after_linux_exec(
13642                || Some(false),
13643                || panic!("must not sample after an exec timeout")
13644            ),
13645            None
13646        );
13647    }
13648}
13649
13650/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
13651/// different executable after the no-delete capture was taken.
13652#[cfg(windows)]
13653pub(crate) fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
13654    if let Err(error) = require_program_search_path(spec) {
13655        return Err(match error {
13656            SuperviseError::Spawn { source, .. } => source,
13657            error => io::Error::other(error),
13658        });
13659    }
13660    let mut program = spec.program.clone();
13661    if program.is_absolute() || program.components().count() > 1 {
13662        // An explicit path can name a PE image with no extension. Try that
13663        // exact file first, as Command does, before falling back to .exe.
13664        match subc_os::ExecutableCapture::open(&program) {
13665            Ok(capture) => return Ok(capture),
13666            Err(error)
13667                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
13668            Err(error) => return Err(error),
13669        }
13670        program.set_extension("exe");
13671        return subc_os::ExecutableCapture::open(&program);
13672    }
13673    if program.extension().is_none() {
13674        program.set_extension("exe");
13675    }
13676    let custom_path = spec
13677        .env
13678        .iter()
13679        .rev()
13680        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
13681    let mut directories = Vec::new();
13682    // These are Windows' default executable search locations. A configured
13683    // PATH still overrides the inherited PATH, not the system directories.
13684    if let Some(parent) = std::env::current_exe()?.parent() {
13685        directories.push(parent.to_path_buf());
13686    }
13687    directories.push(std::env::current_dir()?);
13688    if let Some(windows) = std::env::var_os("SystemRoot") {
13689        let windows = PathBuf::from(windows);
13690        directories.push(windows.join("System32"));
13691        directories.push(windows.clone());
13692    }
13693    let path = custom_path
13694        .map(|(_, value)| std::ffi::OsString::from(value))
13695        .or_else(|| std::env::var_os("PATH"));
13696    if let Some(path) = path {
13697        directories.extend(std::env::split_paths(&path));
13698    }
13699    for directory in directories {
13700        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
13701            Ok(capture) => return Ok(capture),
13702            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
13703            Err(error) => return Err(error),
13704        }
13705    }
13706    Err(io::Error::new(
13707        io::ErrorKind::NotFound,
13708        "resolved executable not found",
13709    ))
13710}
13711
13712async fn spawn_and_mark_running_async(
13713    spec: &ModuleSpec,
13714    runtime: &SupervisorRuntimeConfig,
13715    snapshot: &SharedSnapshot,
13716) -> Result<SupervisedChild, SuperviseError> {
13717    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13718    let registration_watch = watch_registration(&runtime.registry, spec);
13719    #[cfg(unix)]
13720    let joining_group = containment::current(&runtime.child_roster.containment)
13721        .is_some_and(|container| container.group().is_some());
13722    let result = spawn_child_async(spec, runtime).await;
13723    #[cfg(unix)]
13724    let result = if result.is_err()
13725        && joining_group
13726        && !runtime.child_roster.is_closed()
13727        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13728        && runtime
13729            .child_roster
13730            .previous_instance(&spec.module_id)
13731            .is_none()
13732    {
13733        // A failed group join has already ended the old container. Its single
13734        // fresh-container retry belongs to the same restart budget entry.
13735        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13736        spawn_child_async(spec, runtime).await
13737    } else {
13738        result
13739    };
13740    let mut child = result?;
13741    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13742    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13743        containment::terminate_blocking(&runtime.child_roster.containment);
13744        return Err(error);
13745    }
13746    if let Some(spawn) = activity_spawn {
13747        spawn.complete(snapshot);
13748    }
13749    Ok(child)
13750}
13751
13752#[cfg(test)]
13753fn spawn_and_mark_running(
13754    spec: &ModuleSpec,
13755    runtime: &SupervisorRuntimeConfig,
13756    snapshot: &SharedSnapshot,
13757) -> Result<SupervisedChild, SuperviseError> {
13758    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13759    let registration_watch = watch_registration(&runtime.registry, spec);
13760    #[cfg(unix)]
13761    let joining_group = containment::current(&runtime.child_roster.containment)
13762        .is_some_and(|container| container.group().is_some());
13763    let spawn = || {
13764        spawn_child(
13765            spec,
13766            runtime.connection_file_path.as_deref(),
13767            runtime.supervisor_handle.as_ref(),
13768            &runtime.stderr_ring,
13769            runtime.capture_logs_dir.as_deref(),
13770            &runtime.child_roster,
13771            runtime.spawn_delivery(),
13772            #[cfg(target_os = "linux")]
13773            runtime.cgroup_placement.as_ref(),
13774            runtime.register_timeout,
13775        )
13776    };
13777    let result = spawn();
13778    #[cfg(unix)]
13779    let result = if result.is_err()
13780        && joining_group
13781        && !runtime.child_roster.is_closed()
13782        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13783        && runtime
13784            .child_roster
13785            .previous_instance(&spec.module_id)
13786            .is_none()
13787    {
13788        // The failed join already destroyed and verified the old container.
13789        // Try one fresh group as part of this same restart, without spending
13790        // another crash-budget entry. Cgroups and Windows jobs do not retry.
13791        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13792        spawn()
13793    } else {
13794        result
13795    };
13796    let child = result?;
13797    let mut child = child;
13798    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13799    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13800        containment::terminate_blocking(&runtime.child_roster.containment);
13801        return Err(error);
13802    }
13803    if let Some(spawn) = activity_spawn {
13804        spawn.complete(snapshot);
13805    }
13806    Ok(child)
13807}
13808
13809struct RegistrationWatch {
13810    changes: watch::Receiver<u64>,
13811    baseline: u64,
13812}
13813
13814// Capture before spawning: a fast child's accepted HELLO can disappear from the
13815// active registry before the monitor first runs. A pre-existing registration
13816// must not satisfy a new process's deadline either.
13817fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
13818    if spec.protocol != ModuleProtocol::Subc {
13819        return None;
13820    }
13821    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
13822    let baseline = *changes.borrow();
13823    Some(RegistrationWatch { changes, baseline })
13824}
13825
13826fn arm_registration_deadline(
13827    child: &mut SupervisedChild,
13828    budget: Duration,
13829    registration_watch: Option<RegistrationWatch>,
13830) {
13831    child.registration_watch = registration_watch;
13832    child.registration_deadline = (child.protocol == ModuleProtocol::Subc
13833        && child.awaiting_registration())
13834    .then_some(child.spawned_at + budget);
13835}
13836
13837async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
13838    child.retire_registration();
13839    if let Some(container) = child.child.container() {
13840        container.stop_members().await;
13841        child.kill_leader()?;
13842        child.wait().await?;
13843        return Ok(());
13844    }
13845    #[cfg(unix)]
13846    {
13847        // Children lead their own process groups. Signal the whole group so a
13848        // helper stuck before HELLO cannot survive its supervised parent.
13849        let group = unreaped_registration_group(child)?;
13850        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
13851        // Process-group safety rule: only signal a group while its leader is
13852        // our unreaped child. Even a zombie leader reserves its pid/group id;
13853        // reaping it before KILL would let that id name an unrelated group.
13854        // Sleep without waiting so TERM-ignoring helpers get the full grace.
13855        sleep(crate::child_roster::TERM_TO_KILL).await;
13856        let group = unreaped_registration_group(child)?;
13857        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
13858    }
13859    // Windows terminates the containing Job Object as well as the direct
13860    // child. Linux also kills the child's cgroup when placement is enabled.
13861    child.start_kill()?;
13862    child.wait().await?;
13863    Ok(())
13864}
13865
13866#[cfg(unix)]
13867fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
13868    if let Some(container) = child.child.container() {
13869        return container
13870            .group()
13871            .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
13872            .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
13873    }
13874    // The group id comes only from the pid recorded at spawn, when the child
13875    // created its own group. Never admit init's group or a negative group id.
13876    let group = i32::try_from(child.pid)
13877        .ok()
13878        .filter(|pid| *pid > 1)
13879        .and_then(rustix::process::Pid::from_raw)
13880        .ok_or_else(|| {
13881            io::Error::new(
13882                io::ErrorKind::InvalidInput,
13883                "invalid supervised process group",
13884            )
13885        })?;
13886    if child.child.id() != Some(child.pid) {
13887        return Err(io::Error::new(
13888            io::ErrorKind::InvalidInput,
13889            "refusing process-group signal after supervised child was reaped",
13890        ));
13891    }
13892    Ok(group)
13893}
13894
13895enum RegistrationWaitOutcome {
13896    Registered,
13897    Exited(ExitReport),
13898    TimedOut,
13899}
13900
13901struct ReloadRegistrationFailure {
13902    exit_report: ExitReport,
13903    reason: String,
13904}
13905
13906#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13907enum BusyGaugeObservation {
13908    Quiescent,
13909    Busy,
13910    Omitted,
13911}
13912
13913/// What one health answer said about one declared busy gauge.
13914#[derive(Debug, Clone, PartialEq, Eq)]
13915enum GaugeVerdict {
13916    /// The gauge read 0: this gauge is not what holds the drain.
13917    Zero,
13918    /// The gauge read a positive count.
13919    NonZero(u64),
13920    /// The gauge was present but not a non-negative integer (`0.0`, a string,
13921    /// a negative number). It counts as busy, because the daemon cannot tell
13922    /// it is idle. Carries the value as JSON text, cut to a short prefix.
13923    NotACount(String),
13924    /// The health answer had no metrics object, or one without this gauge.
13925    Missing,
13926    /// The probe produced no health answer to read; carries the probe error's
13927    /// evidence label (`no-answer` for a probe that timed out).
13928    ProbeFailed(&'static str),
13929}
13930
13931impl GaugeVerdict {
13932    fn reads_quiet(&self) -> bool {
13933        matches!(self, Self::Zero)
13934    }
13935}
13936
13937impl fmt::Display for GaugeVerdict {
13938    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
13939        match self {
13940            Self::Zero => f.write_str("zero"),
13941            Self::NonZero(value) => write!(f, "non_zero({value})"),
13942            Self::NotACount(value) => write!(f, "not_a_count({value})"),
13943            Self::Missing => f.write_str("missing_from_health_body"),
13944            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
13945        }
13946    }
13947}
13948
13949/// The longest JSON text kept for a gauge value that is not a count, so a
13950/// module reporting a large object as a gauge cannot flood the drain log.
13951const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
13952
13953/// Read each declared gauge out of one health answer's metrics.
13954fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
13955    let metrics = metrics.and_then(Value::as_object);
13956    gauges
13957        .iter()
13958        .map(
13959            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
13960                None => GaugeVerdict::Missing,
13961                Some(value) => match value.as_u64() {
13962                    Some(0) => GaugeVerdict::Zero,
13963                    Some(count) => GaugeVerdict::NonZero(count),
13964                    None => {
13965                        let mut text = value.to_string();
13966                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
13967                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
13968                            while !text.is_char_boundary(end) {
13969                                end -= 1;
13970                            }
13971                            text.truncate(end);
13972                            text.push_str("...");
13973                        }
13974                        GaugeVerdict::NotACount(text)
13975                    }
13976                },
13977            },
13978        )
13979        .collect()
13980}
13981
13982/// The drain's overall reading of one health answer: omitted when any
13983/// declared gauge is missing, quiescent only when every gauge reads 0.
13984fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
13985    if verdicts
13986        .iter()
13987        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
13988    {
13989        BusyGaugeObservation::Omitted
13990    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
13991        BusyGaugeObservation::Quiescent
13992    } else {
13993        BusyGaugeObservation::Busy
13994    }
13995}
13996
13997/// One declared gauge's history over a drain: the latest verdict on it, and
13998/// the latest value actually read for it (which a later failed probe or
13999/// missing gauge does not erase). Times are offsets from the start of the
14000/// drain's wait.
14001#[derive(Debug, Clone, Default, PartialEq, Eq)]
14002struct GaugeTrace {
14003    latest: Option<(GaugeVerdict, Duration)>,
14004    last_value: Option<(GaugeVerdict, Duration)>,
14005}
14006
14007/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
14008/// its ceiling can say in its timeout line which declared gauge held it and
14009/// why. Without it, a gauge that read busy, a gauge the module never
14010/// reported, and a probe that never got an answer all looked the same: a
14011/// drain that ended at its ceiling with nothing in flight.
14012#[derive(Debug, Clone, Default, PartialEq, Eq)]
14013struct DrainGaugeTrace {
14014    /// Health probes the drain sent.
14015    probes: u32,
14016    /// Each declared gauge, in declaration order.
14017    gauges: Vec<(String, GaugeTrace)>,
14018}
14019
14020impl DrainGaugeTrace {
14021    fn new(gauges: &[String]) -> Self {
14022        Self {
14023            probes: 0,
14024            gauges: gauges
14025                .iter()
14026                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
14027                .collect(),
14028        }
14029    }
14030
14031    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
14032    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
14033        self.probes = self.probes.saturating_add(1);
14034        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
14035            trace.latest = Some((verdict.clone(), at));
14036            if matches!(
14037                verdict,
14038                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
14039            ) {
14040                trace.last_value = Some((verdict.clone(), at));
14041            }
14042        }
14043    }
14044}
14045
14046/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
14047/// declared gauge, separated by ` | `. A value is shown as its verdict
14048/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
14049/// and whether that counted as busy.
14050impl fmt::Display for DrainGaugeTrace {
14051    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14052        if self.gauges.is_empty() {
14053            return f.write_str("none declared");
14054        }
14055        for (index, (name, trace)) in self.gauges.iter().enumerate() {
14056            if index > 0 {
14057                f.write_str(" | ")?;
14058            }
14059            write!(f, "{name}: ")?;
14060            match &trace.latest {
14061                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
14062                None => f.write_str("never probed")?,
14063            }
14064            match &trace.last_value {
14065                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
14066                None => f.write_str("; no value ever read")?,
14067            }
14068        }
14069        Ok(())
14070    }
14071}
14072
14073pub(crate) fn declared_busy_gauges(
14074    registry: &Registry,
14075    module_id: &str,
14076) -> Result<Vec<String>, SuperviseError> {
14077    busy_gauges_of(
14078        registry
14079            .get_module(module_id)
14080            .map_err(SuperviseError::Registry)?,
14081    )
14082}
14083
14084/// [`declared_busy_gauges`] for the registration a connection holds, in any
14085/// slot: after cutover the incumbent is no longer the id's active
14086/// registration, and its own manifest is the one that names its gauges.
14087fn declared_busy_gauges_for_connection(
14088    registry: &Registry,
14089    connection_id: ConnectionId,
14090) -> Result<Vec<String>, SuperviseError> {
14091    busy_gauges_of(
14092        registry
14093            .get_module_by_connection(connection_id)
14094            .map_err(SuperviseError::Registry)?,
14095    )
14096}
14097
14098fn busy_gauges_of(
14099    registration: Option<crate::registry::ModuleRegistration>,
14100) -> Result<Vec<String>, SuperviseError> {
14101    let Some(registration) = registration else {
14102        return Ok(Vec::new());
14103    };
14104    let Some(self_signals) = registration.manifest.self_signals else {
14105        return Ok(Vec::new());
14106    };
14107
14108    let mut gauges = Vec::new();
14109    for declaration in self_signals {
14110        if declaration.kind != SelfSignalKind::Busy {
14111            continue;
14112        }
14113        match declaration.anchored_to {
14114            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
14115                gauges.extend(declared)
14116            }
14117            _ => {
14118                // An invalid Busy anchor is fail-safe: the empty name cannot be
14119                // present in a conforming health report, so this drain stays busy.
14120                gauges.push(String::new());
14121            }
14122        }
14123    }
14124    Ok(gauges)
14125}
14126
14127/// Wait for `endpoint` to have nothing in flight and, when the module declares
14128/// busy gauges, for a health probe to report them quiet. The probe is addressed
14129/// by `scope`: a swap's superseded incumbent must be asked about its own
14130/// gauges, and by module id the probe would reach the promoted candidate.
14131/// While the gauges read busy, the probe repeats every
14132/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
14133/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
14134/// timeout line.
14135#[allow(clippy::too_many_arguments)]
14136async fn wait_for_forwarding_quiescence(
14137    forwarding: &ForwardingTable,
14138    module_id: &str,
14139    runtime: &SupervisorRuntimeConfig,
14140    endpoint: crate::ModuleEndpointId,
14141    deadline: Instant,
14142    busy_gauges: &[String],
14143    scope: DrainScope,
14144    trace: &mut DrainGaugeTrace,
14145) -> Result<bool, SuperviseError> {
14146    let wait_started = Instant::now();
14147    let mut gauges_quiescent = busy_gauges.is_empty();
14148    let mut next_probe_at = wait_started;
14149    let mut omission_reported = false;
14150
14151    loop {
14152        let now = Instant::now();
14153        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
14154            let report = match scope {
14155                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
14156                DrainScope::Endpoint(endpoint) => {
14157                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
14158                }
14159            };
14160            let verdicts = match &report {
14161                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
14162                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
14163            };
14164            trace.record(&verdicts, wait_started.elapsed());
14165            gauges_quiescent = match report {
14166                Ok(_) => match busy_gauge_observation(&verdicts) {
14167                    BusyGaugeObservation::Quiescent => true,
14168                    BusyGaugeObservation::Busy => false,
14169                    BusyGaugeObservation::Omitted => {
14170                        // Once per drain: a module that never reports a gauge
14171                        // it declared would otherwise say so on every probe.
14172                        if !omission_reported {
14173                            forwarding
14174                                .counters()
14175                                .increment_drains_with_undeclared_gauge();
14176                            let missing = busy_gauges
14177                                .iter()
14178                                .zip(&verdicts)
14179                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
14180                                .map(|(gauge, _)| gauge.as_str())
14181                                .collect::<Vec<_>>()
14182                                .join(",");
14183                            warn!(
14184                                module_id,
14185                                missing_gauges = %missing,
14186                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
14187                            );
14188                            omission_reported = true;
14189                        }
14190                        false
14191                    }
14192                },
14193                Err(err) => {
14194                    warn!(
14195                        module_id,
14196                        error = %err,
14197                        "drain health.check did not produce declared busy gauges; treating module as busy"
14198                    );
14199                    false
14200                }
14201            };
14202            // Counted from the probe's completion: the probe above was awaited,
14203            // so no second probe can start while one is still outstanding.
14204            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
14205        }
14206
14207        let in_flight = forwarding
14208            .endpoint_in_flight_count(endpoint)
14209            .map_err(SuperviseError::Forwarding)?;
14210        if in_flight == 0 && gauges_quiescent {
14211            return Ok(true);
14212        }
14213
14214        let now = Instant::now();
14215        if now >= deadline {
14216            return Ok(false);
14217        }
14218        let mut wait = deadline
14219            .saturating_duration_since(now)
14220            .min(REGISTRY_RELEASE_POLL);
14221        if !busy_gauges.is_empty() {
14222            wait = wait.min(next_probe_at.saturating_duration_since(now));
14223        }
14224        sleep(wait).await;
14225    }
14226}
14227
14228/// The wait between drain busy-gauge probes: the larger of
14229/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
14230/// drain's own in-flight poll interval.
14231fn drain_busy_reprobe_interval() -> Duration {
14232    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
14233}
14234
14235/// Logs why a drain gave up: the drain waited its full timeout without the
14236/// module going quiet, and teardown is being forced.
14237///
14238/// The line names two kinds of evidence, because a bare "timed out" hides
14239/// causes that need different fixes:
14240/// - `holdouts`: the requests and routes still open when the timeout hit,
14241///   and on which connections. One stuck call is a module bug; a held stream
14242///   on every session is a module that should end its streams when it
14243///   receives `module.draining`.
14244/// - `gauges`: for each busy gauge the module declares, its latest reading
14245///   and how many probes ran. This separates a module that kept reporting
14246///   itself busy, one whose health answer never contained the gauge, and one
14247///   that never answered the probes, which otherwise all look like a timeout
14248///   with nothing in flight.
14249fn warn_drain_timed_out(
14250    module_id: &str,
14251    drain_timeout: Duration,
14252    reason: RouteCloseReason,
14253    holdouts: &crate::forwarding::DrainHoldouts,
14254    gauges: &DrainGaugeTrace,
14255) {
14256    warn!(
14257        module_id = %module_id,
14258        waited = ?drain_timeout,
14259        ?reason,
14260        held_requests = holdouts.requests,
14261        held_routes = holdouts.routes,
14262        total_routes = holdouts.total_routes,
14263        top_connections = ?holdouts.top_connections,
14264        // `module_channel:corr`, so the module can find each held request
14265        // in its own log; capped, so `held_requests` is the full count.
14266        held = %holdouts
14267            .held
14268            .iter()
14269            .map(|(channel, corr)| format!("{channel}:{corr}"))
14270            .collect::<Vec<_>>()
14271            .join(","),
14272        gauge_probes = gauges.probes,
14273        busy_gauges = %gauges,
14274        "route drain timed out before request quiescence; forcing teardown"
14275    );
14276}
14277
14278/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
14279///
14280/// `Ok` is always honest and passed straight through -- the wait actually measured
14281/// in-flight state. `Err` means the wait produced no measurement at all (the
14282/// forwarding table's lock was poisoned), so `false` is reported as the one honest
14283/// constant: the drain did not complete. Never recomputed from route state, never a
14284/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
14285fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
14286    match wait_result {
14287        Ok(drained) => *drained,
14288        Err(_) => false,
14289    }
14290}
14291
14292fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
14293    for released in released_routes {
14294        let frame = match Frame::build_with_version(
14295            released.negotiated_ver,
14296            FrameType::Goodbye,
14297            control_flags(),
14298            released.channel,
14299            released.epoch,
14300            0,
14301            Vec::new(),
14302        ) {
14303            Ok(frame) => frame,
14304            Err(err) => {
14305                warn!(
14306                    route_channel = released.channel,
14307                    error = %err,
14308                    "failed to build supervisor drain route GOODBYE frame"
14309                );
14310                continue;
14311            }
14312        };
14313        if !released.close_on_delivery_failure() {
14314            crate::forwarding::send_module_route_goodbye(
14315                &forwarding.counters(),
14316                &released.sink,
14317                frame,
14318                released.module_id.as_deref(),
14319                "supervisor drain",
14320            );
14321            continue;
14322        }
14323        if let Err(err) = released.sink.try_send(frame) {
14324            warn!(
14325                target_connection_id = released.connection_id.get(),
14326                route_channel = released.channel,
14327                error = %err,
14328                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
14329            );
14330            let _ = forwarding.escalate_client_delivery_failure(
14331                released.connection_id,
14332                released.channel,
14333                released.epoch,
14334                CloseReason::new(
14335                    "route_goodbye_delivery_failed",
14336                    format!(
14337                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
14338                        released.channel
14339                    ),
14340                ),
14341                crate::forwarding::UndeliveredFrame {
14342                    module_id: released.module_id.as_deref(),
14343                    sink: &released.sink,
14344                },
14345            );
14346        }
14347    }
14348}
14349
14350async fn send_module_draining(
14351    module_id: &str,
14352    reason: RouteCloseReason,
14353    deadline_ms: u64,
14354    target: &ModuleDrainTarget,
14355    _wait_for_flush: bool,
14356) -> bool {
14357    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
14358        reason,
14359        deadline_ms,
14360    }) {
14361        Ok(body) => body,
14362        Err(err) => {
14363            warn!(
14364                module_id,
14365                error = %err,
14366                "failed to encode module draining command"
14367            );
14368            return false;
14369        }
14370    };
14371    let frame = match Frame::build_with_version(
14372        target.negotiated_ver,
14373        FrameType::Push,
14374        control_flags(),
14375        0,
14376        0,
14377        0,
14378        body,
14379    ) {
14380        Ok(frame) => frame,
14381        Err(err) => {
14382            warn!(
14383                module_id,
14384                error = %err,
14385                "failed to build module draining command frame"
14386            );
14387            return false;
14388        }
14389    };
14390    #[cfg(unix)]
14391    if _wait_for_flush {
14392        return target.sink.send_flushed(frame).await.is_ok();
14393    }
14394    if let Err(err) = target.sink.try_send(frame) {
14395        warn!(
14396            module_id,
14397            target_connection_id = target.endpoint.connection_id.get(),
14398            error = %err,
14399            "module draining command was not delivered to peer"
14400        );
14401        return false;
14402    }
14403    true
14404}
14405
14406/// The channel-0 GOODBYE that tells a module its stop is planned.
14407fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
14408    match Frame::build_with_version(
14409        negotiated_ver,
14410        FrameType::Goodbye,
14411        control_flags(),
14412        0,
14413        0,
14414        0,
14415        Vec::new(),
14416    ) {
14417        Ok(frame) => Some(frame),
14418        Err(err) => {
14419            warn!(
14420                module_id,
14421                error = %err,
14422                "failed to build module GOODBYE frame"
14423            );
14424            None
14425        }
14426    }
14427}
14428
14429/// Send every registered module connection its module GOODBYE at daemon
14430/// shutdown, then request that connection's close.
14431///
14432/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
14433/// before EOF, so the GOODBYE must reach the socket before the close. A close
14434/// request does not wait for the connection's queued frames: its writer gets a
14435/// bounded grace after the close, is aborted if it overruns it, and the daemon
14436/// process may exit before that grace ends. So with `wait_for_flush`, each
14437/// connection is closed only after its writer has acknowledged writing the
14438/// GOODBYE, or once a short shared budget runs out, so one module that is not
14439/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
14440/// are only queued, for a shutdown the operator has told to stop waiting.
14441/// A connection that is already gone is skipped.
14442#[cfg(unix)]
14443async fn send_module_goodbyes_for_daemon_shutdown(
14444    forwarding: &Arc<ForwardingTable>,
14445    reason: &CloseReason,
14446    wait_for_flush: bool,
14447) {
14448    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
14449    let targets = match forwarding.module_connections() {
14450        Ok(targets) => targets,
14451        Err(err) => {
14452            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
14453            return;
14454        }
14455    };
14456    let deadline = Instant::now() + GOODBYE_BUDGET;
14457    let mut sends = tokio::task::JoinSet::new();
14458    for target in targets {
14459        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
14460            continue;
14461        };
14462        if !wait_for_flush {
14463            if let Err(err) = target.sink.try_send(frame) {
14464                debug!(
14465                    module_id = %target.module_id,
14466                    error = %err,
14467                    "shutdown module GOODBYE was not queued"
14468                );
14469            }
14470            continue;
14471        }
14472        let forwarding = Arc::clone(forwarding);
14473        let reason = reason.clone();
14474        sends.spawn(async move {
14475            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
14476                Ok(Ok(())) => {}
14477                Ok(Err(err)) => debug!(
14478                    module_id = %target.module_id,
14479                    error = %err,
14480                    "module connection closed before its shutdown GOODBYE was written"
14481                ),
14482                Err(_) => warn!(
14483                    module_id = %target.module_id,
14484                    budget = ?GOODBYE_BUDGET,
14485                    "shutdown module GOODBYE was not written within its budget; closing anyway"
14486                ),
14487            }
14488            forwarding.request_connection_close(target.endpoint.connection_id, reason);
14489        });
14490    }
14491    // Every task ends by the shared deadline, so this wait is bounded too.
14492    while sends.join_next().await.is_some() {}
14493}
14494
14495async fn send_module_goodbye(
14496    module_id: &str,
14497    forwarding: &ForwardingTable,
14498    target: &ModuleDrainTarget,
14499    wait_for_flush: bool,
14500) {
14501    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
14502        return;
14503    };
14504    // A channel is shut down synchronously after its drain. Its final frame
14505    // must reach the socket writer first; queue admission alone can lose it.
14506    let sent = if wait_for_flush {
14507        timeout(Duration::from_millis(500), target.sink.send_flushed(frame))
14508            .await
14509            .map_err(|_| "module GOODBYE flush exceeded 500ms".to_owned())
14510            .and_then(|result| result.map_err(|err| err.to_string()))
14511    } else {
14512        target.sink.try_send(frame).map_err(|err| err.to_string())
14513    };
14514    if let Err(err) = sent {
14515        warn!(
14516            module_id,
14517            target_connection_id = target.endpoint.connection_id.get(),
14518            error = %err,
14519            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
14520        );
14521        forwarding.request_connection_close(
14522            target.endpoint.connection_id,
14523            CloseReason::new(
14524                "module_goodbye_delivery_failed",
14525                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
14526            ),
14527        );
14528    }
14529}
14530
14531#[derive(Clone, Copy)]
14532struct ForwardingDrainContext<'a> {
14533    spec: &'a ModuleSpec,
14534    runtime: &'a SupervisorRuntimeConfig,
14535    registry: &'a Registry,
14536    scope: DrainScope,
14537}
14538
14539/// Which process a forwarding drain addresses.
14540#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14541enum DrainScope {
14542    /// Whatever endpoint is active for the module id: every plain stop,
14543    /// restart and reload. Also moves the module's state to `Draining`.
14544    Active,
14545    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
14546    /// module id would resolve to the promoted candidate and leave neither
14547    /// process routable. The module's state is left alone, since the promoted
14548    /// candidate is what it describes and that process is running.
14549    Endpoint(crate::ModuleEndpointId),
14550}
14551
14552/// Whether a child being drained has already been asked to stop by the time
14553/// its drain wait starts.
14554///
14555/// The drain wait is the same budget whatever this says. What it decides is
14556/// whether the supervisor must ask by signal before that wait begins: a child
14557/// that nobody asked will sit out the whole budget and then be SIGKILLed,
14558/// healthy or not.
14559#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14560enum StopNotice {
14561    /// The module was sent `module.draining` and a module GOODBYE over its own
14562    /// registered connection, and stops itself.
14563    SentOverConnection,
14564    /// The forwarding drain found no registered connection for the module: a
14565    /// subc child spawned moments ago that has not sent HELLO yet, or a
14566    /// `protocol: "none"` child, which never registers.
14567    NoConnection,
14568    /// This path sends nothing over the module's connection: the supervisor has
14569    /// no forwarding table, or the caller stops the child without a forwarding
14570    /// drain.
14571    NotSent,
14572}
14573
14574async fn begin_forwarding_drain(
14575    spec: &ModuleSpec,
14576    runtime: &SupervisorRuntimeConfig,
14577    registry: &Registry,
14578    snapshot: &SharedSnapshot,
14579    enabled: Option<bool>,
14580    reason: RouteCloseReason,
14581) -> Result<StopNotice, SuperviseError> {
14582    let Some(forwarding) = runtime.forwarding.as_ref() else {
14583        return Err(SuperviseError::ReloadUnavailable {
14584            module_id: spec.module_id.clone(),
14585            reason: "supervisor was not configured with a forwarding table".to_string(),
14586        });
14587    };
14588
14589    begin_forwarding_drain_with(
14590        forwarding,
14591        ForwardingDrainContext {
14592            spec,
14593            runtime,
14594            registry,
14595            scope: DrainScope::Active,
14596        },
14597        snapshot,
14598        enabled,
14599        reason,
14600        runtime.drain_timeout,
14601    )
14602    .await
14603}
14604
14605async fn begin_forwarding_drain_if_configured(
14606    spec: &ModuleSpec,
14607    runtime: &SupervisorRuntimeConfig,
14608    registry: &Registry,
14609    snapshot: &SharedSnapshot,
14610    enabled: Option<bool>,
14611    reason: RouteCloseReason,
14612) -> Result<StopNotice, SuperviseError> {
14613    begin_forwarding_drain_with_timeout(
14614        spec,
14615        runtime,
14616        registry,
14617        snapshot,
14618        enabled,
14619        reason,
14620        runtime.drain_timeout,
14621    )
14622    .await
14623}
14624
14625/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
14626/// budget, for paths where the operator overrides the module's configured one
14627/// (`supervisor.restart{drain_timeout_ms}`).
14628async fn begin_forwarding_drain_with_timeout(
14629    spec: &ModuleSpec,
14630    runtime: &SupervisorRuntimeConfig,
14631    registry: &Registry,
14632    snapshot: &SharedSnapshot,
14633    enabled: Option<bool>,
14634    reason: RouteCloseReason,
14635    drain_timeout: Duration,
14636) -> Result<StopNotice, SuperviseError> {
14637    let Some(forwarding) = runtime.forwarding.as_ref() else {
14638        return Ok(StopNotice::NotSent);
14639    };
14640
14641    begin_forwarding_drain_with(
14642        forwarding,
14643        ForwardingDrainContext {
14644            spec,
14645            runtime,
14646            registry,
14647            scope: DrainScope::Active,
14648        },
14649        snapshot,
14650        enabled,
14651        reason,
14652        drain_timeout,
14653    )
14654    .await
14655}
14656
14657async fn begin_forwarding_drain_with(
14658    forwarding: &ForwardingTable,
14659    context: ForwardingDrainContext<'_>,
14660    snapshot: &SharedSnapshot,
14661    enabled: Option<bool>,
14662    reason: RouteCloseReason,
14663    drain_timeout: Duration,
14664) -> Result<StopNotice, SuperviseError> {
14665    let ForwardingDrainContext {
14666        spec,
14667        runtime,
14668        registry,
14669        scope,
14670    } = context;
14671    debug_assert_ne!(reason, RouteCloseReason::Crash);
14672    let terminal = matches!(reason, RouteCloseReason::Disable);
14673    let drain_started_at = Instant::now();
14674    let drain_deadline = drain_started_at + drain_timeout;
14675    let deadline_ms =
14676        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
14677    let busy_gauges = match scope {
14678        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
14679        DrainScope::Endpoint(endpoint) => {
14680            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
14681        }
14682    };
14683
14684    // Admission gate first: route.open/commit and route REQUEST admission are closed
14685    // before the first quiescence check, so the outstanding count can only fall.
14686    let gate_started = Instant::now();
14687    let drain_target = match scope {
14688        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
14689        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
14690    }
14691    .map_err(SuperviseError::Forwarding)?;
14692    // The instant admission closed, and how long taking the forwarding write
14693    // lock to close it took. The timeout line reports only the quiescence
14694    // wait, so without this a drain that started late looked like one that
14695    // started on time.
14696    info!(
14697        module_id = %spec.module_id,
14698        ?reason,
14699        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
14700        connected = drain_target.is_some(),
14701        "module drain began; route admission closed"
14702    );
14703    if scope == DrainScope::Active {
14704        update_snapshot(snapshot, Some(&spec.module_id), |state| {
14705            state.state = ModuleState::Draining;
14706            state.draining_to_replace =
14707                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
14708            if let Some(enabled) = enabled {
14709                state.enabled = enabled;
14710            }
14711        })?;
14712    }
14713
14714    let Some(target) = drain_target.as_ref() else {
14715        // Nothing was sent: the module has no registered connection to carry
14716        // `module.draining` or a GOODBYE. The caller must not assume the child
14717        // was asked to stop.
14718        return Ok(StopNotice::NoConnection);
14719    };
14720    let flush_goodbye = runtime.supervisor_handle.as_ref().is_some_and(|handle| {
14721        handle
14722            .registration_spawn(target.endpoint.connection_id)
14723            .is_some()
14724    });
14725    {
14726        send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
14727        #[cfg(feature = "test-support")]
14728        forwarding.wait_before_route_closing_for_test().await;
14729        let routes = forwarding
14730            .endpoint_routes(target.endpoint)
14731            .map_err(SuperviseError::Forwarding)?;
14732        let routes_notified = routes.len();
14733        crate::control::send_route_control_pushes(
14734            forwarding,
14735            routes.clone(),
14736            ClientControlPush::RouteClosing {
14737                module_id: spec.module_id.clone(),
14738                channels: Vec::new(),
14739                reason,
14740            },
14741        );
14742        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
14743
14744        // `route.closing` was just sent above: from here on every return path,
14745        // including an early one, MUST send `route.closed` before propagating
14746        // anything else. A client holds `closing` as a promise that a verdict is
14747        // coming; leaving early without `closed` strands it waiting forever, since
14748        // `closing` carries no timeout of its own.
14749        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
14750        let wait_result = wait_for_forwarding_quiescence(
14751            forwarding,
14752            &spec.module_id,
14753            runtime,
14754            target.endpoint,
14755            drain_deadline,
14756            &busy_gauges,
14757            scope,
14758            &mut gauge_trace,
14759        )
14760        .await;
14761        let drained = drained_after_quiescence_wait(&wait_result);
14762        if let Err(err) = &wait_result {
14763            error!(
14764                module_id = %spec.module_id,
14765                ?reason,
14766                error = %err,
14767                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
14768            );
14769        } else if !drained {
14770            // Read before teardown releases the routes.
14771            let holdouts = forwarding
14772                .endpoint_drain_holdouts(target.endpoint)
14773                .unwrap_or_default();
14774            warn_drain_timed_out(
14775                &spec.module_id,
14776                drain_timeout,
14777                reason,
14778                &holdouts,
14779                &gauge_trace,
14780            );
14781        }
14782        crate::control::send_route_control_pushes(
14783            forwarding,
14784            routes,
14785            ClientControlPush::RouteClosed {
14786                module_id: spec.module_id.clone(),
14787                channels: Vec::new(),
14788                reason,
14789                drained,
14790                abandoned: target.abandoned_bindings.len() as u32,
14791                excluded_subscriptions: target.excluded_subscriptions,
14792                terminal: Some(terminal),
14793            },
14794        );
14795        wait_result?;
14796
14797        // `route.closed` has now been sent unconditionally above. From here the
14798        // remaining steps are cleanup (route + module GOODBYE) rather than a
14799        // promise the client is waiting on, but a lock-poisoned
14800        // `release_module_endpoint_routes` would otherwise skip the module
14801        // GOODBYE silently too -- send it before propagating the error.
14802        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
14803            Ok(routes) => routes,
14804            Err(err) => {
14805                warn!(
14806                    module_id = %spec.module_id,
14807                    ?reason,
14808                    error = %err,
14809                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
14810                );
14811                send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
14812                return Err(SuperviseError::Forwarding(err));
14813            }
14814        };
14815        let route_goodbye_count = released_routes.len();
14816        send_route_goodbyes(forwarding, released_routes);
14817        send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
14818
14819        // The drain's happy path was previously silent: every emission above is
14820        // best-effort with only its failure arm logged, so "were consumers told"
14821        // was unprovable from the daemon log (surfaced by a 30-minute consumer
14822        // hang where the open question was exactly whether teardown notice went
14823        // out). One summary line makes that class decidable in one grep.
14824        info!(
14825            module_id = %spec.module_id,
14826            ?reason,
14827            routes_notified,
14828            route_goodbyes = route_goodbye_count,
14829            abandoned_reservations = target.abandoned_bindings.len(),
14830            excluded_subscriptions = target.excluded_subscriptions,
14831            drained,
14832            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
14833        );
14834    }
14835
14836    Ok(StopNotice::SentOverConnection)
14837}
14838
14839/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
14840/// the only slot a plain (non-swap) spawn can register into.
14841async fn wait_for_registration_after_reload(
14842    registry: &Registry,
14843    module_id: &str,
14844    snapshot: &SharedSnapshot,
14845    child: &mut SupervisedChild,
14846    wait: Duration,
14847) -> Result<RegistrationWaitOutcome, SuperviseError> {
14848    wait_for_slot_registration(
14849        registry,
14850        crate::registry::RegistrationSlot::Active(module_id),
14851        module_id,
14852        snapshot,
14853        child,
14854        wait,
14855    )
14856    .await
14857}
14858
14859/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
14860///
14861/// Keyed on the slot rather than the bare module id because during a swap the
14862/// id's active slot is already held by the incumbent: an id-keyed wait would
14863/// report the incumbent's registration as the candidate's and a candidate that
14864/// never registers would look registered. A swap candidate waits on
14865/// `crate::registry::RegistrationSlot::Candidate`.
14866async fn wait_for_slot_registration(
14867    registry: &Registry,
14868    slot: crate::registry::RegistrationSlot<'_>,
14869    module_id: &str,
14870    snapshot: &SharedSnapshot,
14871    child: &mut SupervisedChild,
14872    wait: Duration,
14873) -> Result<RegistrationWaitOutcome, SuperviseError> {
14874    let deadline = Instant::now() + wait;
14875    loop {
14876        if registry
14877            .registration(slot)
14878            .map_err(SuperviseError::Registry)?
14879            .is_some()
14880        {
14881            return Ok(RegistrationWaitOutcome::Registered);
14882        }
14883
14884        let now = Instant::now();
14885        if now >= deadline {
14886            return Ok(RegistrationWaitOutcome::TimedOut);
14887        }
14888        let remaining = deadline.saturating_duration_since(now);
14889        let poll = remaining.min(REGISTRY_RELEASE_POLL);
14890
14891        tokio::select! {
14892            wait_result = child.wait() => {
14893                let status = wait_result.map_err(|source| SuperviseError::Wait {
14894                    module_id: module_id.to_string(),
14895                    source,
14896                })?;
14897                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
14898                    snapshot,
14899                    child,
14900                    &status,
14901                )));
14902            }
14903            _ = sleep(poll) => {}
14904        }
14905    }
14906}
14907
14908fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
14909    // A process stopped for missing its first HELLO did not provide service,
14910    // even if its TERM handler exits zero. Reload also uses this classification
14911    // when a replacement exits before registering.
14912    if exit_report.kind != ExitKind::DeliberateSeverance {
14913        exit_report.kind = ExitKind::Crash;
14914    }
14915    exit_report
14916}
14917
14918async fn handle_reload_child_registration_failure(
14919    spec: &ModuleSpec,
14920    runtime: &SupervisorRuntimeConfig,
14921    registry: &Registry,
14922    process_liveness: &SupervisorProcessLiveness,
14923    snapshot: &SharedSnapshot,
14924    _child: &mut Option<SupervisedChild>,
14925    failure: ReloadRegistrationFailure,
14926) -> Result<(), SuperviseError> {
14927    let ReloadRegistrationFailure {
14928        exit_report,
14929        reason,
14930    } = failure;
14931    match on_child_exit(
14932        spec,
14933        runtime.restart_policy,
14934        registry,
14935        snapshot,
14936        &runtime.terminal_ring,
14937        &runtime.spawn_events,
14938        &runtime.child_roster,
14939        exit_report,
14940        runtime.recovery_is_daemon,
14941    )
14942    .await
14943    {
14944        NextAction::Stop {
14945            registration_released,
14946        } => {
14947            if registration_released {
14948                process_liveness.untrack_if_current(&spec.module_id, snapshot);
14949            }
14950        }
14951        NextAction::Restart { schedule } => {
14952            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
14953                schedule.delay
14954            });
14955            if let Some(schedule) = schedule {
14956                log_crash_respawn(&spec.module_id, schedule);
14957            }
14958            schedule_respawn(
14959                runtime,
14960                snapshot,
14961                &spec.module_id,
14962                delay,
14963                RespawnKind::Spawn,
14964                true,
14965            )?;
14966        }
14967    }
14968    Err(SuperviseError::ReloadFailed {
14969        module_id: spec.module_id.clone(),
14970        reason,
14971    })
14972}
14973
14974async fn handle_reload_spawn_failure(
14975    spec: &ModuleSpec,
14976    runtime: &SupervisorRuntimeConfig,
14977    process_liveness: &SupervisorProcessLiveness,
14978    snapshot: &SharedSnapshot,
14979    _child: &mut Option<SupervisedChild>,
14980    reason: String,
14981) -> Result<(), SuperviseError> {
14982    let now = Instant::now();
14983    let mut schedule = None;
14984    update_snapshot(snapshot, Some(&spec.module_id), |state| {
14985        clear_current_process_facts(state);
14986        if state.enabled {
14987            schedule = state.next_crash_restart(&runtime.restart_policy, now);
14988            state.state = if schedule.is_some() {
14989                ModuleState::Restarting
14990            } else {
14991                if runtime
14992                    .failure_context(runtime.recovery_is_daemon)
14993                    .may_retry()
14994                {
14995                    state.failed_retry.schedule(&spec.module_id);
14996                }
14997                ModuleState::Failed
14998            };
14999        } else {
15000            state.state = ModuleState::Disabled;
15001        }
15002    })?;
15003    if let Some(schedule) = schedule {
15004        schedule_respawn(
15005            runtime,
15006            snapshot,
15007            &spec.module_id,
15008            schedule.delay,
15009            RespawnKind::Spawn,
15010            true,
15011        )?;
15012    } else {
15013        process_liveness.untrack_if_current(&spec.module_id, snapshot);
15014    }
15015    Err(SuperviseError::ReloadFailed {
15016        module_id: spec.module_id.clone(),
15017        reason,
15018    })
15019}
15020
15021fn control_flags() -> Flags {
15022    Flags::new(false, Priority::Passive, false)
15023}
15024
15025#[allow(clippy::too_many_arguments)]
15026async fn drain_optional_child(
15027    module_id: &str,
15028    protocol: ModuleProtocol,
15029    stop_notice: StopNotice,
15030    registry: &Registry,
15031    forwarding: Option<&ForwardingTable>,
15032    snapshot: &SharedSnapshot,
15033    terminal_ring: &Arc<Mutex<TerminalRing>>,
15034    spawn_events: &SpawnEventFeed,
15035    child: &mut Option<SupervisedChild>,
15036    drain_timeout: Duration,
15037    final_state: ModuleState,
15038    enabled: Option<bool>,
15039    failure: &FailureContext,
15040) -> Result<(), SuperviseError> {
15041    if let Some(child) = child.take() {
15042        drain_child_to_state(
15043            module_id,
15044            protocol,
15045            stop_notice,
15046            registry,
15047            forwarding,
15048            snapshot,
15049            terminal_ring,
15050            spawn_events,
15051            child,
15052            drain_timeout,
15053            final_state,
15054            enabled,
15055            failure,
15056        )
15057        .await
15058    } else {
15059        update_snapshot(snapshot, Some(module_id), |state| {
15060            state.state = final_state;
15061            if final_state == ModuleState::Failed {
15062                state.failed_retry.failed_at = Some(Instant::now());
15063            }
15064            if let Some(enabled) = enabled {
15065                state.enabled = enabled;
15066            }
15067            clear_current_process_facts(state);
15068        })?;
15069        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
15070    }
15071}
15072
15073#[allow(clippy::too_many_arguments)]
15074async fn drain_child_to_state(
15075    module_id: &str,
15076    _protocol: ModuleProtocol,
15077    stop_notice: StopNotice,
15078    registry: &Registry,
15079    forwarding: Option<&ForwardingTable>,
15080    snapshot: &SharedSnapshot,
15081    terminal_ring: &Arc<Mutex<TerminalRing>>,
15082    spawn_events: &SpawnEventFeed,
15083    mut child: SupervisedChild,
15084    drain_timeout: Duration,
15085    final_state: ModuleState,
15086    enabled: Option<bool>,
15087    failure: &FailureContext,
15088) -> Result<(), SuperviseError> {
15089    let protocol = child.protocol;
15090    child.retire_registration();
15091    update_snapshot(snapshot, Some(module_id), |state| {
15092        state.state = ModuleState::Draining;
15093        state.draining_to_replace = final_state == ModuleState::Restarting;
15094        if let Some(enabled) = enabled {
15095            state.enabled = enabled;
15096        }
15097    })?;
15098
15099    // The wait below is the same budget in every case; what differs is
15100    // whether anything has ASKED the child to stop before it starts. Only a
15101    // forwarding drain that reached the module's registered connection has
15102    // (`module.draining`, then a module GOODBYE). Every other child was told
15103    // nothing: a `protocol: "none"` module, which never registers; a subc
15104    // module spawned moments ago that has not sent HELLO yet; or a stop that
15105    // runs no forwarding drain. Without a signal the budget is only a delay
15106    // in front of SIGKILL -- and the not-yet-registered child is the worst
15107    // case, because it registers into a module that is already draining,
15108    // is never told, and is killed while healthy.
15109    if stop_notice != StopNotice::SentOverConnection {
15110        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
15111            info!(
15112                module_id,
15113                pid = child.pid,
15114                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15115                "module has no connection yet; requesting stop by signal"
15116            );
15117        }
15118        request_graceful_stop(module_id, &child);
15119    }
15120
15121    let exit_report = match timeout(drain_timeout, child.wait()).await {
15122        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
15123        Ok(Err(source)) => {
15124            fail_after_wait_error(
15125                module_id,
15126                snapshot,
15127                terminal_ring,
15128                spawn_events,
15129                format!("failed to wait during drain: {source}"),
15130                failure,
15131            );
15132            return Err(SuperviseError::Wait {
15133                module_id: module_id.to_string(),
15134                source,
15135            });
15136        }
15137        Err(_) => {
15138            // Mirror the sibling arm above: state is already `Draining`, and an
15139            // error propagated from here would strand it there -- a state
15140            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
15141            // `Failed | Stopped`), leaving an operator Restart as the only exit.
15142            // `Failed` before `?` keeps the module operator-visible and
15143            // revivable. Trigger is an ESRCH race (process exits between the
15144            // drain timeout firing and the kill) or a post-kill wait failure
15145            // (issue #34).
15146            //
15147            // Logged because the kill is otherwise visible only as signal 9 in
15148            // the terminal ring, and the budget it follows can be long enough
15149            // that consumers see a stretch of refusals with no stated cause.
15150            warn!(
15151                module_id,
15152                pid = child.pid,
15153                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15154                reason = ?final_state,
15155                ?stop_notice,
15156                "drain budget expired before the module exited; killing it"
15157            );
15158            let kill =
15159                if final_state == ModuleState::Restarting && child.child.container().is_some() {
15160                    child.kill_leader()
15161                } else {
15162                    child.start_kill()
15163                };
15164            kill.map_err(|source| {
15165                fail_after_wait_error(
15166                    module_id,
15167                    snapshot,
15168                    terminal_ring,
15169                    spawn_events,
15170                    format!("failed to kill during drain: {source}"),
15171                    failure,
15172                );
15173                SuperviseError::Kill {
15174                    module_id: module_id.to_string(),
15175                    source,
15176                }
15177            })?;
15178            let status = child.wait().await.map_err(|source| {
15179                fail_after_wait_error(
15180                    module_id,
15181                    snapshot,
15182                    terminal_ring,
15183                    spawn_events,
15184                    format!("failed to wait after drain kill: {source}"),
15185                    failure,
15186                );
15187                SuperviseError::Wait {
15188                    module_id: module_id.to_string(),
15189                    source,
15190                }
15191            })?;
15192            classify_reaped_child_exit(snapshot, &child, &status)
15193        }
15194    };
15195
15196    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
15197    publish_after_terminal_record(
15198        module_id,
15199        || {
15200            record_terminal_with_detail(
15201                module_id,
15202                terminal_ring,
15203                spawn_events,
15204                &exit_report,
15205                terminal_disposition(final_state),
15206                detail,
15207            )
15208        },
15209        || {
15210            update_snapshot(snapshot, Some(module_id), |state| {
15211                state.state = final_state;
15212                if final_state == ModuleState::Failed {
15213                    state.failed_retry.failed_at = Some(Instant::now());
15214                }
15215                if let Some(enabled) = enabled {
15216                    state.enabled = enabled;
15217                }
15218                clear_current_process_facts(state);
15219                state.last_exit = Some(exit_report.clone());
15220                if exit_report.kind == ExitKind::DeliberateSeverance {
15221                    state.lifetime_restarts += 1;
15222                }
15223            })
15224        },
15225    )?;
15226    child.drain_stderr(module_id).await;
15227
15228    let released =
15229        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
15230    child
15231        .finish_exit(
15232            released.is_ok()
15233                && final_state == ModuleState::Restarting
15234                && child.spawn_failure.is_none(),
15235        )
15236        .await
15237        .map_err(|source| SuperviseError::Wait {
15238            module_id: module_id.to_owned(),
15239            source,
15240        })?;
15241    released
15242}
15243
15244/// Ask a child that nothing else has asked to stop, by signal.
15245///
15246/// A registered subc module is asked over its own connection: the drain sends
15247/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
15248/// module GOODBYE, and the module stops itself. A module that speaks no subc
15249/// wire receives none of that, and neither does a subc module that has not
15250/// registered yet, so for them the drain budget would be pure delay in front of
15251/// a SIGKILL -- and for a process with a store to flush (JetStream is the
15252/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
15253/// into a recovery on the next start.
15254///
15255/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
15256/// rule rather than an optimisation: that module's graceful stop is already
15257/// running by the time its child is drained, and a signal would race it.
15258///
15259/// Best-effort by construction. A child that has already exited is the ordinary
15260/// case rather than an error (the kill lands on a reaped or exiting pid), so a
15261/// failure is logged at debug and the wait-then-kill below still decides the
15262/// outcome.
15263#[cfg(unix)]
15264fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
15265    let Some(pid) = child
15266        .id()
15267        .and_then(|pid| i32::try_from(pid).ok())
15268        .and_then(rustix::process::Pid::from_raw)
15269    else {
15270        debug!(
15271            module_id,
15272            "no pid to signal for teardown; falling through to the drain wait"
15273        );
15274        return;
15275    };
15276    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
15277        Ok(()) => debug!(
15278            module_id,
15279            "sent SIGTERM to a module nothing else asked to stop"
15280        ),
15281        Err(err) => debug!(
15282            module_id,
15283            error = %err,
15284            "SIGTERM to module failed; the drain wait and kill still apply"
15285        ),
15286    }
15287}
15288
15289/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
15290/// Windows does offer need cooperation this supervisor cannot assume: a console
15291/// control event requires sharing a console with the child, and `WM_CLOSE`
15292/// requires the child to pump a message loop. A supervised server process does
15293/// neither, so there is nothing to send and teardown is the wait followed by the
15294/// kill. Emulating a signal here would mean inventing a stop protocol, which is
15295/// the thing `protocol: "none"` exists to avoid.
15296#[cfg(not(unix))]
15297fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
15298    debug!(
15299        module_id,
15300        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
15301    );
15302}
15303
15304fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
15305    match final_state {
15306        ModuleState::Stopped => TerminalDisposition::Stopped,
15307        ModuleState::Disabled => TerminalDisposition::Disabled,
15308        ModuleState::Restarting => TerminalDisposition::Restarting,
15309        ModuleState::Failed => TerminalDisposition::Failed,
15310        ModuleState::Starting
15311        | ModuleState::Running
15312        | ModuleState::Unresponsive
15313        | ModuleState::Draining => {
15314            unreachable!("terminal exits only finish in terminal or restarting states")
15315        }
15316    }
15317}
15318
15319/// Release a reaped child's registration before allowing another spawn.
15320///
15321/// EOF is not a process-lifetime signal: an inherited socket can stay open
15322/// indefinitely, and serial frame dispatch can be waiting on egress instead of
15323/// reading EOF. After the normal release grace, request connection close (which
15324/// cancels both reads and dispatch), then allow one more release grace for the
15325/// connection guard's forwarding cleanup. Never evict a different connection.
15326async fn release_dead_registration(
15327    registry: &Registry,
15328    forwarding: Option<&ForwardingTable>,
15329    snapshot: &SharedSnapshot,
15330    module_id: &str,
15331    failure: &FailureContext,
15332) -> Result<(), SuperviseError> {
15333    let result = async {
15334        let registration = registry
15335            .get_module(module_id)
15336            .map_err(SuperviseError::Registry)?;
15337        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
15338            Ok(()) => return Ok(()),
15339            Err(SuperviseError::RegistrationStillActive { .. }) => {}
15340            Err(err) => return Err(err),
15341        }
15342        let pid = lock_snapshot(snapshot)?.reaped_pid;
15343        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
15344            warn!(
15345                module_id,
15346                pid,
15347                connection_id = registration.connection_id.get(),
15348                "reaped module registration outlived release grace; closing dead connection"
15349            );
15350            forwarding.request_connection_close(
15351                registration.connection_id,
15352                CloseReason::new(
15353                    "supervised_process_reaped",
15354                    format!("module '{module_id}' pid {pid} exited"),
15355                ),
15356            );
15357            wait_for_slot_registration_release(
15358                registry,
15359                crate::registry::RegistrationSlot::Connection(registration.connection_id),
15360                REGISTRY_RELEASE_TIMEOUT,
15361            )
15362            .await?;
15363        }
15364        wait_for_registration_release(registry, module_id, Duration::ZERO).await
15365    }
15366    .await;
15367    if let Err(err) = &result {
15368        fail_snapshot(snapshot, Some(module_id), None, failure);
15369        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
15370    }
15371    result
15372}
15373
15374/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
15375/// plain stop or restart waits for before it spawns a replacement.
15376async fn wait_for_registration_release(
15377    registry: &Registry,
15378    module_id: &str,
15379    wait: Duration,
15380) -> Result<(), SuperviseError> {
15381    wait_for_slot_registration_release(
15382        registry,
15383        crate::registry::RegistrationSlot::Active(module_id),
15384        wait,
15385    )
15386    .await
15387}
15388
15389/// Wait for the registration in `slot` to go away.
15390///
15391/// Keyed on the slot rather than the bare module id because a successful swap
15392/// never empties the id's active slot (the promoted candidate is in it), so an
15393/// id-keyed wait for the incumbent's release would always time out. Draining a
15394/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
15395/// incumbent's connection instead.
15396async fn wait_for_slot_registration_release(
15397    registry: &Registry,
15398    slot: crate::registry::RegistrationSlot<'_>,
15399    wait: Duration,
15400) -> Result<(), SuperviseError> {
15401    let deadline = Instant::now() + wait;
15402    let mut release_events = registration_release_events().subscribe();
15403    let still_active = |registration: &crate::registry::ModuleRegistration| {
15404        SuperviseError::RegistrationStillActive {
15405            module_id: registration.manifest.module_id.clone(),
15406            waited: wait,
15407        }
15408    };
15409    loop {
15410        let _observed_generation = *release_events.borrow_and_update();
15411        let Some(registration) = registry
15412            .registration(slot)
15413            .map_err(SuperviseError::Registry)?
15414        else {
15415            return Ok(());
15416        };
15417
15418        let now = Instant::now();
15419        if now >= deadline {
15420            return Err(still_active(&registration));
15421        }
15422
15423        let remaining = deadline.saturating_duration_since(now);
15424        match timeout(remaining, release_events.changed()).await {
15425            Ok(Ok(())) | Ok(Err(_)) => {}
15426            Err(_) => return Err(still_active(&registration)),
15427        }
15428    }
15429}
15430
15431#[cfg(test)]
15432mod slot_registration_wait_tests {
15433    use super::*;
15434    use crate::registry::{ConnectionId, RegistrationSlot};
15435    use subc_protocol::manifest::ModuleManifest;
15436
15437    #[tokio::test]
15438    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
15439        let registry = Arc::new(Registry::default());
15440        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
15441        let runtime = supervisor.runtime_config();
15442        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
15443        let spec = ModuleSpec {
15444            module_id: "enable-stale-registration".to_string(),
15445            program: PathBuf::from("/missing/enable-retry-test"),
15446            args: Vec::new(),
15447            env: Vec::new(),
15448            reserved: false,
15449            reserved_prefixes: Vec::new(),
15450            protocol: ModuleProtocol::Subc,
15451            overlap: Default::default(),
15452            carry_containment: false,
15453        };
15454        let connection = ConnectionId::new(90);
15455        registry
15456            .register_with_control_ops(
15457                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
15458                1,
15459                connection,
15460                Vec::new(),
15461            )
15462            .unwrap();
15463        let mut child = None;
15464        let err = set_child_enabled(
15465            &spec,
15466            &runtime,
15467            &registry,
15468            &supervisor.process_liveness,
15469            &snapshot,
15470            &mut child,
15471            true,
15472            &runtime.failure_context(false),
15473        )
15474        .await
15475        .unwrap_err();
15476        assert!(matches!(
15477            err,
15478            SuperviseError::RegistrationStillActive { .. }
15479        ));
15480        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15481        assert!(child.is_none());
15482        registry.deregister_connection(connection).unwrap();
15483        let err = set_child_enabled(
15484            &spec,
15485            &runtime,
15486            &registry,
15487            &supervisor.process_liveness,
15488            &snapshot,
15489            &mut child,
15490            true,
15491            &runtime.failure_context(false),
15492        )
15493        .await
15494        .unwrap_err();
15495        assert!(
15496            matches!(err, SuperviseError::Spawn { .. }),
15497            "second enable must attempt a spawn: {err}"
15498        );
15499        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15500    }
15501
15502    const INCUMBENT: u64 = 1;
15503    const CANDIDATE: u64 = 2;
15504
15505    fn swapped_registry() -> Arc<Registry> {
15506        let registry = Arc::new(Registry::default());
15507        let manifest = ModuleManifest::builder("m", "0.1.0").build();
15508        registry
15509            .register_with_control_ops(
15510                manifest.clone(),
15511                1,
15512                ConnectionId::new(INCUMBENT),
15513                Vec::new(),
15514            )
15515            .unwrap();
15516        registry
15517            .register_candidate_with_control_ops(
15518                manifest,
15519                1,
15520                ConnectionId::new(CANDIDATE),
15521                Vec::new(),
15522            )
15523            .unwrap();
15524        registry
15525    }
15526
15527    /// After a promotion the id's active slot is held by the new process, so an
15528    /// id-keyed wait for the incumbent's release can never succeed; the
15529    /// connection-keyed wait completes as soon as the incumbent deregisters.
15530    #[tokio::test]
15531    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
15532        let registry = swapped_registry();
15533        registry.promote_candidate("m").unwrap().unwrap();
15534
15535        assert!(matches!(
15536            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
15537            Err(SuperviseError::RegistrationStillActive { .. })
15538        ));
15539
15540        // Still held while the incumbent's connection has not deregistered.
15541        assert!(matches!(
15542            wait_for_slot_registration_release(
15543                &registry,
15544                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15545                Duration::from_millis(50),
15546            )
15547            .await,
15548            Err(SuperviseError::RegistrationStillActive { .. })
15549        ));
15550
15551        let releaser = Arc::clone(&registry);
15552        let release = tokio::spawn(async move {
15553            sleep(Duration::from_millis(20)).await;
15554            releaser
15555                .deregister_connection(ConnectionId::new(INCUMBENT))
15556                .unwrap();
15557            notify_registration_release();
15558        });
15559        wait_for_slot_registration_release(
15560            &registry,
15561            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15562            Duration::from_secs(5),
15563        )
15564        .await
15565        .expect("the incumbent's own registration is released");
15566        release.await.unwrap();
15567        assert!(registry.get_module("m").unwrap().is_some());
15568    }
15569
15570    /// The candidate slot is waited on separately from the active slot: the
15571    /// incumbent's registration neither holds up nor stands in for it.
15572    #[tokio::test]
15573    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
15574        let registry = swapped_registry();
15575        assert!(matches!(
15576            wait_for_slot_registration_release(
15577                &registry,
15578                RegistrationSlot::Candidate("m"),
15579                Duration::from_millis(50),
15580            )
15581            .await,
15582            Err(SuperviseError::RegistrationStillActive { .. })
15583        ));
15584        registry
15585            .deregister_connection(ConnectionId::new(CANDIDATE))
15586            .unwrap();
15587        wait_for_slot_registration_release(
15588            &registry,
15589            RegistrationSlot::Candidate("m"),
15590            Duration::from_millis(50),
15591        )
15592        .await
15593        .expect("a candidate slot with no candidate is released");
15594        assert!(registry
15595            .registration(RegistrationSlot::Active("m"))
15596            .unwrap()
15597            .is_some());
15598    }
15599}
15600
15601fn classify_exit(status: &ExitStatus) -> ExitReport {
15602    ExitReport {
15603        kind: if status.success() {
15604            ExitKind::Clean
15605        } else {
15606            ExitKind::Crash
15607        },
15608        code: status.code(),
15609        signal: exit_signal(status),
15610        at_ms: unix_ms_now(),
15611    }
15612}
15613
15614/// The terminal record for a module whose `wait()` call itself errored (e.g. the
15615/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
15616/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
15617/// disposition still must be `Failed` so the terminal ring is not silently missing
15618/// an entry, matching what `fail_snapshot` records for this same arm.
15619fn wait_error_exit_report() -> ExitReport {
15620    ExitReport {
15621        kind: ExitKind::Crash,
15622        code: None,
15623        signal: None,
15624        at_ms: unix_ms_now(),
15625    }
15626}
15627
15628#[cfg(unix)]
15629fn exit_signal(status: &ExitStatus) -> Option<i32> {
15630    use std::os::unix::process::ExitStatusExt;
15631
15632    status.signal()
15633}
15634
15635#[cfg(not(unix))]
15636fn exit_signal(_status: &ExitStatus) -> Option<i32> {
15637    None
15638}
15639
15640/// Give an operator-touched module its full crash budget back.
15641///
15642/// Named for the counter it used to zero; it now empties the in-window ring,
15643/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
15644/// ledger of what happened survives every operator action.
15645fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
15646    update_snapshot(snapshot, Some(module_id), |state| {
15647        state.clear_crash_restarts();
15648    })
15649}
15650
15651fn set_running(
15652    snapshot: &SharedSnapshot,
15653    child: &SupervisedChild,
15654    module_id: &str,
15655    spawn_events: &SpawnEventFeed,
15656) -> Result<(), SuperviseError> {
15657    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15658        module_id: Some(module_id.to_string()),
15659    })?;
15660    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
15661    if std::mem::take(&mut state.coalesced_restart_pending) {
15662        let generation = state.spawn_generation;
15663        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
15664    }
15665    state.drain_disposition_detail = None;
15666    state.spawn_failure = None;
15667    state.admission_refused = None;
15668    state.config_hold_parked = false;
15669    state.config_hold_daemon_initiated = false;
15670    state.config_hold_approval = None;
15671    state.legacy_registration_spawn =
15672        child.registration_spawn.is_some() && child.admission_base.is_none();
15673    state.registration_declaration = child.registration_declaration.clone();
15674    state.stop_reason = None;
15675    state.config_exit_failed_approval = None;
15676    state.waiting_for_previous_pid = None;
15677    state.waiting_for_previous_detail = None;
15678    // Every caller of this is a plain spawn, which always uses the primary key;
15679    // a promoted swap candidate sets the flag itself after this returns.
15680    state.in_alternate_slot = false;
15681    state.configuration_updated_since_spawn = false;
15682    state.spawned_protocol = Some(child.protocol);
15683    state.state = ModuleState::Running;
15684    state.approved_start_failure = None;
15685    state.enabled = true;
15686    state.process_alive = true;
15687    state.pid = child.id();
15688    state.subtree_target = child.subtree_target.clone();
15689    // Probe samples belong to one process. Keep module-level escalation and
15690    // late-answer history, but never charge a new child for its predecessor's misses.
15691    state.health.status = SupervisorHealthStatus::Unknown;
15692    state.last_probe_evidence = None;
15693    state.health.consecutive_failures = 0;
15694    state.health.detail = None;
15695    state.health.metrics = None;
15696    state.health.last_probe_ms = None;
15697    state.failed_retry.healthy_since = None;
15698    state.failed_retry.failed_at = None;
15699    state.failed_retry.running_since = Some(child.spawned_at);
15700    #[cfg(windows)]
15701    {
15702        state.pipe_consumed = child
15703            .nonce_delivery
15704            .as_ref()
15705            .map(|delivery| delivery.consumed());
15706    }
15707    #[cfg(target_os = "macos")]
15708    {
15709        state.report_ready = Some(Arc::clone(&child.report_ready));
15710    }
15711    state.spawned_at_ms = Some(child.spawned_at_ms);
15712    state.exec_outcome = Some(Arc::clone(&child.exec.0));
15713    state.spawned_from = Some(child.spawned_from.clone());
15714    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
15715    state.spawned_file_identity = child.spawned_file_identity.clone();
15716    state.process_start_time = child.process_start_time;
15717    Ok(())
15718}
15719
15720fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
15721    state.legacy_registration_spawn = false;
15722    state.registration_declaration = None;
15723    state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
15724    state.failed_retry.healthy_since = None;
15725    state.failed_retry.running_since = None;
15726    state.waiting_for_previous_pid = None;
15727    state.waiting_for_previous_detail = None;
15728    state.process_alive = false;
15729    state.spawned_protocol = None;
15730    state.pid = None;
15731    #[cfg(windows)]
15732    {
15733        state.pipe_consumed = None;
15734    }
15735    #[cfg(target_os = "macos")]
15736    {
15737        state.report_ready = None;
15738    }
15739    state.spawned_at_ms = None;
15740    state.exec_outcome = None;
15741    state.spawned_from = None;
15742    state.launch_spec_digest = None;
15743    state.spawned_file_identity = None;
15744    state.process_start_time = None;
15745    state.deliberate_severance = None;
15746}
15747
15748#[cfg(test)]
15749fn record_deliberate_severance(
15750    snapshot: &SharedSnapshot,
15751    identity: ProcessIdentity,
15752) -> Result<(), SuperviseError> {
15753    update_snapshot(snapshot, None, |state| {
15754        state.deliberate_severance = Some(identity);
15755    })
15756}
15757
15758fn apply_deliberate_severance_marker(
15759    snapshot: &SharedSnapshot,
15760    exited_identity: Option<ProcessIdentity>,
15761    mut exit_report: ExitReport,
15762) -> ExitReport {
15763    let marker = lock_snapshot(snapshot)
15764        .ok()
15765        .and_then(|mut state| state.deliberate_severance.take());
15766    if marker.is_some() && marker == exited_identity {
15767        exit_report.kind = ExitKind::DeliberateSeverance;
15768    }
15769    exit_report
15770}
15771
15772fn classify_reaped_child_exit(
15773    snapshot: &SharedSnapshot,
15774    child: &SupervisedChild,
15775    status: &ExitStatus,
15776) -> ExitReport {
15777    let _ = update_snapshot(snapshot, None, |state| {
15778        state.reaped_pid = Some(child.pid);
15779        state.spawn_failure = child.spawn_failure.clone();
15780    });
15781    if matches!(
15782        child.spawn_failure.as_deref(),
15783        Some("registration_timeout" | "registration_channel_closed")
15784    ) {
15785        registration_failure_exit_report(classify_exit(status))
15786    } else {
15787        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
15788    }
15789}
15790
15791fn fail_snapshot(
15792    snapshot: &SharedSnapshot,
15793    module_id: Option<&str>,
15794    last_exit: Option<ExitReport>,
15795    failure: &FailureContext,
15796) {
15797    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
15798        state.state = ModuleState::Failed;
15799        state.config_hold_parked = false;
15800        state.config_hold_daemon_initiated = false;
15801        state.config_hold_approval = None;
15802        clear_current_process_facts(state);
15803        if failure.may_retry() && state.enabled {
15804            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
15805        } else if failure.shutdown.is_set() {
15806            state.failed_retry.cancel();
15807        }
15808        if let Some(last_exit) = last_exit {
15809            state.last_exit = Some(last_exit);
15810        }
15811    }) {
15812        error!(error = %err, "failed to mark supervisor state failed");
15813    }
15814}
15815
15816fn update_snapshot(
15817    snapshot: &SharedSnapshot,
15818    module_id: Option<&str>,
15819    update: impl FnOnce(&mut SupervisorSnapshot),
15820) -> Result<(), SuperviseError> {
15821    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15822        module_id: module_id.map(ToOwned::to_owned),
15823    })?;
15824    update(&mut state);
15825    Ok(())
15826}
15827
15828const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
15829
15830fn lock_snapshot_for_control<'a>(
15831    snapshot: &'a SharedSnapshot,
15832    module_id: &str,
15833    caller: &'static str,
15834) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
15835    let started_at = Instant::now();
15836    let guard = lock_snapshot(snapshot)?;
15837    let waited = started_at.elapsed();
15838    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
15839        warn!(
15840            module_id = %module_id,
15841            waited_ms = waited.as_millis() as u64,
15842            caller = %caller,
15843            "slow snapshot lock"
15844        );
15845    }
15846    Ok(guard)
15847}
15848
15849fn lock_snapshot(
15850    snapshot: &SharedSnapshot,
15851) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
15852    snapshot
15853        .lock()
15854        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
15855}
15856
15857#[cfg(test)]
15858mod terminal_history_tests {
15859    include!("supervise_route_open_recovery_tests.rs");
15860    use std::{
15861        path::PathBuf,
15862        sync::Arc,
15863        time::{Duration, Instant},
15864    };
15865
15866    use tokio::{sync::mpsc, time::sleep};
15867
15868    use super::{
15869        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
15870        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
15871        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
15872        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
15873        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
15874        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
15875        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
15876        SupervisorHealthStatus, SupervisorSnapshot,
15877    };
15878    // The supervisor's clock, distinct from the `std::time::Instant` these tests
15879    // use for their own wall-clock deadlines: crash-restart instants must be on
15880    // the same clock the production code stamps them with, which is tokio's (and
15881    // is what `start_paused` tests can move).
15882    use super::Instant as ClockInstant;
15883    use crate::{
15884        registry::Registry,
15885        terminal_ring::{TerminalRing, TerminalRingConfig},
15886    };
15887    use std::sync::Mutex;
15888    use subc_control::TerminalDisposition;
15889
15890    /// See the twin in `control.rs` for why this derives the path from
15891    /// `current_exe()` and why the existence check is here: `--lib` alone does
15892    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
15893    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
15894    pub(super) fn fake_aft_stub_path() -> PathBuf {
15895        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
15896        assert!(
15897            path.exists(),
15898            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15899             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15900            path.display()
15901        );
15902        path
15903    }
15904
15905    #[test]
15906    fn reserved_never_spawned_refuses_every_hello() {
15907        // The canary hole: a reserved id whose module has never spawned had NO
15908        // gate entry and admitted anyone -- the reservation protected the nonce
15909        // holder, not the NAME. Now the entry is present with no legitimate
15910        // holder and refuses all comers.
15911        let supervisor = SupervisorHandle::default();
15912        supervisor.apply_identity_configuration(&ModuleSpec {
15913            module_id: "never-spawned".to_string(),
15914            program: PathBuf::from("/usr/bin/false"),
15915            args: Vec::new(),
15916            env: Vec::new(),
15917            reserved: true,
15918            reserved_prefixes: Vec::new(),
15919            protocol: ModuleProtocol::Subc,
15920            overlap: Default::default(),
15921            carry_containment: false,
15922        });
15923        assert!(
15924            supervisor
15925                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
15926                .is_err(),
15927            "forged nonce must refuse on a reserved never-spawned id"
15928        );
15929        assert!(
15930            supervisor
15931                .reserved_hello_verification("never-spawned", None)
15932                .is_err(),
15933            "absent nonce must refuse on a reserved never-spawned id"
15934        );
15935        // And a real spawn nonce minted later admits exactly that nonce.
15936        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
15937        supervisor.apply_identity_configuration(&ModuleSpec {
15938            module_id: "never-spawned".to_string(),
15939            program: PathBuf::from("/usr/bin/false"),
15940            args: Vec::new(),
15941            env: Vec::new(),
15942            reserved: true,
15943            reserved_prefixes: Vec::new(),
15944            protocol: ModuleProtocol::Subc,
15945            overlap: Default::default(),
15946            carry_containment: false,
15947        });
15948        assert!(supervisor
15949            .reserved_hello_verification("never-spawned", Some("minted"))
15950            .is_ok());
15951        assert!(supervisor
15952            .reserved_hello_verification("never-spawned", Some("forged"))
15953            .is_err());
15954    }
15955
15956    /// Put `count` crash restarts on a snapshot's ring as if they had all just
15957    /// happened, which is what "spent budget" looks like to every reader.
15958    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
15959        let now = ClockInstant::now();
15960        for _ in 0..count {
15961            state.crash_restarts.push_back(now);
15962        }
15963    }
15964
15965    /// Age the oldest recorded restart out of `window`, standing in for the hours
15966    /// that would otherwise have to pass. Injecting the instant is the point: a
15967    /// test that slept a real window would take ten minutes and still prove less.
15968    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
15969        let aged = state
15970            .crash_restarts
15971            .front()
15972            .expect("a crash restart must be recorded before it can be aged")
15973            .checked_sub(window + Duration::from_secs(1))
15974            .expect("the test clock is far enough from its origin to age an instant");
15975        state.crash_restarts[0] = aged;
15976    }
15977
15978    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
15979        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
15980        seed_crash_restarts(&mut state, count);
15981        state
15982    }
15983
15984    #[test]
15985    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
15986        let policy = RestartPolicy::new(3, Duration::ZERO);
15987        let now = ClockInstant::now();
15988        assert!(daemon_will_restart(
15989            &mut snapshot_with_restarts(true, 2),
15990            &policy,
15991            now
15992        ));
15993        assert!(!daemon_will_restart(
15994            &mut snapshot_with_restarts(true, 3),
15995            &policy,
15996            now
15997        ));
15998        assert!(!daemon_will_restart(
15999            &mut snapshot_with_restarts(false, 0),
16000            &policy,
16001            now
16002        ));
16003    }
16004
16005    #[test]
16006    fn crash_restart_backoff_escalates_with_in_window_count() {
16007        let policy = RestartPolicy::new(4, Duration::from_millis(100))
16008            .with_max_backoff(Duration::from_secs(30));
16009        let now = ClockInstant::now();
16010        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16011        let schedules = (0..4)
16012            .map(|_| {
16013                state
16014                    .next_crash_restart(&policy, now)
16015                    .expect("the test policy allows four crash restarts")
16016            })
16017            .collect::<Vec<_>>();
16018
16019        assert_eq!(
16020            schedules
16021                .iter()
16022                .map(|schedule| schedule.restart_in_window)
16023                .collect::<Vec<_>>(),
16024            vec![0, 1, 2, 3]
16025        );
16026        assert_eq!(
16027            schedules
16028                .iter()
16029                .map(|schedule| schedule.delay)
16030                .collect::<Vec<_>>(),
16031            vec![
16032                Duration::from_millis(100),
16033                Duration::from_secs(1),
16034                Duration::from_secs(10),
16035                Duration::from_secs(30),
16036            ]
16037        );
16038    }
16039
16040    #[test]
16041    fn crash_restart_backoff_resets_after_ring_clear() {
16042        let policy = RestartPolicy::new(3, Duration::from_millis(100));
16043        let now = ClockInstant::now();
16044        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16045        assert_eq!(
16046            state.next_crash_restart(&policy, now).unwrap().delay,
16047            Duration::from_millis(100)
16048        );
16049        assert_eq!(
16050            state.next_crash_restart(&policy, now).unwrap().delay,
16051            Duration::from_secs(1)
16052        );
16053
16054        state.clear_crash_restarts();
16055        let schedule = state
16056            .next_crash_restart(&policy, now)
16057            .expect("a cleared ring must allow another restart");
16058        assert_eq!(schedule.restart_in_window, 0);
16059        assert_eq!(schedule.delay, Duration::from_millis(100));
16060    }
16061
16062    #[test]
16063    fn crash_restart_backoff_ignores_aged_restarts() {
16064        let policy = RestartPolicy::new(3, Duration::from_millis(100));
16065        let now = ClockInstant::now();
16066        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16067        state
16068            .next_crash_restart(&policy, now)
16069            .expect("the first restart is allowed");
16070        state
16071            .next_crash_restart(&policy, now)
16072            .expect("the second restart is allowed");
16073        state.crash_restarts[0] = now
16074            .checked_sub(policy.window + Duration::from_secs(1))
16075            .expect("the fake clock can age a restart past the window");
16076
16077        let schedule = state
16078            .next_crash_restart(&policy, now)
16079            .expect("an aged restart must release its slot");
16080        assert_eq!(schedule.restart_in_window, 1);
16081        assert_eq!(schedule.delay, Duration::from_secs(1));
16082        assert_eq!(state.crash_restarts.len(), 2);
16083    }
16084
16085    /// The budget is a rate: the same three spent restarts refuse a respawn
16086    /// while they are recent and allow one once they have aged past the window.
16087    /// Nothing about the module changed in between, which is the whole point.
16088    #[test]
16089    fn a_budget_spent_before_the_window_no_longer_refuses() {
16090        let policy = RestartPolicy::new(3, Duration::ZERO);
16091        let mut state = snapshot_with_restarts(true, 3);
16092        let now = ClockInstant::now();
16093        assert!(!daemon_will_restart(&mut state, &policy, now));
16094
16095        assert!(daemon_will_restart(
16096            &mut state,
16097            &policy,
16098            now + policy.window + Duration::from_secs(1)
16099        ));
16100        assert!(
16101            state.crash_restarts.is_empty(),
16102            "reading the budget must drop the instants that left the window"
16103        );
16104    }
16105
16106    fn module_with_recovery_snapshot(
16107        state: ModuleState,
16108        enabled: bool,
16109        restart_count: u32,
16110    ) -> SupervisedModule {
16111        let registry = Arc::new(Registry::default());
16112        let supervisor =
16113            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
16114        let runtime = supervisor.runtime_config();
16115        let spec = ModuleSpec {
16116            module_id: "recovery-snapshot".to_string(),
16117            program: fake_aft_stub_path(),
16118            args: Vec::new(),
16119            env: Vec::new(),
16120            reserved: false,
16121            reserved_prefixes: Vec::new(),
16122            protocol: ModuleProtocol::Subc,
16123            overlap: Default::default(),
16124            carry_containment: false,
16125        };
16126        let mut snapshot = SupervisorSnapshot::new(state, enabled);
16127        seed_crash_restarts(&mut snapshot, restart_count);
16128        // These tests read synthetic snapshots. A real child and monitor would
16129        // race those reads by replacing the requested state during startup.
16130        let (commands, _rx) = mpsc::channel(4);
16131        let configured_enabled = snapshot.enabled;
16132        SupervisedModule {
16133            inner: Arc::new(SupervisedModuleInner {
16134                module_id: spec.module_id.clone(),
16135                registry,
16136                snapshot: Arc::new(Mutex::new(snapshot)),
16137                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
16138                    spec,
16139                    enabled: configured_enabled,
16140                    health: runtime.health,
16141                    register_timeout: runtime.register_timeout,
16142                    restart_policy: runtime.restart_policy,
16143                })),
16144                stderr_ring: runtime.stderr_ring,
16145                terminal_ring: runtime.terminal_ring,
16146                commands,
16147                monitor: Mutex::new(None),
16148                effective_drain_timeout: runtime.effective_drain_timeout,
16149                provenance_probe: supervisor.provenance_probe.clone(),
16150                retained: runtime.child_roster.retained_view(),
16151            }),
16152        }
16153    }
16154
16155    #[cfg(target_os = "linux")]
16156    #[tokio::test]
16157    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
16158        let supervisor =
16159            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16160                .with_cgroup_placement(None);
16161        let result = supervisor.spawn(ModuleSpec {
16162            module_id: "no-cgroup-placement".to_string(),
16163            program: fake_aft_stub_path(),
16164            args: Vec::new(),
16165            env: Vec::new(),
16166            reserved: false,
16167            reserved_prefixes: Vec::new(),
16168            protocol: ModuleProtocol::Subc,
16169            overlap: Default::default(),
16170            carry_containment: false,
16171        });
16172
16173        assert!(
16174            result.is_ok(),
16175            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
16176        );
16177    }
16178
16179    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16180    async fn undecided_snapshot_uses_shared_restart_predicate() {
16181        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
16182            .will_recover_after_connection_loss()
16183            .unwrap());
16184        assert!(
16185            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
16186                .will_recover_after_connection_loss()
16187                .unwrap()
16188        );
16189    }
16190
16191    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16192    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
16193        assert!(
16194            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
16195                .will_recover_after_connection_loss()
16196                .unwrap()
16197        );
16198    }
16199
16200    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16201    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
16202        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
16203            .will_recover_after_connection_loss()
16204            .unwrap());
16205        assert!(
16206            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
16207                .will_recover_after_connection_loss()
16208                .unwrap()
16209        );
16210    }
16211
16212    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16213    async fn warming_snapshot_is_limited_to_startup_phases() {
16214        for state in [
16215            ModuleState::Starting,
16216            ModuleState::Running,
16217            ModuleState::Restarting,
16218        ] {
16219            assert!(
16220                module_with_recovery_snapshot(state, true, 0)
16221                    .is_warming()
16222                    .unwrap(),
16223                "{state:?} should be warming"
16224            );
16225        }
16226        for state in [
16227            ModuleState::Unresponsive,
16228            ModuleState::Draining,
16229            ModuleState::Stopped,
16230            ModuleState::Failed,
16231            ModuleState::Disabled,
16232        ] {
16233            assert!(
16234                !module_with_recovery_snapshot(state, true, 0)
16235                    .is_warming()
16236                    .unwrap(),
16237                "{state:?} should not be warming"
16238            );
16239        }
16240    }
16241
16242    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16243    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
16244        let registry = Arc::new(Registry::default());
16245        let supervisor =
16246            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
16247        let module = supervisor
16248            .spawn(ModuleSpec {
16249                module_id: "terminal-history".to_string(),
16250                program: fake_aft_stub_path(),
16251                args: Vec::new(),
16252                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16253                reserved: false,
16254                reserved_prefixes: Vec::new(),
16255                protocol: ModuleProtocol::Subc,
16256                overlap: Default::default(),
16257                carry_containment: false,
16258            })
16259            .unwrap();
16260
16261        let deadline = Instant::now() + Duration::from_secs(5);
16262        loop {
16263            let history = module.terminal_history();
16264            if history.entries.len() == 2 {
16265                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
16266                assert_eq!(history.dropped, 0);
16267                assert_eq!(
16268                    history
16269                        .entries
16270                        .iter()
16271                        .map(|entry| entry.exit_code)
16272                        .collect::<Vec<_>>(),
16273                    vec![Some(23), Some(23)]
16274                );
16275                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
16276                return;
16277            }
16278            assert!(
16279                Instant::now() < deadline,
16280                "module did not retain two terminal exits: {history:?}"
16281            );
16282            sleep(Duration::from_millis(10)).await;
16283        }
16284    }
16285
16286    /// A disable issued while a crash respawn is still backing off must preempt
16287    /// that respawn: the operator's stop wins, the disable must not queue behind
16288    /// the backoff, and the module must never come back up afterwards.
16289    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16290    async fn disable_during_crash_backoff_cancels_pending_respawn() {
16291        let backoff = Duration::from_secs(2);
16292        let supervisor = Supervisor::new_for_test(
16293            Arc::new(Registry::default()),
16294            RestartPolicy::new(10, backoff),
16295        );
16296        let module = supervisor
16297            .spawn(ModuleSpec {
16298                module_id: "disable-during-backoff".to_string(),
16299                program: fake_aft_stub_path(),
16300                args: Vec::new(),
16301                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16302                reserved: false,
16303                reserved_prefixes: Vec::new(),
16304                protocol: ModuleProtocol::Subc,
16305                overlap: Default::default(),
16306                carry_containment: false,
16307            })
16308            .unwrap();
16309
16310        // Wait for the first crash to put the module into its backoff window.
16311        let deadline = Instant::now() + Duration::from_secs(5);
16312        loop {
16313            if module.status().unwrap().state == ModuleState::Restarting {
16314                break;
16315            }
16316            assert!(
16317                Instant::now() < deadline,
16318                "module never entered the crash backoff"
16319            );
16320            sleep(Duration::from_millis(10)).await;
16321        }
16322
16323        let started = Instant::now();
16324        module.set_enabled(false).await.unwrap();
16325        let waited = started.elapsed();
16326
16327        assert!(
16328            waited < backoff / 2,
16329            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
16330        );
16331        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
16332
16333        // Outlast the backoff: the respawn it was counting down to must never run.
16334        sleep(backoff + Duration::from_millis(500)).await;
16335        let status = module.status().unwrap();
16336        assert_eq!(status.state, ModuleState::Disabled);
16337        assert_eq!(
16338            status.spawn_generation, 1,
16339            "module respawned after the operator disabled it"
16340        );
16341    }
16342
16343    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
16344    /// the shape of nats-server, the program this rule exists for.
16345    #[cfg(unix)]
16346    fn protocol_none_sigterm_exits_clean_spec(
16347        module_id: &str,
16348        dir: &std::path::Path,
16349    ) -> (ModuleSpec, PathBuf, PathBuf) {
16350        let ready = dir.join("ready");
16351        let marker = dir.join("sigterm");
16352        let spec = ModuleSpec {
16353            module_id: module_id.to_string(),
16354            program: fake_aft_stub_path(),
16355            args: Vec::new(),
16356            env: vec![
16357                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16358                (
16359                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
16360                    marker.display().to_string(),
16361                ),
16362                (
16363                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
16364                    ready.display().to_string(),
16365                ),
16366            ],
16367            reserved: false,
16368            reserved_prefixes: Vec::new(),
16369            protocol: ModuleProtocol::None,
16370            overlap: Default::default(),
16371            carry_containment: false,
16372        };
16373        (spec, ready, marker)
16374    }
16375
16376    /// Wait for a file the child writes, so a signal is never sent before the
16377    /// child's SIGTERM handler is installed (the default disposition would
16378    /// kill it by signal and the exit would not be clean).
16379    #[cfg(unix)]
16380    async fn wait_for_file(path: &std::path::Path) {
16381        let deadline = Instant::now() + Duration::from_secs(10);
16382        while !path.exists() {
16383            assert!(
16384                Instant::now() < deadline,
16385                "{} never appeared",
16386                path.display()
16387            );
16388            sleep(Duration::from_millis(10)).await;
16389        }
16390    }
16391
16392    /// A protocol-none module that exits 0 because something OUTSIDE the
16393    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
16394    /// the crash-path disposition rather than `stopped`.
16395    #[cfg(unix)]
16396    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16397    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
16398        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
16399        let (spec, ready, marker) =
16400            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
16401        let supervisor = Supervisor::new_for_test(
16402            Arc::new(Registry::default()),
16403            RestartPolicy::new(3, Duration::ZERO),
16404        );
16405        let module = supervisor.spawn(spec).unwrap();
16406        wait_for_file(&ready).await;
16407        // The ready file proves the child installed its SIGTERM handler, not
16408        // that the supervisor has processed the privacy trampoline's exec
16409        // acknowledgement. On macOS status withholds the pid until then.
16410        let deadline = Instant::now() + Duration::from_secs(10);
16411        let first_pid = loop {
16412            let status = module.status().unwrap();
16413            if status.state == ModuleState::Running {
16414                if let Some(pid) = status.pid {
16415                    break pid;
16416                }
16417            }
16418            assert!(
16419                Instant::now() < deadline,
16420                "a running module must report its pid after exec confirmation: {status:?}"
16421            );
16422            sleep(Duration::from_millis(10)).await;
16423        };
16424
16425        rustix::process::kill_process(
16426            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
16427            rustix::process::Signal::TERM,
16428        )
16429        .unwrap();
16430
16431        let deadline = Instant::now() + Duration::from_secs(10);
16432        let respawned = loop {
16433            let status = module.status().unwrap();
16434            if status.state == ModuleState::Running
16435                && status.pid.is_some_and(|pid| pid != first_pid)
16436            {
16437                break status;
16438            }
16439            assert!(
16440                Instant::now() < deadline,
16441                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
16442            );
16443            sleep(Duration::from_millis(10)).await;
16444        };
16445        assert_eq!(respawned.spawn_generation, 2);
16446        assert!(
16447            marker.exists(),
16448            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
16449        );
16450
16451        let history = module.terminal_history();
16452        assert_eq!(history.entries.len(), 1, "{history:?}");
16453        let entry = &history.entries[0];
16454        assert_eq!(entry.exit_code, Some(0));
16455        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
16456        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
16457
16458        module.stop().await.unwrap();
16459    }
16460
16461    /// Repeated unrequested clean exits of a protocol-none module spend the
16462    /// restart budget exactly as crashes do, and the module ends `failed` with
16463    /// the budget named.
16464    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16465    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
16466        let supervisor = Supervisor::new_for_test(
16467            Arc::new(Registry::default()),
16468            RestartPolicy::new(1, Duration::ZERO),
16469        );
16470        let module = supervisor
16471            .spawn(ModuleSpec {
16472                module_id: "none-clean-exit-budget".to_string(),
16473                program: fake_aft_stub_path(),
16474                args: Vec::new(),
16475                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
16476                reserved: false,
16477                reserved_prefixes: Vec::new(),
16478                protocol: ModuleProtocol::None,
16479                overlap: Default::default(),
16480                carry_containment: false,
16481            })
16482            .unwrap();
16483
16484        // Failed follows the terminal write; this deadline only bounds a hang,
16485        // not an assumed duration for the two launches or their exit recording.
16486        let deadline = Instant::now() + Duration::from_secs(10);
16487        loop {
16488            let status = module.status().unwrap();
16489            if status.state == ModuleState::Failed {
16490                break;
16491            }
16492            assert!(
16493                Instant::now() < deadline,
16494                "module never exhausted its budget: {status:?} {:?}",
16495                module.terminal_history()
16496            );
16497            sleep(Duration::from_millis(10)).await;
16498        }
16499        let history = module.terminal_history();
16500        assert_eq!(
16501            history
16502                .entries
16503                .iter()
16504                .map(|entry| (entry.exit_code, entry.disposition.clone()))
16505                .collect::<Vec<_>>(),
16506            vec![
16507                (Some(0), TerminalDisposition::Restarting),
16508                (Some(0), TerminalDisposition::Failed),
16509            ]
16510        );
16511        let detail = history.entries[1]
16512            .disposition_detail
16513            .as_deref()
16514            .expect("a budget failure names the budget");
16515        assert!(detail.contains("max_restarts=1"), "{detail}");
16516        assert_eq!(module.status().unwrap().spawn_generation, 2);
16517    }
16518
16519    #[test]
16520    fn restart_budget_failure_is_published_after_its_terminal_record() {
16521        use std::io::{Read, Write};
16522        let supervisor = Supervisor::new_for_test(
16523            Arc::new(Registry::default()),
16524            RestartPolicy::new(0, Duration::ZERO),
16525        );
16526        let runtime = supervisor.runtime_config();
16527        let spec = ModuleSpec {
16528            protocol: ModuleProtocol::None,
16529            ..windowed_crash_spec("budget-publication-order")
16530        };
16531        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16532            ModuleState::Running,
16533            true,
16534        )));
16535        lock_snapshot(&snapshot).unwrap().process_alive = true;
16536        let reaped_snapshot = snapshot.clone();
16537        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16538        let history = runtime.terminal_ring.clone();
16539        let reap = std::thread::spawn(move || {
16540            tokio::runtime::Builder::new_current_thread()
16541                .enable_all()
16542                .build()
16543                .unwrap()
16544                .block_on(on_child_exit(
16545                    &spec,
16546                    runtime.restart_policy,
16547                    &supervisor.registry,
16548                    &reaped_snapshot,
16549                    &runtime.terminal_ring,
16550                    &runtime.spawn_events,
16551                    &runtime.child_roster,
16552                    ExitReport {
16553                        kind: ExitKind::Clean,
16554                        code: Some(0),
16555                        signal: None,
16556                        at_ms: 1,
16557                    },
16558                    true,
16559                ))
16560        });
16561        let mut reached = [0];
16562        barrier.read_exact(&mut reached).unwrap();
16563        assert_eq!(&reached, b"R");
16564        let (before_state, before_alive) = {
16565            let state = lock_snapshot(&snapshot).unwrap();
16566            (state.state, state.process_alive)
16567        };
16568        let before_history = history.lock().unwrap().snapshot();
16569        barrier.write_all(b"X").unwrap();
16570        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
16571        assert!(before_history.entries.is_empty());
16572        assert_ne!(
16573            before_state,
16574            ModuleState::Failed,
16575            "Failed was visible before its terminal record could be written"
16576        );
16577        assert!(before_alive);
16578        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16579        let history = history.lock().unwrap().snapshot();
16580        assert_eq!(history.entries.len(), 1);
16581        assert_eq!(history.entries[0].exit_code, Some(0));
16582        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16583    }
16584
16585    #[test]
16586    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
16587        use std::io::{Read, Write};
16588
16589        let supervisor = Supervisor::new_for_test(
16590            Arc::new(Registry::default()),
16591            RestartPolicy::new(3, Duration::ZERO),
16592        );
16593        let mut runtime = supervisor.runtime_config();
16594        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
16595        let snapshot = module.inner.snapshot.clone();
16596        lock_snapshot(&snapshot).unwrap().process_alive = true;
16597        runtime.terminal_ring = module.inner.terminal_ring.clone();
16598        let spec = windowed_crash_spec(module.module_id());
16599        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16600        let reaped_snapshot = snapshot.clone();
16601        let reap = std::thread::spawn(move || {
16602            tokio::runtime::Builder::new_current_thread()
16603                .enable_all()
16604                .build()
16605                .unwrap()
16606                .block_on(on_child_exit(
16607                    &spec,
16608                    runtime.restart_policy,
16609                    &supervisor.registry,
16610                    &reaped_snapshot,
16611                    &runtime.terminal_ring,
16612                    &runtime.spawn_events,
16613                    &runtime.child_roster,
16614                    ExitReport {
16615                        kind: ExitKind::Crash,
16616                        code: Some(2),
16617                        signal: None,
16618                        at_ms: 1,
16619                    },
16620                    true,
16621                ))
16622        });
16623
16624        // Hold the crash handler between its budget decision and publication.
16625        // Connection cleanup must still see the recovery that this crash chose,
16626        // even when that recovery consumes the last available restart slot.
16627        let mut reached = [0];
16628        barrier.read_exact(&mut reached).unwrap();
16629        assert_eq!(&reached, b"R");
16630        let before_status = module.status().unwrap();
16631        let before_history = module.terminal_history();
16632        let before_recovers = module.will_recover_after_connection_loss().unwrap();
16633        barrier.write_all(b"X").unwrap();
16634        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
16635
16636        assert_eq!(before_status.state, ModuleState::Running);
16637        assert!(before_status.process_alive);
16638        assert_eq!(before_status.restart_count, 3);
16639        assert!(before_history.entries.is_empty());
16640        assert!(
16641            before_recovers,
16642            "connection cleanup would report a terminal close for a scheduled crash restart"
16643        );
16644        let restarting = module.status().unwrap();
16645        assert_eq!(restarting.state, ModuleState::Restarting);
16646        assert_eq!(restarting.restart_count, 3);
16647        assert!(module.will_recover_after_connection_loss().unwrap());
16648        let history = module.terminal_history();
16649        assert_eq!(history.entries.len(), 1);
16650        assert_eq!(
16651            history.entries[0].disposition,
16652            TerminalDisposition::Restarting
16653        );
16654
16655        // Once the replacement is running, its next crash must be judged
16656        // against the exhausted budget rather than the previous recovery.
16657        update_snapshot(&snapshot, Some(module.module_id()), |state| {
16658            state.state = ModuleState::Running;
16659            state.respawn_pending = false;
16660        })
16661        .unwrap();
16662        assert!(!module.will_recover_after_connection_loss().unwrap());
16663        update_snapshot(&snapshot, Some(module.module_id()), |state| {
16664            state.enabled = false;
16665            state.respawn_pending = true;
16666        })
16667        .unwrap();
16668        assert!(!module.will_recover_after_connection_loss().unwrap());
16669    }
16670
16671    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
16672        use std::net::{TcpListener, TcpStream};
16673        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
16674        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
16675        let (barrier, _) = listener.accept().unwrap();
16676        for stream in [&writer, &barrier] {
16677            stream
16678                .set_read_timeout(Some(Duration::from_secs(10)))
16679                .unwrap();
16680            stream
16681                .set_write_timeout(Some(Duration::from_secs(10)))
16682                .unwrap();
16683        }
16684        *events.1.lock().unwrap() = Some(writer);
16685        barrier
16686    }
16687
16688    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
16689        use std::io::{Read, Write};
16690        let barrier = events.1.lock().unwrap().take();
16691        if let Some(mut barrier) = barrier {
16692            barrier.write_all(b"R").unwrap();
16693            let mut release = [0];
16694            barrier.read_exact(&mut release).unwrap();
16695            assert_eq!(&release, b"X");
16696        }
16697    }
16698
16699    #[test]
16700    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
16701        use std::io::{Read, Write};
16702
16703        let supervisor = Supervisor::new_for_test(
16704            Arc::new(Registry::default()),
16705            RestartPolicy::new(0, Duration::ZERO),
16706        );
16707        let mut runtime = supervisor.runtime_config();
16708        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
16709        let snapshot = module.inner.snapshot.clone();
16710        runtime.terminal_ring = module.inner.terminal_ring.clone();
16711        let spec = ModuleSpec {
16712            module_id: module.module_id().into(),
16713            program: fake_aft_stub_path(),
16714            protocol: ModuleProtocol::None,
16715            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16716            ..windowed_crash_spec(module.module_id())
16717        };
16718        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16719        let reap = std::thread::spawn(move || {
16720            tokio::runtime::Builder::new_current_thread()
16721                .enable_all()
16722                .build()
16723                .unwrap()
16724                .block_on(async {
16725                    let mut child =
16726                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16727                    health_restart_child(
16728                        &spec,
16729                        &runtime,
16730                        &supervisor.registry,
16731                        &supervisor.process_liveness,
16732                        &snapshot,
16733                        &mut child,
16734                        SupervisorHealthStatus::Failing,
16735                        Some("health probe did not answer"),
16736                        1,
16737                    )
16738                    .await
16739                    .unwrap();
16740                    assert!(child.is_none());
16741                });
16742        });
16743        // The loopback barrier acknowledges entry into terminal recording by
16744        // health_restart_child's drain, then waits for the test's release byte.
16745        // Status and history are observed while that writer is held, not after
16746        // an elapsed-time guess about when the health action has completed.
16747        let mut reached = [0];
16748        barrier.read_exact(&mut reached).unwrap();
16749        assert_eq!(&reached, b"R");
16750        let before_status = module.status().unwrap();
16751        let before_history = module.terminal_history();
16752        barrier.write_all(b"X").unwrap();
16753        reap.join().unwrap();
16754        assert!(before_history.entries.is_empty());
16755        assert_ne!(
16756            before_status.state,
16757            ModuleState::Failed,
16758            "Failed was visible before its terminal record could be written"
16759        );
16760        assert!(
16761            before_status.process_alive,
16762            "process death was visible before its terminal record could be written"
16763        );
16764        let failed = module.status().unwrap();
16765        assert_eq!(failed.state, ModuleState::Failed);
16766        assert!(!failed.process_alive);
16767        let history = module.terminal_history();
16768        assert_eq!(history.entries.len(), 1);
16769        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16770        assert_eq!(
16771            history.entries[0].disposition_detail.as_deref(),
16772            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
16773        );
16774    }
16775
16776    #[test]
16777    fn observed_exits_publish_process_death_after_terminal_history() {
16778        use std::io::{Read, Write};
16779        for (kind, enabled, shutdown, wait_error, expected_state) in [
16780            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
16781            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
16782            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
16783            (
16784                ExitKind::DeliberateSeverance,
16785                true,
16786                false,
16787                false,
16788                ModuleState::Restarting,
16789            ),
16790            #[cfg(unix)]
16791            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
16792            (ExitKind::Crash, true, false, true, ModuleState::Failed),
16793        ] {
16794            let supervisor = Supervisor::new_for_test(
16795                Arc::new(Registry::default()),
16796                RestartPolicy::new(1, Duration::ZERO),
16797            );
16798            let runtime = supervisor.runtime_config();
16799            #[cfg(unix)]
16800            if shutdown {
16801                runtime.child_roster.close();
16802            }
16803            #[cfg(not(unix))]
16804            let _ = shutdown;
16805            let spec = windowed_crash_spec("exit-publication-order");
16806            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16807                ModuleState::Running,
16808                enabled,
16809            )));
16810            lock_snapshot(&snapshot).unwrap().process_alive = true;
16811            let reaped_snapshot = snapshot.clone();
16812            let history = runtime.terminal_ring.clone();
16813            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16814            let reap = std::thread::spawn(move || {
16815                if wait_error {
16816                    super::fail_after_wait_error(
16817                        &spec.module_id,
16818                        &reaped_snapshot,
16819                        &runtime.terminal_ring,
16820                        &runtime.spawn_events,
16821                        "wait failed".into(),
16822                        &runtime.failure_context(true),
16823                    );
16824                } else {
16825                    tokio::runtime::Builder::new_current_thread()
16826                        .enable_all()
16827                        .build()
16828                        .unwrap()
16829                        .block_on(on_child_exit(
16830                            &spec,
16831                            runtime.restart_policy,
16832                            &supervisor.registry,
16833                            &reaped_snapshot,
16834                            &runtime.terminal_ring,
16835                            &runtime.spawn_events,
16836                            &runtime.child_roster,
16837                            ExitReport {
16838                                kind,
16839                                code: Some(0),
16840                                signal: None,
16841                                at_ms: 1,
16842                            },
16843                            true,
16844                        ));
16845                }
16846            });
16847            let mut reached = [0];
16848            barrier.read_exact(&mut reached).unwrap();
16849            assert_eq!(&reached, b"R");
16850            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
16851            let before_history = history.lock().unwrap().snapshot();
16852            barrier.write_all(b"X").unwrap();
16853            reap.join().unwrap();
16854            assert!(
16855                before_alive,
16856                "{kind:?}: death was published before recording"
16857            );
16858            assert!(before_history.entries.is_empty());
16859            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
16860            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
16861            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
16862        }
16863    }
16864
16865    #[tokio::test]
16866    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
16867        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
16868        let path = dir.join("terminals.jsonl");
16869        std::fs::create_dir(&path).unwrap();
16870        let supervisor = Supervisor::new_for_test(
16871            Arc::new(Registry::default()),
16872            RestartPolicy::new(0, Duration::ZERO),
16873        )
16874        .with_terminal_journal(path, "health-budget-journal-failure".into());
16875        let runtime = supervisor.runtime_config();
16876        let spec = ModuleSpec {
16877            program: fake_aft_stub_path(),
16878            protocol: ModuleProtocol::None,
16879            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16880            ..windowed_crash_spec("health-budget-journal-failure")
16881        };
16882        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16883        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16884        health_restart_child(
16885            &spec,
16886            &runtime,
16887            &supervisor.registry,
16888            &supervisor.process_liveness,
16889            &snapshot,
16890            &mut child,
16891            SupervisorHealthStatus::Failing,
16892            None,
16893            1,
16894        )
16895        .await
16896        .unwrap();
16897        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16898        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
16899        let history = runtime
16900            .terminal_ring
16901            .lock()
16902            .unwrap()
16903            .durable_history(&spec.module_id);
16904        assert!(history.journal_write_failures > 0);
16905        assert_eq!(history.entries.len(), 1);
16906        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16907    }
16908
16909    #[test]
16910    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
16911        let supervisor = Supervisor::new_for_test(
16912            Arc::new(Registry::default()),
16913            RestartPolicy::new(0, Duration::ZERO),
16914        );
16915        let runtime = supervisor.runtime_config();
16916        let spec = ModuleSpec {
16917            program: fake_aft_stub_path(),
16918            protocol: ModuleProtocol::None,
16919            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16920            ..windowed_crash_spec("health-budget-recording-panic")
16921        };
16922        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16923        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
16924            tokio::runtime::Builder::new_current_thread()
16925                .enable_all()
16926                .build()
16927                .unwrap()
16928                .block_on(async {
16929                    let mut child =
16930                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16931                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
16932                    health_restart_child(
16933                        &spec,
16934                        &runtime,
16935                        &supervisor.registry,
16936                        &supervisor.process_liveness,
16937                        &snapshot,
16938                        &mut child,
16939                        SupervisorHealthStatus::Failing,
16940                        None,
16941                        1,
16942                    )
16943                    .await
16944                    .unwrap();
16945                });
16946        }));
16947        let panic = result.expect_err("recording must still unwind");
16948        assert_eq!(
16949            panic.downcast_ref::<String>().map(String::as_str),
16950            Some("spawn event sequence exhausted")
16951        );
16952        let state = lock_snapshot(&snapshot).unwrap();
16953        assert_eq!(state.state, ModuleState::Failed);
16954        assert!(!state.process_alive);
16955        assert!(state.last_exit.is_some());
16956    }
16957
16958    #[tokio::test]
16959    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
16960        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
16961        let path = dir.join("terminals.jsonl");
16962        std::fs::create_dir(&path).unwrap();
16963        let supervisor = Supervisor::new_for_test(
16964            Arc::new(Registry::default()),
16965            RestartPolicy::new(0, Duration::ZERO),
16966        )
16967        .with_terminal_journal(path, "budget-journal-failure".into());
16968        let runtime = supervisor.runtime_config();
16969        let spec = windowed_crash_spec("budget-journal-failure");
16970        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16971        assert!(matches!(
16972            on_child_exit(
16973                &spec,
16974                runtime.restart_policy,
16975                &supervisor.registry,
16976                &snapshot,
16977                &runtime.terminal_ring,
16978                &runtime.spawn_events,
16979                &runtime.child_roster,
16980                crash_exit_report(1),
16981                true,
16982            )
16983            .await,
16984            NextAction::Stop { .. }
16985        ));
16986        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16987        let history = runtime
16988            .terminal_ring
16989            .lock()
16990            .unwrap()
16991            .durable_history(&spec.module_id);
16992        assert!(history.journal_write_failures > 0);
16993        assert_eq!(history.entries.len(), 1);
16994        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16995    }
16996
16997    #[test]
16998    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
16999        let supervisor = Supervisor::new_for_test(
17000            Arc::new(Registry::default()),
17001            RestartPolicy::new(0, Duration::ZERO),
17002        );
17003        let runtime = supervisor.runtime_config();
17004        let spec = windowed_crash_spec("budget-recording-panic");
17005        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17006        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
17007        // Exhausting the event sequence makes emit_exited panic before the
17008        // terminal write, exercising publication on the recording unwind.
17009        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17010        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17011            tokio::runtime::Builder::new_current_thread()
17012                .enable_all()
17013                .build()
17014                .unwrap()
17015                .block_on(on_child_exit(
17016                    &spec,
17017                    runtime.restart_policy,
17018                    &supervisor.registry,
17019                    &snapshot,
17020                    &runtime.terminal_ring,
17021                    &runtime.spawn_events,
17022                    &runtime.child_roster,
17023                    crash_exit_report(1),
17024                    true,
17025                ))
17026        }));
17027        let panic = result.err().expect("recording must still unwind");
17028        assert_eq!(
17029            panic.downcast_ref::<String>().map(String::as_str),
17030            Some("spawn event sequence exhausted")
17031        );
17032        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17033    }
17034
17035    /// A stop the supervisor itself requests still stops a protocol-none
17036    /// module, even though the child answers the SIGTERM with exit 0.
17037    #[cfg(unix)]
17038    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17039    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
17040        for disable in [false, true] {
17041            let label = if disable {
17042                "none-requested-disable"
17043            } else {
17044                "none-requested-stop"
17045            };
17046            let dir = cortexkit_test_support::ScratchDir::new(label);
17047            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
17048            let supervisor = Supervisor::new_for_test(
17049                Arc::new(Registry::default()),
17050                RestartPolicy::new(3, Duration::ZERO),
17051            );
17052            let module = supervisor.spawn(spec).unwrap();
17053            wait_for_file(&ready).await;
17054
17055            if disable {
17056                module.set_enabled(false).await.unwrap();
17057            } else {
17058                module.stop().await.unwrap();
17059            }
17060            assert!(
17061                marker.exists(),
17062                "{label}: the child must have left through its SIGTERM handler with exit 0"
17063            );
17064
17065            // Long enough for a zero-backoff respawn to have happened if the
17066            // exit had been treated as a crash.
17067            sleep(Duration::from_millis(500)).await;
17068            let status = module.status().unwrap();
17069            let expected = if disable {
17070                ModuleState::Disabled
17071            } else {
17072                ModuleState::Stopped
17073            };
17074            assert_eq!(status.state, expected, "{label}");
17075            assert_eq!(
17076                status.spawn_generation, 1,
17077                "{label}: respawned after a requested stop"
17078            );
17079            let history = module.terminal_history();
17080            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
17081            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
17082            assert_ne!(
17083                history.entries[0].disposition,
17084                TerminalDisposition::Restarting,
17085                "{label}"
17086            );
17087        }
17088    }
17089
17090    /// A subc-wire module that exits 0 on its own is still a stop: the
17091    /// protocol-none rule must not reach it.
17092    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17093    async fn subc_wire_clean_exit_is_still_a_stop() {
17094        let supervisor = Supervisor::new_for_test(
17095            Arc::new(Registry::default()),
17096            RestartPolicy::new(3, Duration::ZERO),
17097        );
17098        let module = supervisor
17099            .spawn(ModuleSpec {
17100                module_id: "wire-clean-exit".to_string(),
17101                program: fake_aft_stub_path(),
17102                args: Vec::new(),
17103                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
17104                reserved: false,
17105                reserved_prefixes: Vec::new(),
17106                protocol: ModuleProtocol::Subc,
17107                overlap: Default::default(),
17108                carry_containment: false,
17109            })
17110            .unwrap();
17111
17112        let deadline = Instant::now() + Duration::from_secs(10);
17113        while module.terminal_history().entries.is_empty() {
17114            assert!(Instant::now() < deadline, "module never exited");
17115            sleep(Duration::from_millis(10)).await;
17116        }
17117        // Long enough for a zero-backoff respawn to have happened.
17118        sleep(Duration::from_millis(500)).await;
17119        let status = module.status().unwrap();
17120        assert_eq!(status.state, ModuleState::Stopped);
17121        assert_eq!(status.spawn_generation, 1);
17122        let history = module.terminal_history();
17123        assert_eq!(history.entries.len(), 1, "{history:?}");
17124        assert_eq!(history.entries[0].exit_code, Some(0));
17125        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
17126    }
17127
17128    #[cfg(unix)]
17129    #[tokio::test]
17130    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
17131        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
17132        let record = dir.join("live-children.json");
17133        let supervisor = Supervisor::new_for_test(
17134            Arc::new(Registry::default()),
17135            RestartPolicy::new(0, Duration::ZERO),
17136        );
17137        let mut runtime = supervisor.runtime_config();
17138        runtime.child_roster.record_to(record.clone());
17139        let gate = Arc::new(super::ReloadExitRecordGate::default());
17140        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
17141        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17142        let spec = ModuleSpec {
17143            module_id: "reload-exit-roster".into(),
17144            program: fake_aft_stub_path(),
17145            args: Vec::new(),
17146            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
17147            reserved: false,
17148            reserved_prefixes: Vec::new(),
17149            protocol: ModuleProtocol::Subc,
17150            overlap: Default::default(),
17151            carry_containment: false,
17152        };
17153        let mut child = None;
17154        let reload = super::finish_reload_child(
17155            &spec,
17156            &runtime,
17157            &supervisor.registry,
17158            &supervisor.process_liveness,
17159            &snapshot,
17160            &mut child,
17161        );
17162        tokio::pin!(reload);
17163        tokio::select! {
17164            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
17165            _ = gate.reached.notified() => {}
17166        }
17167        assert!(runtime
17168            .terminal_ring
17169            .lock()
17170            .unwrap()
17171            .snapshot()
17172            .entries
17173            .is_empty());
17174        assert_eq!(
17175            crate::live_children::read_record(&record).unwrap().len(),
17176            1,
17177            "shutdown must still wait for the reaped child until its terminal record exists"
17178        );
17179        runtime.child_roster.close();
17180        gate.resume.notify_one();
17181        assert!(reload.await.is_err());
17182        assert!(crate::live_children::read_record(&record)
17183            .unwrap()
17184            .is_empty());
17185        let history = runtime.terminal_ring.lock().unwrap().snapshot();
17186        assert_eq!(history.entries.len(), 1);
17187        assert_eq!(
17188            history.entries[0].disposition,
17189            TerminalDisposition::DaemonShutdown
17190        );
17191    }
17192
17193    /// Each restart-producing arm has its own state transition. Keeping their
17194    /// lifetime count assertions adjacent prevents a later new arm from silently
17195    /// spending budget without recording the historical restart.
17196    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17197    async fn every_restart_increment_path_advances_lifetime_count() {
17198        let supervisor = Supervisor::new_for_test(
17199            Arc::new(Registry::default()),
17200            RestartPolicy::new(1, Duration::ZERO),
17201        );
17202        let runtime = supervisor.runtime_config();
17203        let spec = ModuleSpec {
17204            module_id: "lifetime-increment-path".to_string(),
17205            program: PathBuf::from("/unused/lifetime-increment-path"),
17206            args: Vec::new(),
17207            env: Vec::new(),
17208            reserved: false,
17209            reserved_prefixes: Vec::new(),
17210            protocol: ModuleProtocol::Subc,
17211            overlap: Default::default(),
17212            carry_containment: false,
17213        };
17214
17215        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17216        assert!(matches!(
17217            on_child_exit(
17218                &spec,
17219                runtime.restart_policy,
17220                &supervisor.registry,
17221                &crash_snapshot,
17222                &runtime.terminal_ring,
17223                &runtime.spawn_events,
17224                &runtime.child_roster,
17225                ExitReport {
17226                    kind: ExitKind::Crash,
17227                    code: Some(1),
17228                    signal: None,
17229                    at_ms: 1,
17230                },
17231                true,
17232            )
17233            .await,
17234            NextAction::Restart { schedule: _ }
17235        ));
17236        let (crash_restarts, crash_lifetime) = {
17237            let state = lock_snapshot(&crash_snapshot).unwrap();
17238            (state.crash_restarts.len(), state.lifetime_restarts)
17239        };
17240        assert_eq!(crash_restarts, 1);
17241        assert_eq!(crash_lifetime, 1);
17242
17243        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17244        let mut health_child = None;
17245        assert!(matches!(
17246            health_restart_child(
17247                &spec,
17248                &runtime,
17249                &supervisor.registry,
17250                &supervisor.process_liveness,
17251                &health_snapshot,
17252                &mut health_child,
17253                SupervisorHealthStatus::Failing,
17254                None,
17255                2,
17256            )
17257            .await,
17258            Ok(())
17259        ));
17260        assert!(health_child.is_none());
17261        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
17262        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
17263        let (health_restarts, health_lifetime) = {
17264            let state = lock_snapshot(&health_snapshot).unwrap();
17265            (state.crash_restarts.len(), state.lifetime_restarts)
17266        };
17267        assert_eq!(health_restarts, 1);
17268        assert_eq!(health_lifetime, 1);
17269
17270        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17271        let mut reload_child = None;
17272        assert!(matches!(
17273            handle_reload_spawn_failure(
17274                &spec,
17275                &runtime,
17276                &supervisor.process_liveness,
17277                &reload_snapshot,
17278                &mut reload_child,
17279                "forced reload spawn failure".to_string(),
17280            )
17281            .await,
17282            Err(SuperviseError::ReloadFailed { .. })
17283        ));
17284        let (reload_restarts, reload_lifetime) = {
17285            let state = lock_snapshot(&reload_snapshot).unwrap();
17286            (state.crash_restarts.len(), state.lifetime_restarts)
17287        };
17288        assert_eq!(reload_restarts, 1);
17289        assert_eq!(reload_lifetime, 1);
17290    }
17291
17292    #[tokio::test]
17293    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
17294        let supervisor = Supervisor::new_for_test(
17295            Arc::new(Registry::default()),
17296            RestartPolicy::new(3, Duration::ZERO),
17297        );
17298        let runtime = supervisor.runtime_config();
17299        let spec = ModuleSpec {
17300            module_id: "deliberately-severed".to_string(),
17301            program: PathBuf::from("/unused/deliberately-severed"),
17302            args: Vec::new(),
17303            env: Vec::new(),
17304            reserved: false,
17305            reserved_prefixes: Vec::new(),
17306            protocol: ModuleProtocol::Subc,
17307            overlap: Default::default(),
17308            carry_containment: false,
17309        };
17310        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17311        let process = ProcessIdentity {
17312            pid: 41,
17313            start_time: 101,
17314        };
17315        record_deliberate_severance(&snapshot, process).unwrap();
17316        let exit_report = apply_deliberate_severance_marker(
17317            &snapshot,
17318            Some(process),
17319            ExitReport {
17320                kind: ExitKind::Crash,
17321                code: Some(1),
17322                signal: None,
17323                at_ms: 1,
17324            },
17325        );
17326        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
17327
17328        assert!(matches!(
17329            on_child_exit(
17330                &spec,
17331                runtime.restart_policy,
17332                &supervisor.registry,
17333                &snapshot,
17334                &runtime.terminal_ring,
17335                &runtime.spawn_events,
17336                &runtime.child_roster,
17337                exit_report,
17338                true,
17339            )
17340            .await,
17341            NextAction::Restart { schedule: _ }
17342        ));
17343        let state = lock_snapshot(&snapshot).unwrap();
17344        assert_eq!(state.lifetime_restarts, 1);
17345        assert_eq!(state.crash_restarts.len(), 0);
17346    }
17347
17348    #[tokio::test]
17349    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
17350        let supervisor = Supervisor::new_for_test(
17351            Arc::new(Registry::default()),
17352            RestartPolicy::new(3, Duration::ZERO),
17353        );
17354        let runtime = supervisor.runtime_config();
17355        let spec = ModuleSpec {
17356            module_id: "genuine-crash".to_string(),
17357            program: PathBuf::from("/unused/genuine-crash"),
17358            args: Vec::new(),
17359            env: Vec::new(),
17360            reserved: false,
17361            reserved_prefixes: Vec::new(),
17362            protocol: ModuleProtocol::Subc,
17363            overlap: Default::default(),
17364            carry_containment: false,
17365        };
17366        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17367
17368        assert!(matches!(
17369            on_child_exit(
17370                &spec,
17371                runtime.restart_policy,
17372                &supervisor.registry,
17373                &snapshot,
17374                &runtime.terminal_ring,
17375                &runtime.spawn_events,
17376                &runtime.child_roster,
17377                ExitReport {
17378                    kind: ExitKind::Crash,
17379                    code: Some(1),
17380                    signal: None,
17381                    at_ms: 1,
17382                },
17383                true,
17384            )
17385            .await,
17386            NextAction::Restart { schedule: _ }
17387        ));
17388        let state = lock_snapshot(&snapshot).unwrap();
17389        assert_eq!(state.lifetime_restarts, 1);
17390        assert_eq!(state.crash_restarts.len(), 1);
17391    }
17392
17393    fn crash_exit_report(at_ms: u64) -> ExitReport {
17394        ExitReport {
17395            kind: ExitKind::Crash,
17396            code: Some(1),
17397            signal: None,
17398            at_ms,
17399        }
17400    }
17401
17402    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
17403        ModuleSpec {
17404            module_id: module_id.to_string(),
17405            program: PathBuf::from("/unused").join(module_id),
17406            args: Vec::new(),
17407            env: Vec::new(),
17408            reserved: false,
17409            reserved_prefixes: Vec::new(),
17410            protocol: ModuleProtocol::Subc,
17411            overlap: Default::default(),
17412            carry_containment: false,
17413        }
17414    }
17415
17416    /// A real crash loop still stops. Three crashes with nothing aging out spend
17417    /// a budget of two and the third respawn is refused, and both surfaces an
17418    /// operator has -- the log line and the retained terminal record -- name the
17419    /// window rather than only the cap, because `max_restarts=2` alone is what
17420    /// this budget used to mean.
17421    #[tokio::test]
17422    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
17423        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
17424        let supervisor = Supervisor::new_for_test(
17425            Arc::new(Registry::default()),
17426            RestartPolicy::new(2, Duration::ZERO),
17427        );
17428        let runtime = supervisor.runtime_config();
17429        let spec = windowed_crash_spec("crash-loop-in-window");
17430        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17431
17432        for attempt in 1..=2 {
17433            assert!(
17434                matches!(
17435                    on_child_exit(
17436                        &spec,
17437                        runtime.restart_policy,
17438                        &supervisor.registry,
17439                        &snapshot,
17440                        &runtime.terminal_ring,
17441                        &runtime.spawn_events,
17442                        &runtime.child_roster,
17443                        crash_exit_report(attempt),
17444                        true,
17445                    )
17446                    .await,
17447                    NextAction::Restart { schedule: _ }
17448                ),
17449                "crash {attempt} is inside the budget and must respawn"
17450            );
17451        }
17452
17453        assert!(matches!(
17454            on_child_exit(
17455                &spec,
17456                runtime.restart_policy,
17457                &supervisor.registry,
17458                &snapshot,
17459                &runtime.terminal_ring,
17460                &runtime.spawn_events,
17461                &runtime.child_roster,
17462                crash_exit_report(3),
17463                true,
17464            )
17465            .await,
17466            NextAction::Stop { .. }
17467        ));
17468
17469        {
17470            let state = lock_snapshot(&snapshot).unwrap();
17471            assert_eq!(state.state, ModuleState::Failed);
17472            assert_eq!(state.crash_restarts.len(), 2);
17473            assert_eq!(state.lifetime_restarts, 2);
17474        }
17475
17476        let history = runtime
17477            .terminal_ring
17478            .lock()
17479            .expect("terminal ring is not poisoned")
17480            .snapshot();
17481        let last = history
17482            .entries
17483            .last()
17484            .expect("the refused crash is retained");
17485        assert_eq!(last.disposition, TerminalDisposition::Failed);
17486        assert_eq!(
17487            last.disposition_detail.as_deref(),
17488            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
17489        );
17490
17491        let captured = crate::router::test_log::captured_logs(&logs);
17492        assert!(
17493            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
17494            "the stop must be logged with its window: {captured}"
17495        );
17496    }
17497
17498    /// The rate, stated as a test: three crashes where the first has aged past
17499    /// the window are two crashes as far as the budget is concerned, so the
17500    /// third respawn is allowed and the ring holds only the two recent ones.
17501    ///
17502    /// This is the case a lifetime counter got wrong -- and the case the daemon
17503    /// now hits routinely, since a module exits non-zero every time its
17504    /// connection to the daemon drops.
17505    #[tokio::test]
17506    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
17507        let supervisor = Supervisor::new_for_test(
17508            Arc::new(Registry::default()),
17509            RestartPolicy::new(2, Duration::ZERO),
17510        );
17511        let runtime = supervisor.runtime_config();
17512        let spec = windowed_crash_spec("crash-across-windows");
17513        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17514
17515        for attempt in 1..=2 {
17516            assert!(matches!(
17517                on_child_exit(
17518                    &spec,
17519                    runtime.restart_policy,
17520                    &supervisor.registry,
17521                    &snapshot,
17522                    &runtime.terminal_ring,
17523                    &runtime.spawn_events,
17524                    &runtime.child_roster,
17525                    crash_exit_report(attempt),
17526                    true,
17527                )
17528                .await,
17529                NextAction::Restart { schedule: _ }
17530            ));
17531        }
17532
17533        // The oldest crash moves out of the window; nothing else about the
17534        // module changes.
17535        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17536            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
17537        })
17538        .unwrap();
17539
17540        assert!(
17541            matches!(
17542                on_child_exit(
17543                    &spec,
17544                    runtime.restart_policy,
17545                    &supervisor.registry,
17546                    &snapshot,
17547                    &runtime.terminal_ring,
17548                    &runtime.spawn_events,
17549                    &runtime.child_roster,
17550                    crash_exit_report(3),
17551                    true,
17552                )
17553                .await,
17554                NextAction::Restart { schedule: _ }
17555            ),
17556            "a crash older than the window must not hold a budget slot"
17557        );
17558
17559        let state = lock_snapshot(&snapshot).unwrap();
17560        assert_eq!(state.state, ModuleState::Restarting);
17561        assert_eq!(
17562            state.crash_restarts.len(),
17563            2,
17564            "the aged instant is dropped and the new one takes its place"
17565        );
17566        assert_eq!(
17567            state.lifetime_restarts, 3,
17568            "the ledger counts every restart, including the ones the window forgot"
17569        );
17570    }
17571
17572    /// An operator restart hands the budget back whole, and the ledger keeps
17573    /// counting. Those are different questions -- "how close is this module to
17574    /// being stopped" and "how many times has it been replaced" -- and the
17575    /// operator action answers only the first.
17576    #[tokio::test]
17577    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
17578        let supervisor = Supervisor::new_for_test(
17579            Arc::new(Registry::default()),
17580            RestartPolicy::new(2, Duration::ZERO),
17581        );
17582        let runtime = supervisor.runtime_config();
17583        let spec = windowed_crash_spec("operator-cleared-budget");
17584        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17585
17586        for attempt in 1..=2 {
17587            assert!(matches!(
17588                on_child_exit(
17589                    &spec,
17590                    runtime.restart_policy,
17591                    &supervisor.registry,
17592                    &snapshot,
17593                    &runtime.terminal_ring,
17594                    &runtime.spawn_events,
17595                    &runtime.child_roster,
17596                    crash_exit_report(attempt),
17597                    true,
17598                )
17599                .await,
17600                NextAction::Restart { schedule: _ }
17601            ));
17602        }
17603
17604        reset_restart_count(&snapshot, &spec.module_id).unwrap();
17605        {
17606            let state = lock_snapshot(&snapshot).unwrap();
17607            assert!(
17608                state.crash_restarts.is_empty(),
17609                "an operator restart returns the full budget"
17610            );
17611            assert_eq!(
17612                state.lifetime_restarts, 2,
17613                "clearing the budget must not unmake the crashes"
17614            );
17615        }
17616
17617        assert!(
17618            matches!(
17619                on_child_exit(
17620                    &spec,
17621                    runtime.restart_policy,
17622                    &supervisor.registry,
17623                    &snapshot,
17624                    &runtime.terminal_ring,
17625                    &runtime.spawn_events,
17626                    &runtime.child_roster,
17627                    crash_exit_report(3),
17628                    true,
17629                )
17630                .await,
17631                NextAction::Restart { schedule: _ }
17632            ),
17633            "the cleared budget must be spendable again"
17634        );
17635        let state = lock_snapshot(&snapshot).unwrap();
17636        assert_eq!(state.crash_restarts.len(), 1);
17637        assert_eq!(state.lifetime_restarts, 3);
17638    }
17639
17640    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17641    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
17642        let severed = ProcessIdentity {
17643            pid: 41,
17644            start_time: 101,
17645        };
17646        let successor = ProcessIdentity {
17647            pid: 41,
17648            start_time: 202,
17649        };
17650        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
17651        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
17652            state.pid = Some(successor.pid);
17653            state.process_start_time = Some(successor.start_time);
17654        })
17655        .unwrap();
17656        assert!(!module.record_deliberate_severance(severed).unwrap());
17657
17658        let exit_report = apply_deliberate_severance_marker(
17659            &module.inner.snapshot,
17660            Some(successor),
17661            ExitReport {
17662                kind: ExitKind::Crash,
17663                code: Some(1),
17664                signal: None,
17665                at_ms: 1,
17666            },
17667        );
17668
17669        assert_eq!(exit_report.kind, ExitKind::Crash);
17670    }
17671
17672    #[tokio::test]
17673    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
17674        let registry = Registry::default();
17675        let supervisor = Supervisor::new_for_test(
17676            Arc::new(Registry::default()),
17677            RestartPolicy::new(3, Duration::ZERO),
17678        );
17679        let runtime = supervisor.runtime_config();
17680        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17681        let spec = ModuleSpec {
17682            module_id: "drain-deliberate-severance".to_string(),
17683            program: fake_aft_stub_path(),
17684            args: Vec::new(),
17685            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17686            reserved: false,
17687            reserved_prefixes: Vec::new(),
17688            protocol: ModuleProtocol::Subc,
17689            overlap: Default::default(),
17690            carry_containment: false,
17691        };
17692        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17693        let process = ProcessIdentity {
17694            pid: 41,
17695            start_time: 101,
17696        };
17697        child.process_identity = Some(process);
17698        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17699            state.pid = Some(process.pid);
17700            state.process_start_time = Some(process.start_time);
17701        })
17702        .unwrap();
17703        record_deliberate_severance(&snapshot, process).unwrap();
17704
17705        drain_child_to_state(
17706            &spec.module_id,
17707            spec.protocol,
17708            // The child exits on its own; no signal may change the exit this
17709            // test classifies.
17710            StopNotice::SentOverConnection,
17711            &registry,
17712            None,
17713            &snapshot,
17714            &runtime.terminal_ring,
17715            &runtime.spawn_events,
17716            child,
17717            Duration::from_secs(1),
17718            ModuleState::Stopped,
17719            Some(false),
17720            &super::FailureContext::operator(),
17721        )
17722        .await
17723        .unwrap();
17724
17725        let state = lock_snapshot(&snapshot).unwrap();
17726        assert_eq!(
17727            state.last_exit.as_ref().map(|exit| exit.kind),
17728            Some(ExitKind::DeliberateSeverance)
17729        );
17730        assert_eq!(state.lifetime_restarts, 1);
17731        assert_eq!(state.crash_restarts.len(), 0);
17732        drop(state);
17733        let history = runtime.terminal_ring.lock().unwrap().snapshot();
17734        assert_eq!(
17735            history.entries[0].exit_kind,
17736            subc_control::TerminalExitKind::DeliberateSeverance
17737        );
17738    }
17739
17740    #[tokio::test]
17741    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
17742        let registry = Registry::default();
17743        let supervisor = Supervisor::new_for_test(
17744            Arc::new(Registry::default()),
17745            RestartPolicy::new(3, Duration::ZERO),
17746        );
17747        let runtime = supervisor.runtime_config();
17748        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17749        let spec = ModuleSpec {
17750            module_id: "ordinary-drain".to_string(),
17751            program: fake_aft_stub_path(),
17752            args: Vec::new(),
17753            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17754            reserved: false,
17755            reserved_prefixes: Vec::new(),
17756            protocol: ModuleProtocol::Subc,
17757            overlap: Default::default(),
17758            carry_containment: false,
17759        };
17760        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17761
17762        drain_child_to_state(
17763            &spec.module_id,
17764            spec.protocol,
17765            // The child exits on its own; no signal may change the exit this
17766            // test classifies.
17767            StopNotice::SentOverConnection,
17768            &registry,
17769            None,
17770            &snapshot,
17771            &runtime.terminal_ring,
17772            &runtime.spawn_events,
17773            child,
17774            Duration::from_secs(1),
17775            ModuleState::Stopped,
17776            Some(false),
17777            &super::FailureContext::operator(),
17778        )
17779        .await
17780        .unwrap();
17781
17782        let state = lock_snapshot(&snapshot).unwrap();
17783        assert_eq!(
17784            state.last_exit.as_ref().map(|exit| exit.kind),
17785            Some(ExitKind::Crash)
17786        );
17787        assert_eq!(state.lifetime_restarts, 0);
17788        assert_eq!(state.crash_restarts.len(), 0);
17789    }
17790
17791    #[test]
17792    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
17793        // The server's generic fatal-routing branch only knows that the
17794        // connection failed; it does not know that the daemon deliberately
17795        // initiated a process-killing severance. Keep this seam explicit so a
17796        // future connection error path cannot silently reintroduce the stale
17797        // exemption that mislabels a later genuine crash.
17798        assert!(!include_str!("server.rs")
17799            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
17800    }
17801
17802    /// The `route.closed` `drained` value must be the quiescence wait's own
17803    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
17804    /// measurement at all and `false` is the one honest constant. This is the exact
17805    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
17806    /// on every return path, including the one that used to return early via `?`
17807    /// with `route.closing` already sent and no `route.closed` ever following.
17808    #[test]
17809    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
17810        assert!(drained_after_quiescence_wait(&Ok(true)));
17811        assert!(!drained_after_quiescence_wait(&Ok(false)));
17812        assert!(!drained_after_quiescence_wait(&Err(
17813            SuperviseError::StatePoisoned { module_id: None }
17814        )));
17815    }
17816
17817    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
17818    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
17819    /// already reaped out-of-band) still leaves a terminal record rather than none
17820    /// at all. Triggering the real `wait()` I/O error from an integration test would
17821    /// need a genuine already-reaped-child race, which is OS-specific and not
17822    /// something this suite attempts elsewhere; this test instead verifies the
17823    /// record produced for that arm end-to-end through the real `TerminalRing`, and
17824    /// the call site itself is verified by inspection to sit in that exact arm.
17825    #[test]
17826    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
17827        let ring = Arc::new(Mutex::new(TerminalRing::new(
17828            TerminalRingConfig::default(),
17829            0,
17830        )));
17831        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
17832
17833        let snapshot = ring.lock().unwrap().snapshot();
17834        assert_eq!(snapshot.entries.len(), 1);
17835        let entry = &snapshot.entries[0];
17836        assert_eq!(entry.exit_code, None);
17837        assert_eq!(entry.exit_signal, None);
17838        assert_eq!(entry.disposition, TerminalDisposition::Failed);
17839    }
17840
17841    #[test]
17842    fn wait_error_exit_path_preserves_spawn_event_density() {
17843        let feed = super::SpawnEventFeed::default();
17844        feed.configure_incarnation("wait-error-density".to_string());
17845        feed.emit_spawned("wait-error", 41, 1);
17846        let ring = Arc::new(Mutex::new(TerminalRing::new(
17847            TerminalRingConfig::default(),
17848            0,
17849        )));
17850
17851        record_wait_error_terminal("wait-error", &ring, &feed);
17852        feed.emit_spawned("after-wait-error", 42, 2);
17853
17854        let state = feed.0.lock().unwrap();
17855        let sequences = state
17856            .events
17857            .iter()
17858            .map(|event| event.cursor.seq)
17859            .collect::<Vec<_>>();
17860        assert_eq!(sequences, vec![1, 2, 3]);
17861        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
17862        assert_eq!(state.events[1].exit_code, None);
17863        assert_eq!(state.events[1].exit_signal, None);
17864    }
17865
17866    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
17867    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
17868    /// not a clean exit it never actually observed.
17869    #[test]
17870    fn wait_error_exit_report_is_classified_as_a_crash() {
17871        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
17872    }
17873}
17874
17875#[cfg(all(test, unix))]
17876mod restart_ordering_tests {
17877    use super::*;
17878    use std::io::Write;
17879    use tokio::io::AsyncReadExt;
17880
17881    #[test]
17882    #[ignore = "child process fixture"]
17883    fn serving_child() {
17884        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
17885            return;
17886        };
17887        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
17888        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
17889        for stream in listener.incoming() {
17890            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
17891        }
17892    }
17893
17894    async fn restart_before_publication(requests: usize) {
17895        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
17896        let address_file = home.path().join("address");
17897        let spec = ModuleSpec {
17898            module_id: "restart-serving-child".into(),
17899            program: std::env::current_exe().unwrap(),
17900            args: vec![
17901                "--exact".into(),
17902                "supervise::restart_ordering_tests::serving_child".into(),
17903                "--ignored".into(),
17904                "--nocapture".into(),
17905            ],
17906            env: vec![(
17907                "RESTART_TEST_ADDRESS_FILE".into(),
17908                address_file.display().to_string(),
17909            )],
17910            reserved: false,
17911            reserved_prefixes: vec![],
17912            protocol: ModuleProtocol::None,
17913            overlap: Default::default(),
17914            carry_containment: false,
17915        };
17916        let mut supervisor =
17917            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
17918        let (spawned_tx, spawned_rx) = oneshot::channel();
17919        let spawned_tx = Mutex::new(Some(spawned_tx));
17920        let (release_tx, release_rx) = std::sync::mpsc::channel();
17921        let release_rx = Mutex::new(release_rx);
17922        // Hold publication, not the child: it must answer a real request before
17923        // the restart enters the queue, even though status still has generation 0.
17924        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
17925            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
17926            release_rx
17927                .lock()
17928                .unwrap()
17929                .recv_timeout(Duration::from_secs(20))
17930                .unwrap();
17931        })));
17932        let registered = supervisor
17933            .register_configured_with_health(
17934                spec.clone(),
17935                true,
17936                HealthConfig::default(),
17937                None,
17938                RestartPolicy::new(3, Duration::ZERO),
17939                DEFAULT_REGISTER_TIMEOUT,
17940            )
17941            .unwrap();
17942        let module = registered.module.clone();
17943        let startup = tokio::task::spawn_blocking(move || {
17944            supervisor.start_registered_configured(registered, None, true)
17945        });
17946        spawned_rx.await.unwrap();
17947        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17948            let address = loop {
17949                if let Ok(address) = std::fs::read_to_string(&address_file) {
17950                    if !address.is_empty() {
17951                        break address;
17952                    }
17953                }
17954                tokio::time::sleep(Duration::from_millis(1)).await;
17955            };
17956            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
17957            let mut reply = String::new();
17958            stream.read_to_string(&mut reply).await.unwrap();
17959            reply.trim().parse::<u32>().unwrap()
17960        })
17961        .await
17962        .expect("child must serve before generation publication");
17963        assert_eq!(
17964            lock_snapshot(&module.inner.snapshot)
17965                .unwrap()
17966                .spawn_generation,
17967            0
17968        );
17969        let mut replies = Vec::new();
17970        for _ in 0..requests {
17971            let mut restart = Box::pin(module.restart(Some(0)));
17972            // Poll the public API through queue admission, leaving only its ACK
17973            // pending. The monitor cannot consume the command until released.
17974            std::future::poll_fn(|cx| {
17975                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
17976                std::task::Poll::Ready(())
17977            })
17978            .await;
17979            replies.push(restart);
17980        }
17981        release_tx.send(()).unwrap();
17982        startup.await.unwrap().unwrap();
17983        for reply in replies {
17984            reply.await.unwrap();
17985        }
17986        // Restart schedules its replacement after completing the handler. Wait
17987        // for that separate actor turn before placing the command barrier.
17988        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
17989            loop {
17990                if lock_snapshot(&module.inner.snapshot)
17991                    .unwrap()
17992                    .spawn_generation
17993                    > 1
17994                {
17995                    break;
17996                }
17997                tokio::time::sleep(Duration::from_millis(1)).await;
17998            }
17999        })
18000        .await;
18001        // Commands are handled serially. The update-spec reply proves every
18002        // restart handler has finished before we inspect the snapshot; restart's
18003        // own ACK only acknowledges initiation, not replacement completion.
18004        module.update_spec_for_test(spec).await.unwrap();
18005        let (generation, pid) = {
18006            let state = lock_snapshot(&module.inner.snapshot).unwrap();
18007            (state.spawn_generation, state.pid)
18008        };
18009        module.stop().await.unwrap();
18010        replacement.expect("restart must publish a replacement for the already-serving child");
18011        assert_ne!(
18012            pid,
18013            Some(old_pid),
18014            "a serving child predating the restart must be replaced"
18015        );
18016        assert_eq!(generation, 2, "queued restarts must share one replacement");
18017    }
18018
18019    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18020    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
18021        restart_before_publication(1).await;
18022    }
18023
18024    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18025    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
18026        restart_before_publication(2).await;
18027    }
18028
18029    #[tokio::test]
18030    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
18031        let supervisor =
18032            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18033        let registered = supervisor
18034            .register_configured_with_health(
18035                ModuleSpec {
18036                    module_id: "queued-before-replacement".into(),
18037                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
18038                    // macOS a module whose running image is not the configured
18039                    // program is refused once its exec is confirmed (see
18040                    // `record_write_stall_tests::sleeping_spec`).
18041                    program: "/bin/sleep".into(),
18042                    args: vec!["600".into()],
18043                    env: vec![],
18044                    reserved: false,
18045                    reserved_prefixes: vec![],
18046                    protocol: ModuleProtocol::None,
18047                    overlap: Default::default(),
18048                    carry_containment: false,
18049                },
18050                true,
18051                HealthConfig::default(),
18052                None,
18053                RestartPolicy::default(),
18054                DEFAULT_REGISTER_TIMEOUT,
18055            )
18056            .unwrap();
18057        let RegisteredConfiguredModule {
18058            mut spec,
18059            mut runtime,
18060            module,
18061            mut commands,
18062        } = registered;
18063        let snapshot = &module.inner.snapshot;
18064        let mut restart = Box::pin(module.restart(Some(0)));
18065        std::future::poll_fn(|cx| {
18066            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18067            std::task::Poll::Ready(())
18068        })
18069        .await;
18070        // Drive the actor explicitly so a replacement is definitely spawned
18071        // after receipt, but before this queued command is dequeued.
18072        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
18073        let replacement_pid = child.as_ref().unwrap().id();
18074        let command = commands.recv().await.unwrap();
18075        assert!(
18076            handle_supervisor_command(
18077                command,
18078                &mut spec,
18079                &mut runtime,
18080                &supervisor.registry,
18081                &supervisor.process_liveness,
18082                snapshot,
18083                &mut child,
18084                &mut commands,
18085                &mut VecDeque::new(),
18086            )
18087            .await
18088        );
18089        restart.await.unwrap();
18090        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
18091        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
18092        if let Some(mut child) = child {
18093            child.start_kill().unwrap();
18094            child.wait().await.unwrap();
18095        }
18096        assert_eq!(
18097            kept_pid, replacement_pid,
18098            "a newer replacement must survive"
18099        );
18100        assert!(
18101            !pending,
18102            "a satisfied restart must not schedule another spawn"
18103        );
18104    }
18105}
18106
18107#[cfg(all(test, unix))]
18108mod health_event_tests {
18109    use super::*;
18110    use cortexkit_test_support::ScratchDir;
18111    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
18112
18113    struct Actor {
18114        module: SupervisedModule,
18115        registry: Arc<Registry>,
18116        forwarding: Arc<ForwardingTable>,
18117        spec: ModuleSpec,
18118        health: HealthConfig,
18119        rx: mpsc::Receiver<crate::router::OutboundFrame>,
18120        roster: ChildRoster,
18121        _home: ScratchDir,
18122    }
18123
18124    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
18125        // OS exec and socket readiness are real I/O. Stay runnable while waiting
18126        // for an observed condition, rather than letting paused time auto-advance
18127        // deadlines or assuming a fixed number of yields finishes the work.
18128        let deadline = std::time::Instant::now() + Duration::from_secs(10);
18129        let tick = Instant::now();
18130        loop {
18131            if let Some(value) = observe() {
18132                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
18133                return value;
18134            }
18135            assert!(
18136                std::time::Instant::now() < deadline,
18137                "timed out waiting for {reason}"
18138            );
18139            tokio::task::yield_now().await;
18140        }
18141    }
18142
18143    impl Actor {
18144        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
18145            Self::start_with_policy(
18146                protocol,
18147                cadence,
18148                DEFAULT_REGISTER_TIMEOUT,
18149                RestartPolicy::new(3, Duration::ZERO),
18150            )
18151            .await
18152        }
18153
18154        async fn start_with_policy(
18155            protocol: ModuleProtocol,
18156            cadence: Duration,
18157            register_timeout: Duration,
18158            policy: RestartPolicy,
18159        ) -> Self {
18160            let home = ScratchDir::new("health-events");
18161            let registry = Arc::new(Registry::default());
18162            let forwarding = Arc::new(ForwardingTable::default());
18163            let health = HealthConfig {
18164                cadence,
18165                deadline: Duration::from_secs(3600),
18166                ..HealthConfig::default()
18167            };
18168            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
18169                .with_forwarding(forwarding.clone())
18170                .with_health_config(health.clone());
18171            let spec = ModuleSpec {
18172                module_id: "event-health-child".into(),
18173                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
18174                // the privacy trampoline refuses a module whose running image
18175                // is not the configured program, and `start` below waits for
18176                // the process to be confirmed (`reported_pid`). The shell form
18177                // passed only while the image check ran before the shell had
18178                // replaced itself with `sleep` (see
18179                // `record_write_stall_tests::sleeping_spec`).
18180                program: PathBuf::from("/bin/sleep"),
18181                // The process stays alive but has no real bus peer. Registrations
18182                // below exercise the same registry writes as accepted HELLOs.
18183                args: vec!["600".into()],
18184                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18185                    .into_iter()
18186                    .map(|key| (key.into(), home.path().display().to_string()))
18187                    .collect(),
18188                reserved: false,
18189                reserved_prefixes: vec![],
18190                protocol,
18191                overlap: Default::default(),
18192                carry_containment: false,
18193            };
18194            let module = supervisor
18195                .supervise_configured_requested(
18196                    spec.clone(),
18197                    true,
18198                    health.clone(),
18199                    Some(0),
18200                    policy,
18201                    register_timeout,
18202                    None,
18203                )
18204                .unwrap();
18205            let (_, rx) = mpsc::channel(8);
18206            let actor = Self {
18207                module,
18208                registry,
18209                forwarding,
18210                spec,
18211                health,
18212                rx,
18213                roster: supervisor.child_roster.clone(),
18214                _home: home,
18215            };
18216            actor
18217                .wait_for_select(|checkpoint| checkpoint.generation > 0)
18218                .await;
18219            actor
18220        }
18221
18222        async fn expire_registration_wait(&self, remaining: Duration) {
18223            tokio::time::advance(remaining).await;
18224            wait_for("registration timeout TERM grace", || {
18225                lock_snapshot(&self.module.inner.snapshot)
18226                    .unwrap()
18227                    .actor_select
18228                    .is_none()
18229                    .then_some(())
18230            })
18231            .await;
18232            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
18233        }
18234
18235        fn turns(&self) -> u64 {
18236            lock_snapshot(&self.module.inner.snapshot)
18237                .unwrap()
18238                .actor_turns
18239        }
18240
18241        async fn wait_for_select(
18242            &self,
18243            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
18244        ) -> ActorSelectCheckpoint {
18245            wait_for("actor select checkpoint", || {
18246                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
18247                state.actor_select.filter(|checkpoint| {
18248                    checkpoint.generation == state.spawn_generation
18249                        && state.state == ModuleState::Running
18250                        && state.process_alive
18251                        && state.reported_pid().is_some()
18252                        && expected(checkpoint)
18253                })
18254            })
18255            .await
18256        }
18257
18258        fn hello(&mut self, connection: u64, health: bool) {
18259            let manifest =
18260                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
18261                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
18262                    .build();
18263            let connection = ConnectionId::new(connection);
18264            let (tx, rx) = mpsc::channel(8);
18265            self.rx = rx;
18266            self.forwarding
18267                .register_module_connection(
18268                    connection,
18269                    self.spec.module_id.clone(),
18270                    subc_protocol::PROTOCOL_VERSION,
18271                    Concurrency::ModuleManaged,
18272                    FrameSink::new(tx),
18273                )
18274                .unwrap();
18275            self.registry
18276                .register_with_control_ops(
18277                    manifest,
18278                    subc_protocol::PROTOCOL_VERSION,
18279                    connection,
18280                    if health {
18281                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
18282                    } else {
18283                        vec![]
18284                    },
18285                )
18286                .unwrap();
18287        }
18288
18289        async fn probe(&mut self) -> crate::router::OutboundFrame {
18290            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
18291                Ok(frame) => Some(frame),
18292                Err(mpsc::error::TryRecvError::Empty) => None,
18293                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
18294            })
18295            .await;
18296            let body: Value = serde_json::from_slice(&frame.body).unwrap();
18297            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
18298            frame
18299        }
18300
18301        fn no_probe(&mut self) {
18302            assert!(matches!(
18303                self.rx.try_recv(),
18304                Err(mpsc::error::TryRecvError::Empty)
18305            ));
18306        }
18307
18308        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
18309            self.forwarding
18310                .complete_module_control_rpc(
18311                    ConnectionId::new(connection),
18312                    frame.header.corr,
18313                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
18314                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
18315                        status: HealthStatus::Ok,
18316                        detail: None,
18317                        metrics: None,
18318                    }),
18319                )
18320                .unwrap();
18321        }
18322    }
18323
18324    #[tokio::test(start_paused = true)]
18325    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
18326        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18327        let initial = actor.module.status().unwrap();
18328        tokio::time::advance(Duration::from_secs(119)).await;
18329        tokio::task::yield_now().await;
18330        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
18331        assert!(actor.module.terminal_history().entries.is_empty());
18332        actor.expire_registration_wait(Duration::from_secs(1)).await;
18333        actor
18334            .wait_for_select(|at| at.generation > initial.spawn_generation)
18335            .await;
18336        let status = actor.module.status().unwrap();
18337        assert_ne!(status.pid, initial.pid);
18338        assert!(!cortexkit_test_support::process_alive(
18339            initial.pid.unwrap() as i32
18340        ));
18341        assert_eq!(status.restart_count, 1);
18342        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
18343        let entries = actor.module.terminal_history().entries;
18344        assert_eq!(entries.len(), 1);
18345        assert_eq!(
18346            entries[0].disposition_detail.as_deref(),
18347            Some("registration_timeout")
18348        );
18349        actor.module.stop().await.unwrap();
18350    }
18351
18352    #[tokio::test(start_paused = true)]
18353    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
18354        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18355        let pid = actor.module.status().unwrap().pid;
18356        tokio::time::advance(Duration::from_secs(119)).await;
18357        actor.hello(1, false);
18358        let registered = actor
18359            .wait_for_select(|at| at.registered_connection.is_some())
18360            .await;
18361        assert_eq!(
18362            registered.wake_after, None,
18363            "HELLO must cancel the deadline"
18364        );
18365        tokio::time::advance(Duration::from_secs(300)).await;
18366        tokio::task::yield_now().await;
18367        assert_eq!(actor.module.status().unwrap().pid, pid);
18368        assert!(actor.module.terminal_history().entries.is_empty());
18369        // A disconnect after the first HELLO must not re-arm a startup timer.
18370        actor
18371            .registry
18372            .deregister_connection(ConnectionId::new(1))
18373            .unwrap();
18374        actor
18375            .wait_for_select(|at| at.registered_connection.is_none())
18376            .await;
18377        tokio::time::advance(Duration::from_secs(300)).await;
18378        tokio::task::yield_now().await;
18379        assert_eq!(actor.module.status().unwrap().pid, pid);
18380        assert!(actor.module.terminal_history().entries.is_empty());
18381        actor.module.stop().await.unwrap();
18382
18383        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18384        let pid = transient.module.status().unwrap().pid;
18385        tokio::time::advance(Duration::from_secs(119)).await;
18386        let turns = transient.turns();
18387        // Both writes happen before the supervisor can refresh the registry.
18388        // An accepted HELLO must remain evidence after its connection is gone.
18389        transient.hello(1, false);
18390        transient
18391            .registry
18392            .deregister_connection(ConnectionId::new(1))
18393            .unwrap();
18394        let parked = transient.wait_for_select(|at| at.turn > turns).await;
18395        assert_eq!(
18396            parked.wake_after, None,
18397            "transient HELLO must cancel the deadline"
18398        );
18399        tokio::time::advance(Duration::from_secs(300)).await;
18400        tokio::task::yield_now().await;
18401        assert_eq!(transient.module.status().unwrap().pid, pid);
18402        assert!(transient.module.terminal_history().entries.is_empty());
18403        transient.module.stop().await.unwrap();
18404    }
18405
18406    #[tokio::test(start_paused = true)]
18407    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
18408        let actor = Actor::start_with_policy(
18409            ModuleProtocol::Subc,
18410            Duration::from_secs(30),
18411            Duration::from_secs(10),
18412            RestartPolicy::new(1, Duration::ZERO),
18413        )
18414        .await;
18415        let generation = actor.module.status().unwrap().spawn_generation;
18416        actor
18417            .expire_registration_wait(Duration::from_secs(10))
18418            .await;
18419        actor.wait_for_select(|at| at.generation > generation).await;
18420        actor
18421            .expire_registration_wait(Duration::from_secs(10))
18422            .await;
18423        let retry = wait_for("failed retry schedule", || {
18424            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
18425            (state.state == ModuleState::Failed)
18426                .then_some(state.failed_retry.pending)
18427                .flatten()
18428        })
18429        .await;
18430        assert_eq!(retry.delay, Duration::from_secs(600));
18431        assert_eq!(retry.attempt, 1);
18432        let entries = actor.module.terminal_history().entries;
18433        assert_eq!(entries.len(), 2);
18434        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
18435        assert!(entries[1]
18436            .disposition_detail
18437            .as_deref()
18438            .unwrap()
18439            .contains("registration_timeout"));
18440        tokio::time::advance(Duration::from_secs(599)).await;
18441        tokio::task::yield_now().await;
18442        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
18443        tokio::time::advance(Duration::from_secs(1)).await;
18444        actor
18445            .wait_for_select(|at| at.generation > generation + 1)
18446            .await;
18447        // The slow retry is another ordinary spawn, with its own HELLO budget.
18448        actor
18449            .expire_registration_wait(Duration::from_secs(10))
18450            .await;
18451        wait_for("slow retry registration timeout", || {
18452            (actor.module.terminal_history().entries.len() == 3).then_some(())
18453        })
18454        .await;
18455        actor.module.stop().await.unwrap();
18456    }
18457
18458    #[tokio::test(start_paused = true)]
18459    async fn registration_timeout_honours_ten_second_override() {
18460        let actor = Actor::start_with_policy(
18461            ModuleProtocol::Subc,
18462            Duration::from_secs(30),
18463            Duration::from_secs(10),
18464            RestartPolicy::new(3, Duration::ZERO),
18465        )
18466        .await;
18467        let generation = actor.module.status().unwrap().spawn_generation;
18468        tokio::time::advance(Duration::from_secs(9)).await;
18469        tokio::task::yield_now().await;
18470        assert!(actor.module.terminal_history().entries.is_empty());
18471        actor.expire_registration_wait(Duration::from_secs(1)).await;
18472        actor.wait_for_select(|at| at.generation > generation).await;
18473        assert_eq!(actor.module.terminal_history().entries.len(), 1);
18474        actor.module.stop().await.unwrap();
18475    }
18476
18477    #[tokio::test]
18478    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
18479        use std::os::unix::process::{CommandExt, ExitStatusExt};
18480
18481        struct Helper(std::process::Child);
18482        impl Drop for Helper {
18483            fn drop(&mut self) {
18484                // This handle still owns the helper until wait returns. Never
18485                // clean up its old process group after the leader is reaped.
18486                if matches!(self.0.try_wait(), Ok(None)) {
18487                    let _ = self.0.kill();
18488                    let _ = self.0.wait();
18489                }
18490            }
18491        }
18492
18493        let home = ScratchDir::new("registration-timeout-helper");
18494        let supervisor = Supervisor::new_for_test(
18495            Arc::new(Registry::default()),
18496            RestartPolicy::new(0, Duration::ZERO),
18497        );
18498        let spec = ModuleSpec {
18499            module_id: "registration-timeout-group".into(),
18500            program: "/bin/sleep".into(),
18501            args: vec!["120".into()],
18502            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18503                .into_iter()
18504                .map(|key| (key.into(), home.path().display().to_string()))
18505                .collect(),
18506            reserved: false,
18507            reserved_prefixes: vec![],
18508            protocol: ModuleProtocol::Subc,
18509            overlap: Default::default(),
18510            carry_containment: false,
18511        };
18512        let module = supervisor
18513            .supervise_configured_requested(
18514                spec,
18515                true,
18516                HealthConfig::default(),
18517                None,
18518                RestartPolicy::new(0, Duration::ZERO),
18519                Duration::from_secs(10),
18520                None,
18521            )
18522            .unwrap();
18523        let leader = timeout(Duration::from_secs(5), async {
18524            loop {
18525                if let Some(pid) = module.status().unwrap().pid {
18526                    break pid;
18527                }
18528                sleep(Duration::from_millis(5)).await;
18529            }
18530        })
18531        .await
18532        .expect("supervised group leader must be reported before the deadline");
18533        assert!(leader > 1);
18534        let ready = home.join("helper.ready");
18535        // Own the helper separately so even a broken group kill can be cleaned
18536        // up by its own unreaped child handle, without signalling a stale pgid.
18537        let mut helper = Helper(
18538            std::process::Command::new("/bin/sh")
18539                .args([
18540                    "-c",
18541                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
18542                    "helper",
18543                ])
18544                .arg(&ready)
18545                .process_group(i32::try_from(leader).unwrap())
18546                .stdout(std::process::Stdio::null())
18547                .stderr(std::process::Stdio::null())
18548                .spawn()
18549                .unwrap(),
18550        );
18551        timeout(Duration::from_secs(5), async {
18552            while !ready.exists() {
18553                assert!(helper.0.try_wait().unwrap().is_none());
18554                sleep(Duration::from_millis(5)).await;
18555            }
18556        })
18557        .await
18558        .expect("helper must install TERM ignore before the registration deadline");
18559        let helper_pid =
18560            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
18561        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
18562        sleep(Duration::from_millis(20)).await;
18563        assert!(
18564            helper.0.try_wait().unwrap().is_none(),
18565            "helper must ignore TERM"
18566        );
18567
18568        timeout(Duration::from_secs(20), async {
18569            while module.state().unwrap() != ModuleState::Failed {
18570                sleep(Duration::from_millis(5)).await;
18571            }
18572        })
18573        .await
18574        .expect("registration timeout must stop and reap the group leader");
18575        let helper_exit = timeout(Duration::from_secs(3), async {
18576            loop {
18577                if let Some(status) = helper.0.try_wait().unwrap() {
18578                    break status;
18579                }
18580                sleep(Duration::from_millis(5)).await;
18581            }
18582        })
18583        .await
18584        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
18585        assert_eq!(helper_exit.signal(), Some(9));
18586        assert!(!cortexkit_test_support::process_alive(
18587            i32::try_from(leader).unwrap()
18588        ));
18589        assert_eq!(
18590            module.status().unwrap().last_exit.unwrap().kind,
18591            ExitKind::Crash
18592        );
18593        assert!(module.terminal_history().entries[0]
18594            .disposition_detail
18595            .as_deref()
18596            .unwrap()
18597            .contains("registration_timeout"));
18598        module.stop().await.unwrap();
18599    }
18600
18601    #[tokio::test(start_paused = true)]
18602    async fn registration_timeout_does_not_count_an_earlier_exit() {
18603        let actor = Actor::start_with_policy(
18604            ModuleProtocol::Subc,
18605            Duration::from_secs(30),
18606            DEFAULT_REGISTER_TIMEOUT,
18607            RestartPolicy::new(0, Duration::ZERO),
18608        )
18609        .await;
18610        let pid = actor.module.status().unwrap().pid.unwrap();
18611        rustix::process::kill_process(
18612            rustix::process::Pid::from_raw(pid as i32).unwrap(),
18613            rustix::process::Signal::TERM,
18614        )
18615        .unwrap();
18616        wait_for("early exit", || {
18617            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
18618        })
18619        .await;
18620        tokio::time::advance(Duration::from_secs(121)).await;
18621        tokio::task::yield_now().await;
18622        let entries = actor.module.terminal_history().entries;
18623        assert_eq!(entries.len(), 1);
18624        assert!(!entries[0]
18625            .disposition_detail
18626            .as_deref()
18627            .unwrap()
18628            .contains("registration_timeout"));
18629        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
18630        actor.module.stop().await.unwrap();
18631    }
18632
18633    #[tokio::test(start_paused = true)]
18634    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
18635        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18636        let pid = actor.module.status().unwrap().pid;
18637        tokio::time::advance(Duration::from_secs(119)).await;
18638        actor.roster.close();
18639        let turns = actor.turns();
18640        tokio::time::advance(Duration::from_secs(300)).await;
18641        let parked = actor.wait_for_select(|at| at.turn > turns).await;
18642        assert_eq!(parked.wake_after, None);
18643        assert_eq!(actor.module.status().unwrap().pid, pid);
18644        assert!(actor.module.terminal_history().entries.is_empty());
18645        actor.module.stop().await.unwrap();
18646        let status = actor.module.status().unwrap();
18647        assert_eq!(status.restart_count, 0);
18648        assert_eq!(
18649            actor.module.terminal_history().entries[0].disposition,
18650            TerminalDisposition::DaemonShutdown
18651        );
18652    }
18653
18654    #[tokio::test(start_paused = true)]
18655    async fn no_health_children_park_for_sixty_seconds() {
18656        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18657        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18658        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18659        unadvertised.hello(1, false);
18660        // Include any one-off registration wake in the budget; receiving a
18661        // no-health HELLO must not turn a parked child into a polling child.
18662        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
18663        // a single 60 s jump would only observe one expired timer.
18664        for _ in 0..6000 {
18665            tokio::time::advance(Duration::from_millis(10)).await;
18666            tokio::task::yield_now().await;
18667        }
18668        for actor in [&none, &unregistered, &unadvertised] {
18669            assert!(
18670                actor.turns() <= 5,
18671                "no-health actor woke {} times",
18672                actor.turns()
18673            );
18674            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
18675        }
18676    }
18677
18678    #[tokio::test(start_paused = true)]
18679    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
18680        // With a zero cadence the first probe is due at once, so this measures
18681        // only how fast the registration notification starts health probing,
18682        // not the normal 30-33 s wait before a first probe (checked below).
18683        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
18684        let tick = Instant::now();
18685        actor.hello(1, true);
18686        actor.probe().await;
18687        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
18688    }
18689
18690    #[tokio::test(start_paused = true)]
18691    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
18692        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18693        let hello_tick = Instant::now();
18694        actor.hello(1, true);
18695        let armed = actor
18696            .wait_for_select(|checkpoint| {
18697                checkpoint.registered_connection == Some(ConnectionId::new(1))
18698                    && checkpoint.next_probe_at.is_some()
18699            })
18700            .await;
18701        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18702        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
18703        let first_deadline = hello_tick + delay;
18704        assert_eq!(armed.next_probe_at, Some(first_deadline));
18705        tokio::time::advance(Duration::from_secs(29)).await;
18706        let turns = actor.turns();
18707        actor
18708            .registry
18709            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
18710            .unwrap();
18711        let updated = actor
18712            .wait_for_select(|checkpoint| checkpoint.turn > turns)
18713            .await;
18714        assert_eq!(
18715            updated.next_probe_at,
18716            Some(first_deadline),
18717            "catalog update must not reset cadence"
18718        );
18719        actor.no_probe();
18720        tokio::time::advance(first_deadline - Instant::now()).await;
18721        let frame = actor.probe().await;
18722        actor.answer(1, frame);
18723        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
18724        let next_deadline = Instant::now() + next;
18725        let rearmed = actor
18726            .wait_for_select(|checkpoint| {
18727                checkpoint
18728                    .next_probe_at
18729                    .is_some_and(|deadline| deadline > first_deadline)
18730            })
18731            .await;
18732        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
18733        tokio::time::advance(next - Duration::from_nanos(1)).await;
18734        actor.no_probe();
18735        tokio::time::advance(Duration::from_nanos(1)).await;
18736        actor.probe().await;
18737    }
18738
18739    #[tokio::test(start_paused = true)]
18740    async fn disconnect_disarms_and_reconnect_rearms_health() {
18741        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18742        actor.hello(1, true);
18743        actor
18744            .wait_for_select(|checkpoint| {
18745                checkpoint.registered_connection == Some(ConnectionId::new(1))
18746                    && checkpoint.next_probe_at.is_some()
18747            })
18748            .await;
18749        let turns = actor.turns();
18750        let tick = Instant::now();
18751        actor
18752            .registry
18753            .deregister_connection(ConnectionId::new(1))
18754            .unwrap();
18755        let disarmed = actor
18756            .wait_for_select(|checkpoint| {
18757                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
18758            })
18759            .await;
18760        assert_eq!(disarmed.next_probe_at, None);
18761        assert_eq!(disarmed.wake_after, None);
18762        assert_eq!(
18763            Instant::now(),
18764            tick,
18765            "disconnect must disarm in the notification tick"
18766        );
18767        tokio::time::advance(Duration::from_secs(60)).await;
18768        actor.no_probe();
18769        let reconnect_tick = Instant::now();
18770        actor.hello(2, true);
18771        let rearmed = actor
18772            .wait_for_select(|checkpoint| {
18773                checkpoint.registered_connection == Some(ConnectionId::new(2))
18774                    && checkpoint.next_probe_at.is_some()
18775            })
18776            .await;
18777        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18778        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
18779        tokio::time::advance(delay).await;
18780        actor.probe().await;
18781    }
18782
18783    #[tokio::test(start_paused = true)]
18784    async fn reregistration_without_health_disarms_the_old_schedule() {
18785        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18786        actor.hello(1, true);
18787        actor
18788            .wait_for_select(|checkpoint| {
18789                checkpoint.registered_connection == Some(ConnectionId::new(1))
18790                    && checkpoint.next_probe_at.is_some()
18791            })
18792            .await;
18793        actor
18794            .registry
18795            .deregister_connection(ConnectionId::new(1))
18796            .unwrap();
18797        actor.hello(2, false);
18798        let disarmed = actor
18799            .wait_for_select(|checkpoint| {
18800                checkpoint.registered_connection == Some(ConnectionId::new(2))
18801            })
18802            .await;
18803        assert_eq!(disarmed.next_probe_at, None);
18804        assert_eq!(disarmed.wake_after, None);
18805        tokio::time::advance(Duration::from_secs(60)).await;
18806        actor.no_probe();
18807    }
18808
18809    #[tokio::test(start_paused = true)]
18810    async fn restart_waits_for_its_own_hello_before_rearming_health() {
18811        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18812        actor.hello(1, true);
18813        let armed = actor
18814            .wait_for_select(|checkpoint| {
18815                checkpoint.registered_connection == Some(ConnectionId::new(1))
18816                    && checkpoint.next_probe_at.is_some()
18817            })
18818            .await;
18819        // Simulate the old process's connection closing before the restart
18820        // tears the process down. The replacement process has started but has
18821        // not sent its HELLO, so it is not registered.
18822        actor
18823            .registry
18824            .deregister_connection(ConnectionId::new(1))
18825            .unwrap();
18826        actor.module.restart(Some(0)).await.unwrap();
18827        let parked = actor
18828            .wait_for_select(|checkpoint| {
18829                checkpoint.generation > armed.generation
18830                    && checkpoint.registered_connection.is_none()
18831                    && checkpoint.next_probe_at.is_none()
18832            })
18833            .await;
18834        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
18835        actor.rx.close();
18836        // Drain the stop notice the restart queued for the old process's connection,
18837        // so it isn't mistaken for traffic to the replacement.
18838        while actor.rx.try_recv().is_ok() {}
18839        let turns = actor.turns();
18840        tokio::time::advance(Duration::from_secs(60)).await;
18841        let after = actor.turns();
18842        eprintln!(
18843            "restart park turns: before={turns} after={after} delta={}",
18844            after - turns
18845        );
18846        assert_eq!(after, turns, "replacement must park before HELLO");
18847        let hello_tick = Instant::now();
18848        actor.hello(2, true);
18849        let rearmed = actor
18850            .wait_for_select(|checkpoint| {
18851                checkpoint.registered_connection == Some(ConnectionId::new(2))
18852                    && checkpoint.next_probe_at.is_some()
18853            })
18854            .await;
18855        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18856        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
18857        tokio::time::advance(delay).await;
18858        actor.probe().await;
18859    }
18860
18861    #[tokio::test(start_paused = true)]
18862    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
18863        use tokio::{
18864            io::{AsyncReadExt, AsyncWriteExt},
18865            net::TcpListener,
18866        };
18867        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18868        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
18869        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
18870        let (started_tx, mut started_rx) = mpsc::channel(8);
18871        let peer = tokio::spawn(async move {
18872            loop {
18873                let (mut socket, _) = listener.accept().await.unwrap();
18874                let mut request = [0; 1024];
18875                assert!(socket.read(&mut request).await.unwrap() > 0);
18876                started_tx.send(Instant::now()).await.unwrap();
18877                socket
18878                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
18879                    .await
18880                    .unwrap();
18881            }
18882        });
18883        actor
18884            .module
18885            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
18886            .await
18887            .unwrap();
18888        let first_tick = Instant::now();
18889        let first = actor
18890            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
18891            .await;
18892        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
18893        assert!(started_rx.try_recv().is_err());
18894
18895        // Restart just before the old deadline: reusing it would give the new
18896        // process only one second to start, rather than a full cadence.
18897        tokio::time::advance(
18898            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
18899        )
18900        .await;
18901        actor.module.restart(Some(0)).await.unwrap();
18902        let spawn_tick = Instant::now();
18903        let replacement = actor
18904            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
18905            .await;
18906        let deadline = replacement.next_probe_at.unwrap();
18907        assert!(
18908            deadline >= spawn_tick + actor.health.cadence,
18909            "a replacement must not inherit its predecessor's probe deadline"
18910        );
18911        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
18912        assert!(
18913            started_rx.try_recv().is_err(),
18914            "no HTTP probe during startup grace"
18915        );
18916        tokio::time::advance(deadline - Instant::now()).await;
18917        let landed = wait_for("first replacement HTTP probe", || {
18918            started_rx.try_recv().ok()
18919        })
18920        .await;
18921        assert!(landed >= spawn_tick + actor.health.cadence);
18922        wait_for("healthy replacement", || {
18923            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
18924                .then_some(())
18925        })
18926        .await;
18927        peer.abort();
18928    }
18929
18930    #[tokio::test(start_paused = true)]
18931    async fn rescan_adds_and_removes_http_health_without_polling() {
18932        use tokio::{
18933            io::{AsyncReadExt, AsyncWriteExt},
18934            net::TcpListener,
18935        };
18936        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18937        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
18938        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
18939        // One millisecond separates successive probes without a polling delay.
18940        actor.health.cadence = Duration::from_millis(1);
18941        let (started_tx, mut started_rx) = mpsc::channel(8);
18942        let peer = tokio::spawn(async move {
18943            loop {
18944                let (mut socket, _) = listener.accept().await.unwrap();
18945                let mut request = [0; 1024];
18946                assert!(socket.read(&mut request).await.unwrap() > 0);
18947                started_tx.send(()).await.unwrap();
18948                socket
18949                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
18950                    .await
18951                    .unwrap();
18952            }
18953        });
18954        let tick = Instant::now();
18955        let turns = actor.turns();
18956        actor
18957            .module
18958            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
18959            .await
18960            .unwrap();
18961        let armed = actor
18962            .wait_for_select(|checkpoint| {
18963                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
18964            })
18965            .await;
18966        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
18967        assert_eq!(Instant::now(), tick);
18968        tokio::time::advance(Duration::from_millis(1)).await;
18969        let mut started = false;
18970        wait_for("successful HTTP probe", || {
18971            started |= started_rx.try_recv().is_ok();
18972            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
18973                .then_some(())
18974        })
18975        .await;
18976        assert!(started, "added HTTP check must start at its first deadline");
18977        assert_eq!(
18978            actor.module.status().unwrap().health.status,
18979            SupervisorHealthStatus::Ok
18980        );
18981        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
18982        actor.health.http = None;
18983        let turns = actor.turns();
18984        actor
18985            .module
18986            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
18987            .await
18988            .unwrap();
18989        let parked = actor
18990            .wait_for_select(|checkpoint| {
18991                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
18992            })
18993            .await;
18994        assert_eq!(parked.wake_after, None);
18995        let turns = actor.turns();
18996        tokio::time::advance(Duration::from_secs(60)).await;
18997        assert_eq!(
18998            actor.turns(),
18999            turns,
19000            "removed HTTP check must park the actor"
19001        );
19002        assert!(
19003            started_rx.try_recv().is_err(),
19004            "removed HTTP check must stay disarmed"
19005        );
19006        assert_eq!(
19007            actor.module.status().unwrap().health,
19008            ModuleHealthStatus::default()
19009        );
19010        peer.abort();
19011    }
19012}
19013
19014#[cfg(test)]
19015mod drain_busy_reprobe_tests {
19016    //! The drain's busy-gauge wait, driven against an in-process module
19017    //! connection whose health answers the test scripts. Time is paused, so
19018    //! every elapsed duration below is exact virtual time.
19019    use super::*;
19020    use std::collections::BTreeMap;
19021    use std::sync::Mutex as StdMutex;
19022    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
19023
19024    const MODULE_ID: &str = "drain-reprobe-module";
19025    const CONNECTION: u64 = 7;
19026    const GAUGE: &str = "runs_in_flight";
19027    /// The production default for both the health cadence and the drain
19028    /// ceiling, which is the pairing that used to cost a full ceiling.
19029    const CADENCE: Duration = Duration::from_secs(30);
19030    const CEILING: Duration = Duration::from_secs(30);
19031
19032    /// How the scripted module answers each `health.check`.
19033    #[derive(Clone, Copy)]
19034    enum Answer {
19035        /// Report the gauge busy until this instant, quiet from then on, after
19036        /// taking `latency` to answer.
19037        BusyUntil { until: Instant, latency: Duration },
19038        /// Answer at once, with metrics that leave the declared gauge out.
19039        OmitsGauge,
19040        /// Never answer.
19041        Silent,
19042    }
19043
19044    struct Drain {
19045        forwarding: Arc<ForwardingTable>,
19046        runtime: SupervisorRuntimeConfig,
19047        endpoint: crate::ModuleEndpointId,
19048        /// When each probe reached the module.
19049        probes: Arc<StdMutex<Vec<Instant>>>,
19050        responder: tokio::task::JoinHandle<()>,
19051    }
19052
19053    impl Drain {
19054        fn start(answer: Answer) -> Self {
19055            let registry = Arc::new(Registry::default());
19056            let forwarding = Arc::new(ForwardingTable::default());
19057            let health = HealthConfig {
19058                cadence: CADENCE,
19059                ..HealthConfig::default()
19060            };
19061            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
19062                .with_forwarding(forwarding.clone())
19063                .with_health_config(health)
19064                .runtime_config();
19065            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
19066            let endpoint = forwarding
19067                .register_module_connection(
19068                    ConnectionId::new(CONNECTION),
19069                    MODULE_ID.to_string(),
19070                    subc_protocol::PROTOCOL_VERSION,
19071                    Concurrency::ModuleManaged,
19072                    FrameSink::new(tx),
19073                )
19074                .unwrap();
19075            let probes = Arc::new(StdMutex::new(Vec::new()));
19076            let responder = tokio::spawn({
19077                let forwarding = forwarding.clone();
19078                let probes = probes.clone();
19079                async move {
19080                    while let Some(frame) = rx.recv().await {
19081                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
19082                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
19083                        probes.lock().unwrap().push(Instant::now());
19084                        let metrics = match answer {
19085                            Answer::BusyUntil { until, latency } => {
19086                                sleep(latency).await;
19087                                let busy = u64::from(Instant::now() < until);
19088                                serde_json::json!({ GAUGE: busy })
19089                            }
19090                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
19091                            Answer::Silent => continue,
19092                        };
19093                        // A probe abandoned at its deadline has no RPC left to
19094                        // complete, so a late answer may be refused.
19095                        let _ = forwarding.complete_module_control_rpc(
19096                            ConnectionId::new(CONNECTION),
19097                            frame.header.corr,
19098                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
19099                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19100                                status: HealthStatus::Ok,
19101                                detail: None,
19102                                metrics: Some(metrics),
19103                            }),
19104                        );
19105                    }
19106                }
19107            });
19108            Self {
19109                forwarding,
19110                runtime,
19111                endpoint,
19112                probes,
19113                responder,
19114            }
19115        }
19116
19117        /// Run one by-id drain wait with `ceiling`, returning its verdict and
19118        /// how long it took.
19119        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
19120            let (drained, elapsed, _) = self.run_traced(ceiling).await;
19121            (drained, elapsed)
19122        }
19123
19124        /// [`Self::run`], also returning what the drain's probes read.
19125        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
19126            let started = Instant::now();
19127            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
19128            let drained = wait_for_forwarding_quiescence(
19129                &self.forwarding,
19130                MODULE_ID,
19131                &self.runtime,
19132                self.endpoint,
19133                started + ceiling,
19134                &[GAUGE.to_string()],
19135                DrainScope::Active,
19136                &mut trace,
19137            )
19138            .await
19139            .unwrap();
19140            (drained, started.elapsed(), trace)
19141        }
19142
19143        fn probes(&self) -> Vec<Instant> {
19144            self.probes.lock().unwrap().clone()
19145        }
19146    }
19147
19148    impl Drop for Drain {
19149        fn drop(&mut self) {
19150            self.responder.abort();
19151        }
19152    }
19153
19154    #[tokio::test(start_paused = true)]
19155    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
19156        let drain = Drain::start(Answer::BusyUntil {
19157            until: Instant::now() + Duration::from_secs(1),
19158            latency: Duration::ZERO,
19159        });
19160        let (drained, elapsed) = drain.run(CEILING).await;
19161        assert!(
19162            drained,
19163            "gauges that read zero during the drain must let it finish drained, \
19164             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
19165        );
19166        assert!(
19167            elapsed < Duration::from_secs(3),
19168            "drain took {elapsed:?} after the gauges went quiet at 1s"
19169        );
19170        let probes = drain.probes();
19171        assert!(
19172            probes.len() >= 2,
19173            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
19174        );
19175    }
19176
19177    /// The same shape in real time on a multi-threaded runtime: the first
19178    /// probe is answered busy at once, within a millisecond of the drain
19179    /// starting, and later probes read idle. The drain must keep probing
19180    /// after that first instant answer and end soon after the gauge falls.
19181    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19182    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
19183        let drain = Drain::start(Answer::BusyUntil {
19184            until: Instant::now() + Duration::from_millis(600),
19185            latency: Duration::ZERO,
19186        });
19187        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
19188        assert!(drained, "the gauge fell at 600ms: {trace}");
19189        assert!(
19190            elapsed < Duration::from_secs(3),
19191            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
19192        );
19193        let probes = drain.probes();
19194        assert!(
19195            probes.len() >= 2 && trace.probes as usize == probes.len(),
19196            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
19197        );
19198    }
19199
19200    #[tokio::test(start_paused = true)]
19201    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
19202        let started = Instant::now();
19203        let drain = Drain::start(Answer::BusyUntil {
19204            until: started + Duration::from_secs(3600),
19205            latency: Duration::ZERO,
19206        });
19207        let (drained, elapsed) = drain.run(CEILING).await;
19208        assert!(!drained, "a module busy throughout must not drain");
19209        assert!(
19210            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
19211            "a busy module is held until the ceiling and no longer: {elapsed:?}"
19212        );
19213        // Re-probing is paced by the interval, not a busy loop: at most one
19214        // probe per interval across the ceiling, plus the first at time zero.
19215        let probes = drain.probes();
19216        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
19217        assert!(
19218            probes.len() as u128 <= max_probes,
19219            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
19220            probes.len()
19221        );
19222        for pair in probes.windows(2) {
19223            assert!(
19224                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
19225                "consecutive probes {:?} apart",
19226                pair[1] - pair[0]
19227            );
19228        }
19229    }
19230
19231    #[tokio::test(start_paused = true)]
19232    async fn a_slow_probe_is_never_overlapped_by_the_next() {
19233        // Each answer takes longer than the re-probe interval; the next probe
19234        // must wait for the previous answer and then the interval.
19235        let latency = Duration::from_secs(1);
19236        let drain = Drain::start(Answer::BusyUntil {
19237            until: Instant::now() + Duration::from_secs(3600),
19238            latency,
19239        });
19240        let (drained, _) = drain.run(Duration::from_secs(5)).await;
19241        assert!(!drained);
19242        let probes = drain.probes();
19243        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
19244        for pair in probes.windows(2) {
19245            assert!(
19246                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
19247                "a probe started {:?} after the previous one, before it could finish",
19248                pair[1] - pair[0]
19249            );
19250        }
19251    }
19252
19253    #[tokio::test(start_paused = true)]
19254    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
19255        let drain = Drain::start(Answer::Silent);
19256        let ceiling = Duration::from_secs(2);
19257        let (drained, elapsed) = drain.run(ceiling).await;
19258        assert!(!drained, "an unanswered probe counts as busy");
19259        assert!(
19260            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
19261            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
19262        );
19263        assert_eq!(
19264            drain.probes().len(),
19265            1,
19266            "the unanswered probe holds the only slot until the ceiling"
19267        );
19268    }
19269
19270    /// Every event logged while it is installed, as `message` plus fields.
19271    #[derive(Clone, Default)]
19272    struct LogCapture {
19273        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
19274    }
19275
19276    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
19277        fn on_event(
19278            &self,
19279            event: &tracing::Event<'_>,
19280            _context: tracing_subscriber::layer::Context<'_, S>,
19281        ) {
19282            struct Fields(BTreeMap<String, String>);
19283            impl tracing::field::Visit for Fields {
19284                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
19285                    self.0
19286                        .insert(field.name().to_string(), format!("{value:?}"));
19287                }
19288            }
19289            let mut fields = Fields(BTreeMap::new());
19290            event.record(&mut fields);
19291            self.events.lock().unwrap().push(fields.0);
19292        }
19293    }
19294
19295    impl LogCapture {
19296        fn install(&self) -> tracing::subscriber::DefaultGuard {
19297            use tracing_subscriber::layer::SubscriberExt;
19298            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
19299        }
19300
19301        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
19302            self.events
19303                .lock()
19304                .unwrap()
19305                .iter()
19306                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
19307                .cloned()
19308                .collect()
19309        }
19310    }
19311
19312    /// A module that declares a busy gauge but leaves it out of its health
19313    /// answers holds a drain to its ceiling. The timeout line must say so by
19314    /// name, and the drain must warn about the missing gauge once, not on
19315    /// every probe.
19316    #[tokio::test(start_paused = true)]
19317    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
19318        let capture = LogCapture::default();
19319        let _guard = capture.install();
19320        let drain = Drain::start(Answer::OmitsGauge);
19321        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
19322        assert!(!drained, "a gauge never reported cannot read quiet");
19323        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
19324
19325        let missing = capture.with_message("lacks a declared busy gauge");
19326        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
19327        assert_eq!(
19328            missing[0].get("missing_gauges").map(String::as_str),
19329            Some(GAUGE)
19330        );
19331
19332        warn_drain_timed_out(
19333            MODULE_ID,
19334            Duration::from_secs(2),
19335            RouteCloseReason::Restart,
19336            &Default::default(),
19337            &trace,
19338        );
19339        let timeouts = capture.with_message("route drain timed out");
19340        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
19341        let gauges = &timeouts[0]["busy_gauges"];
19342        assert!(
19343            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
19344            "the timeout line must name the gauge and why it read busy: {gauges}"
19345        );
19346        assert!(
19347            gauges.ends_with("; no value ever read"),
19348            "no value was ever read for the gauge: {gauges}"
19349        );
19350        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
19351    }
19352
19353    /// The trace names a busy value and when it was read. For a probe that
19354    /// never got an answer, it names the probe's error kind and says that no
19355    /// value was ever read.
19356    #[tokio::test(start_paused = true)]
19357    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
19358        let busy = Drain::start(Answer::BusyUntil {
19359            until: Instant::now() + Duration::from_secs(3600),
19360            latency: Duration::ZERO,
19361        });
19362        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
19363        let rendered = trace.to_string();
19364        assert!(
19365            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
19366                && rendered.contains("; last value non_zero(1) @"),
19367            "{rendered}"
19368        );
19369
19370        let silent = Drain::start(Answer::Silent);
19371        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
19372        // The probe is capped at the 1 s ceiling, and its reading is stamped
19373        // when it gave up.
19374        assert_eq!(
19375            trace.to_string(),
19376            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
19377        );
19378        assert_eq!(trace.probes, 1);
19379    }
19380
19381    /// A gauge value that is not a count reads busy and is shown as sent.
19382    #[test]
19383    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
19384        let gauges = [GAUGE.to_string()];
19385        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
19386        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
19387        assert_eq!(
19388            busy_gauge_observation(&verdicts),
19389            BusyGaugeObservation::Busy
19390        );
19391        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
19392        assert_eq!(
19393            busy_gauge_observation(&zero),
19394            BusyGaugeObservation::Quiescent
19395        );
19396        let none = gauge_verdicts(None, &gauges);
19397        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
19398    }
19399}
19400
19401#[cfg(test)]
19402mod health_evidence_tests {
19403    use super::{HealthProbeError, HealthProbeEvidence};
19404    use std::collections::HashSet;
19405
19406    /// The evidential asymmetry, asserted rather than described.
19407    ///
19408    /// Exactly ONE observation is proof a module cannot serve, and the one that
19409    /// fires under CPU starvation is not it. Before the split, all fifteen
19410    /// construction sites collapsed into a single String, so a timeout carried the
19411    /// same weight as a dead lane -- which is how a healthy module was restarted
19412    /// three times in one day.
19413    #[test]
19414    fn only_a_dead_lane_is_proof_of_death() {
19415        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
19416        // Three non-proof classes, each for a different reason: silence is
19417        // consistent with health, a bad answer proves the module ALIVE, and a
19418        // daemon-side fault never reached the module at all.
19419        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
19420        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
19421        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
19422    }
19423
19424    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
19425    ///
19426    /// A shared label renders two different observations identically in the line an
19427    /// operator reads after an unexplained restart -- the exact confusion this
19428    /// change removes.
19429    #[test]
19430    fn every_evidence_class_has_a_distinct_label() {
19431        let labels = [
19432            HealthProbeError::lane_dead("").label(),
19433            HealthProbeError::no_answer("").label(),
19434            HealthProbeError::bad_answer("").label(),
19435            HealthProbeError::misconfigured("").label(),
19436        ];
19437        let unique: HashSet<_> = labels.iter().collect();
19438        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
19439    }
19440
19441    /// The class is additional information, not a replacement.
19442    ///
19443    /// An operator needs both "this was silence" and the specific text saying how
19444    /// long we waited; a classification that swallowed the message would trade one
19445    /// missing distinction for another.
19446    #[test]
19447    fn classification_preserves_the_original_message() {
19448        let err = HealthProbeError::no_answer("module did not answer within 5s");
19449        assert_eq!(err.to_string(), "module did not answer within 5s");
19450        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19451    }
19452}
19453
19454#[cfg(test)]
19455mod health_tombstone_tests {
19456    use std::{path::PathBuf, sync::Arc, time::Duration};
19457
19458    use subc_protocol::{
19459        manifest::Concurrency,
19460        session::{HealthStatus, ModuleControlResponse},
19461    };
19462    use tokio::sync::mpsc;
19463
19464    use super::{
19465        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
19466        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
19467    };
19468    use crate::{
19469        control::ControlHandler,
19470        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
19471        registry::{ConnectionId, Registry},
19472        router::FrameSink,
19473    };
19474
19475    struct ProbeHarness {
19476        spec: ModuleSpec,
19477        runtime: SupervisorRuntimeConfig,
19478        forwarding: Arc<ForwardingTable>,
19479        module_connection: ConnectionId,
19480        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19481        handler: ControlHandler,
19482        module: super::SupervisedModule,
19483    }
19484
19485    fn probe_harness() -> ProbeHarness {
19486        let registry = Arc::new(Registry::default());
19487        let forwarding = Arc::new(ForwardingTable::default());
19488        let supervisor_handle = super::SupervisorHandle::new();
19489        let health = HealthConfig {
19490            http: None,
19491            cadence: Duration::from_secs(30),
19492            deadline: Duration::from_secs(5),
19493            failure_threshold: 3,
19494            on_degraded: HealthAction::Report,
19495            on_failing: HealthAction::Report,
19496            critical: false,
19497        };
19498        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
19499            .with_forwarding(Arc::clone(&forwarding))
19500            .with_handle(supervisor_handle.clone())
19501            .with_health_config(health);
19502        let spec = ModuleSpec {
19503            module_id: "late-health-module".to_string(),
19504            program: PathBuf::from("disabled-module"),
19505            args: Vec::new(),
19506            env: Vec::new(),
19507            reserved: false,
19508            reserved_prefixes: Vec::new(),
19509            protocol: ModuleProtocol::Subc,
19510            overlap: Default::default(),
19511            carry_containment: false,
19512        };
19513        let module = supervisor
19514            .supervise_configured(spec.clone(), false)
19515            .unwrap();
19516        let runtime = supervisor.runtime_config();
19517        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
19518            .with_supervisor(supervisor_handle);
19519        let module_connection = ConnectionId::new(700);
19520        let (module_tx, module_rx) = mpsc::channel(8);
19521        forwarding
19522            .register_module_connection(
19523                module_connection,
19524                spec.module_id.clone(),
19525                subc_protocol::PROTOCOL_VERSION,
19526                Concurrency::ModuleManaged,
19527                FrameSink::new(module_tx),
19528            )
19529            .unwrap();
19530
19531        ProbeHarness {
19532            spec,
19533            runtime,
19534            forwarding,
19535            module_connection,
19536            module_rx,
19537            handler,
19538            module,
19539        }
19540    }
19541
19542    async fn finish_after(
19543        harness: &mut ProbeHarness,
19544        stall: Duration,
19545    ) -> ModuleControlRpcCompletion {
19546        assert!(stall > harness.runtime.health.deadline);
19547        let deadline = harness.runtime.health.deadline;
19548        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19549        let answer = async {
19550            let frame = harness.module_rx.recv().await.expect("health.check frame");
19551            tokio::time::advance(deadline).await;
19552            tokio::task::yield_now().await;
19553            tokio::time::advance(stall - deadline).await;
19554            harness
19555                .forwarding
19556                .complete_module_control_rpc(
19557                    harness.module_connection,
19558                    frame.header.corr,
19559                    Some("health.check"),
19560                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19561                        status: HealthStatus::Ok,
19562                        detail: None,
19563                        metrics: None,
19564                    }),
19565                )
19566                .unwrap()
19567        };
19568        let (probe_result, completion) = tokio::join!(probe, answer);
19569        let err = probe_result.expect_err("probe must miss its deadline");
19570        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19571        completion
19572    }
19573
19574    async fn time_out_without_answer(harness: &mut ProbeHarness) {
19575        let deadline = harness.runtime.health.deadline;
19576        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19577        let exhaust_deadline = async {
19578            let _frame = harness.module_rx.recv().await.expect("health.check frame");
19579            tokio::time::advance(deadline).await;
19580            tokio::task::yield_now().await;
19581        };
19582        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
19583        let err = probe_result.expect_err("probe must miss its deadline");
19584        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19585    }
19586
19587    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
19588        let registry = Arc::clone(&harness.module.inner.registry);
19589        let snapshot = Arc::clone(&harness.module.inner.snapshot);
19590        let process_liveness = super::SupervisorProcessLiveness::default();
19591        let mut child = None;
19592        let cycle = super::run_health_probe_cycle(
19593            &harness.spec,
19594            &harness.runtime,
19595            &registry,
19596            &process_liveness,
19597            &snapshot,
19598            &mut child,
19599        );
19600        let peer = async {
19601            let frame = harness.module_rx.recv().await.expect("health.check frame");
19602            if answer {
19603                harness
19604                    .forwarding
19605                    .complete_module_control_rpc(
19606                        harness.module_connection,
19607                        frame.header.corr,
19608                        Some("health.check"),
19609                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19610                            status: HealthStatus::Ok,
19611                            detail: None,
19612                            metrics: Some(serde_json::json!({"ready": true})),
19613                        }),
19614                    )
19615                    .unwrap();
19616            } else {
19617                tokio::time::advance(harness.runtime.health.deadline).await;
19618                tokio::task::yield_now().await;
19619            }
19620        };
19621        tokio::join!(cycle, peer);
19622    }
19623
19624    #[tokio::test(start_paused = true)]
19625    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
19626        let mut harness = probe_harness();
19627        // Drive the probe cycle directly with an in-memory wire peer. Stop the
19628        // disabled module's monitor so only this test owns lifecycle transitions;
19629        // no OS process is launched, and a restart is observed at scheduling.
19630        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
19631        monitor.abort();
19632        let _ = monitor.await;
19633        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
19634            state.enabled = true;
19635            state.state = super::ModuleState::Running;
19636            state.process_alive = true;
19637        })
19638        .unwrap();
19639
19640        run_probe_cycle(&mut harness, true).await;
19641        assert_eq!(
19642            harness.module.status().unwrap().health.status,
19643            super::SupervisorHealthStatus::Ok
19644        );
19645
19646        for failures in 1..harness.runtime.health.failure_threshold {
19647            run_probe_cycle(&mut harness, false).await;
19648            let status = harness.module.status().unwrap();
19649            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19650            assert_eq!(status.health.consecutive_failures, failures);
19651            assert!(status.health.last_probe_ms.is_some());
19652            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
19653            assert!(status.health.metrics.is_none());
19654            assert_eq!(status.state, super::ModuleState::Running);
19655            assert!(status.process_alive);
19656            assert_eq!(status.restart_count, 0);
19657            assert_eq!(status.lifetime_restarts, 0);
19658            assert!(status.health.last_action.is_none());
19659            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19660        }
19661
19662        run_probe_cycle(&mut harness, true).await;
19663        let recovered = harness.module.status().unwrap();
19664        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
19665        assert_eq!(recovered.health.consecutive_failures, 0);
19666        assert!(recovered.health.detail.is_none());
19667        assert_eq!(
19668            recovered.health.metrics,
19669            Some(serde_json::json!({"ready": true}))
19670        );
19671        assert_eq!(recovered.lifetime_restarts, 0);
19672
19673        for failures in 1..=harness.runtime.health.failure_threshold {
19674            run_probe_cycle(&mut harness, false).await;
19675            let status = harness.module.status().unwrap();
19676            assert_eq!(status.health.consecutive_failures, failures);
19677            if failures < harness.runtime.health.failure_threshold {
19678                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19679                assert_eq!(status.state, super::ModuleState::Running);
19680                assert_eq!(status.lifetime_restarts, 0);
19681                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19682            } else {
19683                assert_eq!(
19684                    status.health.status,
19685                    super::SupervisorHealthStatus::Unresponsive
19686                );
19687                assert_eq!(status.state, super::ModuleState::Restarting);
19688                assert_eq!(status.restart_count, 1);
19689                assert_eq!(status.lifetime_restarts, 1);
19690                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
19691                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
19692            }
19693        }
19694    }
19695
19696    #[tokio::test(start_paused = true)]
19697    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
19698        let mut harness = probe_harness();
19699
19700        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
19701        let first_latency = match &first {
19702            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19703            other => panic!("late answer was not retained: {other:?}"),
19704        };
19705        assert!(harness.handler.observe_module_control_completion(first));
19706
19707        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
19708        let second_latency = match &second {
19709            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19710            other => panic!("late answer was not retained: {other:?}"),
19711        };
19712        assert!(harness.handler.observe_module_control_completion(second));
19713
19714        assert_eq!(first_latency, Duration::from_secs(8));
19715        assert_eq!(
19716            second_latency - first_latency,
19717            Duration::from_secs(3),
19718            "latency must grow linearly with the additional stall"
19719        );
19720        let health = harness.module.status().unwrap().health;
19721        assert_eq!(health.late_answer_count, 2);
19722        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
19723    }
19724
19725    /// A module that answers every probe late must never march to the kill
19726    /// threshold: the late answer proves it is alive, so it must clear the miss
19727    /// streak the timeout recorded. Without the reset, a CPU-starved module
19728    /// that serves every probe seconds past the deadline accumulates
19729    /// `consecutive_failures` to the threshold and is killed — the exact
19730    /// sequence from the 2026-08-14 aft disable, where the daemon logged
19731    /// "proves the module is alive" five times while counting five misses.
19732    #[tokio::test(start_paused = true)]
19733    async fn late_answer_clears_the_consecutive_failure_streak() {
19734        let mut harness = probe_harness();
19735
19736        // Timeout recorded first: the probe path saw no answer in time.
19737        time_out_without_answer(&mut harness).await;
19738        harness
19739            .module
19740            .record_health_probe_failure_for_test("[no-answer] test miss")
19741            .unwrap();
19742        assert_eq!(
19743            harness.module.status().unwrap().health.consecutive_failures,
19744            1,
19745            "precondition: the miss must be on the streak before the late answer"
19746        );
19747
19748        // The stalled reply then lands: proof of life.
19749        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
19750        assert!(matches!(
19751            late,
19752            ModuleControlRpcCompletion::LateHealthAnswer { .. }
19753        ));
19754        assert!(harness.handler.observe_module_control_completion(late));
19755
19756        let health = harness.module.status().unwrap().health;
19757        assert_eq!(
19758            health.consecutive_failures, 0,
19759            "a late answer is an answer: the streak must reset"
19760        );
19761        assert_eq!(health.late_answer_count, 1);
19762    }
19763
19764    #[tokio::test(start_paused = true)]
19765    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
19766        let mut harness = probe_harness();
19767
19768        for _ in 0..20 {
19769            time_out_without_answer(&mut harness).await;
19770            assert_eq!(
19771                harness.forwarding.health_probe_tombstone_count().unwrap(),
19772                1
19773            );
19774        }
19775    }
19776}
19777
19778#[cfg(test)]
19779mod child_env_tests {
19780    #[cfg(windows)]
19781    use super::{
19782        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
19783    };
19784    #[cfg(windows)]
19785    use std::sync::{Arc, OnceLock};
19786    #[cfg(windows)]
19787    #[test]
19788    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
19789        let spec = spec(Vec::new());
19790        let mut command = Command::new("unused");
19791        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
19792            .unwrap()
19793            .unwrap();
19794        let name = command
19795            .as_std()
19796            .get_envs()
19797            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
19798            .and_then(|(_, value)| value)
19799            .unwrap();
19800        assert_eq!(name, OsStr::new(handoff.name()));
19801        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
19802        assert!(command.as_std().get_envs().any(|(key, value)| key
19803            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
19804            && value == Some(OsStr::new("env"))));
19805        assert!(command
19806            .as_std()
19807            .get_envs()
19808            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
19809    }
19810
19811    #[cfg(windows)]
19812    #[test]
19813    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
19814        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
19815        snapshot.process_alive = true;
19816        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
19817        assert_eq!(
19818            observed_launch_nonce_source(&snapshot).as_deref(),
19819            Some("env")
19820        );
19821        let consumed = Arc::new(OnceLock::new());
19822        snapshot.pipe_consumed = Some(consumed.clone());
19823        assert_eq!(
19824            observed_launch_nonce_source(&snapshot).as_deref(),
19825            Some("env")
19826        );
19827        consumed.set(()).unwrap();
19828        assert_eq!(
19829            observed_launch_nonce_source(&snapshot).as_deref(),
19830            Some("pipe")
19831        );
19832        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
19833        assert_eq!(
19834            observed_launch_nonce_source(&snapshot).as_deref(),
19835            Some("env")
19836        );
19837        clear_current_process_facts(&mut snapshot);
19838        assert_eq!(observed_launch_nonce_source(&snapshot), None);
19839    }
19840    use super::{
19841        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
19842        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
19843        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
19844    };
19845    use std::{ffi::OsStr, path::PathBuf};
19846    use tokio::process::Command;
19847
19848    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
19849        ModuleSpec {
19850            module_id: "env-plan".to_string(),
19851            program: PathBuf::from("/nonexistent"),
19852            args: Vec::new(),
19853            env,
19854            reserved: false,
19855            reserved_prefixes: Vec::new(),
19856            protocol: ModuleProtocol::Subc,
19857            overlap: Default::default(),
19858            carry_containment: false,
19859        }
19860    }
19861
19862    #[test]
19863    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
19864        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
19865        let digest = LaunchSpecDigest::from_spec(&base);
19866        let mut reordered = base.clone();
19867        reordered.env.reverse();
19868        reordered.module_id = "different-id".into();
19869        reordered.reserved = true;
19870        reordered.overlap = Default::default();
19871        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
19872        let mut delimiter_collision = base.clone();
19873        delimiter_collision.env[0] = ("A=B".into(), "C".into());
19874        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
19875        let mut changed = base.clone();
19876        changed.protocol = ModuleProtocol::None;
19877        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
19878        changed = base.clone();
19879        changed.args.push("--flag".into());
19880        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
19881        changed = base.clone();
19882        changed.program = PathBuf::from("/other");
19883        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
19884        changed = base.clone();
19885        changed.env[1].1 = "other".into();
19886        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
19887        assert!(!format!("{digest:?}").contains("B=C"));
19888        assert!(!format!("{digest:?}").contains("[u8"));
19889    }
19890
19891    #[cfg(unix)]
19892    #[test]
19893    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
19894        use std::os::unix::ffi::OsStringExt;
19895        let mut left = spec(Vec::new());
19896        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
19897        let mut right = left.clone();
19898        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
19899        assert_eq!(
19900            left.program.to_string_lossy(),
19901            right.program.to_string_lossy()
19902        );
19903        assert_ne!(
19904            LaunchSpecDigest::from_spec(&left),
19905            LaunchSpecDigest::from_spec(&right)
19906        );
19907    }
19908
19909    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
19910    /// one still gets its own.
19911    ///
19912    /// This is the narrow goal `env_clear()` was reached for, and the reason the
19913    /// fix is `env_remove` rather than deleting the line: an operator's ambient
19914    /// filter silently becoming an unconfigured module's log level is a real
19915    /// defect, just a much smaller one than clearing the environment.
19916    ///
19917    /// Asserted on the command plan rather than a spawned child because proving
19918    /// the ABSENCE of an inherited variable needs the parent's environment
19919    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
19920    /// removal as `(key, None)`, which is exactly the distinction wanted: not
19921    /// "absent because nobody set it" but "explicitly unset for the child".
19922    #[test]
19923    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
19924        let mut command = Command::new("/nonexistent");
19925        apply_child_env(&mut command, &spec(Vec::new()));
19926        let removed = command
19927            .as_std()
19928            .get_envs()
19929            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
19930        assert!(
19931            removed,
19932            "ambient CK_LOG must be explicitly removed for an unconfigured module"
19933        );
19934
19935        let mut configured = Command::new("/nonexistent");
19936        apply_child_env(
19937            &mut configured,
19938            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
19939        );
19940        let effective = configured
19941            .as_std()
19942            .get_envs()
19943            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
19944            .last()
19945            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
19946        assert_eq!(
19947            effective,
19948            Some(Some("debug".to_string())),
19949            "a module's configured CK_LOG must survive the ambient removal"
19950        );
19951    }
19952
19953    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
19954    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
19955    /// the same reason as the CK_LOG test above.
19956    ///
19957    /// The argument is the load-bearing half: a stock binary exits on an
19958    /// unknown flag before it listens, so with `--subc` appended the mode
19959    /// could not supervise the one process it exists for. Found by the first
19960    /// conformance run (nats-server: `flag provided but not defined: -subc`).
19961    #[test]
19962    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
19963        let connection_file = std::path::Path::new("/run/subc-connection.json");
19964        let handle = SupervisorHandle::new();
19965
19966        let mut none_spec = spec(Vec::new());
19967        none_spec.protocol = ModuleProtocol::None;
19968        let mut none = Command::new("/nonexistent");
19969        let none_handoff =
19970            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
19971                .expect("protocol-none spawn args apply");
19972        assert!(
19973            none_handoff.is_none(),
19974            "protocol:none spawn must not receive a nonce descriptor"
19975        );
19976        #[cfg(windows)]
19977        assert!(!none.as_std().get_envs().any(|(key, value)| key
19978            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
19979            && value.is_some()));
19980        assert!(
19981            !none.as_std().get_envs().any(|(key, value)| key
19982                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
19983                && value.is_some()),
19984            "protocol:none spawn must not name a nonce descriptor"
19985        );
19986        let none_args: Vec<String> = none
19987            .as_std()
19988            .get_args()
19989            .map(|a| a.to_string_lossy().into_owned())
19990            .collect();
19991        assert!(
19992            !none_args.iter().any(|a| a == SUBC_ARG),
19993            "protocol:none argv must not carry --subc; got {none_args:?}"
19994        );
19995        let none_has_nonce = none
19996            .as_std()
19997            .get_envs()
19998            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
19999        assert!(
20000            !none_has_nonce,
20001            "protocol:none spawn must not receive a launch nonce"
20002        );
20003        let none_has_module_id = none
20004            .as_std()
20005            .get_envs()
20006            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
20007        assert!(
20008            none_has_module_id,
20009            "SUBC_MODULE_ID is inert and stays on every path"
20010        );
20011        assert!(
20012            handle.spawn_nonce(&none_spec.module_id).is_none(),
20013            "no nonce record for a process that will never present one"
20014        );
20015
20016        // Control: the subc-wire path is unchanged by the branch above.
20017        let wire_spec = spec(Vec::new());
20018        let mut wire = Command::new("/nonexistent");
20019        let wire_handoff =
20020            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
20021                .expect("subc-wire spawn args apply");
20022        let wire_fd_env = wire
20023            .as_std()
20024            .get_envs()
20025            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
20026            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20027        #[cfg(unix)]
20028        assert_eq!(
20029            wire_fd_env,
20030            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
20031            "a subc-wire spawn names the pipe it will receive at descriptor 3"
20032        );
20033        #[cfg(windows)]
20034        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
20035        #[cfg(not(any(unix, windows)))]
20036        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
20037        let wire_args: Vec<String> = wire
20038            .as_std()
20039            .get_args()
20040            .map(|a| a.to_string_lossy().into_owned())
20041            .collect();
20042        assert_eq!(
20043            wire_args,
20044            vec![
20045                SUBC_ARG.to_string(),
20046                connection_file.to_string_lossy().into_owned()
20047            ],
20048            "a subc-wire spawn still carries --subc <path>"
20049        );
20050        assert_eq!(
20051            wire.as_std()
20052                .get_envs()
20053                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
20054            !cfg!(unix),
20055            "only Windows supplies the environment nonce"
20056        );
20057        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
20058    }
20059
20060    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
20061    /// spec tries to set it; only a swap candidate carries it.
20062    ///
20063    /// "Set it only on candidates" is not enough, because spawn applies the
20064    /// spec's env verbatim and the daemon's own environment is inherited: either
20065    /// could hand a plain restart the swap role, and a module reading it would
20066    /// warm on its long swap budget while callers wait. Asserted as an explicit
20067    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
20068    /// test above gives.
20069    #[test]
20070    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
20071        let role = |command: &Command| {
20072            command
20073                .as_std()
20074                .get_envs()
20075                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
20076                .last()
20077                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
20078        };
20079        let forged = spec(vec![(
20080            SUBC_SPAWN_ROLE_ENV.to_string(),
20081            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
20082        )]);
20083
20084        let mut plain = Command::new("/nonexistent");
20085        apply_child_env(&mut plain, &forged);
20086        apply_spawn_role(&mut plain, SpawnRole::Plain);
20087        assert_eq!(
20088            role(&plain),
20089            Some(None),
20090            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
20091        );
20092
20093        let mut candidate = Command::new("/nonexistent");
20094        apply_child_env(&mut candidate, &spec(Vec::new()));
20095        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
20096        assert_eq!(
20097            role(&candidate),
20098            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
20099        );
20100    }
20101
20102    /// Daemon-private capture retention keys never reach the child.
20103    ///
20104    /// cortexkit-log exposes retention as a Rust struct with no environment
20105    /// names, so these entries are supervisor metadata. Passing them through
20106    /// would invent a public child-process contract by accident.
20107    #[test]
20108    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
20109        let mut command = Command::new("/nonexistent");
20110        apply_child_env(
20111            &mut command,
20112            &spec(vec![
20113                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
20114                ("KEPT".to_string(), "yes".to_string()),
20115            ]),
20116        );
20117        let keys: Vec<String> = command
20118            .as_std()
20119            .get_envs()
20120            .filter(|(_, value)| value.is_some())
20121            .map(|(key, _)| key.to_string_lossy().into_owned())
20122            .collect();
20123        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
20124        assert!(
20125            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
20126            "daemon-private capture key leaked to the child: {keys:?}"
20127        );
20128    }
20129}
20130
20131#[cfg(test)]
20132mod jitter_tests {
20133    use super::jittered_health_delay;
20134    use std::{collections::HashSet, time::Duration};
20135
20136    /// Module ids drawn from a real fleet, so the dispersal claim is about names
20137    /// that actually occur rather than invented ones.
20138    ///
20139    /// This is a SAMPLE, not a registry: the property under test is that distinct
20140    /// ids disperse, which holds for any set of distinct strings. Several entries
20141    /// are already historical (modules get renamed), and that costs nothing here --
20142    /// but it means a reader must not mistake this for the live module set, and a
20143    /// rename sweep will match it without there being anything to change.
20144    const FLEET: [&str; 14] = [
20145        "aft",
20146        "alfonso-core",
20147        "magic-context",
20148        "broca",
20149        "thalamus",
20150        "quota",
20151        "engram",
20152        "plexus",
20153        "cerebellum",
20154        "astrocyte",
20155        "synapse",
20156        "subc-mcp",
20157        "cortexkit-credentials",
20158        "subc-federation",
20159    ];
20160
20161    /// Probes must not converge after a fleet-wide restart.
20162    ///
20163    /// This is the property the jitter exists for: every module reconnects at
20164    /// once, and without dispersal all fourteen would then probe on the same
20165    /// tick forever. Nothing failed visibly when this went untested -- a
20166    /// convergent fleet still probes correctly, just in a burst, so the symptom
20167    /// is a periodic load spike that looks like whatever else is running.
20168    #[test]
20169    fn probe_delays_disperse_across_the_fleet() {
20170        let cadence = Duration::from_secs(30);
20171        let delays: HashSet<Duration> = FLEET
20172            .iter()
20173            .map(|id| jittered_health_delay(id, 0, cadence))
20174            .collect();
20175        assert_eq!(
20176            delays.len(),
20177            FLEET.len(),
20178            "every supervised module must land on its own probe offset"
20179        );
20180    }
20181
20182    /// The offset may only ever DELAY a probe, never bring it forward.
20183    ///
20184    /// A delay below the cadence would probe a module more often than
20185    /// configured, which is the opposite of what an operator asked for and
20186    /// would tighten the failure budget without anyone changing it.
20187    #[test]
20188    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
20189        let cadence = Duration::from_secs(30);
20190        let span = cadence / 10;
20191        for id in FLEET {
20192            for probe_index in 0..8 {
20193                let delay = jittered_health_delay(id, probe_index, cadence);
20194                assert!(
20195                    delay >= cadence,
20196                    "{id}#{probe_index}: jitter must not shorten the cadence"
20197                );
20198                assert!(
20199                    delay < cadence + span,
20200                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
20201                );
20202            }
20203        }
20204    }
20205
20206    /// A module keeps its offset across daemon restarts.
20207    ///
20208    /// The delay is derived rather than randomised precisely so a restart does
20209    /// not re-roll every module into a fresh chance of collision. A random
20210    /// source would satisfy the dispersal test above and quietly lose this.
20211    #[test]
20212    fn a_module_offset_is_stable_across_restarts() {
20213        let cadence = Duration::from_secs(30);
20214        for id in FLEET {
20215            assert_eq!(
20216                jittered_health_delay(id, 0, cadence),
20217                jittered_health_delay(id, 0, cadence),
20218                "{id}: the same module and probe index must produce the same offset"
20219            );
20220        }
20221    }
20222
20223    /// A zero cadence disables probing rather than producing a busy loop.
20224    #[test]
20225    fn zero_cadence_yields_zero_delay() {
20226        assert_eq!(
20227            jittered_health_delay("aft", 0, Duration::ZERO),
20228            Duration::ZERO
20229        );
20230    }
20231}
20232
20233#[cfg(all(test, target_os = "linux"))]
20234mod cgroup_placement_tests {
20235    use super::{
20236        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
20237        SuperviseError, SupervisedChild,
20238    };
20239    use crate::stderr_tail::{StderrRing, StderrTailConfig};
20240    use cortexkit_test_support::ScratchDir;
20241    use std::{
20242        fs, io,
20243        path::{Path, PathBuf},
20244        sync::{Arc, Mutex},
20245    };
20246    use tokio::process::Command;
20247
20248    #[tokio::test]
20249    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
20250        use super::*;
20251        let dir = ScratchDir::new("unique-spawn-cgroups");
20252        let root = PathBuf::from(format!(
20253            "/sys/fs/cgroup/subc-unique-test-{}-{}",
20254            std::process::id(),
20255            unix_ms_now()
20256        ));
20257        if let Err(error) = fs::create_dir(&root) {
20258            assert!(
20259                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
20260                "required cgroup test cannot execute: {error}"
20261            );
20262            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
20263            return;
20264        }
20265        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
20266        let group_count = || {
20267            fs::read_dir(root.join("subc-modules"))
20268                .unwrap()
20269                .map(|entry| entry.unwrap().file_type().unwrap())
20270                .filter(|kind| kind.is_dir())
20271                .count()
20272        };
20273        let supervisor =
20274            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
20275                .with_cgroup_placement(Some(placement.clone()));
20276        let runtime = supervisor.runtime_config();
20277        let mut spec = ModuleSpec {
20278            module_id: "unique-spawn".into(),
20279            program: PathBuf::from("/bin/sleep"),
20280            args: vec!["60".into()],
20281            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
20282                .into_iter()
20283                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
20284                .collect(),
20285            reserved: false,
20286            reserved_prefixes: vec![],
20287            protocol: ModuleProtocol::None,
20288            overlap: Default::default(),
20289            carry_containment: false,
20290        };
20291        let spawn = |spec: &ModuleSpec| {
20292            spawn_child(
20293                spec,
20294                None,
20295                None,
20296                &runtime.stderr_ring,
20297                None,
20298                &runtime.child_roster,
20299                runtime.spawn_delivery(),
20300                Some(&placement),
20301                runtime.register_timeout,
20302            )
20303            .unwrap()
20304        };
20305        let mut live = spawn(&spec);
20306        for _ in 0..3 {
20307            // A new process can enter the old slot while retirement is pending.
20308            let next = spawn(&spec);
20309            assert_ne!(live.module_id, next.module_id);
20310            live.start_kill().unwrap();
20311            live.wait().await.unwrap();
20312            live = next;
20313            assert!(
20314                live.child.try_wait().unwrap().is_none(),
20315                "retiring the old slot must not kill the replacement"
20316            );
20317            assert_eq!(
20318                group_count(),
20319                1,
20320                "only the live spawn's cgroup should remain"
20321            );
20322        }
20323        supervisor.begin_daemon_shutdown();
20324        let reap = tokio::spawn(async move {
20325            live.wait().await.unwrap();
20326        });
20327        supervisor
20328            .end_children_for_daemon_shutdown(false, std::future::pending())
20329            .await;
20330        reap.await.unwrap();
20331        assert_eq!(group_count(), 0);
20332        // A normal exit uses the same tree-cleanup path as a killed spawn.
20333        spec.program = PathBuf::from("/bin/true");
20334        spec.args.clear();
20335        let fresh_roster = ChildRoster::default();
20336        let mut short = spawn_child(
20337            &spec,
20338            None,
20339            None,
20340            &runtime.stderr_ring,
20341            None,
20342            &fresh_roster,
20343            None,
20344            Some(&placement),
20345            runtime.register_timeout,
20346        )
20347        .unwrap();
20348        short.wait().await.unwrap();
20349        assert_eq!(group_count(), 0);
20350        spec.module_id = "_".repeat(255);
20351        let mut long_id = spawn_child(
20352            &spec,
20353            None,
20354            None,
20355            &runtime.stderr_ring,
20356            None,
20357            &fresh_roster,
20358            None,
20359            Some(&placement),
20360            runtime.register_timeout,
20361        )
20362        .unwrap();
20363        long_id.wait().await.unwrap();
20364        assert_eq!(
20365            group_count(),
20366            0,
20367            "valid long module IDs must not exceed cgroup NAME_MAX"
20368        );
20369        fs::remove_dir(root.join("subc-modules")).unwrap();
20370        fs::remove_dir(root).unwrap();
20371        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
20372    }
20373
20374    #[test]
20375    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
20376        let path = Path::new("/definitely-missing-subc-cgroup");
20377        let mut command = Command::new("true");
20378        let error = apply_cgroup_placement(
20379            &mut command,
20380            &ModuleSpec {
20381                module_id: "broken-cgroup".to_string(),
20382                program: PathBuf::from("true"),
20383                args: Vec::new(),
20384                env: Vec::new(),
20385                reserved: false,
20386                reserved_prefixes: Vec::new(),
20387                protocol: ModuleProtocol::Subc,
20388                overlap: Default::default(),
20389                carry_containment: false,
20390            },
20391            path,
20392        )
20393        .expect_err("a parent cgroup open failure must reject the supervised spawn");
20394        let reason = error.to_string();
20395
20396        assert!(
20397            matches!(error, SuperviseError::Cgroup { .. }),
20398            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
20399        );
20400        assert!(
20401            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
20402            "parent cgroup open failure must name cgroup.procs: {reason}"
20403        );
20404    }
20405
20406    #[tokio::test]
20407    async fn reaping_a_child_removes_its_empty_module_cgroup() {
20408        let root = ScratchDir::new("supervisor-reap-cgroup");
20409        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20410        let placement = subc_cgroup::prepare_at(&root)
20411            .expect("prepare scratch cgroup root")
20412            .expect("scratch root has a cgroup.procs marker");
20413        let module_id = "reaped-module";
20414        let module = placement
20415            .module_path(module_id)
20416            .expect("create scratch module cgroup");
20417        let child = Command::new("true")
20418            .env("XDG_DATA_HOME", root.path())
20419            .env("XDG_RUNTIME_DIR", root.path())
20420            .env("XDG_CONFIG_HOME", root.path())
20421            .spawn()
20422            .expect("spawn short-lived child");
20423        let pid = child.id().expect("spawned child has pid");
20424        let mut child = SupervisedChild {
20425            registration_channel: None,
20426            registration_spawn: None,
20427            live_program_binding: None,
20428            registered_config_connection: None,
20429            admission_base: None,
20430            registration_declaration: None,
20431            _program_binding: None,
20432            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
20433            child: crate::supervise::containment::ProcessChild::plain(child, pid),
20434            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
20435            protocol: ModuleProtocol::Subc,
20436            module_id: module_id.to_string(),
20437            cgroup_placement: Some(placement),
20438            stdout_pump: None,
20439            stderr_pump: None,
20440            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
20441            spawned_at: tokio::time::Instant::now(),
20442            config_delivery: None,
20443            registration_deadline: None,
20444            registration_watch: None,
20445            spawned_at_ms: 0,
20446            spawned_from: PathBuf::from("true"),
20447            launch_spec_digest: LaunchSpecDigest([0; 32]),
20448            spawned_file_identity: None,
20449            process_start_time: None,
20450            process_identity: None,
20451            pid,
20452            roster_guard: None,
20453            #[cfg(target_os = "macos")]
20454            privacy_exec: None,
20455            spawn_failure: None,
20456            admission_kill_sent: false,
20457        };
20458
20459        child.wait().await.expect("reap short-lived child");
20460
20461        assert!(
20462            !module.exists(),
20463            "waiting for a non-carried child must remove its empty cgroup before returning"
20464        );
20465        child.finish_exit(false).await.unwrap();
20466    }
20467
20468    #[test]
20469    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
20470        let root = ScratchDir::new("supervisor-non-empty-cgroup");
20471        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20472        let placement = subc_cgroup::prepare_at(&root)
20473            .expect("prepare scratch cgroup root")
20474            .expect("scratch root has a cgroup.procs marker");
20475        let module = placement
20476            .module_path("surviving-module")
20477            .expect("create scratch module cgroup");
20478        fs::write(module.join("surviving-process"), b"still present")
20479            .expect("make scratch cgroup non-empty");
20480        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
20481
20482        remove_module_cgroup(&placement, "surviving-module");
20483
20484        let logs = crate::router::test_log::captured_logs(&logs);
20485        assert!(
20486            module.exists(),
20487            "failed removal must leave the cgroup intact"
20488        );
20489        assert!(
20490            logs.contains("could not remove module cgroup after process exit; continuing teardown")
20491                && logs.contains("surviving-module"),
20492            "best-effort removal must report the failure without returning it: {logs}"
20493        );
20494    }
20495
20496    #[test]
20497    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
20498        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
20499        let reason = SuperviseError::Spawn {
20500            program: PathBuf::from("/bin/true"),
20501            source: io::Error::from_raw_os_error(13),
20502            cgroup_path: Some(cgroup_path.clone()),
20503        }
20504        .to_string();
20505
20506        assert!(
20507            reason.contains(&cgroup_path.display().to_string()),
20508            "a pre_exec spawn failure must name the cgroup path: {reason}"
20509        );
20510    }
20511}
20512
20513#[cfg(test)]
20514mod spawn_subscriber_lag_tests {
20515    use super::*;
20516
20517    /// A subscriber whose connection stops draining is dropped once its frame
20518    /// channel fills. The client must learn that from a terminal Error frame
20519    /// after the frames already queued for it, not from a stream that simply
20520    /// goes quiet.
20521    #[tokio::test]
20522    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
20523        let feed = SpawnEventFeed::default();
20524        feed.configure_incarnation("lag-incarnation".to_string());
20525        // A one-slot connection queue that nobody reads until the emits are
20526        // done: the forwarder parks on it and the subscriber channel fills.
20527        let (tx, mut rx) = mpsc::channel(1);
20528        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
20529            .expect("subscribe");
20530        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
20531        for index in 0..emitted {
20532            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
20533            // Let the forwarder take what it can so the fill point is the
20534            // subscriber channel, not a scheduling accident.
20535            tokio::task::yield_now().await;
20536        }
20537        assert_eq!(
20538            feed.subscriber_count(),
20539            0,
20540            "the lagged subscriber must be removed"
20541        );
20542
20543        let mut data = Vec::new();
20544        let mut last = None;
20545        loop {
20546            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
20547                .await
20548                .expect("the forwarder must finish once the subscriber is dropped");
20549            let Some(outbound) = next else { break };
20550            let frame = outbound.frame;
20551            if frame.header.ty == FrameType::StreamData {
20552                assert!(last.is_none(), "no data may follow the terminal frame");
20553                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
20554                data.push(event.cursor.seq);
20555            } else {
20556                assert!(last.is_none(), "exactly one terminal frame");
20557                last = Some(frame);
20558            }
20559        }
20560        assert!(!data.is_empty(), "queued frames drain before the terminal");
20561        for pair in data.windows(2) {
20562            assert_eq!(
20563                pair[1],
20564                pair[0] + 1,
20565                "queued frames arrive dense and in order"
20566            );
20567        }
20568        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
20569        assert_eq!(terminal.header.ty, FrameType::Error);
20570        assert_eq!(terminal.header.corr, 7);
20571        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
20572        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
20573        let detail = body.detail.expect("lagged error carries detail");
20574        assert_eq!(
20575            detail["first_undelivered_cursor"]["seq"],
20576            data.last().unwrap() + 1,
20577            "the named cursor is the first event the subscriber did not receive"
20578        );
20579        assert_eq!(
20580            detail["first_undelivered_cursor"]["daemon_incarnation"],
20581            "lag-incarnation"
20582        );
20583    }
20584}
20585
20586#[cfg(test)]
20587mod terminal_history_read_concurrency_tests {
20588    use super::*;
20589    use crate::terminal_journal::read_pause;
20590    use cortexkit_test_support::ScratchDir;
20591    use std::sync::mpsc as std_mpsc;
20592
20593    fn journaled_ring(
20594        journal: &Arc<crate::terminal_journal::TerminalJournal>,
20595    ) -> Arc<Mutex<TerminalRing>> {
20596        Arc::new(Mutex::new(
20597            TerminalRing::new(TerminalRingConfig::default(), 1)
20598                .with_journal(Some(Arc::clone(journal))),
20599        ))
20600    }
20601
20602    fn crash(at_ms: u64) -> ExitReport {
20603        ExitReport {
20604            kind: ExitKind::Crash,
20605            code: Some(1),
20606            signal: None,
20607            at_ms,
20608        }
20609    }
20610
20611    /// Record an exit on another thread and report whether it finished within
20612    /// `bound`. The recorder thread is left running if it did not.
20613    fn record_within(
20614        module_id: &'static str,
20615        ring: &Arc<Mutex<TerminalRing>>,
20616        at_ms: u64,
20617        bound: Duration,
20618    ) -> bool {
20619        let ring = Arc::clone(ring);
20620        let (done, done_rx) = std_mpsc::channel();
20621        std::thread::spawn(move || {
20622            record_terminal(
20623                module_id,
20624                &ring,
20625                &SpawnEventFeed::default(),
20626                &crash(at_ms),
20627                TerminalDisposition::Restarting,
20628            );
20629            let _ = done.send(());
20630        });
20631        done_rx.recv_timeout(bound).is_ok()
20632    }
20633
20634    /// A history read in progress must not hold the journal writer (which every
20635    /// module's exit recording needs) or the module's own ring. Exits recorded
20636    /// while the read is paused complete promptly; the paused read answers as of
20637    /// the moment it started, and the next read has each exit exactly once.
20638    #[test]
20639    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
20640        let dir = ScratchDir::new("terminal-history-concurrent-read");
20641        let path = dir.join("terminals.jsonl");
20642        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
20643            path.clone(),
20644            "daemon".into(),
20645        ));
20646        let reader_ring = journaled_ring(&journal);
20647        let other_ring = journaled_ring(&journal);
20648        assert!(record_within(
20649            "reader-module",
20650            &reader_ring,
20651            10,
20652            Duration::from_secs(5)
20653        ));
20654
20655        let (started, release) = read_pause::install(&path);
20656        let reading = {
20657            let ring = Arc::clone(&reader_ring);
20658            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
20659        };
20660        started
20661            .recv_timeout(Duration::from_secs(5))
20662            .expect("the history read reached its pause");
20663
20664        let bound = Duration::from_secs(1);
20665        assert!(
20666            record_within("other-module", &other_ring, 20, bound),
20667            "another module's exit waited on a history read (journal writer held)"
20668        );
20669        assert!(
20670            record_within("reader-module", &reader_ring, 30, bound),
20671            "the read module's own exit waited on its history read (ring held)"
20672        );
20673
20674        drop(release);
20675        let paused = reading.join().unwrap();
20676        assert_eq!(
20677            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20678            vec![10],
20679            "an exit recorded after the read began lands in neither half of it"
20680        );
20681        assert_eq!(paused.journal_skipped_lines, 0);
20682        assert_eq!(paused.journal_read_errors, 0);
20683
20684        let after = durable_terminal_history_of(&reader_ring, "reader-module");
20685        assert_eq!(
20686            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20687            vec![10, 30],
20688            "the next read merges ring and journal with no duplicate"
20689        );
20690        assert_eq!(after.journal_skipped_lines, 0);
20691    }
20692}
20693
20694/// What a restart does with the exited process's output readers. These drive
20695/// the same settle functions the supervisor calls, with a reader the test
20696/// holds, so a reader that has not been scheduled by the bound is a controlled
20697/// input rather than something only a loaded machine produces.
20698#[cfg(test)]
20699mod stderr_settle_tests {
20700    use std::{
20701        future::Future,
20702        io,
20703        pin::Pin,
20704        sync::{Arc, Mutex},
20705        task::{Context, Poll},
20706        time::Duration,
20707    };
20708
20709    use tokio::{
20710        io::{AsyncRead, ReadBuf},
20711        sync::oneshot,
20712        time::Instant,
20713    };
20714
20715    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
20716    use crate::stderr_tail::{
20717        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
20718        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
20719    };
20720
20721    const BOUND: Duration = Duration::from_millis(250);
20722
20723    /// Yields `before`, then stays pending until the gate is released, then
20724    /// yields `after` and reaches EOF. The bytes after the gate were written
20725    /// by a process that has already exited; only the reader is behind.
20726    struct HeldReader {
20727        before: Option<Vec<u8>>,
20728        gate: Option<oneshot::Receiver<()>>,
20729        after: io::Cursor<Vec<u8>>,
20730    }
20731
20732    impl AsyncRead for HeldReader {
20733        fn poll_read(
20734            mut self: Pin<&mut Self>,
20735            cx: &mut Context<'_>,
20736            buf: &mut ReadBuf<'_>,
20737        ) -> Poll<io::Result<()>> {
20738            if let Some(bytes) = self.before.take() {
20739                buf.put_slice(&bytes);
20740                return Poll::Ready(Ok(()));
20741            }
20742            if let Some(gate) = self.gate.as_mut() {
20743                match Pin::new(gate).poll(cx) {
20744                    Poll::Pending => return Poll::Pending,
20745                    Poll::Ready(_) => self.gate = None,
20746                }
20747            }
20748            Pin::new(&mut self.after).poll_read(cx, buf)
20749        }
20750    }
20751
20752    struct DiscardSink;
20753
20754    impl OutputSink for DiscardSink {
20755        fn write_line(&mut self, _line: &[u8]) {}
20756    }
20757
20758    fn line(text: &str) -> TailEntry {
20759        TailEntry::Line {
20760            text: text.to_string(),
20761            truncated: false,
20762            at_ms: None,
20763        }
20764    }
20765
20766    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
20767        ring.lock().unwrap()
20768    }
20769
20770    /// Start a reader for a new process generation that delivers `before`
20771    /// immediately and `after` only once the returned sender fires (or is
20772    /// dropped).
20773    fn held_pump(
20774        ring: &Arc<Mutex<StderrRing>>,
20775        before: &str,
20776        after: &str,
20777    ) -> (StderrPump, oneshot::Sender<()>) {
20778        let generation = lock(ring).begin_process();
20779        let (release, gate) = oneshot::channel();
20780        let reader = HeldReader {
20781            before: Some(before.as_bytes().to_vec()),
20782            gate: Some(gate),
20783            after: io::Cursor::new(after.as_bytes().to_vec()),
20784        };
20785        let task = tokio::spawn(pump_stderr_to(
20786            reader,
20787            Arc::clone(ring),
20788            generation,
20789            DiscardSink,
20790        ));
20791        (StderrPump { task, generation }, release)
20792    }
20793
20794    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
20795        for _ in 0..1000 {
20796            if done(&lock(ring)) {
20797                return;
20798            }
20799            tokio::time::sleep(Duration::from_millis(1)).await;
20800        }
20801        panic!(
20802            "ring never reached the expected state: {:?}",
20803            lock(ring).snapshot(None, None)
20804        );
20805    }
20806
20807    #[tokio::test(start_paused = true)]
20808    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
20809        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
20810        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
20811
20812        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
20813        let before_release = lock(&ring).snapshot(None, None);
20814        assert!(
20815            matches!(before_release.capture, CaptureState::Incomplete { .. }),
20816            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
20817        );
20818
20819        // The restart: the next process starts and writes before the old
20820        // reader catches up.
20821        let next = lock(&ring).begin_process();
20822        lock(&ring).push_line_from(next, "next process booting");
20823        release.send(()).unwrap();
20824        wait_until(&ring, |ring| {
20825            ring.snapshot(None, None).capture == CaptureState::Captured
20826        })
20827        .await;
20828
20829        assert_eq!(
20830            untimed(lock(&ring).snapshot(None, None).entries),
20831            vec![
20832                line("booting"),
20833                line("config error: missing storage"),
20834                TailEntry::ProcessStart,
20835                line("next process booting"),
20836            ],
20837            "the crash's last line must survive a slow reader and stay in the crashed process's section"
20838        );
20839    }
20840
20841    #[tokio::test(start_paused = true)]
20842    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
20843    ) {
20844        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
20845        // `_held` is never fired: a descendant keeps the pipe open for the
20846        // whole test.
20847        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
20848
20849        let started = Instant::now();
20850        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
20851        assert_eq!(
20852            started.elapsed(),
20853            BOUND,
20854            "the restart must wait exactly the bound for a pipe that stays open, no longer"
20855        );
20856
20857        let next = lock(&ring).begin_process();
20858        lock(&ring).push_line_from(next, "next process booting");
20859        tokio::time::sleep(Duration::from_secs(60)).await;
20860
20861        let snapshot = lock(&ring).snapshot(None, None);
20862        match &snapshot.capture {
20863            CaptureState::Incomplete { reason } => assert!(
20864                reason.contains("had not reached EOF") && reason.contains("250ms"),
20865                "the reason must say what is missing and after how long: {reason}"
20866            ),
20867            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
20868        }
20869        assert_eq!(
20870            untimed(snapshot.entries),
20871            vec![
20872                line("parent exiting"),
20873                TailEntry::ProcessStart,
20874                line("next process booting"),
20875            ]
20876        );
20877    }
20878
20879    #[tokio::test(start_paused = true)]
20880    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
20881        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
20882        let (pump, release) = held_pump(&ring, "one\n", "two\n");
20883        release.send(()).unwrap();
20884
20885        settle_stderr_pump("clean", &ring, pump, BOUND).await;
20886
20887        let snapshot = lock(&ring).snapshot(None, None);
20888        assert_eq!(snapshot.capture, CaptureState::Captured);
20889        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
20890    }
20891
20892    #[tokio::test(start_paused = true)]
20893    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
20894        const BURST: usize = 150;
20895        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
20896        let path = capture.join("two-pipe.stderr.log");
20897        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
20898        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
20899        let generation = lock(&ring).begin_process();
20900        let stdout = (0..BURST)
20901            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
20902            .collect::<String>();
20903        let stderr = (0..BURST)
20904            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
20905            .collect::<String>();
20906        // Model a child that has written both bursts and exited. Hold the stdout
20907        // reader after one line and a partial prefix, as if pipe readiness or
20908        // capture-file writes had delayed it. The final line also needs EOF to
20909        // flush: neither complete nor partial buffered output may be discarded.
20910        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
20911        let (release, gate) = oneshot::channel();
20912        let reader = HeldReader {
20913            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
20914            gate: Some(gate),
20915            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
20916        };
20917        let (finished, completion) = oneshot::channel();
20918        let stdout_sink = sink.clone();
20919        let pump = tokio::spawn(async move {
20920            pump_stdout_to(reader, stdout_sink).await;
20921            finished.send(()).unwrap();
20922        });
20923        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
20924
20925        let started = Instant::now();
20926        settle_stdout_pump("two-pipe", pump, BOUND).await;
20927        assert_eq!(
20928            started.elapsed(),
20929            BOUND,
20930            "a late reader must not hold up restart"
20931        );
20932        assert_eq!(
20933            std::fs::read_to_string(&path).unwrap().lines().count(),
20934            BURST + 1
20935        );
20936
20937        release
20938            .send(())
20939            .expect("stdout reader must survive the exit drain bound");
20940        tokio::time::timeout(Duration::from_secs(1), completion)
20941            .await
20942            .expect("released stdout reader must reach EOF")
20943            .expect("stdout capture must finish at EOF, not be cancelled");
20944        let contents = std::fs::read_to_string(&path).unwrap();
20945        let mut lines = contents
20946            .lines()
20947            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
20948            .collect::<Vec<_>>();
20949        let mut expected = (0..BURST)
20950            .flat_map(|i| {
20951                [
20952                    format!("out-{i:04}-{}", "o".repeat(64)),
20953                    format!("err-{i:04}-{}", "e".repeat(64)),
20954                ]
20955            })
20956            .collect::<Vec<_>>();
20957        lines.sort_unstable();
20958        expected.sort_unstable();
20959        assert_eq!(
20960            lines, expected,
20961            "both pipes must retain every complete line exactly once"
20962        );
20963    }
20964}
20965
20966/// Containment of a module's process tree (issue #109).
20967///
20968/// The behaviour these defend against is a module helper surviving its module:
20969/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
20970/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
20971/// compounds it.
20972///
20973/// They run against the SUPERVISOR rather than the job-object crate because the
20974/// claim is about teardown: a crate-level test proves a job can reap a tree, not
20975/// that the daemon's drain path reaches it.
20976///
20977/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
20978/// lane there is a separate containment path with its own tests.
20979#[cfg(all(test, windows))]
20980mod job_containment_tests {
20981    use super::*;
20982    use cortexkit_test_support::ScratchDir;
20983    use std::{
20984        path::{Path, PathBuf},
20985        sync::{Arc, Mutex},
20986        time::{Duration, Instant},
20987    };
20988
20989    /// The stub, expected beside this test executable.
20990    ///
20991    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
20992    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
20993    /// failure then reads as a broken test rather than an unbuilt dependency.
20994    fn stub_path() -> PathBuf {
20995        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
20996        assert!(
20997            path.exists(),
20998            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
20999             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
21000            path.display()
21001        );
21002        path
21003    }
21004
21005    /// Poll for the grandchild pid the stub records, and parse it.
21006    fn read_grandchild_pid(path: &Path) -> u32 {
21007        let deadline = Instant::now() + Duration::from_secs(10);
21008        loop {
21009            if let Ok(contents) = std::fs::read_to_string(path) {
21010                if let Ok(pid) = contents.trim().parse() {
21011                    return pid;
21012                }
21013            }
21014            assert!(
21015                Instant::now() < deadline,
21016                "the stub never recorded a grandchild pid at {}",
21017                path.display()
21018            );
21019            std::thread::sleep(Duration::from_millis(10));
21020        }
21021    }
21022
21023    /// Everything one fixture run needs, so the two tests below differ in exactly
21024    /// one place: whether the child is contained.
21025    struct Fixture {
21026        _dir: ScratchDir,
21027        _roster: ChildRoster,
21028        module_id: String,
21029        grandchild: u32,
21030        child: Option<SupervisedChild>,
21031        registry: Arc<Registry>,
21032        snapshot: Arc<Mutex<SupervisorSnapshot>>,
21033        terminal_ring: Arc<Mutex<TerminalRing>>,
21034        spawn_events: SpawnEventFeed,
21035    }
21036
21037    fn fixture(label: &str, module_id: &str) -> Fixture {
21038        fixture_with_carry(label, module_id, false)
21039    }
21040
21041    fn fixture_with_carry(label: &str, module_id: &str, carry_containment: bool) -> Fixture {
21042        let dir = ScratchDir::new(label);
21043        let pid_file = dir.join("grandchild.pid");
21044        let supervisor = Supervisor::new_for_test(
21045            Arc::new(Registry::default()),
21046            RestartPolicy::new(3, Duration::ZERO),
21047        );
21048        let runtime = supervisor.runtime_config();
21049        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21050        let spec = ModuleSpec {
21051            module_id: module_id.to_string(),
21052            program: stub_path(),
21053            // Zero args deliberately: a `--subc` argument would make the stub dial
21054            // a daemon that is not there, and the failure would land in the same
21055            // stderr ring this fixture exists to keep quiet.
21056            args: Vec::new(),
21057            env: vec![
21058                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
21059                (
21060                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
21061                    pid_file.display().to_string(),
21062                ),
21063            ],
21064            reserved: false,
21065            reserved_prefixes: Vec::new(),
21066            protocol: ModuleProtocol::Subc,
21067            overlap: Default::default(),
21068            carry_containment,
21069        };
21070        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
21071            .expect("spawn the supervised fixture");
21072        let grandchild = read_grandchild_pid(&pid_file);
21073        Fixture {
21074            _dir: dir,
21075            _roster: runtime.child_roster.clone(),
21076            module_id: module_id.to_string(),
21077            grandchild,
21078            child: Some(child),
21079            registry: Arc::new(Registry::default()),
21080            snapshot,
21081            terminal_ring: Arc::clone(&runtime.terminal_ring),
21082            spawn_events: SpawnEventFeed::default(),
21083        }
21084    }
21085
21086    async fn assert_admission_kill_reaps_grandchild(carry: bool) {
21087        let label = if carry {
21088            "admission-kill-carried-tree"
21089        } else {
21090            "admission-kill-plain-tree"
21091        };
21092        let mut fixture = fixture_with_carry(label, label, carry);
21093        let descendant = subc_os::Process::open(fixture.grandchild)
21094            .unwrap()
21095            .expect("the stub descendant must be alive before admission kill");
21096        assert!(descendant.observe().is_some());
21097        let child = fixture.child.as_mut().unwrap();
21098        assert_eq!(child.child.container().is_some(), carry);
21099        assert_eq!(child.job.is_some(), !carry);
21100
21101        child.start_admission_kill().unwrap();
21102        assert!(child.admission_kill_sent);
21103        let status = timeout(Duration::from_secs(10), child.wait())
21104            .await
21105            .unwrap()
21106            .unwrap();
21107        assert_eq!(
21108            status.code(),
21109            Some(admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE),
21110            "containment teardown must preserve the leader's distinctive status"
21111        );
21112        // Keep the child and its job/container alive: later exit finalization or
21113        // handle drop must not be what makes the descendant-death claim pass.
21114        assert!(
21115            descendant.wait_for_exit(Duration::from_secs(10)).unwrap(),
21116            "admission kill left a descendant running before exit finalization"
21117        );
21118        assert_eq!(child.child.container().is_some(), carry);
21119        assert_eq!(child.job.is_some(), !carry);
21120        child.finish_exit(false).await.unwrap();
21121    }
21122
21123    #[tokio::test]
21124    async fn admission_kill_reaps_grandchild_with_plain_job() {
21125        assert_admission_kill_reaps_grandchild(false).await;
21126    }
21127
21128    #[tokio::test]
21129    async fn admission_kill_reaps_grandchild_with_carried_job() {
21130        assert_admission_kill_reaps_grandchild(true).await;
21131    }
21132
21133    #[tokio::test]
21134    async fn admission_kill_retires_registration_before_wait() {
21135        let mut fixture = fixture("admission-registration", "admission-registration");
21136        let handle = SupervisorHandle::new();
21137        let spawn =
21138            RegistrationSpawn::new(&handle, "admission-registration", SpawnRole::Plain, None);
21139        spawn.ready();
21140        let child = fixture.child.as_mut().unwrap();
21141        child.registration_spawn = Some(spawn.clone());
21142        child.start_admission_kill().unwrap();
21143        assert!(
21144            !spawn.launch_ready(),
21145            "admission kill must retire before its caller can await the child"
21146        );
21147        timeout(Duration::from_secs(10), child.wait())
21148            .await
21149            .unwrap()
21150            .unwrap();
21151        child.finish_exit(false).await.unwrap();
21152    }
21153
21154    impl Fixture {
21155        /// Drain through the supervisor's own teardown path.
21156        async fn drain(&mut self) {
21157            let child = self
21158                .child
21159                .take()
21160                .expect("the fixture child is still present");
21161            drain_child_to_state(
21162                &self.module_id,
21163                ModuleProtocol::Subc,
21164                // No forwarding table in this fixture, so nothing reaches the
21165                // child over a connection.
21166                StopNotice::NotSent,
21167                &self.registry,
21168                None,
21169                &self.snapshot,
21170                &self.terminal_ring,
21171                &self.spawn_events,
21172                child,
21173                Duration::from_millis(500),
21174                ModuleState::Stopped,
21175                Some(false),
21176                &FailureContext::operator(),
21177            )
21178            .await
21179            .expect("drain the supervised fixture");
21180        }
21181    }
21182
21183    /// Teardown reaps the grandchild, not merely the direct child.
21184    ///
21185    /// This is the assertion the change exists for. Before containment the
21186    /// grandchild survived: it is a separate process, and `start_kill` is
21187    /// `TerminateProcess` scoped to one pid.
21188    #[tokio::test]
21189    async fn teardown_reaps_the_grandchild() {
21190        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
21191        let grandchild = fixture.grandchild;
21192
21193        assert!(
21194            subc_jobobject::process_exists(grandchild),
21195            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
21196        );
21197
21198        fixture.drain().await;
21199
21200        assert!(
21201            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21202            "grandchild {grandchild} outlived module teardown: the tree was not contained"
21203        );
21204    }
21205
21206    /// The mutation control: with containment withheld, the grandchild survives
21207    /// the same kill.
21208    ///
21209    /// This is the defect reproduction from #109 — a direct-child kill reaches
21210    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
21211    /// supervisor because `spawn_and_mark_running` now always contains on
21212    /// Windows, which is the point: there is no longer a path that spawns
21213    /// uncontained, so the control has to construct one.
21214    ///
21215    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
21216    /// grandchild ever dies here, that test is passing for a reason unrelated to
21217    /// the job object and the containment claim is unproven.
21218    #[test]
21219    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
21220        let dir = ScratchDir::new("teardown-uncontained");
21221        let pid_file = dir.join("grandchild.pid");
21222        let mut child = std::process::Command::new(stub_path())
21223            .env("FAKE_AFT_NEVER_CONNECT", "1")
21224            .env(
21225                "FAKE_AFT_GRANDCHILD_PID_FILE",
21226                pid_file.display().to_string(),
21227            )
21228            .stdin(std::process::Stdio::null())
21229            .stdout(std::process::Stdio::null())
21230            .stderr(std::process::Stdio::null())
21231            .spawn()
21232            .expect("spawn the uncontained fixture");
21233        let grandchild = read_grandchild_pid(&pid_file);
21234
21235        // Exactly what the pre-fix teardown did: kill the direct child.
21236        child.kill().expect("kill the direct child");
21237        let _ = child.wait();
21238
21239        assert!(
21240            subc_jobobject::process_exists(grandchild),
21241            "grandchild {grandchild} died with the direct child, so this control no longer \
21242             distinguishes contained from uncontained teardown and the regression test is \
21243             passing vacuously"
21244        );
21245
21246        // The orphan this control demonstrates is the leak the fix prevents, so
21247        // the control must not leave one behind.
21248        kill_tree(grandchild);
21249    }
21250
21251    /// Crash durability: closing the containment handle reaps the tree with no
21252    /// teardown code running at all.
21253    ///
21254    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
21255    /// call anything — and it is why containment is a kernel property of the
21256    /// handle rather than a step in the drain. Discovered by getting the
21257    /// mutation control wrong: clearing `job` to "disable" containment instead
21258    /// killed the tree, which is the guarantee, not a mistake.
21259    #[tokio::test]
21260    async fn dropping_containment_reaps_the_grandchild() {
21261        let mut fixture = fixture("drop-containment", "tree-drop");
21262        let grandchild = fixture.grandchild;
21263
21264        assert!(subc_jobobject::process_exists(grandchild));
21265
21266        // No `drain` call, no kill: dropping the handle is the entire mechanism.
21267        fixture.child.as_mut().expect("child present").job = None;
21268
21269        assert!(
21270            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21271            "grandchild {grandchild} survived the containment handle closing, so a daemon \
21272             crash would leave the tree behind"
21273        );
21274    }
21275
21276    /// Kill a pid and its tree, then confirm it is gone.
21277    fn kill_tree(pid: u32) {
21278        let _ = std::process::Command::new("taskkill.exe")
21279            .args(["/PID", &pid.to_string(), "/T", "/F"])
21280            .stdin(std::process::Stdio::null())
21281            .stdout(std::process::Stdio::null())
21282            .stderr(std::process::Stdio::null())
21283            .status();
21284        assert!(
21285            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
21286            "could not clean up grandchild {pid}"
21287        );
21288    }
21289}
21290
21291#[cfg(test)]
21292mod privacy_trampoline_configuration_tests {
21293    #[tokio::test]
21294    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21295    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
21296        #[cfg(target_os = "macos")]
21297        {
21298            let supervisor = super::Supervisor::new(
21299                std::sync::Arc::new(crate::Registry::default()),
21300                super::RestartPolicy::default(),
21301            );
21302            let error = supervisor.spawn(spec()).unwrap_err();
21303            assert!(
21304                error
21305                    .to_string()
21306                    .contains("no privacy trampoline configured"),
21307                "{error}"
21308            );
21309        }
21310    }
21311
21312    #[tokio::test]
21313    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21314    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
21315        #[cfg(target_os = "macos")]
21316        {
21317            let supervisor = super::Supervisor::new(
21318                std::sync::Arc::new(crate::Registry::default()),
21319                super::RestartPolicy::default(),
21320            )
21321            .with_privacy_trampoline(std::env::current_exe().unwrap());
21322            let error = supervisor.spawn(spec()).unwrap_err();
21323            assert!(
21324                error
21325                    .to_string()
21326                    .contains("binary does not implement the privacy trampoline protocol"),
21327                "{error}"
21328            );
21329        }
21330    }
21331
21332    #[cfg(target_os = "macos")]
21333    fn spec() -> super::ModuleSpec {
21334        super::ModuleSpec {
21335            module_id: "privacy-configuration".into(),
21336            program: "/bin/sleep".into(),
21337            args: vec!["30".into()],
21338            env: vec![],
21339            reserved: false,
21340            reserved_prefixes: vec![],
21341            protocol: subc_control::ModuleProtocol::None,
21342            overlap: super::ModuleOverlap::Exclusive,
21343            carry_containment: false,
21344        }
21345    }
21346}
21347
21348#[cfg(test)]
21349mod privacy_exec_boundary_tests {
21350    #[cfg(target_os = "macos")]
21351    use super::*;
21352    #[cfg(target_os = "macos")]
21353    use std::{
21354        io::{Read, Write},
21355        net::{TcpListener, TcpStream},
21356    };
21357
21358    /// Unit-test-only pause at the actual early image read, not at a later
21359    /// status read. Production supervisors never inspect this environment key.
21360    #[cfg(target_os = "macos")]
21361    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
21362        if let Some((_, path)) = spec
21363            .env
21364            .iter()
21365            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
21366        {
21367            let mut barrier = TcpStream::connect(path).unwrap();
21368            barrier
21369                .set_read_timeout(Some(Duration::from_secs(30)))
21370                .unwrap();
21371            barrier.write_all(&pid.to_ne_bytes()).unwrap();
21372            let mut release = [0];
21373            barrier.read_exact(&mut release).unwrap();
21374            assert_eq!(&release, b"X");
21375        }
21376    }
21377
21378    #[cfg(target_os = "macos")]
21379    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
21380        ModuleSpec {
21381            module_id: "privacy-boundary".into(),
21382            program: program.into(),
21383            args: args.iter().map(|arg| (*arg).into()).collect(),
21384            env: vec![],
21385            reserved: false,
21386            reserved_prefixes: vec![],
21387            protocol: ModuleProtocol::None,
21388            overlap: ModuleOverlap::Exclusive,
21389            carry_containment: false,
21390        }
21391    }
21392
21393    #[cfg(target_os = "macos")]
21394    async fn accept(listener: TcpListener) -> TcpStream {
21395        // Socket readiness, not elapsed time, establishes both pause points.
21396        let listener = tokio::net::TcpListener::from_std({
21397            listener.set_nonblocking(true).unwrap();
21398            listener
21399        })
21400        .unwrap();
21401        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
21402            .await
21403            .unwrap()
21404            .unwrap();
21405        let stream = stream.into_std().unwrap();
21406        stream.set_nonblocking(false).unwrap();
21407        stream
21408            .set_read_timeout(Some(Duration::from_secs(30)))
21409            .unwrap();
21410        stream
21411    }
21412
21413    #[tokio::test]
21414    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21415    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
21416        #[cfg(target_os = "macos")]
21417        {
21418            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
21419            // Loopback sockets also work when the replay adapter's TMPDIR is
21420            // longer than Darwin's Unix-domain socket path limit.
21421            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21422            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21423            let record = root.join("live-children.json");
21424            let supervisor =
21425                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21426                    .with_live_children_record(&record);
21427            let runtime = supervisor.runtime_config();
21428            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21429            let mut spec = spec("/bin/sleep", &["30"]);
21430            spec.env = vec![
21431                (
21432                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21433                    exec_listener.local_addr().unwrap().to_string(),
21434                ),
21435                (
21436                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
21437                    sample_listener.local_addr().unwrap().to_string(),
21438                ),
21439            ];
21440            let spawn = tokio::task::spawn_blocking(move || {
21441                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
21442            });
21443            let mut sample = accept(sample_listener).await;
21444            let mut pid = [0; 4];
21445            sample.read_exact(&mut pid).unwrap();
21446            let pid = u32::from_ne_bytes(pid);
21447            let mut exec = accept(exec_listener).await;
21448            let mut ready = [0];
21449            exec.read_exact(&mut ready).unwrap();
21450            assert_eq!(&ready, b"R");
21451            // The early read is guaranteed to see a real, non-null trampoline
21452            // image: the fixture has reached its barrier and cannot exec yet.
21453            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
21454            assert_eq!(
21455                observe_spawned_image(pid).unwrap().executable,
21456                Some(trampoline)
21457            );
21458            sample.write_all(b"X").unwrap();
21459            let mut child = spawn.await.unwrap();
21460            let early = crate::live_children::read_record(&record).unwrap();
21461            assert_eq!(early.len(), 1);
21462            assert_eq!(early[0].pid, pid);
21463            assert_eq!(
21464                early[0].executable, None,
21465                "unconfirmed trampoline image entered the roster"
21466            );
21467            assert!(child.report_ready.get().is_none());
21468            // The barrier's duration is unrelated to the production five-second
21469            // exec budget. Start the test's confirmation budget upon release.
21470            child.privacy_exec.as_mut().unwrap().deadline =
21471                tokio::time::Instant::now() + Duration::from_secs(30);
21472            exec.write_all(b"X").unwrap();
21473            child.confirm_privacy_exec().await;
21474            assert_eq!(child.spawn_failure, None);
21475            assert!(child.report_ready.get().is_some());
21476            let confirmed = crate::live_children::read_record(&record).unwrap();
21477            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
21478            assert_ne!(module, trampoline);
21479            assert_eq!(confirmed[0].executable, Some(module.into()));
21480            child.start_kill().unwrap();
21481            child.wait().await.unwrap();
21482            child.release_roster();
21483        }
21484    }
21485
21486    #[tokio::test]
21487    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21488    async fn macos_unstamped_reexec_records_the_observed_native_image() {
21489        #[cfg(target_os = "macos")]
21490        {
21491            let root = cortexkit_test_support::ScratchDir::new("privacy-reexec");
21492            let record = root.join("live-children.json");
21493            let supervisor =
21494                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21495                    .with_live_children_record(&record);
21496            let runtime = supervisor.runtime_config();
21497            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21498            let fixture = crate::config_gate::program::native_fixture_for_test();
21499            let mut spec = spec(
21500                "/bin/sh",
21501                &["-c", "exec \"$1\" 600", "reexec", fixture.to_str().unwrap()],
21502            );
21503            spec.env
21504                .push(("SUBC_TEST_PRIVACY_IMAGE_DELAY_MS".into(), "5".into()));
21505            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21506            let native_image = subc_os::file_identity(&fixture).unwrap();
21507            // Native image installation, not shell startup speed, establishes
21508            // that this launch has crossed the re-exec boundary before sampling.
21509            let reexec = tokio::time::timeout(Duration::from_secs(5), async {
21510                while observe_spawned_image(child.pid).and_then(|image| image.executable)
21511                    != Some(native_image)
21512                {
21513                    tokio::time::sleep(Duration::from_millis(1)).await;
21514                }
21515            })
21516            .await;
21517            child.confirm_privacy_exec().await;
21518            let failure = child.spawn_failure.clone();
21519            let ready = child.report_ready.get().is_some();
21520            let confirmed = crate::live_children::read_record(&record).unwrap();
21521            child.start_kill().unwrap();
21522            tokio::time::timeout(Duration::from_secs(5), child.wait())
21523                .await
21524                .unwrap()
21525                .unwrap();
21526            child.release_roster();
21527
21528            reexec.expect("the shell must re-exec the native fixture");
21529            assert_eq!(
21530                failure, None,
21531                "an unstamped program may re-exec before observation"
21532            );
21533            assert!(ready);
21534            assert_eq!(confirmed.len(), 1);
21535            assert_eq!(confirmed[0].executable, Some(native_image.into()));
21536        }
21537    }
21538
21539    #[tokio::test]
21540    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21541    async fn macos_unstamped_ack_without_exec_still_refuses_the_trampoline() {
21542        #[cfg(target_os = "macos")]
21543        {
21544            let listener = TcpListener::bind("127.0.0.1:0").unwrap();
21545            let supervisor =
21546                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
21547            let runtime = supervisor.runtime_config();
21548            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21549            let fixture = crate::config_gate::program::native_fixture_for_test();
21550            let mut spec = spec(fixture.to_str().unwrap(), &["600"]);
21551            spec.env.push((
21552                "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21553                listener.local_addr().unwrap().to_string(),
21554            ));
21555            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21556            let mut exec = accept(listener).await;
21557            let mut ready = [0];
21558            exec.read_exact(&mut ready).unwrap();
21559            assert_eq!(&ready, b"R");
21560            assert_eq!(
21561                observe_spawned_image(child.pid).unwrap().executable,
21562                subc_os::file_identity(&test_privacy_trampoline())
21563            );
21564            // Simulate a broken trampoline that acknowledges exec but keeps
21565            // running its own image, held at the fixture's pre-exec barrier.
21566            let (reader, writer) = std::io::pipe().unwrap();
21567            drop(writer);
21568            child.privacy_exec.as_mut().unwrap().reader =
21569                tokio::io::unix::AsyncFd::new(reader).unwrap();
21570            child.confirm_privacy_exec().await;
21571            let failure = child.spawn_failure.clone();
21572            let ready = child.report_ready.get().is_some();
21573            child.start_kill().unwrap();
21574            tokio::time::timeout(Duration::from_secs(5), child.wait())
21575                .await
21576                .unwrap()
21577                .unwrap();
21578            child.release_roster();
21579            assert_eq!(
21580                failure.as_deref(),
21581                Some("privacy identity trampoline did not exec the module")
21582            );
21583            assert!(!ready);
21584        }
21585    }
21586
21587    #[tokio::test]
21588    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21589    async fn macos_resumed_121_is_a_module_exit_not_a_trampoline_refusal() {
21590        #[cfg(target_os = "macos")]
21591        {
21592            let registry = Arc::new(Registry::default());
21593            let policy = RestartPolicy::new(0, Duration::ZERO);
21594            let fixture = crate::config_gate::program::native_fixture_for_test();
21595            let spec = spec(fixture.to_str().unwrap(), &["exit121"]);
21596            let scratch = cortexkit_test_support::ScratchDir::new("resumed-exit");
21597            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy)
21598                .with_config_runtime(suspended_spawn_tests::gated_owner(&scratch, &spec, true));
21599            let runtime = supervisor.runtime_config();
21600            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21601            // A module cannot exit before suspended admission. After resume,
21602            // retain the real exit status even when it resembles a refusal.
21603            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21604            child.confirm_privacy_exec().await;
21605            assert_eq!(child.spawn_failure, None);
21606            assert!(child.report_ready.get().is_some());
21607            let pid = child.pid;
21608            tokio::task::spawn_blocking(move || {
21609                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
21610            })
21611            .await
21612            .unwrap()
21613            .unwrap();
21614            let status = child.wait().await.unwrap();
21615            assert_eq!(status.code(), Some(121));
21616            assert!(child.privacy_exec.is_none());
21617            let report = classify_reaped_child_exit(&snapshot, &child, &status);
21618            on_child_exit(
21619                &spec,
21620                policy,
21621                &registry,
21622                &snapshot,
21623                &runtime.terminal_ring,
21624                &runtime.spawn_events,
21625                &runtime.child_roster,
21626                report,
21627                true,
21628            )
21629            .await;
21630            let state = lock_snapshot(&snapshot).unwrap();
21631            assert_eq!(state.state, ModuleState::Failed);
21632            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
21633            assert_eq!(state.reported_pid(), None);
21634            drop(state);
21635            let history = runtime.terminal_ring.lock().unwrap().snapshot();
21636            assert_eq!(history.entries.len(), 1);
21637            let terminal = &history.entries[0];
21638            assert_eq!(terminal.exit_code, Some(121));
21639            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
21640            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
21641            assert_eq!(
21642                terminal.disposition_detail.as_deref(),
21643                Some(policy.budget_exhausted_detail().as_str()),
21644                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
21645            );
21646            assert_eq!(child.spawn_failure, None);
21647            child.release_roster();
21648        }
21649    }
21650}
21651
21652#[cfg(all(test, target_os = "macos"))]
21653#[path = "supervise/suspended_spawn_tests.rs"]
21654mod suspended_spawn_tests;
21655
21656/// The daemon's real spawn path hands a subc-wire child its launch nonce on
21657/// descriptor 3, without an environment copy. The shell records the nonce
21658/// and its environment after exec so these tests observe the real handover.
21659#[cfg(all(test, unix))]
21660mod launch_nonce_descriptor_tests {
21661    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
21662    use crate::stderr_tail::{StderrRing, StderrTailConfig};
21663    use cortexkit_test_support::ScratchDir;
21664    use std::{
21665        path::PathBuf,
21666        sync::{Arc, Mutex},
21667        time::{Duration, Instant},
21668    };
21669
21670    async fn probe(role: super::SpawnRole) {
21671        let scratch = ScratchDir::new("launch-nonce-descriptor");
21672        let fd_copy = scratch.join("from-descriptor");
21673        let env_copy = scratch.join("environment");
21674        let script = format!(
21675            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
21676            fd = fd_copy.display(), env = env_copy.display(),
21677        );
21678        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
21679        let spec = ModuleSpec {
21680            module_id: "nonce-descriptor-probe".to_string(),
21681            program: PathBuf::from("/bin/sh"),
21682            args: vec!["-c".to_string(), script],
21683            env: vec![
21684                xdg("XDG_DATA_HOME"),
21685                xdg("XDG_RUNTIME_DIR"),
21686                xdg("XDG_CONFIG_HOME"),
21687                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
21688            ],
21689            reserved: true,
21690            reserved_prefixes: Vec::new(),
21691            protocol: ModuleProtocol::Subc,
21692            overlap: Default::default(),
21693            carry_containment: false,
21694        };
21695        let handle = SupervisorHandle::new();
21696        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21697        let roster = ChildRoster::default();
21698        #[cfg(target_os = "macos")]
21699        {
21700            let path = super::test_privacy_trampoline();
21701            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
21702        }
21703        let child = super::spawn_child_in_slot(
21704            &spec,
21705            None,
21706            Some(&handle),
21707            &ring,
21708            None,
21709            &roster,
21710            None,
21711            #[cfg(target_os = "linux")]
21712            None,
21713            role,
21714            matches!(role, super::SpawnRole::SwapCandidate),
21715            super::DEFAULT_REGISTER_TIMEOUT,
21716        )
21717        .expect("spawn probe");
21718        let deadline = Instant::now() + Duration::from_secs(10);
21719        while !(fd_copy.exists() && env_copy.exists()) {
21720            assert!(Instant::now() < deadline, "probe never wrote its copies");
21721            tokio::time::sleep(Duration::from_millis(20)).await;
21722        }
21723        let nonce = std::fs::read_to_string(fd_copy).unwrap();
21724        assert!(!nonce.is_empty());
21725        let environment = std::fs::read_to_string(env_copy).unwrap();
21726        assert!(environment
21727            .lines()
21728            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
21729        let copy = environment
21730            .lines()
21731            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
21732        assert_eq!(
21733            copy, None,
21734            "Unix children must never receive the environment nonce"
21735        );
21736        if matches!(role, super::SpawnRole::Plain) {
21737            assert_eq!(
21738                handle.spawn_nonce(&spec.module_id).as_deref(),
21739                Some(nonce.as_str())
21740            );
21741        }
21742        drop(child);
21743    }
21744
21745    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
21746    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
21747        probe(super::SpawnRole::Plain).await;
21748    }
21749
21750    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
21751    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
21752        probe(super::SpawnRole::SwapCandidate).await;
21753    }
21754}
21755
21756#[cfg(all(test, target_os = "linux"))]
21757#[path = "supervise/cgroup_reap_tests.rs"]
21758mod cgroup_reap_tests;
21759
21760#[cfg(all(test, target_os = "linux"))]
21761mod cgroup_containment_tests {
21762    use super::*;
21763    use cortexkit_test_support::ScratchDir;
21764
21765    #[tokio::test]
21766    async fn linux_subtree_memory_covers_module_children() {
21767        let dir = ScratchDir::new("subtree-memory");
21768        let root = PathBuf::from(format!(
21769            "/sys/fs/cgroup/subc-memory-test-{}-{}",
21770            std::process::id(),
21771            unix_ms_now()
21772        ));
21773        if let Err(error) = std::fs::create_dir(&root) {
21774            assert!(
21775                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
21776                "required cgroup test cannot execute: {error}"
21777            );
21778            eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
21779            return;
21780        }
21781        let placement = subc_cgroup::prepare_at(&root)
21782            .unwrap()
21783            .expect("delegated cgroup");
21784        let supervisor = Supervisor::new_for_test(
21785            Arc::new(Registry::default()),
21786            RestartPolicy::new(3, Duration::ZERO),
21787        )
21788        .with_cgroup_placement(Some(placement));
21789        let ready = dir.join("ready");
21790        let module = supervisor
21791            .spawn(ModuleSpec {
21792                module_id: "memory-tree".into(),
21793                program: PathBuf::from("/bin/sh"),
21794                args: vec![
21795                    "-c".into(),
21796                    "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
21797                    "fixture".into(),
21798                    ready.display().to_string(),
21799                ],
21800                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
21801                    .into_iter()
21802                    .map(|key| (key.into(), dir.display().to_string()))
21803                    .collect(),
21804                reserved: false,
21805                reserved_prefixes: Vec::new(),
21806                protocol: ModuleProtocol::None,
21807                overlap: Default::default(),
21808                carry_containment: false,
21809            })
21810            .unwrap();
21811        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
21812        while !ready.exists() {
21813            assert!(
21814                tokio::time::Instant::now() < deadline,
21815                "children never started"
21816            );
21817            tokio::time::sleep(Duration::from_millis(10)).await;
21818        }
21819        let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
21820        let reading = &readings[0];
21821        assert!(
21822            reading.process_count.is_some_and(|count| count >= 4),
21823            "{reading:?}"
21824        );
21825        assert!(
21826            reading.memory_bytes.is_some_and(|bytes| bytes > 0),
21827            "{reading:?}"
21828        );
21829        module.stop().await.unwrap();
21830        drop(module);
21831        std::fs::remove_dir(root.join("subc-modules")).unwrap();
21832        std::fs::remove_dir(&root).unwrap();
21833        eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
21834    }
21835
21836    fn running(pid: u32) -> bool {
21837        // An orphan can remain a zombie until the container init reaps it.
21838        std::fs::read_to_string(format!("/proc/{pid}/stat"))
21839            .ok()
21840            .and_then(|stat| {
21841                stat.rsplit_once(") ")
21842                    .map(|(_, rest)| rest.starts_with('Z'))
21843            })
21844            .is_some_and(|zombie| !zombie)
21845    }
21846
21847    #[tokio::test]
21848    async fn linux_teardown_reaps_the_grandchild() {
21849        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
21850    }
21851
21852    #[tokio::test]
21853    async fn linux_shutdown_straggler_reaps_the_grandchild() {
21854        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
21855    }
21856
21857    async fn teardown_tree(test_name: &str, shutdown: bool) {
21858        let dir = ScratchDir::new(test_name);
21859        let root = PathBuf::from(format!(
21860            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
21861            std::process::id(),
21862            unix_ms_now()
21863        ));
21864        if let Err(error) = std::fs::create_dir(&root) {
21865            assert!(
21866                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
21867                "required cgroup test cannot execute: {error}"
21868            );
21869            eprintln!(
21870                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
21871                root.display()
21872            );
21873            return;
21874        }
21875        let placement = subc_cgroup::prepare_at(&root)
21876            .expect("prepare isolated kernel cgroup")
21877            .expect("isolated cgroup is delegated");
21878        let module_id = "tree-teardown";
21879        let module = placement
21880            .module_path(module_id)
21881            .expect("create isolated module cgroup");
21882        if !module.join("cgroup.kill").exists() {
21883            std::fs::remove_dir(&module).unwrap();
21884            std::fs::remove_dir(root.join("subc-modules")).unwrap();
21885            std::fs::remove_dir(&root).unwrap();
21886            assert!(
21887                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
21888                "required cgroup.kill interface unavailable"
21889            );
21890            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
21891            return;
21892        }
21893        let supervisor = Supervisor::new_for_test(
21894            Arc::new(Registry::default()),
21895            RestartPolicy::new(3, Duration::ZERO),
21896        )
21897        .with_cgroup_placement(Some(placement));
21898        let mut runtime = supervisor.runtime_config();
21899        runtime.child_roster = runtime
21900            .child_roster
21901            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
21902        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21903        let pid_file = dir.join("grandchild.pid");
21904        let spec = ModuleSpec {
21905            module_id: module_id.to_string(),
21906            program: PathBuf::from("/bin/sh"),
21907            args: vec![
21908                "-c".into(),
21909                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
21910                "fixture".into(),
21911                pid_file.display().to_string(),
21912            ],
21913            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
21914                .into_iter()
21915                .map(|key| (key.to_string(), dir.display().to_string()))
21916                .collect(),
21917            reserved: false,
21918            reserved_prefixes: Vec::new(),
21919            protocol: ModuleProtocol::None,
21920            overlap: Default::default(),
21921            carry_containment: false,
21922        };
21923        let child =
21924            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
21925        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
21926        let grandchild: u32 = loop {
21927            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
21928                if let Ok(pid) = contents.trim().parse() {
21929                    break pid;
21930                }
21931            }
21932            assert!(
21933                tokio::time::Instant::now() < deadline,
21934                "grandchild pid was not recorded"
21935            );
21936            tokio::time::sleep(Duration::from_millis(10)).await;
21937        };
21938        assert!(
21939            running(grandchild),
21940            "grandchild must be alive before teardown"
21941        );
21942        if shutdown {
21943            let mut child = child;
21944            crate::child_roster::end_children_for_daemon_shutdown(
21945                &runtime.child_roster,
21946                false,
21947                std::future::pending(),
21948            )
21949            .await;
21950            child.wait().await.expect("reap shutdown straggler");
21951        } else {
21952            drain_child_to_state(
21953                module_id,
21954                ModuleProtocol::None,
21955                StopNotice::NotSent,
21956                &Registry::default(),
21957                None,
21958                &snapshot,
21959                &runtime.terminal_ring,
21960                &SpawnEventFeed::default(),
21961                child,
21962                Duration::from_millis(100),
21963                ModuleState::Stopped,
21964                Some(false),
21965                &FailureContext::operator(),
21966            )
21967            .await
21968            .expect("real supervisor teardown");
21969        }
21970        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
21971        while running(grandchild) && tokio::time::Instant::now() < deadline {
21972            tokio::time::sleep(Duration::from_millis(10)).await;
21973        }
21974        let survived = running(grandchild);
21975        // Kill a surviving grandchild so a failed test does not leave it behind.
21976        if survived {
21977            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
21978            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
21979            tokio::time::sleep(Duration::from_millis(100)).await;
21980        }
21981        if module.exists() {
21982            std::fs::remove_dir(&module).expect("remove empty module cgroup");
21983        }
21984        std::fs::remove_dir(root.join("subc-modules")).unwrap();
21985        std::fs::remove_dir(&root).unwrap();
21986        assert!(
21987            !survived,
21988            "grandchild {grandchild} outlived module teardown"
21989        );
21990        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
21991    }
21992}
21993
21994/// A slow live-children record write on the restart path must not stop the
21995/// daemon from answering the replacement module's authentication handshake.
21996#[cfg(all(test, unix))]
21997mod record_write_stall_tests {
21998    use super::*;
21999    use std::sync::atomic::{AtomicBool, Ordering};
22000    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
22001
22002    /// `subc-client-rs` gives the whole handshake this long before the module
22003    /// exits with `Timeout { stage: ServerProof }`.
22004    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
22005    /// How long the replacement's admission write is held: longer than the
22006    /// module's deadline, standing in for the delay a busy disk can add
22007    /// across the two syncs in that write.
22008    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
22009
22010    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
22011    /// On macOS every module is launched through the privacy trampoline, and
22012    /// a module that is not a `#!` script is admitted only if its running
22013    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
22014    /// the shell soon replaces itself with `/bin/sleep`, which does not match
22015    /// the configured `/bin/sh`. The image is read only after the spawn's
22016    /// live-children record write, and here that write lasted long enough for
22017    /// the shell to have exec'd every time, so each process was refused and
22018    /// killed right after spawn, and the supervisor crash-looped it with
22019    /// backoff until its restart budget ran out. The restart this test
22020    /// measures would then never produce a module at all.
22021    fn sleeping_spec() -> ModuleSpec {
22022        ModuleSpec {
22023            module_id: "record-write-stall".into(),
22024            program: "/bin/sleep".into(),
22025            args: vec!["600".into()],
22026            env: vec![],
22027            reserved: false,
22028            reserved_prefixes: vec![],
22029            protocol: ModuleProtocol::None,
22030            overlap: Default::default(),
22031            carry_containment: false,
22032        }
22033    }
22034
22035    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22036    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
22037        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
22038        let supervisor =
22039            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
22040                .with_live_children_record(home.join("live-children.json"));
22041        let module = supervisor.spawn(sleeping_spec()).unwrap();
22042        // `reported_pid` names a process only once it is confirmed as the
22043        // module (on macOS, once the trampoline's image check has passed), so
22044        // a process the daemon is about to refuse and kill never counts as
22045        // the running module, here or in the final wait below.
22046        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
22047            loop {
22048                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
22049                    .unwrap()
22050                    .reported_pid()
22051                {
22052                    return pid;
22053                }
22054                tokio::time::sleep(Duration::from_millis(10)).await;
22055            }
22056        })
22057        .await
22058        .expect("the first process was admitted as the module");
22059
22060        // The daemon's listener, served by this runtime's workers like the
22061        // supervisor's own tasks.
22062        let key = vec![0x42; 32];
22063        let daemon_id = [0x24; 16];
22064        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22065        let port = listener.local_addr().unwrap().port();
22066        let server = tokio::spawn(crate::server::serve_listener(
22067            listener,
22068            Arc::new(crate::router::Router::with_default_self_handler()),
22069            crate::server::ServerAuth::with_limits(
22070                key.clone(),
22071                daemon_id,
22072                "test",
22073                crate::server::DEFAULT_AUTH_DEADLINE,
22074                4,
22075            ),
22076        ));
22077
22078        // Hold the write that admits the replacement, once, and say when it
22079        // has started.
22080        let (held_tx, held_rx) = std::sync::mpsc::channel();
22081        let armed = AtomicBool::new(true);
22082        supervisor
22083            .child_roster
22084            .set_record_write_hook_for_test(move |live| {
22085                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
22086                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
22087                    let _ = held_tx.send(());
22088                    std::thread::sleep(HELD_RECORD_WRITE);
22089                }
22090            });
22091
22092        // The replacement module, as a separate process would be: its own
22093        // thread and runtime, so only the daemon's runtime can be stalled.
22094        let module_process = std::thread::spawn(move || {
22095            held_rx
22096                .recv_timeout(Duration::from_secs(30))
22097                .expect("the restart reached the replacement's record write");
22098            let conn = ConnectionInfo {
22099                schema: SCHEMA_VERSION,
22100                wire_version: None,
22101                endpoints: vec![Endpoint {
22102                    host: "127.0.0.1".to_owned(),
22103                    port,
22104                }],
22105                key,
22106                daemon_id,
22107                pid: std::process::id(),
22108                daemon_ver: "test".to_owned(),
22109            };
22110            tokio::runtime::Builder::new_current_thread()
22111                .enable_all()
22112                .build()
22113                .unwrap()
22114                .block_on(async {
22115                    let started = Instant::now();
22116                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
22117                        .await
22118                        .unwrap();
22119                    let result =
22120                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
22121                    (result, started.elapsed())
22122                })
22123        });
22124
22125        module.restart(Some(0)).await.unwrap();
22126        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
22127            .await
22128            .unwrap();
22129        assert!(
22130            auth.is_ok(),
22131            "the daemon must answer a module's handshake while a restart waits on the \
22132             live-children record write; got {auth:?} after {elapsed:?}"
22133        );
22134
22135        tokio::time::timeout(Duration::from_secs(20), async {
22136            while lock_snapshot(&module.inner.snapshot)
22137                .unwrap()
22138                .reported_pid()
22139                .is_none_or(|pid| pid == old_pid)
22140            {
22141                tokio::time::sleep(Duration::from_millis(10)).await;
22142            }
22143        })
22144        .await
22145        .expect("the restart published its replacement");
22146        module.stop().await.unwrap();
22147        server.abort();
22148    }
22149}
22150
22151#[cfg(all(test, unix))]
22152mod retained_status_tests {
22153    use super::*;
22154
22155    #[tokio::test]
22156    async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
22157        use crate::live_children::{ContainerIdentity, RetainedReason};
22158        for (reason, carried, expected) in [
22159            (
22160                RetainedReason::ContainerSurvivors(vec![4242]),
22161                true,
22162                Some("waiting: container survivors (pids [4242])".to_owned()),
22163            ),
22164            (
22165                RetainedReason::ContainerUnverifiable("member inspection denied".into()),
22166                true,
22167                Some(
22168                    "waiting: container survivors could not be verified: member inspection denied"
22169                        .to_owned(),
22170                ),
22171            ),
22172            (RetainedReason::Survived, false, None),
22173        ] {
22174            let supervisor =
22175                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
22176            let spec = ModuleSpec {
22177                module_id: "retained-status-fixture".into(),
22178                program: PathBuf::from("/not-spawned"),
22179                args: Vec::new(),
22180                env: Vec::new(),
22181                reserved: false,
22182                reserved_prefixes: Vec::new(),
22183                protocol: ModuleProtocol::None,
22184                overlap: ModuleOverlap::Exclusive,
22185                carry_containment: carried,
22186            };
22187            let module = supervisor
22188                .supervise_configured(spec.clone(), false)
22189                .unwrap();
22190            // No process is launched. Invalid group zero is deliberate: if a
22191            // status read wrongly rechecks, verification changes the reason
22192            // and fails the assertions without signalling any real process.
22193            let guard = supervisor
22194                .child_roster
22195                .admit(
22196                    spec.module_id,
22197                    0,
22198                    ModuleProtocol::None,
22199                    None,
22200                    crate::child_roster::RecordedIdentity {
22201                        container: carried.then_some(ContainerIdentity {
22202                            pgid: Some(0),
22203                            created_start_time: Some(0),
22204                            cgroup_path: None,
22205                            ..ContainerIdentity::test_metadata()
22206                        }),
22207                        ..crate::child_roster::RecordedIdentity::default()
22208                    },
22209                )
22210                .unwrap();
22211            guard.retain(reason);
22212            let status = module.status().unwrap();
22213            assert_eq!(status.waiting_for_previous_detail, expected.clone());
22214            assert_eq!(status.health.detail, expected);
22215            assert!(!status.process_alive);
22216        }
22217    }
22218}
22219
22220fn registration_child_is_running(child: &mut SupervisedChild) -> bool {
22221    child.try_wait().is_ok_and(|status| status.is_none())
22222}