Skip to main content

subc_daemon/
supervise.rs

1//! Identity policy updates read the running-protocol snapshot before taking the
2//! nonce locks. Nested identity locks are taken in this order: spawn nonces,
3//! reserved nonces, then `SupervisorHandle::implicit_protocol_none`.
4
5#[cfg(all(test, target_os = "linux"))]
6#[path = "supervise/audit_identity_tests.rs"]
7mod audit_identity_tests;
8mod resource_launch;
9pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
10
11use std::{
12    collections::{HashMap, HashSet, VecDeque},
13    error::Error,
14    fmt, io,
15    path::PathBuf,
16    process::{ExitStatus, Stdio},
17    sync::{Arc, Mutex, OnceLock},
18    time::{Duration, SystemTime, UNIX_EPOCH},
19};
20
21use cortexkit_log::Retention;
22use serde_json::Value;
23use sha2::{Digest, Sha256};
24use subc_control::{
25    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
26    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
27};
28use subc_protocol::{
29    manifest::{SelfSignalKind, SignalAnchor},
30    session::{
31        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
32        MODULE_CONTROL_OP_HEALTH_CHECK,
33    },
34    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
35};
36use tokio::{
37    process::{Child, Command},
38    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
39    task::JoinHandle,
40    time::{sleep, sleep_until, timeout, timeout_at, Instant},
41};
42use tracing::{debug, error, info, warn};
43
44use crate::{
45    child_roster::ChildRoster,
46    daemon_config::{
47        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
48    },
49    forwarding::{
50        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
51        ModuleDrainTarget, PendingModuleControlRpc,
52    },
53    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
54    registry::{ConnectionId, RegistrationEndReason, RegistryError},
55    stderr_tail::{
56        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
57        StderrTailSnapshot,
58    },
59    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
60    Frame, FrameSink, Registry,
61};
62
63#[cfg(not(windows))]
64use crate::provenance::spawned_file_identity;
65
66#[path = "supervise_swap.rs"]
67mod swap;
68
69#[path = "supervise_activity.rs"]
70mod activity;
71
72#[cfg(unix)]
73#[path = "supervise/admission.rs"]
74mod admission;
75mod admission_exit;
76#[cfg(all(test, target_os = "linux"))]
77mod async_spawn_tests;
78#[cfg(all(test, windows))]
79mod carried_job_tests;
80mod config_delivery;
81mod content_binding;
82mod program_binding;
83mod registration;
84/// Each launch nonce, mapped to its launch's module id and the launch.
85type NonceLaunches = HashMap<String, (String, std::sync::Weak<RegistrationSpawn>)>;
86pub(crate) use registration::NonceAttestation;
87#[cfg(all(test, unix))]
88pub(crate) use registration::RegistrationProof;
89pub(crate) use registration::RegistrationSpawn;
90pub(crate) use registration::REGISTRATION_CHANNEL_CLOSED;
91#[cfg(all(test, unix))]
92pub(crate) mod unix_channel_placement_tests;
93use program_binding::ProgramBindingPublication;
94mod config_exit_failure;
95#[cfg(all(test, any(target_os = "linux", windows)))]
96mod missing_path_tests;
97#[cfg(all(test, unix))]
98mod program_binding_tests;
99#[cfg(all(test, windows))]
100mod program_binding_windows_tests;
101#[cfg(all(test, windows))]
102mod windows_channel_tests;
103pub use config_exit_failure::ModuleStopReason;
104#[cfg(all(test, unix))]
105mod config_exit_hold_tests;
106pub(crate) mod containment;
107#[cfg(all(test, unix))]
108mod containment_tests;
109mod slow_retry;
110use slow_retry::{FailedRetry, FailureContext};
111mod host_health;
112use host_health::HostHealth;
113
114#[cfg(all(test, unix))]
115mod mark_readiness_tests;
116#[cfg(all(test, unix))]
117mod slow_retry_tests;
118
119/// Command-line flag used by supervised modules to find subc.
120///
121/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
122/// The path points at the TCP+key connection file; it is not an ambient signal and
123/// is never inherited by standalone children.
124pub const SUBC_ARG: &str = "--subc";
125
126const DEFAULT_MAX_RESTARTS: u32 = 3;
127const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
128const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
129/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
130/// to contain a real crash loop (which respawns in seconds) and short enough
131/// that unrelated crashes hours apart never accumulate into a permanent stop.
132const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
133/// How long a drain waits for already-dispatched requests to finalize before
134/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
135/// not RPC-scale: the original 2s value silently cut nearly every real tool
136/// call at the fence, making the wait-for-finalize design decorative for the
137/// workloads it existed for. Quiescence short-circuits, so an idle module
138/// restarts immediately regardless of this value; the budget is spent only
139/// when a genuine in-flight request is worth finishing. Per-module override:
140/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
141/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
142/// where a stuck request will never settle).
143pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
144pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
145const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
146const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
147/// How long a drain waits after one busy-gauge probe completes before it asks
148/// again.
149///
150/// A drain re-probes on this short interval of its own rather than on the
151/// module's health cadence. The default cadence (30 s) equals the default
152/// drain ceiling, so a cadence-paced drain that found the module busy at its
153/// first probe never asked a second time: it waited out the whole ceiling and
154/// forced GOODBYE even after the gauges had read zero. The interval is counted
155/// from the moment the previous probe finished, and probes run one at a time,
156/// so a slow module is never asked more often than it can answer. Periodic
157/// health checks outside a drain keep using the configured cadence.
158const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
159/// How long a restart waits for the exited process's output readers.
160///
161/// The restart does not depend on either output reader finishing. A stderr
162/// reader still running at this bound is left running, and its later output goes
163/// into the exited process's own section of the stderr ring (see
164/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
165/// bound no longer decides whether a crash's last lines are kept: under load
166/// the reader may simply not have been scheduled yet, and cutting it there
167/// discarded exactly the lines that explained the crash.
168///
169/// What the bound still decides is when the tail starts reporting
170/// `Incomplete`: a pipe open past it usually means a descendant of the exited
171/// process still holds it, and the tail cannot claim to be whole until that
172/// pipe closes. That is also the only case in which the wait costs the restart
173/// anything, because a pipe with no other holder reaches EOF when the process
174/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
175/// `Captured` at EOF with nothing lost.
176///
177/// The stdout reader carries no ring, only the capture file. It also continues
178/// to EOF after this bound: child exit closes the writer but does not mean the
179/// reader has consumed all buffered bytes. Late output from either pipe can
180/// therefore reach the shared capture file after the successor starts.
181const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
182/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
183pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
184const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
185/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
186/// dropped because the subscriber stopped draining its frames. The detail's
187/// `first_undelivered_cursor` names the first event it did not receive; the
188/// client resubscribes from the last cursor it did receive.
189pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
190
191#[cfg(target_os = "macos")]
192struct PrivacyExec {
193    content_check: Option<tokio::task::JoinHandle<Result<(), crate::config_gate::GateError>>>,
194    gated: bool,
195    admission_binding: Option<ProgramBindingPublication>,
196    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
197    deadline: tokio::time::Instant,
198    expected: Option<subc_os::FileIdentity>,
199    trampoline: Option<subc_os::FileIdentity>,
200    signature: Option<Arc<subc_os::privacy_identity::SuspendedSignature>>,
201    module_id: String,
202    #[cfg(test)]
203    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
204    #[cfg(test)]
205    image_probe: Option<fn(u32) -> Option<subc_os::Observation>>,
206    #[cfg(test)]
207    image_read_delay: Option<Duration>,
208}
209
210#[cfg(all(test, target_os = "macos"))]
211pub(crate) fn test_privacy_trampoline() -> PathBuf {
212    // The fixture bin lives in the profile folder the unit-test executable was
213    // built into. This honors custom CARGO_TARGET_DIR too.
214    let path = cortexkit_test_support::cargo_profile_bin("privacy-trampoline-fixture");
215    // Without the fixture every macOS spawn is refused, and the tests that
216    // spawn fail later as a module in state Failed, which names the wrong
217    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
218    assert!(
219        path.exists(),
220        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
221         --bins --features test-support` or `cargo test -p subc-daemon` first",
222        path.display()
223    );
224    path
225}
226
227#[cfg(target_os = "macos")]
228fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
229    use std::io::Read;
230    let mut probe = subc_os::child_spawn::spawn(
231        std::process::Command::new(path)
232            .args(["__disclaim-exec", "--probe-suspended"])
233            .stdin(Stdio::null())
234            .stdout(Stdio::piped())
235            .stderr(Stdio::piped()),
236    )
237    .map_err(|error| {
238        format!(
239            "privacy trampoline probe failed for {}: {error}",
240            path.display()
241        )
242    })?;
243    let deadline = std::time::Instant::now() + Duration::from_secs(5);
244    let status = loop {
245        match probe.try_wait() {
246            Ok(Some(status)) => break status,
247            Ok(None) if std::time::Instant::now() < deadline => {
248                std::thread::sleep(Duration::from_millis(5))
249            }
250            result => {
251                let _ = probe.kill();
252                let _ = probe.wait();
253                return Err(format!(
254                    "privacy trampoline probe failed or timed out for {}: {result:?}",
255                    path.display()
256                ));
257            }
258        }
259    };
260    let mut answer = String::new();
261    if let Some(stdout) = probe.stdout.take() {
262        let _ = stdout.take(256).read_to_string(&mut answer);
263    }
264    if status.success() && answer.trim() == subc_os::privacy_identity::SUSPENDED_TRAMPOLINE_PROBE {
265        return Ok(());
266    }
267    let mut diagnostic = String::new();
268    if let Some(stderr) = probe.stderr.take() {
269        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
270    }
271    let cause = diagnostic
272        .trim()
273        .strip_prefix("ck-subc: own privacy identity refused: ")
274        .unwrap_or("binary does not implement the privacy trampoline protocol");
275    Err(format!(
276        "{cause}: probe of {} exited {status}",
277        path.display()
278    ))
279}
280
281#[cfg(target_os = "macos")]
282fn privacy_command(
283    spec: &ModuleSpec,
284    binding: &SpawnProgramBinding,
285    roster: &ChildRoster,
286    gated: bool,
287) -> Result<
288    (
289        Command,
290        Option<PrivacyExec>,
291        subc_os::privacy_identity::ExecAcknowledgement,
292    ),
293    SuperviseError,
294> {
295    let failure = |cause: String| {
296        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
297        SuperviseError::Spawn {
298            program: spec.program.clone(),
299            source: io::Error::other(cause),
300            cgroup_path: None,
301        }
302    };
303    let trampoline = roster.privacy_trampoline().map_err(failure)?;
304    let expected = Some(binding.identity);
305    let trampoline_image = subc_os::file_identity(&trampoline);
306    let signature = gated
307        .then(|| {
308            subc_os::privacy_identity::SuspendedSignature::capture(&binding._file, &binding.path)
309                .map(Arc::new)
310        })
311        .transpose()
312        .map_err(|error| failure(error.to_string()))?;
313    if expected == trampoline_image {
314        return Err(failure(
315            "privacy identity module executable is missing or is the trampoline itself".to_string(),
316        ));
317    }
318    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
319        .map_err(|error| failure(error.to_string()))?;
320    let reader =
321        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
322    let mut command = Command::new(&trampoline);
323    command
324        .arg(if gated {
325            "__disclaim-exec-suspended"
326        } else {
327            "__disclaim-exec"
328        })
329        .arg(ack.fd().to_string())
330        .arg(&binding.path);
331    Ok((
332        command,
333        Some(PrivacyExec {
334            gated,
335            admission_binding: None,
336            content_check: None,
337            reader,
338            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
339            expected,
340            trampoline: trampoline_image,
341            signature,
342            module_id: spec.module_id.clone(),
343            #[cfg(test)]
344            image_probe: None,
345            #[cfg(test)]
346            image_read_delay: spec
347                .env
348                .iter()
349                .find(|(key, _)| key == "SUBC_TEST_PRIVACY_IMAGE_DELAY_MS")
350                .map(|(_, millis)| Duration::from_millis(millis.parse().unwrap())),
351            #[cfg(test)]
352            confirmation_pause: spec
353                .env
354                .iter()
355                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
356                .map(|(_, address)| (address.clone(), None)),
357        }),
358        ack,
359    ))
360}
361
362#[cfg(unix)]
363type SpawnRegistrationChannel = std::os::unix::net::UnixStream;
364#[cfg(windows)]
365type SpawnRegistrationChannel = subc_os::registration_channel::RegistrationPipe;
366#[cfg(test)]
367type CapturedRegistrationChannels =
368    Arc<Mutex<Option<Vec<(Arc<RegistrationSpawn>, CapturedRegistrationEndpoint)>>>>;
369#[cfg(test)]
370struct CapturedRegistrationEndpoint {
371    _channel: SpawnRegistrationChannel,
372}
373#[cfg(test)]
374impl fmt::Debug for CapturedRegistrationEndpoint {
375    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
376        f.write_str("CapturedRegistrationEndpoint")
377    }
378}
379
380struct SupervisedChild {
381    registration_spawn: Option<Arc<RegistrationSpawn>>,
382    /// Owned daemon endpoint, retained for the per-spawn registration service.
383    #[cfg(any(unix, windows))]
384    registration_channel: Option<SpawnRegistrationChannel>,
385    live_program_binding: Option<Arc<content_binding::LiveProgramBinding>>,
386    registered_config_connection: Option<ConnectionId>,
387    admission_base: Option<subc_config::Digest>,
388    registration_declaration: Option<String>,
389    _program_binding: Option<ProgramBindingPublication>,
390    subtree_target: crate::subtree_resources::Target,
391    child: containment::ProcessChild,
392    exec: crate::activity::ExecOwner,
393    #[cfg(windows)]
394    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
395    #[cfg(target_os = "macos")]
396    privacy_exec: Option<PrivacyExec>,
397    /// Set once this launch's exec acknowledgement confirms the module image.
398    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
399    /// `subc_os::privacy_identity`), which then replaces itself with the
400    /// module. The supervisor owns and can kill that pid from spawn, but
401    /// status readers report it only after this latch is set, so nothing
402    /// reports the trampoline's image as the module's.
403    #[cfg(target_os = "macos")]
404    report_ready: Arc<OnceLock<()>>,
405    /// Supervisor failure detail retained for the terminal record, including
406    /// image refusals and first-HELLO timeouts after a successful spawn.
407    spawn_failure: Option<String>,
408    /// Set only after a kill requested because launch admission failed was
409    /// accepted, never for an operator, health, or shutdown kill.
410    admission_kill_sent: bool,
411    /// The protocol this process was launched with. A reload can store a new
412    /// launch spec with a different protocol, but that takes effect only at the
413    /// next spawn, so this process keeps being handled by the protocol it
414    /// actually speaks.
415    protocol: ModuleProtocol,
416    /// This container's cgroup name: a bounded module/slot label followed by a
417    /// unique suffix (when cgroup placement is on). Carried replacements reuse
418    /// the name; ordinary launches each get a fresh container. A
419    /// retired process in a slot may still be draining when a later one is
420    /// spawned into that slot, so the suffix keeps the later process out of
421    /// the retired one's cgroup, which is the domain a kill applies to.
422    #[cfg(target_os = "linux")]
423    module_id: String,
424    #[cfg(target_os = "linux")]
425    cgroup_placement: Option<subc_cgroup::Placement>,
426    /// The job for an ordinary, non-carried launch. For carried
427    /// launches the actor owns it and the child holds only a weak container link.
428    ///
429    /// Dropping this handle is what reaps a surviving tree when no supervisor
430    /// code runs — a daemon crash — because the job carries
431    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
432    ///
433    /// That limit is not crash-only, and the difference is worth knowing: a
434    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
435    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
436    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
437    /// module at once. Before this change they survived that, saw EOF on the
438    /// control socket, and ran their own teardown; Unix keeps that path
439    /// deliberately, so a module can seal a WAL or close a capture rather than
440    /// be killed mid-write. So this trades graceful teardown on every Windows
441    /// daemon stop for containment on a crash, which is the right way round
442    /// today: orphaned GPU workers are a reported, recurring problem, and the
443    /// modules that write most heavily do not run on Windows.
444    ///
445    /// The fix is a real Windows stop path — the daemon draining before it
446    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
447    /// reaches only what the drain left behind, which is what it should reach.
448    #[cfg(windows)]
449    job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
450    stdout_pump: Option<JoinHandle<()>>,
451    stderr_pump: Option<StderrPump>,
452    stderr_ring: Arc<Mutex<StderrRing>>,
453    /// Conservative monotonic bound sampled before `command.spawn()`, not when
454    /// identity recording finishes or the spawn generation is published.
455    spawned_at: Instant,
456    config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
457    /// Ordinary spawns must send their first HELLO within this budget. Once
458    /// accepted, a later disconnect must not restart the registration clock.
459    registration_deadline: Option<Instant>,
460    registration_watch: Option<RegistrationWatch>,
461    spawned_at_ms: u64,
462    spawned_from: PathBuf,
463    launch_spec_digest: LaunchSpecDigest,
464    spawned_file_identity: Option<SpawnedFileIdentity>,
465    process_start_time: Option<u64>,
466    process_identity: Option<ProcessIdentity>,
467    pid: u32,
468    /// This process's entry in the daemon's child roster, released when the
469    /// process is reaped or this handle is dropped.
470    roster_guard: Option<crate::child_roster::RosterGuard>,
471}
472
473impl SupervisedChild {
474    fn retire_registration(&self) {
475        // All exit observers enter here before cleanup can await. The retired
476        // program-binding flag and the registration's Dead state make repeated
477        // observations harmless, even when another task observes the group
478        // leader retained unreaped to reserve its process-group id.
479        if let Some(spawn) = &self.registration_spawn {
480            spawn.retire();
481        } else if let Some(binding) = &self._program_binding {
482            binding.retire();
483        }
484        #[cfg(unix)]
485        if let Some(channel) = &self.registration_channel {
486            let _ = channel.shutdown(std::net::Shutdown::Both);
487        }
488        #[cfg(windows)]
489        if let Some(channel) = &self.registration_channel {
490            if let Ok(shutdown) = channel.shutdown_handle() {
491                shutdown.shutdown();
492            }
493        }
494    }
495    fn registered_since_spawn(&self) -> bool {
496        self.registration_watch
497            .as_ref()
498            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
499    }
500
501    fn awaiting_registration(&self) -> bool {
502        // Admission failure retires the channel before its kill becomes
503        // waitable. That closure must not replace the admission failure with
504        // a registration failure and spend crash restart budget.
505        let admission_failed = self.admission_kill_sent && self.spawn_failure.is_some();
506        !admission_failed && !self.registered_since_spawn()
507    }
508
509    fn id(&self) -> Option<u32> {
510        Some(self.pid)
511    }
512
513    fn process_identity(&self) -> Option<ProcessIdentity> {
514        self.process_identity
515    }
516
517    fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
518        let result = self.child.try_wait();
519        if !matches!(result, Ok(None)) {
520            self.retire_registration();
521            #[cfg(any(unix, windows))]
522            self.registration_channel.take();
523            if let Some(binding) = &self._program_binding {
524                binding.retire();
525            }
526        }
527        result
528    }
529
530    async fn wait(&mut self) -> io::Result<ExitStatus> {
531        #[cfg(target_os = "macos")]
532        self.confirm_privacy_exec().await;
533        // The roster entry is NOT released here. A daemon shutdown waits for the
534        // roster to empty and then exits the process, so releasing at the reap
535        // let it exit before the exit handler wrote this child's terminal record
536        // (the stderr drain and snapshot update sit in between), and the
537        // shutdown's own `daemon_shutdown` record was intermittently lost. The
538        // caller releases it after recording the exit (`release_roster`), and
539        // dropping the handle releases it too.
540        let result = self.child.wait().await;
541        self.retire_registration();
542        #[cfg(any(unix, windows))]
543        self.registration_channel.take();
544        // All plain and swap waits pass here. Revoke the reaped child's nonce
545        // before cgroup cleanup, stderr draining or exit observation can await.
546        if let Some(binding) = &self._program_binding {
547            binding.retire();
548        }
549        #[cfg(all(test, target_os = "linux"))]
550        if let Some(binding) = &self._program_binding {
551            binding.pause_after_wait_for_test().await;
552        }
553        #[cfg(target_os = "linux")]
554        if result.is_ok() && self.child.container().is_none() {
555            if let Some(placement) = self.cgroup_placement.as_ref() {
556                // Ordinary children still own a per-spawn cgroup: returning
557                // from wait must leave it empty and removed, as before carry.
558                // Keep cleanup ownership across cancellation until it finishes.
559                cleanup_reaped_cgroup(placement, &self.module_id).await;
560                self.cgroup_placement = None;
561            }
562        }
563        result
564    }
565
566    async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
567        self.retire_registration();
568        // Idempotent fallback for cleanup paths that did not call wait.
569        if let Some(binding) = self._program_binding.take() {
570            binding.retire();
571        }
572        if self.child.container().is_some() {
573            if !carry {
574                self.child.terminate_container().await;
575            }
576        } else {
577            #[cfg(target_os = "linux")]
578            if let Some(placement) = self.cgroup_placement.as_ref() {
579                cleanup_reaped_cgroup(placement, &self.module_id).await;
580                self.cgroup_placement = None;
581            }
582        }
583        self.child.reap().await?;
584        self.release_roster();
585        Ok(())
586    }
587
588    #[cfg(target_os = "macos")]
589    async fn confirm_privacy_exec(&mut self) {
590        if self
591            .privacy_exec
592            .as_ref()
593            .is_some_and(|pending| pending.content_check.is_some())
594        {
595            self.finish_privacy_content_check().await;
596            return;
597        }
598        let Some(pending) = &mut self.privacy_exec else {
599            return;
600        };
601        let result = tokio::time::timeout_at(pending.deadline, async {
602            let mut record = Vec::new();
603            loop {
604                let mut ready = pending.reader.readable().await?;
605                let read = ready.try_io(|reader| {
606                    use std::io::Read;
607                    let mut reader = reader.get_ref();
608                    let mut buffer = [0u8; 256];
609                    reader.read(&mut buffer).map(|count| (count, buffer))
610                });
611                match read {
612                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
613                    Ok(Ok((count, buffer))) => {
614                        if record.len() + count > 1024 {
615                            return Err(io::Error::other(
616                                "privacy exec refusal record is too long",
617                            ));
618                        }
619                        record.extend_from_slice(&buffer[..count]);
620                    }
621                    Ok(Err(error)) => return Err(error),
622                    Err(_) => continue,
623                }
624            }
625        })
626        .await;
627        #[cfg(test)]
628        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
629            use tokio::io::{AsyncReadExt, AsyncWriteExt};
630            if stream.is_none() {
631                *stream = Some(
632                    tokio::net::TcpStream::connect(address.as_str())
633                        .await
634                        .unwrap(),
635                );
636                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
637                stream
638                    .as_mut()
639                    .unwrap()
640                    .write_all(&self.pid.to_ne_bytes())
641                    .await
642                    .unwrap();
643            }
644            let mut reply = [0];
645            stream
646                .as_mut()
647                .unwrap()
648                .read_exact(&mut reply)
649                .await
650                .unwrap();
651            pending.confirmation_pause = None;
652            if &reply == b"F" {
653                Ok(Err(io::Error::other("injected late exec refusal")))
654            } else {
655                assert_eq!(&reply, b"X");
656                result
657            }
658        } else {
659            result
660        };
661        // Keep the reader in self across await: select cancellation must not
662        // discard the handshake or reset its original five-second deadline.
663        // Use try_wait so a child reaped during confirmation has its program
664        // binding retired before another command can use its nonce.
665        let waited = if matches!(&result, Ok(Ok(record)) if record.is_empty()) {
666            Some(self.try_wait())
667        } else {
668            None
669        };
670        let pending = self.privacy_exec.as_ref().expect("pending exec");
671        let cause = match result {
672            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
673            Ok(Err(error)) => Some(format!(
674                "privacy identity exec acknowledgement failed: {error}"
675            )),
676            Ok(Ok(record)) if !record.is_empty() => Some(
677                std::str::from_utf8(&record)
678                    .ok()
679                    .and_then(|record| {
680                        record
681                            .trim()
682                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
683                    })
684                    .filter(|cause| !cause.is_empty())
685                    .unwrap_or("invalid privacy exec refusal record")
686                    .to_string(),
687            ),
688            Ok(Ok(_)) if !pending.gated => match waited.expect("exec EOF checked child") {
689                Ok(Some(_)) => None,
690                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
691                Ok(None) => {
692                    #[cfg(test)]
693                    if let Some(delay) = pending.image_read_delay {
694                        tokio::time::sleep(delay).await;
695                    }
696                    let image = observe_spawned_image(self.pid);
697                    #[cfg(test)]
698                    let image = pending.image_probe.map_or(image, |probe| probe(self.pid));
699                    // An unstamped module can re-exec before we wake to read its
700                    // image. Only the trampoline image proves exec never happened;
701                    // retain the observed image for later orphan identification.
702                    if let Some(image) = image.filter(|image| {
703                        image.executable.is_some() && image.executable != pending.trampoline
704                    }) {
705                        match self
706                            .roster_guard
707                            .as_ref()
708                            .map(|guard| guard.confirm_executable(image))
709                            .transpose()
710                        {
711                            Err(error) => Some(format!(
712                                "could not durably record confirmed module image: {error}"
713                            )),
714                            Ok(_) => {
715                                let _ = self.report_ready.set(());
716                                None
717                            }
718                        }
719                    } else if image.is_none()
720                        || image.is_some_and(|image| image.executable.is_none())
721                    {
722                        // Empty EOF acknowledged exec. An image disappearing
723                        // during observation is an ordinary module exit.
724                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
725                            Ok(result) => {
726                                self.retire_registration();
727                                self.registration_channel.take();
728                                if let Some(binding) = &self._program_binding {
729                                    binding.retire();
730                                }
731                                result.err().map(|error| format!("privacy identity trampoline wait failed: {error}"))
732                            }
733                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
734                        }
735                    } else {
736                        Some("privacy identity trampoline did not exec the module".to_string())
737                    }
738                }
739            },
740            Ok(Ok(_)) => match waited.expect("exec EOF checked child") {
741                Ok(Some(_)) => {
742                    Some("privacy identity target exited before suspended admission".to_string())
743                }
744                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
745                Ok(None) => {
746                    // CLOEXEC closes the ack just before the kernel reports the
747                    // suspended target. EOF alone must never admit a running image.
748                    let stopped = tokio::time::timeout_at(pending.deadline, async {
749                        loop {
750                            if subc_os::privacy_identity::SuspendedSignature::is_suspended(
751                                self.pid,
752                            )? {
753                                return Ok::<_, io::Error>(());
754                            }
755                            tokio::time::sleep(Duration::from_millis(1)).await;
756                        }
757                    })
758                    .await;
759                    match stopped {
760                        Err(_) => {
761                            Some("privacy identity target was not suspended within 5s".to_string())
762                        }
763                        Ok(Err(error)) => Some(format!(
764                            "privacy identity suspended target unavailable: {error}"
765                        )),
766                        Ok(Ok(())) => {
767                            let image = observe_spawned_image(self.pid);
768                            if image.is_some_and(|image| {
769                                image.executable.is_some()
770                                    && image.executable != pending.trampoline
771                                    && image.executable == pending.expected
772                            }) {
773                                None
774                            } else {
775                                Some("privacy identity trampoline executable mismatch: expected the suspended module program".to_string())
776                            }
777                        }
778                    }
779                }
780            },
781        };
782        #[cfg(test)]
783        if let Some(binding) = &self._program_binding {
784            binding.pause_after_wait_for_test().await;
785        }
786        if cause.is_none() && pending.gated {
787            let publication = pending.admission_binding.clone();
788            let live = self.live_program_binding.clone();
789            let signature = Arc::clone(pending.signature.as_ref().expect("gated signature"));
790            let pid = self.pid;
791            let gated = pending.gated;
792            self.privacy_exec
793                .as_mut()
794                .expect("pending exec")
795                .content_check = Some(tokio::task::spawn_blocking(move || {
796                signature.verify(pid).map_err(|error| {
797                    crate::config_gate::GateError::new("program_refused", error)
798                })?;
799                if !gated {
800                    return Ok(());
801                }
802                match (publication, live) {
803                    (Some(publication), _) => publication.verify(),
804                    (None, Some(live)) => live.check(),
805                    _ => Err(crate::config_gate::GateError::new(
806                        "config_delivery_deferred",
807                        "live program witness is unavailable",
808                    )),
809                }
810            }));
811            self.finish_privacy_content_check().await;
812        } else {
813            self.finish_privacy_exec(cause);
814        }
815    }
816
817    #[cfg(target_os = "macos")]
818    async fn finish_privacy_content_check(&mut self) {
819        let result = self
820            .privacy_exec
821            .as_mut()
822            .expect("pending exec")
823            .content_check
824            .as_mut()
825            .expect("pending content check")
826            .await;
827        let cause = match result {
828            Ok(Ok(())) => None,
829            Ok(Err(error)) => Some(format!("{}: {}", error.code, error.detail)),
830            Err(error) => Some(format!(
831                "config_delivery_deferred: content check stopped: {error}"
832            )),
833        };
834        self.finish_privacy_exec(cause);
835    }
836
837    #[cfg(target_os = "macos")]
838    fn finish_privacy_exec(&mut self, mut cause: Option<String>) {
839        let mut pending = self.privacy_exec.take().expect("pending exec");
840        if cause.is_none() && pending.gated {
841            let admission = (|| -> io::Result<()> {
842                let image = observe_spawned_image(self.pid)
843                    .filter(|image| image.executable == pending.expected)
844                    .ok_or_else(|| io::Error::other("suspended executable mismatch"))?;
845                if let Some(guard) = &self.roster_guard {
846                    guard.confirm_executable(image)?;
847                }
848                pending
849                    .signature
850                    .as_ref()
851                    .expect("gated signature")
852                    .resume(self.pid)?;
853                let _ = self.report_ready.set(());
854                info!(module_id = %pending.module_id, pid = self.pid,
855                    "verified suspended module resumed with own privacy identity");
856                Ok(())
857            })();
858            if let Err(error) = admission {
859                cause = Some(format!("program_refused: {error}"));
860            }
861        }
862        if cause.is_some() {
863            if let Some(spawn) = &self.registration_spawn {
864                spawn.refuse();
865            }
866            if let Some(binding) = &mut pending.admission_binding {
867                binding.reject();
868            }
869        } else if let Some(spawn) = &self.registration_spawn {
870            spawn.ready();
871        }
872        // Exec confirmation describes launch success, not roster retirement.
873        // Keep even a reaped child recorded until its terminal history exists.
874        self.exec.0.finish(if cause.is_some() {
875            crate::activity::ExecState::Refused
876        } else {
877            crate::activity::ExecState::CommittedExec
878        });
879        if let Some(cause) = cause {
880            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
881            self.spawn_failure = Some(
882                if pending.gated
883                    && (cause.contains("executable mismatch")
884                        || cause.starts_with("program_refused:"))
885                {
886                    format!("program_refused: {cause}")
887                } else if pending.gated {
888                    format!("config_delivery_deferred: {cause}")
889                } else {
890                    cause
891                },
892            );
893            // No resume signal is sent on failure; kill the entire fresh group
894            // even if a faulty trampoline did not honor the suspended protocol.
895            let _ = self.start_admission_kill();
896        }
897    }
898
899    /// Releases the leader's roster entry after recording and finalizing its
900    /// exit. A carried container retains its separate roster entry and, for a
901    /// Unix process group, its unreaped first leader that reserves the group ID.
902    fn release_roster(&mut self) {
903        self.roster_guard = None;
904    }
905
906    /// Kill the child and, where containment is available, its process tree.
907    ///
908    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
909    /// helper process leaked the helper — the Synapse embedding module's CUDA
910    /// worker holds the GPU allocation, so the leak cost VRAM until the next
911    /// restart of something else. Terminating the job reaches grandchildren that
912    /// a tree walk cannot, including one whose parent has already exited and
913    /// been reparented away.
914    ///
915    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
916    /// direct-child kill still decides the outcome, so containment can never
917    /// change whether a module is reported as stopped.
918    fn start_kill(&mut self) -> io::Result<()> {
919        self.retire_registration();
920        if let Some(container) = self.child.container() {
921            container.hard_kill();
922            return self.kill_leader();
923        }
924        #[cfg(windows)]
925        if let Some(job) = &self.job {
926            if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
927                debug!(
928                    error = %error,
929                    "job termination failed; the direct-child kill still owns the outcome"
930                );
931            }
932        }
933        #[cfg(target_os = "linux")]
934        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
935        self.child.start_kill()
936    }
937
938    fn kill_leader(&mut self) -> io::Result<()> {
939        self.retire_registration();
940        self.child.start_kill()
941    }
942
943    fn start_admission_kill(&mut self) -> io::Result<()> {
944        self.retire_registration();
945        #[cfg(windows)]
946        let result = {
947            // Preserve the leader's distinctive status before ending the whole
948            // container/job; descendants must not wait for exit finalization.
949            let result = self
950                .spawned_file_identity
951                .as_ref()
952                .and_then(|identity| identity.windows_image.as_ref())
953                .ok_or_else(|| io::Error::other("admission child handle unavailable"))
954                .and_then(|image| {
955                    image.process().terminate_with_exit_code(
956                        admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE as u32,
957                    )
958                });
959            if let Some(container) = self.child.container() {
960                container.hard_kill();
961            } else if let Some(job) = &self.job {
962                if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
963                    debug!(%error, "job termination failed after admission leader termination");
964                }
965            }
966            result
967        };
968        #[cfg(not(windows))]
969        let result = self.start_kill().map(|()| true);
970        #[cfg(unix)]
971        let result = {
972            // A leader can exit before its helpers. Still end the fresh group
973            // even when the direct-child kill reports that the leader is gone.
974            let group_sent = self.child.container().is_none()
975                && rustix::process::Pid::from_raw(self.pid as i32).is_some_and(|group| {
976                    rustix::process::kill_process_group(group, rustix::process::Signal::KILL)
977                        .is_ok()
978                });
979            if group_sent {
980                Ok(true)
981            } else {
982                result
983            }
984        };
985        self.admission_kill_sent |= result?;
986        Ok(())
987    }
988
989    async fn drain_stderr(&mut self, module_id: &str) {
990        if let Some(pump) = self.stdout_pump.take() {
991            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
992        }
993
994        let Some(pump) = self.stderr_pump.take() else {
995            return;
996        };
997        settle_stderr_pump(
998            module_id,
999            &self.stderr_ring,
1000            pump,
1001            STDERR_PUMP_DRAIN_TIMEOUT,
1002        )
1003        .await;
1004    }
1005}
1006
1007/// Bound the restart's wait, not the lifetime of the exited child's stdout
1008/// reader. Slow pipe reads or capture writes must not turn the bound into lost
1009/// output; dropping the handle lets the reader finish at EOF, like stderr.
1010async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
1011    match timeout(bound, &mut pump).await {
1012        Ok(Ok(())) => {}
1013        Ok(Err(error)) => {
1014            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
1015        }
1016        Err(_) => {
1017            drop(pump);
1018            warn!(
1019                module_id,
1020                waited = ?bound,
1021                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
1022            );
1023        }
1024    }
1025}
1026
1027/// The reader task for one process's stderr, with the ring generation its
1028/// lines are attributed to.
1029struct StderrPump {
1030    task: JoinHandle<()>,
1031    generation: u64,
1032}
1033
1034/// Retire an exited process's stderr reader and wait up to `bound` for it to
1035/// reach EOF. A reader still running at the bound is detached, not stopped: it
1036/// keeps filling the exited process's section of the ring until its pipe
1037/// closes, and the tail reads `Incomplete` until then. See
1038/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
1039async fn settle_stderr_pump(
1040    module_id: &str,
1041    ring: &Arc<Mutex<StderrRing>>,
1042    pump: StderrPump,
1043    bound: Duration,
1044) {
1045    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
1046    let StderrPump {
1047        mut task,
1048        generation,
1049    } = pump;
1050    lock().retire_pump(generation);
1051    match timeout(bound, &mut task).await {
1052        Ok(Ok(())) => {}
1053        Ok(Err(err)) => {
1054            let mut ring = lock();
1055            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
1056            ring.finish_pump(generation);
1057            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
1058        }
1059        Err(_) => {
1060            // Dropping the handle detaches the task; it ends at EOF on its pipe.
1061            drop(task);
1062            lock().mark_pump_late(
1063                generation,
1064                format!(
1065                    "stderr of the exited process had not reached EOF {bound:?} after it was \
1066                     retired (a descendant may still hold the pipe open); lines it still \
1067                     writes are kept in that process's section"
1068                ),
1069            );
1070            warn!(
1071                module_id,
1072                waited = ?bound,
1073                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
1074            );
1075        }
1076    }
1077}
1078
1079fn registration_release_events() -> &'static watch::Sender<u64> {
1080    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
1081    EVENTS.get_or_init(|| {
1082        let (sender, _receiver) = watch::channel(0);
1083        sender
1084    })
1085}
1086
1087pub(crate) fn notify_registration_release() {
1088    let events = registration_release_events();
1089    let next_generation = (*events.borrow()).wrapping_add(1);
1090    events.send_replace(next_generation);
1091}
1092
1093/// How to launch one singleton module process.
1094#[derive(Debug, Clone, PartialEq, Eq)]
1095pub struct ModuleSpec {
1096    pub module_id: String,
1097    pub program: PathBuf,
1098    pub args: Vec<String>,
1099    pub env: Vec<(String, String)>,
1100    /// When true this is a reserved module: each spawn gets a fresh one-time launch
1101    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
1102    /// process can register this module_id (a security-boundary module like the
1103    /// credential vault must not be impersonable while it is down/restarting).
1104    pub reserved: bool,
1105    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
1106    /// Prefixes come from daemon config and must end in `:` before they reach the
1107    /// supervisor; the owner module's current spawn nonce authorizes claims under
1108    /// each prefix.
1109    pub reserved_prefixes: Vec<String>,
1110    /// The wire protocol this module speaks, as DECLARED in daemon config.
1111    ///
1112    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
1113    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
1114    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
1115    /// and NO launch nonce, and a clean exit the daemon did not request is
1116    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
1117    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
1118    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
1119    /// because a process ignores an environment variable it does not read.
1120    ///
1121    /// The argument is the part that cannot be "harmless to a process that
1122    /// ignores it": a stock binary exits on an unknown flag before it listens
1123    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
1124    /// first conformance run against this mode found it. The nonce is withheld
1125    /// because a process that will never present it gains nothing from holding
1126    /// it, and a secret in the environment of a process that does not need it is
1127    /// a leak surface for no benefit.
1128    pub protocol: ModuleProtocol,
1129    /// Whether two processes of this module may run at once, which is what a
1130    /// blue/green swap does for the length of its overlap. Declared in daemon
1131    /// config because the daemon must be able to answer it while the module is
1132    /// down, and so a module cannot talk itself into it after registering.
1133    pub overlap: ModuleOverlap,
1134    /// Keep module-started helpers in the same container across replacements.
1135    /// This is a daemon-config policy, fixed for the lifetime of a container;
1136    /// stopping the module or losing its supervisor still destroys the container.
1137    pub carry_containment: bool,
1138}
1139
1140/// Only the configured launch inputs enter this value; spawned connection paths,
1141/// nonces and inherited environment never do. Its bytes remain daemon-private.
1142#[derive(Clone, PartialEq, Eq)]
1143pub(crate) struct LaunchSpecDigest([u8; 32]);
1144
1145impl fmt::Debug for LaunchSpecDigest {
1146    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1147        f.write_str("LaunchSpecDigest([redacted])")
1148    }
1149}
1150
1151impl LaunchSpecDigest {
1152    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
1153        fn field(hash: &mut Sha256, bytes: &[u8]) {
1154            hash.update((bytes.len() as u64).to_le_bytes());
1155            hash.update(bytes);
1156        }
1157        let mut hash = Sha256::new();
1158        if let Some(program) = spec.program.to_str() {
1159            field(&mut hash, b"utf8-program");
1160            field(&mut hash, program.as_bytes());
1161        } else {
1162            // The daemon config is UTF-8, but in-process callers may provide
1163            // native paths: lossy conversion would collapse distinct programs.
1164            #[cfg(unix)]
1165            {
1166                use std::os::unix::ffi::OsStrExt;
1167                field(&mut hash, b"unix-program");
1168                field(&mut hash, spec.program.as_os_str().as_bytes());
1169            }
1170            #[cfg(windows)]
1171            {
1172                use std::os::windows::ffi::OsStrExt;
1173                field(&mut hash, b"windows-program");
1174                let bytes = spec
1175                    .program
1176                    .as_os_str()
1177                    .encode_wide()
1178                    .flat_map(u16::to_le_bytes)
1179                    .collect::<Vec<_>>();
1180                field(&mut hash, &bytes);
1181            }
1182            #[cfg(not(any(unix, windows)))]
1183            field(&mut hash, spec.program.to_string_lossy().as_bytes());
1184        }
1185        hash.update((spec.args.len() as u64).to_le_bytes());
1186        for arg in &spec.args {
1187            field(&mut hash, arg.as_bytes());
1188        }
1189        let mut env = spec.env.iter().collect::<Vec<_>>();
1190        env.sort_unstable();
1191        hash.update((env.len() as u64).to_le_bytes());
1192        for (key, value) in env {
1193            field(&mut hash, key.as_bytes());
1194            field(&mut hash, value.as_bytes());
1195        }
1196        field(
1197            &mut hash,
1198            match spec.protocol {
1199                ModuleProtocol::Subc => b"subc",
1200                ModuleProtocol::None => b"none",
1201            },
1202        );
1203        Self(hash.finalize().into())
1204    }
1205}
1206
1207/// Whether a module tolerates a second process of itself running alongside.
1208///
1209/// Most modules are single-writer on their store (a WAL, a capture log, a
1210/// resident index behind a writer barrier), and two processes on one store
1211/// corrupt it. So a swap, which overlaps the old and new process by design,
1212/// is refused unless the module's config opts in with `overlap: "safe"`.
1213#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1214pub enum ModuleOverlap {
1215    /// Never run two processes of this module at once. The default.
1216    #[default]
1217    Exclusive,
1218    /// The module has said a second process of itself is harmless for the
1219    /// length of a swap.
1220    ///
1221    /// Declare it only if a second instance can run for a few seconds without
1222    /// touching ANY single-writer store: every database, WAL, index, projector
1223    /// and scheduled job the module owns. A lease on part of that state is not
1224    /// enough. broca's session lease guards WAL appends while its run index, its
1225    /// store projector and its archive fold timer (which unlinks live WAL files)
1226    /// stay single-writer, so broca is exclusive despite holding a lease. The
1227    /// refusal only fires after this has been decided, so the decision is the
1228    /// check.
1229    Safe,
1230}
1231
1232impl ModuleOverlap {
1233    pub fn as_str(self) -> &'static str {
1234        match self {
1235            Self::Exclusive => "exclusive",
1236            Self::Safe => "safe",
1237        }
1238    }
1239}
1240
1241/// Environment variable telling a spawned module which case it was started
1242/// for, before it sends HELLO. Only a swap candidate carries it, as
1243/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
1244///
1245/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
1246/// longer because nobody waits on it, while a plain restart must flip ready
1247/// quickly because callers see `module_warming` until it does. Absence means
1248/// plain restart, the safe reading. The daemon trusts nothing about it; the
1249/// candidate is proven by its launch nonce at HELLO.
1250pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
1251/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
1252pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
1253/// How long a swap waits for its candidate to register and declare itself
1254/// ready when the operator does not say. A module warming as a swap candidate
1255/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
1256/// daemon allows that plus time to start the process and send HELLO.
1257pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
1258
1259/// Bounded restart policy for crash exits.
1260///
1261/// `max_restarts` is the number of replacement processes allowed after the
1262/// initial spawn WITHIN `window`. After that many crash restarts inside one
1263/// window the module enters [`ModuleState::Failed`] and the supervisor stops
1264/// the crash loop.
1265///
1266/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
1267/// and that only survived because crashes were rare: a module that crashed
1268/// three times across a week was disabled forever by crashes that had nothing
1269/// to do with each other. That stopped being survivable once modules began
1270/// exiting non-zero whenever the daemon's connection to them drops, because
1271/// then every daemon-side connection drop spends a unit of the same budget and
1272/// one flappy hour permanently stops a healthy module. Restarts older than
1273/// `window` release their slot, so a module that crashed twice yesterday has a
1274/// full budget today, while a genuine crash loop -- which is fast by
1275/// definition -- still reaches the cap and stops.
1276#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1277pub struct RestartPolicy {
1278    pub max_restarts: u32,
1279    /// Base delay before a crash replacement. The actual delay escalates with
1280    /// the number of recent crash replacements and is capped by `max_backoff`.
1281    pub backoff: Duration,
1282    /// Maximum delay before a crash replacement.
1283    pub max_backoff: Duration,
1284    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
1285    /// budget effectively infinite (nothing is ever in-window), which is why
1286    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
1287    pub window: Duration,
1288}
1289
1290impl RestartPolicy {
1291    /// A policy with the default crash window. Callers that care about the
1292    /// window say so with [`Self::with_window`]; the ones that do not are
1293    /// asking for the standard rate limit, not for no limit.
1294    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
1295        Self {
1296            max_restarts,
1297            backoff,
1298            max_backoff: DEFAULT_MAX_BACKOFF,
1299            window: DEFAULT_RESTART_WINDOW,
1300        }
1301    }
1302
1303    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
1304        self.max_backoff = max_backoff;
1305        self
1306    }
1307
1308    pub fn with_window(mut self, window: Duration) -> Self {
1309        self.window = window;
1310        self
1311    }
1312
1313    /// Calculate the capped exponential delay for the next crash replacement.
1314    /// `restart_in_window` is zero for the first replacement after an operator
1315    /// action (restart, reload, re-enable) cleared the crash ring, or after all
1316    /// older crash replacements have aged out of the window.
1317    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
1318        if self.backoff.is_zero() || self.max_backoff.is_zero() {
1319            return Duration::ZERO;
1320        }
1321
1322        let mut delay = self.backoff;
1323        for _ in 0..restart_in_window {
1324            if delay >= self.max_backoff {
1325                return self.max_backoff;
1326            }
1327            delay = delay
1328                .checked_mul(10)
1329                .unwrap_or(self.max_backoff)
1330                .min(self.max_backoff);
1331        }
1332        delay.min(self.max_backoff)
1333    }
1334
1335    /// The one sentence that explains a budget-exhausted stop, used for both the
1336    /// log line and the terminal record so the two cannot drift. It names the
1337    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
1338    /// exactly what this budget is not.
1339    fn budget_exhausted_detail(&self) -> String {
1340        format!(
1341            "crash budget exhausted: max_restarts={} within window_secs={}",
1342            self.max_restarts,
1343            self.window.as_secs()
1344        )
1345    }
1346}
1347
1348impl Default for RestartPolicy {
1349    fn default() -> Self {
1350        Self {
1351            max_restarts: DEFAULT_MAX_RESTARTS,
1352            backoff: DEFAULT_BACKOFF,
1353            max_backoff: DEFAULT_MAX_BACKOFF,
1354            window: DEFAULT_RESTART_WINDOW,
1355        }
1356    }
1357}
1358
1359#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1360struct CrashRestartSchedule {
1361    restart_in_window: u32,
1362    delay: Duration,
1363}
1364
1365/// Whether the daemon itself will bring this module back after the exit being
1366/// handled: it is enabled AND its in-window crash restarts are below the cap.
1367///
1368/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
1369/// the window are dropped here rather than by a timer, so the count is right
1370/// the moment somebody asks and no bookkeeping runs for idle modules.
1371fn daemon_will_restart(
1372    state: &mut SupervisorSnapshot,
1373    policy: &RestartPolicy,
1374    now: Instant,
1375) -> bool {
1376    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1377}
1378
1379const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1380const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1381const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1382const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1383
1384#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1385pub enum HealthAction {
1386    Report,
1387    Restart,
1388    Alert,
1389}
1390
1391impl fmt::Display for HealthAction {
1392    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1393        f.write_str(match self {
1394            Self::Report => "report",
1395            Self::Restart => "restart",
1396            Self::Alert => "alert",
1397        })
1398    }
1399}
1400
1401#[derive(Debug, Clone, PartialEq, Eq)]
1402pub struct HealthConfig {
1403    /// Optional loopback HTTP endpoint for a managed non-wire process.
1404    /// Changing it applies live on rescan; the process protocol changes only
1405    /// at its next spawn.
1406    pub http: Option<String>,
1407    pub cadence: Duration,
1408    pub deadline: Duration,
1409    pub failure_threshold: u32,
1410    pub on_degraded: HealthAction,
1411    pub on_failing: HealthAction,
1412    pub critical: bool,
1413}
1414
1415impl Default for HealthConfig {
1416    fn default() -> Self {
1417        Self {
1418            http: None,
1419            cadence: DEFAULT_HEALTH_CADENCE,
1420            deadline: DEFAULT_HEALTH_DEADLINE,
1421            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1422            on_degraded: HealthAction::Report,
1423            on_failing: HealthAction::Report,
1424            critical: false,
1425        }
1426    }
1427}
1428
1429/// The supervisor's view of one module's health, relayed to clients over
1430/// channel-0 and rendered by `ck health`.
1431///
1432/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1433/// stated here rather than only at the wire type a consumer reads. A reader can
1434/// look up what `None` means; only a writer can silently change it, and the
1435/// writer has no reason to go looking at a downstream contract before editing.
1436///
1437/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1438/// back to `None` on re-registration precisely so a respawned module does not
1439/// carry its predecessor's timestamp — so an old value and an absent one call for
1440/// opposite readings, and anything that defaulted this to a number would make a
1441/// never-probed module indistinguishable from one probed at the epoch.
1442///
1443/// `detail` and `metrics` are `None` when the module published none on this
1444/// probe, which does not mean it reported nothing wrong — it is also the shape
1445/// when the probe never reached it. `last_probe_ms` is what separates those.
1446#[derive(Debug, Clone, PartialEq)]
1447pub struct ModuleHealthStatus {
1448    pub status: SupervisorHealthStatus,
1449    pub last_probe_ms: Option<u64>,
1450    pub detail: Option<String>,
1451    pub metrics: Option<Value>,
1452    pub consecutive_failures: u32,
1453    /// Number of replies received after a recurring health probe's deadline.
1454    /// Unlike a timeout, every increment proves the module was alive.
1455    pub late_answer_count: u64,
1456    /// End-to-end latency of the newest late reply, measured from probe start.
1457    pub last_late_answer_latency_ms: Option<u64>,
1458    pub last_action: Option<String>,
1459    /// Set together with `last_action`; the pair moves as one, and both being
1460    /// absent means no escalation has ever been taken rather than that the last
1461    /// one succeeded.
1462    pub last_action_ms: Option<u64>,
1463}
1464
1465impl Default for ModuleHealthStatus {
1466    fn default() -> Self {
1467        Self {
1468            status: SupervisorHealthStatus::Unknown,
1469            last_probe_ms: None,
1470            detail: None,
1471            metrics: None,
1472            consecutive_failures: 0,
1473            late_answer_count: 0,
1474            last_late_answer_latency_ms: None,
1475            last_action: None,
1476            last_action_ms: None,
1477        }
1478    }
1479}
1480
1481/// Typed lifecycle state for a supervised module.
1482#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1483pub enum ModuleState {
1484    Starting,
1485    Running,
1486    Unresponsive,
1487    Restarting,
1488    Draining,
1489    Stopped,
1490    Failed,
1491    Disabled,
1492}
1493
1494impl fmt::Display for ModuleState {
1495    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1496        f.write_str(match self {
1497            Self::Starting => "starting",
1498            Self::Running => "running",
1499            Self::Unresponsive => "unresponsive",
1500            Self::Restarting => "restarting",
1501            Self::Draining => "draining",
1502            Self::Stopped => "stopped",
1503            Self::Failed => "failed",
1504            Self::Disabled => "disabled",
1505        })
1506    }
1507}
1508
1509/// Supervisor classification of a child-process exit.
1510#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1511pub enum ExitKind {
1512    Clean,
1513    Crash,
1514    DeliberateSeverance,
1515}
1516
1517impl From<ExitKind> for TerminalExitKind {
1518    fn from(kind: ExitKind) -> Self {
1519        match kind {
1520            ExitKind::Clean => Self::Clean,
1521            ExitKind::Crash => Self::Crash,
1522            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1523        }
1524    }
1525}
1526
1527/// Exact process identity retained when a supervised module registers its
1528/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1529#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1530pub(crate) struct ProcessIdentity {
1531    pub(crate) pid: u32,
1532    pub(crate) start_time: u64,
1533}
1534
1535/// Last observed child exit, if any.
1536#[derive(Debug, Clone, PartialEq, Eq)]
1537pub struct ExitReport {
1538    pub kind: ExitKind,
1539    pub code: Option<i32>,
1540    pub signal: Option<i32>,
1541    pub at_ms: u64,
1542}
1543
1544/// Point-in-time module status answerable by subc without forwarding to the
1545/// module process.
1546#[derive(Debug, Clone, PartialEq)]
1547pub struct ModuleStatus {
1548    pub(crate) admission_refused: Option<String>,
1549    pub(crate) legacy_registration_spawn: bool,
1550    pub(crate) registration_declaration: Option<String>,
1551    pub module_id: String,
1552    /// Retained configuration-exit infrastructure failure. Cleared only after a
1553    /// successful launch; an ordinary stop has no infrastructure stop reason.
1554    pub stop_reason: Option<ModuleStopReason>,
1555    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1556    pub waiting_for_previous_pid: Option<u32>,
1557    /// Why the previous instance still prevents a new start.
1558    pub waiting_for_previous_detail: Option<String>,
1559    /// The daemon's Windows per-spawn record: `pipe` after the process it
1560    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1561    /// List/status reports use this record, not the source the module sends
1562    /// in its registration message's manifest.provenance.launch_nonce_source.
1563    pub launch_nonce_source: Option<String>,
1564    pub state: ModuleState,
1565    pub enabled: bool,
1566    pub process_alive: bool,
1567    pub registration_active: bool,
1568    /// The module's declared wire protocol, carried beside `live` because it is
1569    /// what makes `live` readable: the two fields answer one question together.
1570    /// While a process is alive this is its launch declaration, not a later
1571    /// pending-reload edit. When down it is the configured next launch protocol.
1572    pub protocol: ModuleProtocol,
1573    /// Whether the module is serving, under the strongest definition the daemon
1574    /// can assert for its protocol.
1575    ///
1576    /// A subc module must also be REGISTERED: its process being alive says
1577    /// nothing about whether it can take a request. A `protocol: "none"` module
1578    /// never registers, so that term is dropped and this falls back to "enabled,
1579    /// running, and the process the daemon launched is alive" -- which is all
1580    /// the daemon observes about a process that speaks no subc wire. It stays a
1581    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1582    /// rather than printing it bare.
1583    pub live: bool,
1584    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1585    /// restarts have already released their slot, so this count can go down
1586    /// without anybody touching the module.
1587    pub restart_count: u32,
1588    /// Replacement processes spawned over this module's entire supervisor lifetime;
1589    /// unlike `restart_count`, this value is never reset by an operator action
1590    /// and never falls out of a window.
1591    pub lifetime_restarts: u32,
1592    pub spawn_generation: u64,
1593    /// Retained daemon-local outcome; the legacy control response omits it.
1594    pub swap_outcome: Option<SwapOutcome>,
1595    /// The budget `restart_count` is spent against. Carried alongside the count
1596    /// because the count alone does not say how close the module is to being
1597    /// disabled, and reporting one without the other is what makes an
1598    /// about-to-be-retired module look ordinary.
1599    pub max_restarts: u32,
1600    /// The span `restart_count` is counted over. Carried with the pair above for
1601    /// the same reason they are carried together: "2 of 3" means one thing for a
1602    /// ten-minute window and something else entirely for a lifetime.
1603    pub restart_window: Duration,
1604    /// Effective drain and restart timing policy used by this running module.
1605    /// These values are carried together with the restart budget so status
1606    /// readers can compare configured intent with what the supervisor applied.
1607    pub drain_timeout: Duration,
1608    pub restart_backoff: Duration,
1609    pub restart_max_backoff: Duration,
1610    /// The module's process. On macOS this stays absent while the `ck-subc`
1611    /// launch trampoline is still running in that pid, and appears once the
1612    /// exec acknowledgement confirms the module image has replaced it. Launch
1613    /// time and the supervisor's own hold on the process are unaffected.
1614    pub pid: Option<u32>,
1615    pub spawned_at_ms: Option<u64>,
1616    pub spawned_from: Option<PathBuf>,
1617    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1618    pub process_start_time: Option<u64>,
1619    pub last_exit: Option<ExitReport>,
1620    pub health: ModuleHealthStatus,
1621}
1622
1623#[cfg(test)]
1624#[derive(Debug, Clone, Copy, PartialEq)]
1625struct ActorSelectCheckpoint {
1626    generation: u64,
1627    turn: u64,
1628    registered_connection: Option<ConnectionId>,
1629    next_probe_at: Option<Instant>,
1630    wake_after: Option<Duration>,
1631}
1632
1633#[derive(Debug, Clone, PartialEq)]
1634struct SupervisorSnapshot {
1635    admission_refused: Option<String>,
1636    /// Distinguishes an admission-parked actor from an unrelated terminal exit.
1637    config_hold_parked: bool,
1638    /// Preserve daemon attribution for a paused start so a failing release can
1639    /// schedule the next slow retry instead of treating the start as operator work.
1640    config_hold_daemon_initiated: bool,
1641    /// The approval that owns the parked attempt, not whichever approval is latest.
1642    config_hold_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1643    /// Retain an approved launch failure across competing reconciliation passes.
1644    approved_start_failure: Option<(crate::config_gate::runtime::ApprovalIdentity, String)>,
1645    legacy_registration_spawn: bool,
1646    registration_declaration: Option<String>,
1647    stop_reason: Option<ModuleStopReason>,
1648    config_exit_failed_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1649    subtree_target: crate::subtree_resources::Target,
1650    failed_retry: FailedRetry,
1651    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1652    #[cfg(windows)]
1653    pipe_consumed: Option<Arc<OnceLock<()>>>,
1654    /// Counts running-child loop turns, not executor polls of a parked wait.
1655    #[cfg(test)]
1656    actor_turns: u64,
1657    /// Running select after exec confirmation and with no unconsumed registry
1658    /// event. Recording a spawn or starting a loop turn is not this barrier.
1659    #[cfg(test)]
1660    actor_select: Option<ActorSelectCheckpoint>,
1661    state: ModuleState,
1662    enabled: bool,
1663    process_alive: bool,
1664    spawned_protocol: Option<ModuleProtocol>,
1665    spawn_failure: Option<String>,
1666    waiting_for_previous_pid: Option<u32>,
1667    waiting_for_previous_detail: Option<String>,
1668    /// When each crash restart was spent, oldest first. This IS the crash
1669    /// budget: its in-window length is the count an operator sees and the count
1670    /// the restart decision is made against, so there is no second counter that
1671    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1672    /// operator actions that used to zero the old lifetime counter.
1673    crash_restarts: VecDeque<Instant>,
1674    lifetime_restarts: u32,
1675    /// Successful child spawns in this daemon incarnation.
1676    ///
1677    /// `lifetime_restarts` was considered and rejected: it starts at zero
1678    /// (line 640), successful initial/operator spawns in `set_running` do not
1679    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1680    /// increments before a successful replacement exists (lines 604, 3846,
1681    /// and 3921), so a failed spawn can consume it. This counter moves only
1682    /// when a live PID is accepted below.
1683    spawn_generation: u64,
1684    swap_outcome: Option<SwapOutcome>,
1685    pid: Option<u32>,
1686    #[cfg(target_os = "macos")]
1687    report_ready: Option<Arc<OnceLock<()>>>,
1688    /// Last reaped child, retained after current process facts are cleared.
1689    reaped_pid: Option<u32>,
1690    /// Whether a replacement has already been scheduled for the current process.
1691    respawn_pending: bool,
1692    /// A second restart is waiting for the replacement already scheduled.
1693    coalesced_restart_pending: bool,
1694    spawned_at_ms: Option<u64>,
1695    spawned_from: Option<PathBuf>,
1696    launch_spec_digest: Option<LaunchSpecDigest>,
1697    spawned_file_identity: Option<SpawnedFileIdentity>,
1698    process_start_time: Option<u64>,
1699    deliberate_severance: Option<ProcessIdentity>,
1700    last_exit: Option<ExitReport>,
1701    /// Diagnostic attached to the next drain's terminal record, if any.
1702    drain_disposition_detail: Option<String>,
1703    health: ModuleHealthStatus,
1704    last_probe_evidence: Option<HealthProbeEvidence>,
1705    /// Whether the current process was started as a swap candidate and so
1706    /// lives in the module's alternate cgroup. The next swap's candidate takes
1707    /// the other one, so the two processes of a swap never share a cgroup. A
1708    /// plain spawn always uses the primary cgroup.
1709    in_alternate_slot: bool,
1710    /// Whether the current `Draining` state ends in a replacement process
1711    /// (restart, reload, health restart) rather than a stop. Only meaningful
1712    /// while `state` is `Draining`; every entry into that state rewrites it.
1713    /// It is what lets route.open answer the retryable `module_reloading` to a
1714    /// consumer that reaches a still-registered process mid-restart, instead of
1715    /// the `supervisor_not_live` a stop or disable deserves.
1716    draining_to_replace: bool,
1717    /// Whether a configuration update has been applied since the current
1718    /// process was spawned, so that process runs an older spec than the one
1719    /// the supervisor now holds. A queued restart is only coalesced into a
1720    /// fresher process when this is false: a restart requested to pick up a
1721    /// new configuration must not be satisfied by a process that predates it.
1722    configuration_updated_since_spawn: bool,
1723}
1724
1725impl SupervisorSnapshot {
1726    /// The pid that status, provenance and resource readings may report. While
1727    /// the launch trampoline still runs in the pid, reading its executable or
1728    /// resource use would describe `ck-subc`, not the module, so none is
1729    /// reported until the exec acknowledgement confirms the module image.
1730    fn reported_pid(&self) -> Option<u32> {
1731        #[cfg(target_os = "macos")]
1732        if self
1733            .report_ready
1734            .as_ref()
1735            .is_some_and(|ready| ready.get().is_none())
1736        {
1737            return None;
1738        }
1739        self.pid
1740    }
1741
1742    fn starting() -> Self {
1743        Self::new(ModuleState::Starting, true)
1744    }
1745
1746    fn disabled() -> Self {
1747        Self::new(ModuleState::Disabled, false)
1748    }
1749
1750    /// Crash restarts still inside `window`, having dropped the ones that are
1751    /// not. Pruning on read is what makes the budget a rate: an instant older
1752    /// than the window stops holding a slot the moment anybody counts.
1753    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1754        while let Some(oldest) = self.crash_restarts.front() {
1755            if now.duration_since(*oldest) > window {
1756                self.crash_restarts.pop_front();
1757            } else {
1758                break;
1759            }
1760        }
1761        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1762    }
1763
1764    /// Spend one unit of the crash budget and record the restart in the ledger.
1765    ///
1766    /// The ring is bounded by the cap because more than `max_restarts` in-window
1767    /// instants can never be reached (the caller refuses the restart first), so
1768    /// anything beyond that is an unbounded queue waiting to happen.
1769    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1770        self.crash_restarts.push_back(now);
1771        while self.crash_restarts.len() > policy.max_restarts as usize {
1772            self.crash_restarts.pop_front();
1773        }
1774        self.lifetime_restarts += 1;
1775    }
1776
1777    /// Reserve one crash-restart slot and calculate the delay before respawning.
1778    /// The count is captured before recording this restart, so the first retry
1779    /// uses the base delay and each later in-window retry escalates once.
1780    fn next_crash_restart(
1781        &mut self,
1782        policy: &RestartPolicy,
1783        now: Instant,
1784    ) -> Option<CrashRestartSchedule> {
1785        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1786        if restart_in_window >= policy.max_restarts {
1787            return None;
1788        }
1789        self.record_crash_restart(policy, now);
1790        Some(CrashRestartSchedule {
1791            restart_in_window,
1792            delay: policy.delay_for_restart(restart_in_window),
1793        })
1794    }
1795
1796    /// Give the module its full budget back, as an operator restart, reload, or
1797    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1798    /// ledger of what actually happened, and an operator action does not unmake
1799    /// the crashes.
1800    fn clear_crash_restarts(&mut self) {
1801        self.crash_restarts.clear();
1802    }
1803
1804    fn new(state: ModuleState, enabled: bool) -> Self {
1805        Self {
1806            admission_refused: None,
1807            config_hold_parked: false,
1808            config_hold_daemon_initiated: false,
1809            config_hold_approval: None,
1810            approved_start_failure: None,
1811            legacy_registration_spawn: false,
1812            registration_declaration: None,
1813            stop_reason: None,
1814            config_exit_failed_approval: None,
1815            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1816            failed_retry: FailedRetry::default(),
1817            exec_outcome: None,
1818            #[cfg(test)]
1819            actor_turns: 0,
1820            #[cfg(test)]
1821            actor_select: None,
1822            state,
1823            enabled,
1824            process_alive: false,
1825            #[cfg(windows)]
1826            pipe_consumed: None,
1827            spawned_protocol: None,
1828            spawn_failure: None,
1829            waiting_for_previous_pid: None,
1830            waiting_for_previous_detail: None,
1831            crash_restarts: VecDeque::new(),
1832            lifetime_restarts: 0,
1833            spawn_generation: 0,
1834            swap_outcome: None,
1835            pid: None,
1836            #[cfg(target_os = "macos")]
1837            report_ready: None,
1838            reaped_pid: None,
1839            respawn_pending: false,
1840            coalesced_restart_pending: false,
1841            spawned_at_ms: None,
1842            spawned_from: None,
1843            launch_spec_digest: None,
1844            spawned_file_identity: None,
1845            process_start_time: None,
1846            deliberate_severance: None,
1847            last_exit: None,
1848            drain_disposition_detail: None,
1849            health: ModuleHealthStatus::default(),
1850            last_probe_evidence: None,
1851            in_alternate_slot: false,
1852            draining_to_replace: false,
1853            configuration_updated_since_spawn: false,
1854        }
1855    }
1856}
1857
1858type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1859
1860type SpawnSubscriberKey = (ConnectionId, u64);
1861
1862#[derive(Debug)]
1863struct SpawnSubscriber {
1864    version: u8,
1865    frames: mpsc::Sender<Frame>,
1866    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1867    /// from which event. The full frame channel cannot carry that news, so it
1868    /// travels beside it; see `SpawnEventFeed::subscribe`.
1869    lagged: Option<oneshot::Sender<SpawnCursor>>,
1870}
1871
1872#[derive(Debug)]
1873struct SpawnEventState {
1874    daemon_incarnation: String,
1875    seq: u64,
1876    capacity: usize,
1877    live: HashMap<String, LiveSpawn>,
1878    generations: HashMap<String, u64>,
1879    events: VecDeque<SpawnEvent>,
1880    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1881    process_changes: HashMap<String, tokio::sync::watch::Sender<u64>>,
1882}
1883
1884impl Default for SpawnEventState {
1885    fn default() -> Self {
1886        Self {
1887            daemon_incarnation: "unconfigured".to_string(),
1888            seq: 0,
1889            capacity: SPAWN_EVENT_RING_CAPACITY,
1890            live: HashMap::new(),
1891            generations: HashMap::new(),
1892            events: VecDeque::new(),
1893            subscribers: HashMap::new(),
1894            process_changes: HashMap::new(),
1895        }
1896    }
1897}
1898
1899#[derive(Debug, Clone, Default)]
1900struct SpawnEventFeed(
1901    Arc<Mutex<SpawnEventState>>,
1902    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1903);
1904
1905#[derive(Debug, Clone, PartialEq, Eq)]
1906pub(crate) enum SpawnSubscribeRefusal {
1907    ForeignIncarnation { current: String },
1908    TooOld { oldest: SpawnCursor },
1909    Frame(String),
1910}
1911
1912impl SpawnEventFeed {
1913    fn configure_incarnation(&self, daemon_incarnation: String) {
1914        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1915        state.daemon_incarnation = daemon_incarnation;
1916        state.seq = 0;
1917        state.live.clear();
1918        state.generations.clear();
1919        state.events.clear();
1920        state.subscribers.clear();
1921        for changed in state.process_changes.values() {
1922            changed.send_replace(0);
1923        }
1924    }
1925
1926    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1927        SpawnCursor {
1928            daemon_incarnation: state.daemon_incarnation.clone(),
1929            seq: state.seq,
1930        }
1931    }
1932
1933    fn snapshot(&self) -> SpawnSnapshot {
1934        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1935        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1936        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1937        SpawnSnapshot {
1938            cursor: Self::cursor(&state),
1939            ring_bound: state.capacity as u64,
1940            live,
1941        }
1942    }
1943
1944    fn allocate_generation_locked(state: &mut SpawnEventState, module_id: &str) -> u64 {
1945        let generation = state
1946            .generations
1947            .get(module_id)
1948            .copied()
1949            .unwrap_or(0)
1950            .checked_add(1)
1951            .expect("spawn generation exhausted");
1952        state.generations.insert(module_id.to_string(), generation);
1953        generation
1954    }
1955
1956    /// Allocate the spawn generation of a launch that is about to start, with
1957    /// the daemon incarnation it belongs to. The number names the launch from
1958    /// this moment, before its process runs; a launch that never runs keeps
1959    /// its number, so the generations a subscriber sees may skip.
1960    fn allocate_generation(&self, module_id: &str) -> (String, u64) {
1961        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1962        let generation = Self::allocate_generation_locked(&mut state, module_id);
1963        (state.daemon_incarnation.clone(), generation)
1964    }
1965
1966    /// Publish a running process. `allocated` is the generation its launch
1967    /// was given before it started; a launch without one (a process with no
1968    /// registration channel) is numbered here.
1969    fn emit_spawned(
1970        &self,
1971        module_id: &str,
1972        allocated: Option<u64>,
1973        pid: u32,
1974        spawned_at_ms: u64,
1975    ) -> u64 {
1976        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1977        let generation = match allocated {
1978            Some(generation) => generation,
1979            None => Self::allocate_generation_locked(&mut state, module_id),
1980        };
1981        let live = LiveSpawn {
1982            module_id: module_id.to_string(),
1983            spawn_generation: generation,
1984            pid,
1985            spawned_at_ms,
1986        };
1987        state.live.insert(module_id.to_string(), live);
1988        Self::emit_locked(
1989            &mut state,
1990            SpawnEventKind::Spawned,
1991            module_id.to_string(),
1992            generation,
1993            pid,
1994            None,
1995            None,
1996        );
1997        generation
1998    }
1999
2000    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
2001        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2002        let Some(live) = state.live.remove(module_id) else {
2003            warn!(
2004                module_id,
2005                "terminal record had no live spawn event identity"
2006            );
2007            return;
2008        };
2009        Self::emit_locked(
2010            &mut state,
2011            SpawnEventKind::Exited,
2012            module_id.to_string(),
2013            live.spawn_generation,
2014            live.pid,
2015            exit_code,
2016            exit_signal,
2017        );
2018    }
2019
2020    /// Report the exit of a process that a swap has already replaced.
2021    ///
2022    /// `emit_exited` removes the module's live entry, which after a swap's
2023    /// cutover describes the promoted candidate, not the old process now
2024    /// exiting. This emits the old generation's exit and leaves the live entry
2025    /// alone unless it still names that generation.
2026    fn emit_superseded_exited(
2027        &self,
2028        module_id: &str,
2029        spawn_generation: u64,
2030        pid: u32,
2031        exit_code: Option<i32>,
2032        exit_signal: Option<i32>,
2033    ) {
2034        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2035        if state
2036            .live
2037            .get(module_id)
2038            .is_some_and(|live| live.spawn_generation == spawn_generation)
2039        {
2040            state.live.remove(module_id);
2041        }
2042        Self::emit_locked(
2043            &mut state,
2044            SpawnEventKind::Exited,
2045            module_id.to_string(),
2046            spawn_generation,
2047            pid,
2048            exit_code,
2049            exit_signal,
2050        );
2051    }
2052
2053    #[allow(clippy::too_many_arguments)]
2054    fn emit_locked(
2055        state: &mut SpawnEventState,
2056        kind: SpawnEventKind,
2057        module_id: String,
2058        spawn_generation: u64,
2059        pid: u32,
2060        exit_code: Option<i32>,
2061        exit_signal: Option<i32>,
2062    ) {
2063        state.seq = state
2064            .seq
2065            .checked_add(1)
2066            .expect("spawn event sequence exhausted");
2067        let event = SpawnEvent {
2068            cursor: Self::cursor(state),
2069            kind,
2070            module_id,
2071            spawn_generation,
2072            pid,
2073            exit_code,
2074            exit_signal,
2075        };
2076        state
2077            .process_changes
2078            .retain(|_, changed| changed.receiver_count() > 0);
2079        if let Some(changed) = state.process_changes.get(&event.module_id) {
2080            changed.send_replace(state.seq);
2081        }
2082        state.events.push_back(event.clone());
2083        while state.events.len() > state.capacity {
2084            state.events.pop_front();
2085        }
2086        let body = match serde_json::to_vec(&event) {
2087            Ok(body) => body,
2088            Err(error) => {
2089                error!(%error, "failed to serialize supervisor spawn event");
2090                return;
2091            }
2092        };
2093        state.subscribers.retain(|(connection_id, corr), subscriber| {
2094            let frame = Frame::build_with_version(
2095                subscriber.version,
2096                FrameType::StreamData,
2097                control_flags(),
2098                0,
2099                0,
2100                *corr,
2101                body.clone(),
2102            );
2103            match frame {
2104                Ok(frame) => {
2105                    if subscriber.frames.try_send(frame).is_ok() {
2106                        true
2107                    } else {
2108                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
2109                        if let Some(lagged) = subscriber.lagged.take() {
2110                            let _ = lagged.send(event.cursor.clone());
2111                        }
2112                        false
2113                    }
2114                }
2115                Err(error) => {
2116                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
2117                    false
2118                }
2119            }
2120        });
2121    }
2122
2123    fn subscribe(
2124        &self,
2125        connection_id: ConnectionId,
2126        corr: u64,
2127        version: u8,
2128        since: Option<SpawnCursor>,
2129        sink: FrameSink,
2130    ) -> Result<(), SpawnSubscribeRefusal> {
2131        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
2132        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
2133        {
2134            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2135            let replay = if let Some(since) = since {
2136                if since.daemon_incarnation != state.daemon_incarnation {
2137                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
2138                        current: state.daemon_incarnation.clone(),
2139                    });
2140                }
2141                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
2142                    if since.seq < oldest.seq.saturating_sub(1) {
2143                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
2144                    }
2145                }
2146                state
2147                    .events
2148                    .iter()
2149                    .filter(|event| event.cursor.seq > since.seq)
2150                    .cloned()
2151                    .collect::<Vec<_>>()
2152            } else {
2153                Vec::new()
2154            };
2155            for event in replay {
2156                let body = serde_json::to_vec(&event)
2157                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2158                let frame = Frame::build_with_version(
2159                    version,
2160                    FrameType::StreamData,
2161                    control_flags(),
2162                    0,
2163                    0,
2164                    corr,
2165                    body,
2166                )
2167                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2168                frames
2169                    .try_send(frame)
2170                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2171            }
2172            state.subscribers.insert(
2173                (connection_id, corr),
2174                SpawnSubscriber {
2175                    version,
2176                    frames,
2177                    lagged: Some(lagged),
2178                },
2179            );
2180        }
2181        // The lagged terminal is sent here, by the forwarder, rather than by
2182        // the emitter: at the moment of the drop the subscriber's own channel
2183        // is full, and writing to the connection sink directly from the emitter
2184        // would put the Error AHEAD of the events still queued in that channel
2185        // (and the emitter holds the feed lock, so it cannot await the sink).
2186        // Dropping the subscriber drops the only sender, so `recv` drains every
2187        // queued event and then returns `None`; only then is the Error sent, so
2188        // the client sees each event it can keep, then the reason it was cut.
2189        // Cancel and connection removal drop the oneshot unsent, so they end
2190        // the stream with no Error.
2191        tokio::spawn(async move {
2192            while let Some(frame) = receiver.recv().await {
2193                if sink.send(frame).await.is_err() {
2194                    return;
2195                }
2196            }
2197            let Ok(first_undelivered) = lagged_rx.try_recv() else {
2198                return;
2199            };
2200            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
2201                Ok(frame) => {
2202                    let _ = sink.send(frame).await;
2203                }
2204                Err(error) => {
2205                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
2206                }
2207            }
2208        });
2209        Ok(())
2210    }
2211
2212    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
2213        let Some(subscriber) = self
2214            .0
2215            .lock()
2216            .unwrap_or_else(|p| p.into_inner())
2217            .subscribers
2218            .remove(&(connection_id, corr))
2219        else {
2220            return false;
2221        };
2222        if let Ok(frame) = Frame::build_with_version(
2223            subscriber.version,
2224            FrameType::StreamEnd,
2225            control_flags(),
2226            0,
2227            0,
2228            corr,
2229            Vec::new(),
2230        ) {
2231            tokio::spawn(async move {
2232                let _ = subscriber.frames.send(frame).await;
2233            });
2234        }
2235        true
2236    }
2237
2238    fn remove_connection(&self, connection_id: ConnectionId) {
2239        self.0
2240            .lock()
2241            .unwrap_or_else(|p| p.into_inner())
2242            .subscribers
2243            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
2244    }
2245
2246    #[cfg(any(test, feature = "test-support"))]
2247    fn set_capacity(&self, capacity: usize) {
2248        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
2249    }
2250
2251    #[cfg(any(test, feature = "test-support"))]
2252    fn subscriber_count(&self) -> usize {
2253        self.0
2254            .lock()
2255            .unwrap_or_else(|p| p.into_inner())
2256            .subscribers
2257            .len()
2258    }
2259}
2260
2261/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
2262/// The terminal Error a lagged spawn subscriber receives after its queued events.
2263fn spawn_subscriber_lagged_frame(
2264    version: u8,
2265    corr: u64,
2266    first_undelivered: SpawnCursor,
2267) -> Result<Frame, String> {
2268    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
2269    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
2270        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
2271        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
2272            .to_string(),
2273        detail: Some(serde_json::json!({
2274            "first_undelivered_cursor": first_undelivered
2275        })),
2276    })
2277    .map_err(|error| error.to_string())?;
2278    Frame::build_with_version(
2279        version,
2280        FrameType::Error,
2281        crate::router::daemon_error_flags(),
2282        0,
2283        0,
2284        corr,
2285        body,
2286    )
2287    .map_err(|error| error.to_string())
2288}
2289
2290pub trait ModuleProcessLiveness: Send + Sync {
2291    fn process_live(&self, module_id: &str) -> Option<bool>;
2292
2293    /// Whether the supervisor is replacing this module's process right now: an
2294    /// operator restart or reload, a health restart, or a crash respawn whose
2295    /// backoff is running. A module in that state is not live, but a consumer
2296    /// refused now should retry shortly rather than treat the target as gone.
2297    /// Stopped, failed, and disabled modules are not replacing.
2298    fn process_replacing(&self, _module_id: &str) -> bool {
2299        false
2300    }
2301}
2302
2303/// Shared process-liveness registry keyed by supervised `module_id`.
2304#[derive(Debug, Clone, Default)]
2305pub struct SupervisorProcessLiveness {
2306    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
2307}
2308
2309impl SupervisorProcessLiveness {
2310    pub fn new() -> Self {
2311        Self::default()
2312    }
2313
2314    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
2315        let mut snapshots = self
2316            .snapshots
2317            .lock()
2318            .unwrap_or_else(|poisoned| poisoned.into_inner());
2319        snapshots.insert(module_id, snapshot);
2320    }
2321
2322    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
2323        let mut snapshots = self
2324            .snapshots
2325            .lock()
2326            .unwrap_or_else(|poisoned| poisoned.into_inner());
2327        let is_current = snapshots
2328            .get(module_id)
2329            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
2330            .unwrap_or(false);
2331        if is_current {
2332            snapshots.remove(module_id);
2333        }
2334    }
2335}
2336
2337impl ModuleProcessLiveness for SupervisorProcessLiveness {
2338    fn process_live(&self, module_id: &str) -> Option<bool> {
2339        let snapshot = {
2340            let snapshots = self
2341                .snapshots
2342                .lock()
2343                .unwrap_or_else(|poisoned| poisoned.into_inner());
2344            snapshots.get(module_id).cloned()
2345        }?;
2346        let snapshot = snapshot
2347            .lock()
2348            .unwrap_or_else(|poisoned| poisoned.into_inner());
2349        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
2350    }
2351
2352    fn process_replacing(&self, module_id: &str) -> bool {
2353        let Some(snapshot) = self
2354            .snapshots
2355            .lock()
2356            .unwrap_or_else(|poisoned| poisoned.into_inner())
2357            .get(module_id)
2358            .cloned()
2359        else {
2360            return false;
2361        };
2362        let snapshot = snapshot
2363            .lock()
2364            .unwrap_or_else(|poisoned| poisoned.into_inner());
2365        snapshot.enabled
2366            && match snapshot.state {
2367                ModuleState::Restarting => true,
2368                ModuleState::Draining => snapshot.draining_to_replace,
2369                ModuleState::Starting
2370                | ModuleState::Running
2371                | ModuleState::Unresponsive
2372                | ModuleState::Stopped
2373                | ModuleState::Failed
2374                | ModuleState::Disabled => false,
2375            }
2376    }
2377}
2378
2379#[cfg(test)]
2380#[derive(Debug, Default)]
2381struct ReloadExitRecordGate {
2382    reached: tokio::sync::Notify,
2383    resume: tokio::sync::Notify,
2384}
2385
2386#[derive(Debug, Clone, Copy)]
2387enum RespawnKind {
2388    Spawn,
2389    Reload,
2390}
2391
2392#[derive(Debug, Clone)]
2393struct PendingRespawn {
2394    deadline: Instant,
2395    kind: RespawnKind,
2396    daemon_initiated: bool,
2397    approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
2398}
2399
2400impl PendingRespawn {
2401    fn failure_context(&self, runtime: &SupervisorRuntimeConfig) -> FailureContext {
2402        FailureContext {
2403            approval: self.approval.clone(),
2404            ..runtime.failure_context(self.daemon_initiated)
2405        }
2406    }
2407}
2408
2409type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
2410
2411#[derive(Debug, Clone)]
2412struct SupervisorRuntimeConfig {
2413    registry: Arc<Registry>,
2414    config_delivery: Option<config_delivery::ConfigDelivery>,
2415    /// Retained across an operation's deferred backoff, until its child starts.
2416    recovery_is_daemon: bool,
2417    start_requester: Option<Principal>,
2418    pending_activity_restart: bool,
2419    provenance_probe: ExecutableIdentityProbe,
2420    /// Restart operations hand their backoff to the loop, which keeps serving commands.
2421    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2422    /// A reload acknowledges completion only after its replacement registers.
2423    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2424    restart_policy: RestartPolicy,
2425    register_timeout: Duration,
2426    /// This module's RESOLVED drain budget: per-module config when present,
2427    /// else `default_drain_timeout`.
2428    drain_timeout: Duration,
2429    /// Shared with the status handle so the attested value changes atomically
2430    /// when a rescan updates the running drain policy.
2431    effective_drain_timeout: Arc<Mutex<Duration>>,
2432    /// The supervisor-wide fallback, kept so a configuration update that
2433    /// REMOVES the per-module override can re-resolve to it.
2434    default_drain_timeout: Duration,
2435    health: HealthConfig,
2436    host_health: HostHealth,
2437    connection_file_path: Option<PathBuf>,
2438    capture_logs_dir: Option<PathBuf>,
2439    forwarding: Option<Arc<ForwardingTable>>,
2440    /// The shared handle, so every spawn path (initial, restart, reload) records the
2441    /// reserved-module launch nonce the HELLO verifier checks against.
2442    supervisor_handle: Option<SupervisorHandle>,
2443    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
2444    /// status queries.
2445    ///
2446    /// One ring per module, held across every respawn. The lines explaining an exit
2447    /// are written BEFORE that exit, so a ring recreated per process would be empty
2448    /// exactly when it is asked for.
2449    stderr_ring: Arc<Mutex<StderrRing>>,
2450    terminal_ring: Arc<Mutex<TerminalRing>>,
2451    spawn_events: SpawnEventFeed,
2452    child_roster: ChildRoster,
2453    #[cfg(target_os = "linux")]
2454    cgroup_placement: Option<subc_cgroup::Placement>,
2455    #[cfg(test)]
2456    test_seed_stale_facts_before_enable_spawn: bool,
2457    #[cfg(test)]
2458    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2459}
2460
2461impl SupervisorRuntimeConfig {
2462    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2463        FailureContext {
2464            daemon_initiated,
2465            approval: None,
2466            shutdown: self.child_roster.shutdown_flag(),
2467        }
2468    }
2469}
2470
2471#[derive(Debug, Clone, PartialEq, Eq)]
2472struct SupervisedConfiguration {
2473    spec: ModuleSpec,
2474    enabled: bool,
2475    health: HealthConfig,
2476    register_timeout: Duration,
2477    restart_policy: RestartPolicy,
2478}
2479
2480/// Shared daemon lookup table for supervised module handles.
2481///
2482/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2483/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2484/// launch nonces recorded at spawn are checked by the same daemon instance.
2485#[derive(Debug, Clone, Default)]
2486pub struct SupervisorHandle {
2487    #[cfg(test)]
2488    resource_commit_pause: Arc<Mutex<Option<resource_launch::ResourceCommitPause>>>,
2489    #[cfg(test)]
2490    registration_test_channels: CapturedRegistrationChannels,
2491    registration_service: Arc<Mutex<Option<registration::RegistrationService>>>,
2492    registration_connections: Arc<Mutex<HashMap<ConnectionId, Arc<RegistrationSpawn>>>>,
2493    registration_launches: Arc<Mutex<Vec<std::sync::Weak<RegistrationSpawn>>>>,
2494    gate_runtime: Arc<Mutex<Option<std::sync::Weak<crate::config_gate::runtime::RuntimeOwner>>>>,
2495    program_bindings: program_binding::ProgramBindings,
2496    #[cfg(test)]
2497    identity_sample_pause: Arc<Mutex<Option<IdentitySamplePause>>>,
2498    #[cfg(all(test, unix))]
2499    plain_identity_policy_pause: Arc<Mutex<Option<(String, IdentitySamplePause)>>>,
2500    host_health: HostHealth,
2501    restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2502    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2503    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2504    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2505    /// Module ids the supervisor has taken on. An id is added BEFORE the
2506    /// module's first process is spawned and removed only when the module
2507    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2508    /// the keys of `modules`.
2509    ///
2510    /// `modules` cannot answer "is this module configured?" on its own: a
2511    /// [`SupervisedModule`] only exists once its process has been spawned, and
2512    /// a fast child can connect, register, sync its scopes and ask about them
2513    /// before the supervisor has inserted it. Answering "not configured" in that
2514    /// gap makes scope admission refuse with the terminal "will never sync"
2515    /// instead of the retryable "has not synced yet".
2516    configured_ids: Arc<Mutex<HashSet<String>>>,
2517    spawn_events: SpawnEventFeed,
2518    /// The current expected launch nonce for each reserved module_id. Set when the
2519    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2520    /// non-reserved module never has an entry here and is never nonce-checked.
2521    /// Reserved module ids and the nonce that authorizes their next HELLO.
2522    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2523    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2524    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2525    /// had NO entry and admitted anyone: the reservation protected the nonce
2526    /// holder, not the NAME (found live by CKCRED's canary probe registering
2527    /// against a reserved scratch id).
2528    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2529    /// Reservations installed only to prevent HELLO for a protocol:none process.
2530    /// Replacement by a subc process releases them; explicitly reserved ids are
2531    /// never placed in this set and still require an operator release.
2532    implicit_protocol_none: Arc<Mutex<HashSet<String>>>,
2533    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2534    ///
2535    /// This is deliberately in-memory only: subc is state-free across daemon
2536    /// restarts, and the tombstone only explains the hours-after-removal window
2537    /// while this executing daemon is still alive. Do not persist it in a store.
2538    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2539    /// The current launch nonce for every supervised spawn. This is separate from
2540    /// reserved_nonces because consumer route.open attestation applies to all spawned
2541    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2542    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2543    /// The launch each live launch nonce was minted for, keyed by nonce, with
2544    /// the launch's module id, so a nonce-attested consumer or registration
2545    /// resolves to that launch's authority record. An entry whose launch has
2546    /// ended stays while its nonce still authorizes, so a nonce whose record
2547    /// is gone is told apart from one that never had a record.
2548    nonce_launches: Arc<Mutex<NonceLaunches>>,
2549    /// Reserved namespace prefixes mapped to the supervised owner module whose
2550    /// current spawn nonce authorizes HELLO claims below the prefix.
2551    ///
2552    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2553    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2554    /// accidental collisions and lower-trust processes from squatting protected
2555    /// namespaces.
2556    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2557    /// Blue/green swaps in progress, by module id. An entry exists from just
2558    /// before the candidate process is spawned until the swap has failed, or
2559    /// has cut over and the old process is gone. While it exists, HELLO for the
2560    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2561    /// consumer attestation accepts both processes' nonces.
2562    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2563    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2564    promotion_observer: PromotionObserverSlot,
2565    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2566    /// this daemon-wide ordering, a rescan could retire or update a module while a
2567    /// concurrent reload still held its old handle and launch specification.
2568    operation_lock: Arc<AsyncMutex<()>>,
2569}
2570
2571#[cfg(test)]
2572#[derive(Debug)]
2573struct IdentitySamplePause {
2574    reached: std::sync::mpsc::Sender<()>,
2575    release: std::sync::mpsc::Receiver<()>,
2576}
2577
2578/// Told when a swap has promoted its candidate to be the module's active
2579/// registration.
2580///
2581/// An ordinary HELLO runs the control plane's registration side effects (the
2582/// capability cache, the deny census, the requirement recompute) as it
2583/// registers. A swap candidate's HELLO does not, because it is not routable;
2584/// promotion is when those must run instead, and promotion happens in the
2585/// supervisor, which has no other way into the control handler.
2586pub(crate) trait SwapPromotionObserver: Send + Sync {
2587    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2588}
2589
2590/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2591/// control handler) owns this handle, so a strong reference back would be a
2592/// cycle that keeps both alive.
2593#[derive(Clone, Default)]
2594struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2595
2596impl fmt::Debug for PromotionObserverSlot {
2597    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2598        f.write_str("PromotionObserverSlot")
2599    }
2600}
2601
2602/// The nonces of one open swap.
2603#[derive(Debug, Clone)]
2604struct OpenSwap {
2605    /// Legacy candidate token. A gated candidate has an empty lifecycle
2606    /// placeholder instead; only its own channel admits its HELLO.
2607    candidate_nonce: String,
2608    /// Incumbent token accepted before cutover, then cleared by promotion even
2609    /// while the incumbent's process is still draining.
2610    incumbent_nonce: Option<String>,
2611    /// Set once a HELLO has been admitted with the swap token, so the token
2612    /// admits one registration and cannot be replayed after cutover empties
2613    /// the candidate slot.
2614    candidate_admitted: bool,
2615}
2616
2617/// What the swap gate says about a HELLO. See
2618/// [`SupervisorHandle::swap_hello_admission`].
2619#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2620pub(crate) enum SwapHelloAdmission {
2621    /// No swap is open for the id (or the HELLO carries the incumbent's own
2622    /// nonce); the ordinary gates decide.
2623    NotSwapping,
2624    /// The HELLO carries the swap token: register it into the candidate slot.
2625    Candidate,
2626    /// A swap is open and the HELLO carries a nonce the supervisor did not
2627    /// mint for this id, no nonce, or a token already used.
2628    Refused,
2629    /// The id belongs to a supervised `protocol: none` module. Such a module
2630    /// never registers, so no HELLO may claim its id, with or without a nonce.
2631    ProtocolNone,
2632    /// The id is gated. Its supervised child registers only over the
2633    /// registration channel it inherited at launch, never by HELLO alone.
2634    ChannelRequired,
2635}
2636
2637#[derive(Debug, Clone, PartialEq, Eq)]
2638pub(crate) enum ReservedHelloRejection {
2639    Exact {
2640        module_id: String,
2641    },
2642    Prefix {
2643        prefix: String,
2644        owner_module_id: String,
2645    },
2646}
2647
2648/// Identity facts established by the reserved-module policy for one HELLO.
2649#[derive(Debug, Default)]
2650pub(crate) struct ReservedHelloVerification {
2651    pub(crate) nonce_verified: bool,
2652    pub(crate) prefix_owner: Option<String>,
2653}
2654
2655#[derive(Debug)]
2656struct DaemonRestartIntent {
2657    expires: Option<Duration>,
2658    window: Duration,
2659}
2660
2661impl Default for DaemonRestartIntent {
2662    fn default() -> Self {
2663        Self {
2664            expires: None,
2665            window: Duration::from_secs(120),
2666        }
2667    }
2668}
2669
2670impl SupervisorHandle {
2671    pub(crate) fn module_is_gated(&self, module: &str) -> bool {
2672        let runtime = {
2673            let Ok(runtime) = self.gate_runtime.lock() else {
2674                return true;
2675            };
2676            runtime.clone()
2677        };
2678        runtime.is_some_and(|runtime| {
2679            runtime
2680                .upgrade()
2681                .is_none_or(|owner| owner.module_is_gated(module))
2682        })
2683    }
2684    pub(crate) fn announce_daemon_restart(&self) -> u64 {
2685        let mut intent = self
2686            .restart_intent
2687            .lock()
2688            .unwrap_or_else(|p| p.into_inner());
2689        intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2690        unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2691    }
2692
2693    #[cfg(feature = "test-support")]
2694    pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2695        self.restart_intent
2696            .lock()
2697            .unwrap_or_else(|p| p.into_inner())
2698            .window = window;
2699    }
2700
2701    #[cfg(unix)]
2702    fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2703        let expiry = self
2704            .restart_intent
2705            .lock()
2706            .unwrap_or_else(|p| p.into_inner())
2707            .expires
2708            .take();
2709        if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2710            RouteCloseReason::Restart
2711        } else {
2712            RouteCloseReason::Disable
2713        }
2714    }
2715
2716    pub fn new() -> Self {
2717        Self::default()
2718    }
2719
2720    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2721        self.spawn_events.snapshot()
2722    }
2723
2724    /// Subscribe before reading the bound process. A coalesced wake is enough:
2725    /// readers compare its exact generation and PID, not the last event kind.
2726    pub(crate) fn subscribe_process_changes(
2727        &self,
2728        module: &str,
2729    ) -> tokio::sync::watch::Receiver<u64> {
2730        let mut state = self
2731            .spawn_events
2732            .0
2733            .lock()
2734            .unwrap_or_else(|p| p.into_inner());
2735        state
2736            .process_changes
2737            .retain(|_, changed| changed.receiver_count() > 0);
2738        let seq = state.seq;
2739        state
2740            .process_changes
2741            .entry(module.into())
2742            .or_insert_with(|| tokio::sync::watch::channel(seq).0)
2743            .subscribe()
2744    }
2745
2746    pub(crate) fn current_spawn(&self, module: &str) -> Option<LiveSpawn> {
2747        self.spawn_events
2748            .0
2749            .lock()
2750            .unwrap_or_else(|p| p.into_inner())
2751            .live
2752            .get(module)
2753            .cloned()
2754    }
2755
2756    pub(crate) fn subscribe_spawns(
2757        &self,
2758        connection_id: ConnectionId,
2759        corr: u64,
2760        version: u8,
2761        since: Option<SpawnCursor>,
2762        sink: FrameSink,
2763    ) -> Result<(), SpawnSubscribeRefusal> {
2764        self.spawn_events
2765            .subscribe(connection_id, corr, version, since, sink)
2766    }
2767
2768    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2769        self.spawn_events.cancel(connection_id, corr)
2770    }
2771
2772    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2773        self.spawn_events.remove_connection(connection_id);
2774    }
2775
2776    #[cfg(any(test, feature = "test-support"))]
2777    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2778        assert!(capacity > 0, "spawn event capacity must be non-zero");
2779        self.spawn_events.set_capacity(capacity);
2780    }
2781
2782    #[cfg(any(test, feature = "test-support"))]
2783    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2784        self.spawn_events.subscriber_count()
2785    }
2786
2787    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2788    /// a respawn invalidates stale consumer identities.
2789    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2790        self.spawn_nonces
2791            .lock()
2792            .unwrap_or_else(|poisoned| poisoned.into_inner())
2793            .insert(module_id.to_string(), nonce);
2794        self.refresh_program_binding_nonces(module_id);
2795    }
2796
2797    /// Record the launch nonce expected from the next HELLO for a reserved module,
2798    /// replacing any prior nonce (a respawn invalidates the previous one).
2799    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2800        {
2801            let mut reserved = self
2802                .reserved_nonces
2803                .lock()
2804                .unwrap_or_else(|p| p.into_inner());
2805            self.implicit_protocol_none
2806                .lock()
2807                .unwrap_or_else(|p| p.into_inner())
2808                .remove(module_id);
2809            reserved.insert(module_id.to_string(), Some(nonce));
2810        }
2811        self.refresh_program_binding_nonces(module_id);
2812    }
2813
2814    /// Record namespace prefixes owned by a supervised module.
2815    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2816        let mut owners = self
2817            .reserved_prefix_owners
2818            .lock()
2819            .unwrap_or_else(|poisoned| poisoned.into_inner());
2820        owners.retain(|_, owner| owner != owner_module_id);
2821        for prefix in prefixes {
2822            owners.insert(prefix.clone(), owner_module_id.to_string());
2823        }
2824    }
2825
2826    /// The launch nonce most recently minted for a module's spawn, if any.
2827    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2828        if self.module_is_gated(module_id) {
2829            return None;
2830        }
2831        self.spawn_nonces
2832            .lock()
2833            .unwrap_or_else(|poisoned| poisoned.into_inner())
2834            .get(module_id)
2835            .cloned()
2836    }
2837
2838    #[cfg(all(test, unix))]
2839    pub(crate) fn legacy_nonce_present_for_test(&self, module: &str) -> bool {
2840        self.spawn_nonces.lock().unwrap().contains_key(module)
2841    }
2842
2843    pub(crate) fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2844        // Prefix ownership follows staged configuration, not the running
2845        // child's launch settings. A respawn may still carry an older spec.
2846        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2847        let protocol = self
2848            .get(&spec.module_id)
2849            .and_then(|module| {
2850                let snapshot = module.inner.snapshot.lock().ok()?;
2851                snapshot
2852                    .process_alive
2853                    .then_some(snapshot.spawned_protocol.unwrap_or(spec.protocol))
2854            })
2855            .unwrap_or(spec.protocol);
2856        self.apply_identity_policy(spec, protocol);
2857    }
2858
2859    #[cfg(all(test, unix))]
2860    pub(crate) fn pause_plain_identity_policy_for_test(
2861        &self,
2862        module: &str,
2863    ) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
2864        let (reached, waiting) = std::sync::mpsc::channel();
2865        let (release, paused) = std::sync::mpsc::channel();
2866        *self.plain_identity_policy_pause.lock().unwrap() = Some((
2867            module.to_owned(),
2868            IdentitySamplePause {
2869                reached,
2870                release: paused,
2871            },
2872        ));
2873        (waiting, release)
2874    }
2875
2876    fn apply_identity_policy(&self, spec: &ModuleSpec, protocol: ModuleProtocol) {
2877        let channel_only = self.module_is_gated(&spec.module_id);
2878        let mut current_nonces = self
2879            .spawn_nonces
2880            .lock()
2881            .unwrap_or_else(|poisoned| poisoned.into_inner());
2882        if protocol == ModuleProtocol::None || channel_only {
2883            current_nonces.remove(&spec.module_id);
2884        }
2885        let spawn_nonce = current_nonces.get(&spec.module_id).cloned();
2886        #[cfg(test)]
2887        if let Some(pause) = self.identity_sample_pause.lock().unwrap().take() {
2888            pause.reached.send(()).unwrap();
2889            pause.release.recv().unwrap();
2890        }
2891        let mut reserved_nonces = self
2892            .reserved_nonces
2893            .lock()
2894            .unwrap_or_else(|poisoned| poisoned.into_inner());
2895        let mut implicit = self
2896            .implicit_protocol_none
2897            .lock()
2898            .unwrap_or_else(|p| p.into_inner());
2899        if spec.reserved {
2900            implicit.remove(&spec.module_id);
2901        } else if protocol == ModuleProtocol::None {
2902            if !reserved_nonces.contains_key(&spec.module_id) {
2903                implicit.insert(spec.module_id.clone());
2904            }
2905        } else if implicit.remove(&spec.module_id) {
2906            reserved_nonces.remove(&spec.module_id);
2907        }
2908        if spec.reserved || protocol == ModuleProtocol::None {
2909            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2910            // reserved name whose module has never spawned has no legitimate
2911            // holder, and the entry's absence is what used to leave the name
2912            // open to the first claimant.
2913            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2914        }
2915        drop(implicit);
2916        drop(reserved_nonces);
2917        drop(current_nonces);
2918        self.refresh_program_binding_nonces(&spec.module_id);
2919        // A later unreserved declaration must not silently unreserve an id that
2920        // was retained after its reserved configuration was removed. The explicit
2921        // release ceremony is the only operation that retires that gate.
2922        self.removal_tombstones
2923            .lock()
2924            .unwrap_or_else(|poisoned| poisoned.into_inner())
2925            .remove(&spec.module_id);
2926    }
2927
2928    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2929    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2930    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2931    /// with no matching prefix are always authorized.
2932    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2933        self.reserved_hello_verification(module_id, presented)
2934            .is_ok()
2935    }
2936
2937    /// Whether HELLO passed a nonce check that was actually required by reserved
2938    /// module policy, or the refusal explaining why a required check failed.
2939    pub(crate) fn reserved_hello_verification(
2940        &self,
2941        module_id: &str,
2942        presented: Option<&str>,
2943    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2944        if self.module_is_gated(module_id) {
2945            return Err(ReservedHelloRejection::Exact {
2946                module_id: module_id.into(),
2947            });
2948        }
2949        self.hello_verification(module_id, presented, false)
2950    }
2951
2952    pub(crate) fn channel_hello_verification(
2953        &self,
2954        module_id: &str,
2955    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2956        if self
2957            .get(module_id)
2958            .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
2959        {
2960            return Err(ReservedHelloRejection::Exact {
2961                module_id: module_id.into(),
2962            });
2963        }
2964        let mut verification = self.hello_verification(module_id, None, true)?;
2965        verification.nonce_verified = true;
2966        Ok(verification)
2967    }
2968
2969    pub(crate) fn prefix_claim_authorized(&self, owner: &str, module_id: &str) -> bool {
2970        self.prefix_claim_publication(owner, module_id).is_some()
2971    }
2972
2973    #[cfg(all(test, unix))]
2974    pub(crate) fn prefix_publication_locked_for_test(&self) -> bool {
2975        matches!(
2976            self.reserved_prefix_owners.try_lock(),
2977            Err(std::sync::TryLockError::WouldBlock)
2978        )
2979    }
2980
2981    // Keep this guard through registry/forwarding publication. Rescans replace
2982    // prefix owners without stopping their launches, so the spawn guard alone
2983    // cannot fence a reservation transfer.
2984    pub(crate) fn prefix_claim_publication(
2985        &self,
2986        owner: &str,
2987        module_id: &str,
2988    ) -> Option<std::sync::MutexGuard<'_, HashMap<String, String>>> {
2989        if subc_protocol::manifest::validate_module_id(module_id).is_err()
2990            || self.module_is_gated(module_id)
2991            || self
2992                .reserved_nonces
2993                .lock()
2994                .unwrap_or_else(|p| p.into_inner())
2995                .contains_key(module_id)
2996        {
2997            return None;
2998        }
2999        let owners = self
3000            .reserved_prefix_owners
3001            .lock()
3002            .unwrap_or_else(|p| p.into_inner());
3003        let authorized = owners
3004            .iter()
3005            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3006            .max_by_key(|(prefix, _)| prefix.len())
3007            .is_some_and(|(prefix, holder)| holder == owner && module_id.len() > prefix.len());
3008        authorized.then_some(owners)
3009    }
3010
3011    fn hello_verification(
3012        &self,
3013        module_id: &str,
3014        presented: Option<&str>,
3015        channel: bool,
3016    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
3017        let nonces = self
3018            .reserved_nonces
3019            .lock()
3020            .unwrap_or_else(|poisoned| poisoned.into_inner());
3021        if let Some(expected) = nonces.get(module_id) {
3022            // `None` = reserved with no legitimate holder: refuse every
3023            // presentation, because no process can hold a nonce that was never
3024            // minted. Only a real minted nonce admits, in constant time.
3025            let authorized = channel
3026                || match expected {
3027                    Some(expected) => presented.is_some_and(|p| {
3028                        !p.is_empty() && constant_time_eq(expected.as_bytes(), p.as_bytes())
3029                    }),
3030                    None => false,
3031                };
3032            if authorized {
3033                return Ok(ReservedHelloVerification {
3034                    nonce_verified: true,
3035                    prefix_owner: None,
3036                });
3037            }
3038            return Err(ReservedHelloRejection::Exact {
3039                module_id: module_id.to_string(),
3040            });
3041        }
3042        drop(nonces);
3043
3044        let matched_prefix = self
3045            .reserved_prefix_owners
3046            .lock()
3047            .unwrap_or_else(|poisoned| poisoned.into_inner())
3048            .iter()
3049            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3050            .max_by_key(|(prefix, _)| prefix.len())
3051            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
3052        let Some((prefix, owner_module_id)) = matched_prefix else {
3053            return Ok(ReservedHelloVerification::default());
3054        };
3055
3056        if module_id.len() == prefix.len()
3057            || (!channel
3058                && (self.module_is_gated(&owner_module_id) || self.protocol_none(&owner_module_id)))
3059        {
3060            return Err(ReservedHelloRejection::Prefix {
3061                prefix,
3062                owner_module_id,
3063            });
3064        }
3065
3066        let authorized = if channel {
3067            owner_module_id == module_id
3068        } else {
3069            presented.is_some_and(|presented| {
3070                self.spawn_nonces
3071                .lock()
3072                .unwrap_or_else(|poisoned| poisoned.into_inner())
3073                .get(&owner_module_id)
3074                .is_some_and(|expected| !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes()))
3075                // While the owner is being swapped, children started by
3076                // either of its two processes hold that process's nonce.
3077                || self.swap_nonce_matches(&owner_module_id, presented)
3078            })
3079        };
3080        if authorized {
3081            Ok(ReservedHelloVerification {
3082                nonce_verified: true,
3083                prefix_owner: (owner_module_id != module_id).then_some(owner_module_id),
3084            })
3085        } else {
3086            Err(ReservedHelloRejection::Prefix {
3087                prefix,
3088                owner_module_id,
3089            })
3090        }
3091    }
3092
3093    /// Whether a consumer connection proved it came from a daemon-spawned module.
3094    ///
3095    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
3096    /// accepted only for module ids the supervisor has spawned.
3097    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
3098        if self.module_is_gated(module_id) || self.protocol_none(module_id) {
3099            return false;
3100        }
3101        let nonces = self
3102            .spawn_nonces
3103            .lock()
3104            .unwrap_or_else(|poisoned| poisoned.into_inner());
3105        let current = nonces.get(module_id).is_some_and(|expected| {
3106            !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes())
3107        });
3108        drop(nonces);
3109        // Before cutover both launches may attest. Promotion removes the
3110        // incumbent's token even if its process still needs time to drain.
3111        current || self.swap_nonce_matches(module_id, presented)
3112    }
3113
3114    /// Whether `presented` is either nonce of an open swap for `module_id`.
3115    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
3116        if self.module_is_gated(module_id) || self.protocol_none(module_id) || presented.is_empty()
3117        {
3118            return false;
3119        }
3120        let swaps = self
3121            .swaps
3122            .lock()
3123            .unwrap_or_else(|poisoned| poisoned.into_inner());
3124        swaps.get(module_id).is_some_and(|swap| {
3125            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
3126                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
3127                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
3128                })
3129        })
3130    }
3131
3132    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
3133    /// Called before the candidate process exists.
3134    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
3135        let incumbent_nonce = self
3136            .spawn_nonces
3137            .lock()
3138            .unwrap_or_else(|poisoned| poisoned.into_inner())
3139            .get(module_id)
3140            .cloned();
3141        self.swaps
3142            .lock()
3143            .unwrap_or_else(|poisoned| poisoned.into_inner())
3144            .insert(
3145                module_id.to_string(),
3146                OpenSwap {
3147                    candidate_nonce,
3148                    incumbent_nonce,
3149                    candidate_admitted: false,
3150                },
3151            );
3152        self.refresh_program_binding_nonces(module_id);
3153    }
3154
3155    /// Close the swap for `module_id`, releasing whichever nonce is no longer
3156    /// the module's recorded one.
3157    pub(crate) fn close_swap(&self, module_id: &str) {
3158        self.swaps
3159            .lock()
3160            .unwrap_or_else(|poisoned| poisoned.into_inner())
3161            .remove(module_id);
3162        self.refresh_program_binding_nonces(module_id);
3163    }
3164
3165    /// Install the observer told about swap promotions, replacing any earlier
3166    /// one.
3167    pub(crate) fn set_swap_promotion_observer(
3168        &self,
3169        observer: std::sync::Weak<dyn SwapPromotionObserver>,
3170    ) {
3171        *self
3172            .promotion_observer
3173            .0
3174            .lock()
3175            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
3176    }
3177
3178    /// Tell the installed observer, if it is still alive, that a swap promoted
3179    /// `registration`.
3180    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
3181        let observer = self
3182            .promotion_observer
3183            .0
3184            .lock()
3185            .unwrap_or_else(|poisoned| poisoned.into_inner())
3186            .as_ref()
3187            .and_then(std::sync::Weak::upgrade);
3188        if let Some(observer) = observer {
3189            observer.swap_promoted(registration);
3190        }
3191    }
3192
3193    /// Whether a swap is open for `module_id`.
3194    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
3195        self.swaps
3196            .lock()
3197            .unwrap_or_else(|poisoned| poisoned.into_inner())
3198            .contains_key(module_id)
3199    }
3200
3201    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
3202    /// respawn would, once cutover has made the candidate the module's process.
3203    /// Promotion revokes the incumbent's tokens; the swap record stays open
3204    /// only for lifecycle cleanup while the old process drains.
3205    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
3206        let candidate_nonce = self
3207            .swaps
3208            .lock()
3209            .unwrap_or_else(|poisoned| poisoned.into_inner())
3210            .get_mut(module_id)
3211            .map(|swap| {
3212                swap.incumbent_nonce = None;
3213                swap.candidate_nonce.clone()
3214            });
3215        self.spawn_nonces
3216            .lock()
3217            .unwrap_or_else(|p| p.into_inner())
3218            .remove(module_id);
3219        {
3220            let mut nonces = self
3221                .reserved_nonces
3222                .lock()
3223                .unwrap_or_else(|p| p.into_inner());
3224            if reserved || nonces.contains_key(module_id) {
3225                nonces.insert(module_id.into(), None);
3226            }
3227        }
3228        if self.module_is_gated(module_id) {
3229            return;
3230        }
3231        let Some(nonce) = candidate_nonce.filter(|nonce| !nonce.is_empty()) else {
3232            return;
3233        };
3234        self.set_spawn_nonce(module_id, nonce.clone());
3235        if reserved {
3236            self.set_reserved_nonce(module_id, nonce);
3237        }
3238    }
3239
3240    /// The swap gate for a HELLO claiming `module_id`.
3241    ///
3242    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
3243    /// presents the candidate nonce, which the reserved gate (holding the
3244    /// incumbent's nonce) would refuse as `reserved_module` before swap
3245    /// admission was ever reached. And it applies to unreserved ids too: for an
3246    /// unreserved id the only thing that ever stopped a second process claiming
3247    /// a live id was the `duplicate_module_id` refusal, which is exactly the
3248    /// refusal a swap lifts for its candidate.
3249    ///
3250    /// The incumbent's own nonce falls through to the ordinary gates, which
3251    /// treat it as they always have (a live incumbent is refused as a
3252    /// duplicate). Anything else while a swap is open is refused, including an
3253    /// absent nonce.
3254    pub(crate) fn swap_hello_admission(
3255        &self,
3256        module_id: &str,
3257        presented: Option<&str>,
3258    ) -> SwapHelloAdmission {
3259        if self.protocol_none(module_id) {
3260            return SwapHelloAdmission::ProtocolNone;
3261        }
3262        if self.module_is_gated(module_id) {
3263            return SwapHelloAdmission::ChannelRequired;
3264        }
3265        let swaps = self
3266            .swaps
3267            .lock()
3268            .unwrap_or_else(|poisoned| poisoned.into_inner());
3269        let Some(swap) = swaps.get(module_id) else {
3270            return SwapHelloAdmission::NotSwapping;
3271        };
3272        let Some(presented) = presented.filter(|nonce| !nonce.is_empty()) else {
3273            return SwapHelloAdmission::Refused;
3274        };
3275        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
3276            return if swap.candidate_admitted {
3277                SwapHelloAdmission::Refused
3278            } else {
3279                SwapHelloAdmission::Candidate
3280            };
3281        }
3282        if swap
3283            .incumbent_nonce
3284            .as_deref()
3285            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
3286        {
3287            return SwapHelloAdmission::NotSwapping;
3288        }
3289        SwapHelloAdmission::Refused
3290    }
3291
3292    fn protocol_none(&self, module: &str) -> bool {
3293        self.get(module)
3294            .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3295    }
3296
3297    /// Exclude swap closure through the short, synchronous candidate publication.
3298    pub(crate) fn with_swap_candidate_admission<T>(
3299        &self,
3300        module_id: &str,
3301        publish: impl FnOnce() -> (T, bool),
3302    ) -> Option<T> {
3303        let mut swaps = self.swaps.lock().unwrap_or_else(|p| p.into_inner());
3304        let swap = swaps
3305            .get_mut(module_id)
3306            .filter(|swap| !swap.candidate_admitted)?;
3307        let (result, admitted) = publish();
3308        swap.candidate_admitted = admitted;
3309        Some(result)
3310    }
3311
3312    /// Test/support lookup for the current launch nonce of a supervised spawn.
3313    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3314        self.spawn_nonce(module_id)
3315    }
3316
3317    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
3318    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3319        if self.module_is_gated(module_id) {
3320            return None;
3321        }
3322        self.reserved_nonces
3323            .lock()
3324            .unwrap_or_else(|poisoned| poisoned.into_inner())
3325            .get(module_id)
3326            .cloned()
3327            .flatten()
3328    }
3329
3330    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
3331        // Normally already marked before the process was spawned; marking here
3332        // too keeps `configured_ids` a superset of the roster for any caller
3333        // that inserts a module directly.
3334        self.mark_configured(module.module_id());
3335        let mut modules = self
3336            .modules
3337            .lock()
3338            .unwrap_or_else(|poisoned| poisoned.into_inner());
3339        modules.insert(module.module_id().to_string(), module)
3340    }
3341
3342    /// Record that the supervisor has taken on `module_id`. Called before the
3343    /// module's first process is spawned, so that by the time that process can
3344    /// register, [`Self::is_configured`] already answers true.
3345    fn mark_configured(&self, module_id: &str) {
3346        self.update_configured_ids(|ids| {
3347            ids.insert(module_id.to_string());
3348        });
3349    }
3350
3351    fn publish_configured_ids(&self, ids: Option<&HashSet<String>>) {
3352        let owner = self
3353            .gate_runtime
3354            .lock()
3355            .unwrap_or_else(|p| p.into_inner())
3356            .clone()
3357            .and_then(|owner| owner.upgrade());
3358        if let Some(owner) = owner {
3359            owner.observe_supervisor_configured_ids(ids, Arc::downgrade(&self.configured_ids));
3360        }
3361    }
3362
3363    fn update_configured_ids(&self, update: impl FnOnce(&mut HashSet<String>)) {
3364        match self.configured_ids.lock() {
3365            Ok(mut ids) => {
3366                update(&mut ids);
3367                // Publish while this roster mutation is serialized, so an
3368                // older snapshot can never overwrite a later configured id.
3369                self.publish_configured_ids(Some(&ids));
3370            }
3371            Err(poisoned) => {
3372                update(&mut poisoned.into_inner());
3373                self.publish_configured_ids(None);
3374            }
3375        }
3376    }
3377
3378    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
3379    /// before it was ever put on the roster. A module already on the roster
3380    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
3381    fn unmark_configured_unless_rostered(&self, module_id: &str) {
3382        let modules = self
3383            .modules
3384            .lock()
3385            .unwrap_or_else(|poisoned| poisoned.into_inner());
3386        if !modules.contains_key(module_id) {
3387            self.update_configured_ids(|ids| {
3388                ids.remove(module_id);
3389            });
3390        }
3391    }
3392
3393    /// Whether `module_id` is a module this daemon supervises: on the roster,
3394    /// or about to be (its process is being spawned right now).
3395    ///
3396    /// This, not `get(..).is_some()`, is what "the owner is configured" means
3397    /// for scopes: a supervised module's process can register and sync before
3398    /// [`Self::get`] can return it, and in that window it is still a module
3399    /// that will sync, not one that never will.
3400    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
3401        self.configured_ids
3402            .lock()
3403            .unwrap_or_else(|poisoned| poisoned.into_inner())
3404            .contains(module_id)
3405    }
3406
3407    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
3408        let modules = self
3409            .modules
3410            .lock()
3411            .unwrap_or_else(|poisoned| poisoned.into_inner());
3412        modules.get(module_id).cloned()
3413    }
3414
3415    pub(crate) fn record_late_health_answer(
3416        &self,
3417        module_id: &str,
3418        latency_ms: u64,
3419    ) -> Result<bool, SuperviseError> {
3420        let Some(module) = self.get(module_id) else {
3421            return Ok(false);
3422        };
3423        self.host_health.record_late_module(module_id);
3424        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
3425            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
3426            state.health.last_late_answer_latency_ms = Some(latency_ms);
3427            // A late answer is an answer: the module served the probe, just past
3428            // the deadline. Leaving the miss streak in place while logging
3429            // "proves the module is alive" is how a CPU-starved module that
3430            // answers every probe a few seconds late still marches to the
3431            // threshold and gets killed — the exact kill class `NoAnswer` is
3432            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
3433            // is degradation, and degradation reports; it does not restart.
3434            state.health.consecutive_failures = 0;
3435        })?;
3436        Ok(true)
3437    }
3438
3439    pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
3440        self.host_health.spawn_monitor()
3441    }
3442
3443    /// Arm the one-shot marker for the module process that this caller
3444    /// deliberately initiated severance against. Generic connection teardown
3445    /// must not call this:
3446    /// a surviving process would otherwise retain an exemption for a later
3447    /// genuine crash.
3448    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
3449        let Some(module) = self.get(module_id) else {
3450            return Ok(false);
3451        };
3452        let snapshot = lock_snapshot(&module.inner.snapshot)?;
3453        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
3454            return Ok(false);
3455        };
3456        drop(snapshot);
3457        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
3458    }
3459
3460    pub fn list(&self) -> Vec<SupervisedModule> {
3461        let modules = self
3462            .modules
3463            .lock()
3464            .unwrap_or_else(|poisoned| poisoned.into_inner());
3465        let mut modules = modules.values().cloned().collect::<Vec<_>>();
3466        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
3467        modules
3468    }
3469
3470    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
3471        self.activity_images
3472            .lock()
3473            .unwrap_or_else(|p| p.into_inner())
3474            .remove(module_id);
3475        self.spawn_nonces
3476            .lock()
3477            .unwrap_or_else(|poisoned| poisoned.into_inner())
3478            .remove(module_id);
3479        self.close_swap(module_id);
3480        let mut reserved_nonces = self
3481            .reserved_nonces
3482            .lock()
3483            .unwrap_or_else(|poisoned| poisoned.into_inner());
3484        if reserved_nonces.contains_key(module_id) {
3485            // The old nonce must die with the removed process, but the exact-id
3486            // gate remains until an operator explicitly releases it.
3487            reserved_nonces.insert(module_id.to_string(), None);
3488        }
3489        drop(reserved_nonces);
3490        self.reserved_prefix_owners
3491            .lock()
3492            .unwrap_or_else(|poisoned| poisoned.into_inner())
3493            .retain(|_, owner| owner != module_id);
3494        let removed = self
3495            .modules
3496            .lock()
3497            .unwrap_or_else(|poisoned| poisoned.into_inner())
3498            .remove(module_id);
3499        self.update_configured_ids(|ids| {
3500            ids.remove(module_id);
3501        });
3502        self.remove_program_binding_records(module_id);
3503        removed
3504    }
3505
3506    /// Remember a module removed by a non-preview rescan so route.open can
3507    /// distinguish that intentional removal from an unknown id.
3508    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
3509        self.removal_tombstones
3510            .lock()
3511            .unwrap_or_else(|poisoned| poisoned.into_inner())
3512            .insert(module_id.to_string(), unix_ms_now());
3513    }
3514
3515    /// Return how long ago a rescan removed this module in milliseconds.
3516    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
3517        self.removal_tombstones
3518            .lock()
3519            .unwrap_or_else(|poisoned| poisoned.into_inner())
3520            .get(module_id)
3521            .copied()
3522            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
3523    }
3524
3525    /// Retire a reserved-id gate only after its module has left supervision.
3526    ///
3527    /// A retained gate has no live nonce (`None`), so releasing any other entry
3528    /// would weaken a currently configured or otherwise active reservation.
3529    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
3530        if self.get(module_id).is_some() {
3531            return false;
3532        }
3533        let mut reserved_nonces = self
3534            .reserved_nonces
3535            .lock()
3536            .unwrap_or_else(|poisoned| poisoned.into_inner());
3537        if !matches!(reserved_nonces.get(module_id), Some(None)) {
3538            return false;
3539        }
3540        reserved_nonces.remove(module_id);
3541        self.implicit_protocol_none
3542            .lock()
3543            .unwrap_or_else(|p| p.into_inner())
3544            .remove(module_id);
3545        true
3546    }
3547
3548    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
3549        Arc::clone(&self.operation_lock)
3550    }
3551}
3552
3553/// Process supervisor for subc-owned singleton modules.
3554#[derive(Debug, Clone)]
3555pub struct Supervisor {
3556    registry: Arc<Registry>,
3557    config_delivery: Option<config_delivery::ConfigDelivery>,
3558    restart_policy: RestartPolicy,
3559    drain_timeout: Duration,
3560    connection_file_path: Option<PathBuf>,
3561    capture_logs_dir: Option<PathBuf>,
3562    forwarding: Option<Arc<ForwardingTable>>,
3563    process_liveness: Arc<SupervisorProcessLiveness>,
3564    supervisor_handle: Option<SupervisorHandle>,
3565    health: HealthConfig,
3566    host_health: HostHealth,
3567    daemon_start_clock: crate::clock::StartClock,
3568    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
3569    spawn_events: SpawnEventFeed,
3570    provenance_probe: ExecutableIdentityProbe,
3571    /// Every process spawned through this supervisor (and its clones) and not
3572    /// yet reaped, so daemon shutdown can end them.
3573    child_roster: ChildRoster,
3574    #[cfg(target_os = "linux")]
3575    cgroup_placement: Option<subc_cgroup::Placement>,
3576    #[cfg(test)]
3577    test_after_first_spawn: AfterFirstSpawnHook,
3578}
3579
3580/// A roster entry whose monitor and first process have not started yet.
3581/// Keeping the command receiver here lets discovery serve status immediately
3582/// while start/stop requests wait until bootstrap has published the connection file.
3583pub(crate) struct RegisteredConfiguredModule {
3584    spec: ModuleSpec,
3585    runtime: SupervisorRuntimeConfig,
3586    module: SupervisedModule,
3587    commands: mpsc::Receiver<SupervisorCommand>,
3588}
3589
3590/// Test-only hook run on the path that takes on a new module, right after its
3591/// first `spawn_child` returns (the process exists and could already be
3592/// registering) and before that process is handed to the module's supervise
3593/// loop and put on the roster. Lets a test observe what a fast child would see
3594/// in that window without racing a real one.
3595#[cfg(test)]
3596#[derive(Clone, Default)]
3597struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
3598
3599#[cfg(test)]
3600type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
3601
3602#[cfg(test)]
3603impl fmt::Debug for AfterFirstSpawnHook {
3604    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3605        f.write_str("AfterFirstSpawnHook")
3606    }
3607}
3608
3609#[cfg(test)]
3610impl AfterFirstSpawnHook {
3611    fn run(&self, module_id: &str) {
3612        if let Some(hook) = &self.0 {
3613            hook(module_id);
3614        }
3615    }
3616}
3617
3618impl Supervisor {
3619    #[cfg(all(test, target_os = "linux"))]
3620    pub(crate) fn set_admission_hook_for_test(
3621        &self,
3622        hook: impl Fn(&str, &'static str, Option<u32>) -> bool + Send + Sync + 'static,
3623    ) {
3624        self.child_roster.set_admission_hook_for_test(hook);
3625    }
3626
3627    #[cfg(test)]
3628    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
3629        let supervisor = Self::new(registry, policy);
3630        #[cfg(target_os = "macos")]
3631        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
3632        supervisor
3633    }
3634    /// Verify the trampoline once when configured. Missing private OS support
3635    /// refuses every macOS launch by name but does not stop the daemon's control
3636    /// server. Embedders must explicitly provide a binary with the subc-os hidden
3637    /// entry point; the library must not exec an arbitrary hosting program.
3638    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
3639        let path = path.into();
3640        #[cfg(target_os = "macos")]
3641        {
3642            let result = probe_privacy_trampoline(&path).map(|()| path);
3643            if let Err(cause) = &result {
3644                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
3645            }
3646            self.child_roster.set_privacy_trampoline(result);
3647        }
3648        #[cfg(not(target_os = "macos"))]
3649        let _ = path;
3650        self
3651    }
3652    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
3653    /// capability-requirement log to read.
3654    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
3655        self.child_roster.shutdown_flag()
3656    }
3657
3658    /// The first step of an announced daemon shutdown, before the notice and
3659    /// before any connection is closed.
3660    ///
3661    /// Sets the daemon-shutdown flag first: from here on no module is
3662    /// respawned (crash restart, operator restart, or swap), and every child
3663    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
3664    /// the module exits on the EOF this shutdown gives it or is signalled by a
3665    /// service manager that kills the whole cgroup. Then writes the journal's
3666    /// shutdown marker, which records the instant and closes this daemon
3667    /// incarnation's stretch of the journal.
3668    #[cfg(unix)]
3669    pub(crate) fn begin_daemon_shutdown(&self) {
3670        self.child_roster.close();
3671        if let Some(journal) = &self.terminal_journal {
3672            journal.stamp_shutdown();
3673        }
3674    }
3675
3676    /// Announce a cut while established connections can still carry replies.
3677    /// These budgets bound the attempt to deliver notices and the wait for
3678    /// routes to drain; neither guarantees delivery or child completion, and
3679    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
3680    #[cfg(unix)]
3681    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
3682        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
3683        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
3684        let Some(forwarding) = &self.forwarding else {
3685            return Ok(());
3686        };
3687        let notice_started = Instant::now();
3688        let modules = forwarding
3689            .begin_daemon_drain()
3690            .map_err(SuperviseError::Forwarding)?;
3691        let deadline_ms =
3692            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
3693        let shutdown_reason = self
3694            .supervisor_handle
3695            .as_ref()
3696            .map(SupervisorHandle::take_daemon_shutdown_reason)
3697            .unwrap_or(RouteCloseReason::Disable);
3698        let mut modules_told = 0;
3699        let mut notices = tokio::task::JoinSet::new();
3700        let mut drains = Vec::new();
3701        for connection in forwarding
3702            .module_connections()
3703            .map_err(SuperviseError::Forwarding)?
3704        {
3705            let target = ModuleDrainTarget {
3706                endpoint: connection.endpoint,
3707                sink: connection.sink,
3708                negotiated_ver: connection.negotiated_ver,
3709                abandoned_bindings: Vec::new(),
3710                excluded_subscriptions: 0,
3711            };
3712            notices.spawn(async move {
3713                (
3714                    true,
3715                    send_module_draining(
3716                        &connection.module_id,
3717                        shutdown_reason,
3718                        deadline_ms,
3719                        &target,
3720                        true,
3721                    )
3722                    .await,
3723                )
3724            });
3725        }
3726        for (module_id, endpoint) in modules {
3727            let Some(target) = forwarding
3728                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
3729                .map_err(SuperviseError::Forwarding)?
3730            else {
3731                continue;
3732            };
3733            let routes = forwarding
3734                .endpoint_routes(target.endpoint)
3735                .map_err(SuperviseError::Forwarding)?;
3736            // Only an announced restart promises that modules are coming back.
3737            // Client route notices retain their existing reconnect semantics.
3738            let mut recipients = Vec::new();
3739            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3740            for route in routes {
3741                let client = route.goodbye_target;
3742                if let Some((_, channels)) = clients
3743                    .iter_mut()
3744                    .find(|(existing, _)| existing.connection_id == client.connection_id)
3745                {
3746                    channels.push(client.channel);
3747                } else {
3748                    let channel = client.channel;
3749                    clients.push((client, vec![channel]));
3750                }
3751            }
3752            for (client, mut channels) in clients {
3753                channels.sort_unstable();
3754                channels.dedup();
3755                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3756                    module_id: module_id.clone(),
3757                    channels,
3758                    reason: RouteCloseReason::Restart,
3759                })
3760                .expect("route closing serializes");
3761                recipients.push((client.sink, client.negotiated_ver, closing));
3762            }
3763            for (sink, version, body) in recipients {
3764                notices.spawn(async move {
3765                    let frame = Frame::build_with_version(
3766                        version,
3767                        FrameType::Push,
3768                        control_flags(),
3769                        0,
3770                        0,
3771                        0,
3772                        body,
3773                    )
3774                    .expect("bounded lifecycle notice frame builds");
3775                    (false, sink.send_flushed(frame).await.is_ok())
3776                });
3777            }
3778            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3779            drains.push((module_id, target.endpoint, gauges));
3780        }
3781        // A quiet forwarding table is not proof that queued notices reached the
3782        // socket. Wait for writer flush acknowledgements before testing quiescence.
3783        let notice_deadline = Instant::now() + NOTICE_BUDGET;
3784        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3785            match result {
3786                Ok((true, true)) => modules_told += 1,
3787                Ok((false, true)) => {}
3788                result => warn!(?result, "daemon shutdown notice delivery failed"),
3789            }
3790        }
3791        notices.abort_all();
3792        info!(
3793            ?shutdown_reason,
3794            modules_told, "daemon shutdown module notices delivered"
3795        );
3796        let notice_elapsed = notice_started.elapsed();
3797        let drain_started = Instant::now();
3798        let deadline = drain_started + DRAIN_BUDGET;
3799        let mut waits = tokio::task::JoinSet::new();
3800        for (module_id, endpoint, gauges) in drains {
3801            let forwarding = Arc::clone(forwarding);
3802            let mut runtime = self.runtime_config();
3803            runtime.health.cadence = Duration::from_millis(100);
3804            waits.spawn(async move {
3805                let wait_result = wait_for_forwarding_quiescence(
3806                    &forwarding,
3807                    &module_id,
3808                    &runtime,
3809                    endpoint,
3810                    deadline,
3811                    &gauges,
3812                    DrainScope::Endpoint(endpoint),
3813                    &mut DrainGaugeTrace::new(&gauges),
3814                )
3815                .await;
3816                // Store the measurement before the task returns: quiescence
3817                // may be reached just as the deadline stops the join loop from
3818                // collecting results. Unfinished or failed waits report false.
3819                forwarding
3820                    .record_daemon_drain_result(
3821                        endpoint,
3822                        drained_after_quiescence_wait(&wait_result),
3823                    )
3824                    .map_err(SuperviseError::Forwarding)?;
3825                wait_result
3826            });
3827        }
3828        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3829            if !matches!(result, Ok(Ok(true))) {
3830                warn!(?result, "daemon shutdown drain did not reach quiescence");
3831            }
3832        }
3833        // Measure only the forwarding drain, not child teardown or the caller's
3834        // signal delivery and process-reaping delays.
3835        info!(
3836            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3837            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3838            "daemon shutdown forwarding drain complete"
3839        );
3840        Ok(())
3841    }
3842
3843    /// The last step of a graceful daemon shutdown, after the notice and the
3844    /// drain: send every registered module a module GOODBYE, the same planned
3845    /// stop signal `ck module stop` gives, then close every connection so each
3846    /// subc module sees EOF and starts its own teardown, then end every
3847    /// supervised child that has not exited
3848    /// by its own deadline (its drain budget, capped). Modules lead their own
3849    /// process groups, so a
3850    /// service manager's group kill no longer reaches them; without this a
3851    /// child that does not stop on EOF (every `protocol: "none"` child, which
3852    /// has no connection) would outlive the daemon. Every wait is bounded (see
3853    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
3854    #[cfg(unix)]
3855    pub(crate) async fn end_children_for_daemon_shutdown(
3856        &self,
3857        already_escalated: bool,
3858        escalate: impl std::future::Future<Output = ()>,
3859    ) {
3860        tokio::pin!(escalate);
3861        let mut escalated = already_escalated;
3862        if let Some(forwarding) = &self.forwarding {
3863            let reason = CloseReason::new(
3864                "daemon_shutdown",
3865                "the daemon is exiting after its shutdown notice and drain",
3866            );
3867            if escalated {
3868                // The operator asked to stop waiting: queue the GOODBYEs but
3869                // do not wait for them to be written.
3870                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3871            } else {
3872                tokio::select! {
3873                    biased;
3874                    _ = escalate.as_mut() => {
3875                        info!("second SIGTERM: abandoning module GOODBYE delivery");
3876                        escalated = true;
3877                    }
3878                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3879                }
3880            }
3881            let closed = forwarding.close_all_connections(&reason);
3882            debug!(closed, "closed established connections for daemon shutdown");
3883        }
3884        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3885        // already completed and must not be polled again; the child shutdown
3886        // wait is told it is escalated and gets a future that never fires.
3887        let escalated_here = escalated && !already_escalated;
3888        let remaining_escalate = async move {
3889            if escalated_here {
3890                std::future::pending::<()>().await;
3891            } else {
3892                escalate.await;
3893            }
3894        };
3895        crate::child_roster::end_children_for_daemon_shutdown(
3896            &self.child_roster,
3897            escalated,
3898            remaining_escalate,
3899        )
3900        .await;
3901    }
3902
3903    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3904        Self {
3905            registry,
3906            config_delivery: None,
3907            restart_policy,
3908            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3909            connection_file_path: None,
3910            capture_logs_dir: None,
3911            forwarding: None,
3912            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3913            supervisor_handle: None,
3914            health: HealthConfig::default(),
3915            host_health: HostHealth::default(),
3916            daemon_start_clock: crate::clock::StartClock::capture(),
3917            terminal_journal: None,
3918            spawn_events: SpawnEventFeed::default(),
3919            provenance_probe: ExecutableIdentityProbe::default(),
3920            child_roster: ChildRoster::default(),
3921            #[cfg(target_os = "linux")]
3922            cgroup_placement: None,
3923            #[cfg(test)]
3924            test_after_first_spawn: AfterFirstSpawnHook::default(),
3925        }
3926    }
3927
3928    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3929        self.drain_timeout = drain_timeout;
3930        self
3931    }
3932
3933    /// The drain budget a module gets when its config entry sets none.
3934    pub(crate) fn default_drain_timeout(&self) -> Duration {
3935        self.drain_timeout
3936    }
3937
3938    pub fn with_process_liveness(
3939        mut self,
3940        process_liveness: Arc<SupervisorProcessLiveness>,
3941    ) -> Self {
3942        self.process_liveness = process_liveness;
3943        self
3944    }
3945
3946    /// Connect configuration delivery to observed process lifecycle events.
3947    pub fn with_config_runtime(
3948        mut self,
3949        owner: Arc<crate::config_gate::runtime::RuntimeOwner>,
3950    ) -> Self {
3951        if let Some(handle) = &self.supervisor_handle {
3952            handle.install_binding_retirement(&owner);
3953        }
3954        self.config_delivery = Some(config_delivery::ConfigDelivery(owner));
3955        self
3956    }
3957
3958    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3959        self.connection_file_path = Some(connection_file_path.into());
3960        self
3961    }
3962
3963    /// Enables daemon-owned capture files for supervised stdout and stderr.
3964    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3965        self.capture_logs_dir = Some(logs_dir.into());
3966        self
3967    }
3968
3969    /// Names this daemon lifetime in spawn events, independently of whether a
3970    /// terminal journal is configured.
3971    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3972        // A millisecond start stamp can repeat after clock rollback or a rapid
3973        // restart. Use the connection file's random daemon_id instead: it already
3974        // identifies this daemon lifetime independently of the wall clock.
3975        self.spawn_events.configure_incarnation(daemon_incarnation);
3976        self
3977    }
3978
3979    /// Enables best-effort history shared by every supervised module. Without
3980    /// it, terminal history is kept only in each module's in-memory ring.
3981    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3982        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3983        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3984            path,
3985            daemon_incarnation,
3986        )));
3987        this
3988    }
3989
3990    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3991        self.child_roster.counters = forwarding.counters();
3992        self.forwarding = Some(forwarding);
3993        self
3994    }
3995
3996    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3997        if let Some(delivery) = &self.config_delivery {
3998            supervisor_handle.install_binding_retirement(&delivery.0);
3999        }
4000        self.host_health = supervisor_handle.host_health.clone();
4001        self.spawn_events = supervisor_handle.spawn_events.clone();
4002        self.supervisor_handle = Some(supervisor_handle);
4003        self
4004    }
4005
4006    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
4007        self.health = health;
4008        self
4009    }
4010
4011    /// Keeps a record of every live child at `path`, rewritten on each spawn and
4012    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
4013    /// record is kept.
4014    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
4015        self.child_roster.record_to(path.into());
4016        self
4017    }
4018
4019    #[cfg(target_os = "linux")]
4020    pub fn with_cgroup_placement(
4021        mut self,
4022        cgroup_placement: Option<subc_cgroup::Placement>,
4023    ) -> Self {
4024        self.child_roster
4025            .set_cgroup_placement(cgroup_placement.clone());
4026        self.cgroup_placement = cgroup_placement;
4027        self
4028    }
4029
4030    /// Spawn `spec.program` and start monitoring it.
4031    ///
4032    /// The child is expected to parse `--subc <connection-file-path>`, read the
4033    /// TCP+key connection file, authenticate to the already-running listener, and
4034    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
4035    ///
4036    /// This synchronous embedder API blocks on launch admission and store I/O.
4037    /// Do not call it on an async runtime thread; use [`Self::spawn_async`].
4038    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4039        validate_spec(&spec)?;
4040        self.establish_identity(&spec);
4041
4042        let runtime = self.runtime_config();
4043        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4044        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4045        let registration_watch = watch_registration(&self.registry, &spec);
4046        let spawned = spawn_child(
4047            &spec,
4048            runtime.connection_file_path.as_deref(),
4049            self.supervisor_handle.as_ref(),
4050            &runtime.stderr_ring,
4051            runtime.capture_logs_dir.as_deref(),
4052            &runtime.child_roster,
4053            runtime.spawn_delivery(),
4054            #[cfg(target_os = "linux")]
4055            runtime.cgroup_placement.as_ref(),
4056            runtime.register_timeout,
4057        );
4058        #[cfg(test)]
4059        self.test_after_first_spawn.run(&spec.module_id);
4060        let mut child = match spawned {
4061            Ok(child) => child,
4062            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4063                defer_config_spawn(
4064                    &spec,
4065                    &runtime,
4066                    &snapshot,
4067                    RespawnKind::Spawn,
4068                    &runtime.failure_context(runtime.recovery_is_daemon),
4069                )?;
4070                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4071            }
4072            Err(SuperviseError::ConfigHeld { .. }) => {
4073                hold_config_spawn(
4074                    &spec,
4075                    &runtime,
4076                    &snapshot,
4077                    &runtime.failure_context(runtime.recovery_is_daemon),
4078                )?;
4079                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4080            }
4081            Err(err) => {
4082                // Unlike the configured paths, a failed `spawn` leaves nothing
4083                // on the roster, so the module must not stay marked configured.
4084                self.abandon_unrostered(&spec.module_id);
4085                return Err(err);
4086            }
4087        };
4088        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4089        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4090        if let Some(spawn) = activity_spawn {
4091            spawn.complete(&snapshot);
4092        }
4093
4094        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4095    }
4096
4097    /// Spawn without running configuration integrity or keychain checks on the
4098    /// runtime thread. Cancellation during preparation cannot create a child.
4099    pub async fn spawn_async(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4100        validate_spec(&spec)?;
4101        self.establish_identity(&spec);
4102        let mut identity = AsyncInitialIdentity {
4103            supervisor: self,
4104            module: spec.module_id.clone(),
4105            committed: false,
4106        };
4107        let runtime = self.runtime_config();
4108        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4109        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4110        let registration_watch = watch_registration(&self.registry, &spec);
4111        let spawned = spawn_child_async(&spec, &runtime).await;
4112        #[cfg(test)]
4113        self.test_after_first_spawn.run(&spec.module_id);
4114        let mut child = match spawned {
4115            Ok(child) => child,
4116            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4117                defer_config_spawn(
4118                    &spec,
4119                    &runtime,
4120                    &snapshot,
4121                    RespawnKind::Spawn,
4122                    &runtime.failure_context(runtime.recovery_is_daemon),
4123                )?;
4124                identity.committed = true;
4125                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4126            }
4127            Err(SuperviseError::ConfigHeld { .. }) => {
4128                hold_config_spawn(
4129                    &spec,
4130                    &runtime,
4131                    &snapshot,
4132                    &runtime.failure_context(runtime.recovery_is_daemon),
4133                )?;
4134                identity.committed = true;
4135                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4136            }
4137            Err(err) => {
4138                self.abandon_unrostered(&spec.module_id);
4139                return Err(err);
4140            }
4141        };
4142        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4143        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4144        if let Some(spawn) = activity_spawn {
4145            spawn.complete(&snapshot);
4146        }
4147        identity.committed = true;
4148        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4149    }
4150
4151    /// Make `spec`'s module count as configured, with its identity gates
4152    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
4153    /// exists.
4154    ///
4155    /// Every path that takes on a new module calls this before `spawn_child`.
4156    /// The order is the point: the child can connect, register, sync its
4157    /// scopes and ask about them as soon as it is spawned, and the module is
4158    /// only put on the roster after `spawn_child` returns. Were the mark set
4159    /// with the roster entry, a fast child would see its own owner reported
4160    /// as not configured, and a scoped `route.open` in that window would be
4161    /// refused as terminal `scope_not_live` ("will never sync") instead of
4162    /// retryable `scope_not_synced`.
4163    fn establish_identity(&self, spec: &ModuleSpec) {
4164        if let Some(supervisor_handle) = &self.supervisor_handle {
4165            supervisor_handle.apply_identity_configuration(spec);
4166            supervisor_handle.mark_configured(&spec.module_id);
4167        }
4168    }
4169
4170    /// Take back [`Self::establish_identity`]'s configured mark when the
4171    /// module will not be put on the roster after all.
4172    fn abandon_unrostered(&self, module_id: &str) {
4173        if let Some(supervisor_handle) = &self.supervisor_handle {
4174            supervisor_handle.unmark_configured_unless_rostered(module_id);
4175        }
4176    }
4177
4178    /// Record a freshly spawned first process as running. On failure the
4179    /// module never reaches the roster, so its configured mark is taken back.
4180    fn mark_first_process_running(
4181        &self,
4182        spec: &ModuleSpec,
4183        runtime: &SupervisorRuntimeConfig,
4184        snapshot: &SharedSnapshot,
4185        child: &mut SupervisedChild,
4186    ) -> Result<(), SuperviseError> {
4187        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
4188            containment::terminate_blocking(&runtime.child_roster.containment);
4189            self.abandon_unrostered(&spec.module_id);
4190            return Err(err);
4191        }
4192        self.process_liveness
4193            .track(spec.module_id.clone(), Arc::clone(snapshot));
4194        Ok(())
4195    }
4196
4197    /// Start supervising a module declared in daemon configuration.
4198    ///
4199    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
4200    /// failures in the supervisor handle so operator-facing `supervisor.list`
4201    /// reflects every configured module while daemon startup continues.
4202    pub fn supervise_configured(
4203        &self,
4204        spec: ModuleSpec,
4205        enabled: bool,
4206    ) -> Result<SupervisedModule, SuperviseError> {
4207        self.supervise_configured_requested(
4208            spec,
4209            enabled,
4210            self.health.clone(),
4211            None,
4212            self.restart_policy,
4213            DEFAULT_REGISTER_TIMEOUT,
4214            None,
4215        )
4216    }
4217
4218    /// Supervise a configured module with its own health, drain, and crash
4219    /// budget. The restart policy is per-module because the config file is:
4220    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
4221    /// module that is expensive to restart should not be forced onto the same
4222    /// budget as one that is cheap.
4223    pub fn supervise_configured_with_health(
4224        &self,
4225        spec: ModuleSpec,
4226        enabled: bool,
4227        health: HealthConfig,
4228        drain_timeout_ms: Option<u64>,
4229        restart_policy: RestartPolicy,
4230    ) -> Result<SupervisedModule, SuperviseError> {
4231        self.supervise_configured_requested(
4232            spec,
4233            enabled,
4234            health,
4235            drain_timeout_ms,
4236            restart_policy,
4237            DEFAULT_REGISTER_TIMEOUT,
4238            None,
4239        )
4240    }
4241
4242    #[allow(clippy::too_many_arguments)]
4243    pub(crate) fn supervise_configured_requested(
4244        &self,
4245        spec: ModuleSpec,
4246        enabled: bool,
4247        health: HealthConfig,
4248        drain_timeout_ms: Option<u64>,
4249        restart_policy: RestartPolicy,
4250        register_timeout: Duration,
4251        requester: Option<Principal>,
4252    ) -> Result<SupervisedModule, SuperviseError> {
4253        let registered = self.register_configured_with_health(
4254            spec,
4255            enabled,
4256            health,
4257            drain_timeout_ms,
4258            restart_policy,
4259            register_timeout,
4260        )?;
4261        let daemon_initiated = requester.is_none();
4262        self.start_registered_configured(registered, requester, daemon_initiated)
4263    }
4264
4265    #[allow(clippy::too_many_arguments)]
4266    pub(crate) async fn supervise_configured_requested_async(
4267        &self,
4268        spec: ModuleSpec,
4269        enabled: bool,
4270        health: HealthConfig,
4271        drain_timeout_ms: Option<u64>,
4272        restart_policy: RestartPolicy,
4273        register_timeout: Duration,
4274        requester: Option<Principal>,
4275    ) -> Result<SupervisedModule, SuperviseError> {
4276        let registered = self.register_configured_with_health(
4277            spec,
4278            enabled,
4279            health,
4280            drain_timeout_ms,
4281            restart_policy,
4282            register_timeout,
4283        )?;
4284        let daemon_initiated = requester.is_none();
4285        self.start_registered_configured_async(registered, requester, daemon_initiated)
4286            .await
4287    }
4288
4289    /// Record a configured module as starting or disabled, without launching a
4290    /// process or running its monitor. Bootstrap registers the entire roster
4291    /// before publishing discovery, then activates it once children can read
4292    /// the connection file passed in their `--subc` argument.
4293    pub(crate) fn register_configured_with_health(
4294        &self,
4295        spec: ModuleSpec,
4296        enabled: bool,
4297        health: HealthConfig,
4298        drain_timeout_ms: Option<u64>,
4299        restart_policy: RestartPolicy,
4300        register_timeout: Duration,
4301    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
4302        validate_spec(&spec)?;
4303        self.establish_identity(&spec);
4304
4305        let mut runtime = self.runtime_config();
4306        runtime.health = health;
4307        runtime.restart_policy = restart_policy;
4308        runtime.register_timeout = register_timeout;
4309        if let Some(ms) = drain_timeout_ms {
4310            runtime.drain_timeout = Duration::from_millis(ms);
4311            *runtime
4312                .effective_drain_timeout
4313                .lock()
4314                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
4315        }
4316        let snapshot = if enabled {
4317            SupervisorSnapshot::starting()
4318        } else {
4319            SupervisorSnapshot::disabled()
4320        };
4321        let snapshot = {
4322            let mut snapshot = snapshot;
4323            if enabled {
4324                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
4325                {
4326                    snapshot.waiting_for_previous_pid = Some(pid);
4327                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
4328                }
4329            }
4330            snapshot
4331        };
4332        let (module, commands) =
4333            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
4334        Ok(RegisteredConfiguredModule {
4335            spec,
4336            runtime,
4337            module,
4338            commands,
4339        })
4340    }
4341
4342    /// Activate a previously registered module. Queued control requests are
4343    /// served by the monitor after the initial configured spawn has settled.
4344    pub(crate) fn start_registered_configured(
4345        &self,
4346        registered: RegisteredConfiguredModule,
4347        requester: Option<Principal>,
4348        daemon_initiated: bool,
4349    ) -> Result<SupervisedModule, SuperviseError> {
4350        let RegisteredConfiguredModule {
4351            spec,
4352            mut runtime,
4353            module,
4354            commands,
4355        } = registered;
4356        runtime.start_requester = requester;
4357        runtime.recovery_is_daemon = daemon_initiated;
4358        let snapshot = &module.inner.snapshot;
4359        let should_spawn = {
4360            let state = lock_snapshot(snapshot)?;
4361            state.enabled && state.waiting_for_previous_pid.is_none()
4362        };
4363        let child = if should_spawn {
4364            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4365            let registration_watch = watch_registration(&self.registry, &spec);
4366            let spawned = spawn_child(
4367                &spec,
4368                runtime.connection_file_path.as_deref(),
4369                self.supervisor_handle.as_ref(),
4370                &runtime.stderr_ring,
4371                runtime.capture_logs_dir.as_deref(),
4372                &runtime.child_roster,
4373                runtime.spawn_delivery(),
4374                #[cfg(target_os = "linux")]
4375                runtime.cgroup_placement.as_ref(),
4376                runtime.register_timeout,
4377            );
4378            #[cfg(test)]
4379            self.test_after_first_spawn.run(&spec.module_id);
4380            match spawned {
4381                Ok(mut child) => {
4382                    arm_registration_deadline(
4383                        &mut child,
4384                        runtime.register_timeout,
4385                        registration_watch,
4386                    );
4387                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4388                    if let Some(spawn) = activity_spawn {
4389                        spawn.complete(snapshot);
4390                    }
4391                    Some(child)
4392                }
4393                Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4394                    defer_config_spawn(
4395                        &spec,
4396                        &runtime,
4397                        snapshot,
4398                        RespawnKind::Spawn,
4399                        &runtime.failure_context(daemon_initiated),
4400                    )?;
4401                    None
4402                }
4403                Err(mut err) => {
4404                    if matches!(err, SuperviseError::ConfigHeld { .. }) {
4405                        match hold_config_spawn(
4406                            &spec,
4407                            &runtime,
4408                            snapshot,
4409                            &runtime.failure_context(daemon_initiated),
4410                        ) {
4411                            Ok(()) => {
4412                                self.start_monitor(spec, runtime, &module, None, commands);
4413                                return Ok(module);
4414                            }
4415                            Err(parking_error) => err = parking_error,
4416                        }
4417                    }
4418                    if runtime.health.critical {
4419                        error!(
4420                            module_id = %spec.module_id,
4421                            program = %spec.program.display(),
4422                            error = %err,
4423                            "critical configured module failed to spawn; marking failed and alerting"
4424                        );
4425                    } else {
4426                        error!(
4427                            module_id = %spec.module_id,
4428                            program = %spec.program.display(),
4429                            error = %err,
4430                            "configured module failed to spawn; marking failed and continuing"
4431                        );
4432                    }
4433                    fail_snapshot(
4434                        snapshot,
4435                        Some(&spec.module_id),
4436                        None,
4437                        &runtime.failure_context(daemon_initiated),
4438                    );
4439                    None
4440                }
4441            }
4442        } else {
4443            None
4444        };
4445        self.start_monitor(spec, runtime, &module, child, commands);
4446        Ok(module)
4447    }
4448
4449    pub(crate) async fn start_registered_configured_async(
4450        &self,
4451        registered: RegisteredConfiguredModule,
4452        requester: Option<Principal>,
4453        daemon_initiated: bool,
4454    ) -> Result<SupervisedModule, SuperviseError> {
4455        self.start_registered_with_failure_policy(registered, requester, daemon_initiated, None)
4456            .await
4457    }
4458
4459    pub(crate) async fn start_registered_approved_async(
4460        &self,
4461        registered: RegisteredConfiguredModule,
4462        requester: Principal,
4463        generation: crate::config_gate::runtime::ApprovalIdentity,
4464    ) -> Result<SupervisedModule, SuperviseError> {
4465        self.start_registered_with_failure_policy(
4466            registered,
4467            Some(requester),
4468            false,
4469            Some(generation),
4470        )
4471        .await
4472    }
4473
4474    async fn start_registered_with_failure_policy(
4475        &self,
4476        registered: RegisteredConfiguredModule,
4477        requester: Option<Principal>,
4478        daemon_initiated: bool,
4479        approved_generation: Option<crate::config_gate::runtime::ApprovalIdentity>,
4480    ) -> Result<SupervisedModule, SuperviseError> {
4481        let RegisteredConfiguredModule {
4482            spec,
4483            mut runtime,
4484            module,
4485            commands,
4486        } = registered;
4487        runtime.start_requester = requester;
4488        runtime.recovery_is_daemon = daemon_initiated;
4489        let snapshot = &module.inner.snapshot;
4490        let mut startup_error = None;
4491        let failure = FailureContext {
4492            approval: approved_generation.clone(),
4493            ..runtime.failure_context(daemon_initiated)
4494        };
4495        let should_spawn = {
4496            let state = lock_snapshot(snapshot)?;
4497            state.enabled && state.waiting_for_previous_pid.is_none()
4498        };
4499        let child = if should_spawn {
4500            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4501            let registration_watch = watch_registration(&self.registry, &spec);
4502            let spawned = spawn_child_async(&spec, &runtime).await;
4503            #[cfg(test)]
4504            self.test_after_first_spawn.run(&spec.module_id);
4505            match spawned {
4506                Ok(mut child) => {
4507                    arm_registration_deadline(
4508                        &mut child,
4509                        runtime.register_timeout,
4510                        registration_watch,
4511                    );
4512                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4513                    if let Some(spawn) = activity_spawn {
4514                        spawn.complete(snapshot);
4515                    }
4516                    Some(child)
4517                }
4518                Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4519                    defer_config_spawn(&spec, &runtime, snapshot, RespawnKind::Spawn, &failure)?;
4520                    None
4521                }
4522                Err(mut err) => {
4523                    if matches!(err, SuperviseError::ConfigHeld { .. }) {
4524                        match hold_config_spawn(&spec, &runtime, snapshot, &failure) {
4525                            Ok(()) => {
4526                                self.start_monitor(spec, runtime, &module, None, commands);
4527                                return Ok(module);
4528                            }
4529                            Err(parking_error) => err = parking_error,
4530                        }
4531                    }
4532                    if runtime.health.critical {
4533                        error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4534                            "critical configured module failed to spawn; marking failed and alerting");
4535                    } else {
4536                        error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4537                            "configured module failed to spawn; marking failed and continuing");
4538                    }
4539                    fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
4540                    if let Some(generation) = approved_generation {
4541                        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4542                            state.approved_start_failure = Some((generation, err.to_string()));
4543                        });
4544                        startup_error = Some(err);
4545                    }
4546                    None
4547                }
4548            }
4549        } else {
4550            None
4551        };
4552        self.start_monitor(spec, runtime, &module, child, commands);
4553        // Applying an operator-approved configuration must report a spawn error
4554        // without removing the failed module or its monitor. Bootstrap instead
4555        // tolerates failed starts so other configured modules can still run.
4556        startup_error.map_or(Ok(module), Err)
4557    }
4558
4559    fn runtime_config(&self) -> SupervisorRuntimeConfig {
4560        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
4561        SupervisorRuntimeConfig {
4562            registry: self.registry.clone(),
4563            config_delivery: self.config_delivery.clone(),
4564            recovery_is_daemon: true,
4565            start_requester: None,
4566            pending_activity_restart: false,
4567            provenance_probe: self.provenance_probe.clone(),
4568            scheduled_respawn: Arc::default(),
4569            deferred_reload_reply: Arc::default(),
4570            restart_policy: self.restart_policy,
4571            register_timeout: DEFAULT_REGISTER_TIMEOUT,
4572            drain_timeout: self.drain_timeout,
4573            // Shared with this module's roster copy: daemon shutdown waits on
4574            // each child for the module's own drain budget, as resolved now.
4575            child_roster: self
4576                .child_roster
4577                .for_module(Arc::clone(&effective_drain_timeout)),
4578            effective_drain_timeout,
4579            default_drain_timeout: self.drain_timeout,
4580            health: self.health.clone(),
4581            host_health: self.host_health.clone(),
4582            connection_file_path: self.connection_file_path.clone(),
4583            capture_logs_dir: self.capture_logs_dir.clone(),
4584            forwarding: self.forwarding.clone(),
4585            supervisor_handle: self.supervisor_handle.clone(),
4586            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
4587            terminal_ring: Arc::new(Mutex::new(
4588                TerminalRing::new(
4589                    TerminalRingConfig::default(),
4590                    self.daemon_start_clock.started_at_ms(),
4591                )
4592                .with_start_clock(self.daemon_start_clock)
4593                .with_journal(self.terminal_journal.clone())
4594                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
4595            )),
4596            spawn_events: self.spawn_events.clone(),
4597            #[cfg(target_os = "linux")]
4598            cgroup_placement: self.cgroup_placement.clone(),
4599            #[cfg(test)]
4600            test_seed_stale_facts_before_enable_spawn: false,
4601            #[cfg(test)]
4602            test_reload_exit_record_gate: None,
4603        }
4604    }
4605
4606    fn supervised_module(
4607        &self,
4608        spec: ModuleSpec,
4609        runtime: SupervisorRuntimeConfig,
4610        snapshot: SharedSnapshot,
4611        child: Option<SupervisedChild>,
4612    ) -> SupervisedModule {
4613        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
4614        self.start_monitor(spec, runtime, &module, child, commands);
4615        module
4616    }
4617
4618    fn register_module(
4619        &self,
4620        spec: &ModuleSpec,
4621        runtime: &SupervisorRuntimeConfig,
4622        snapshot: SharedSnapshot,
4623    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
4624        runtime.host_health.track(spec, &snapshot);
4625        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
4626            spec: spec.clone(),
4627            enabled: snapshot.lock().unwrap_or_else(|p| p.into_inner()).enabled,
4628            health: runtime.health.clone(),
4629            register_timeout: runtime.register_timeout,
4630            restart_policy: runtime.restart_policy,
4631        }));
4632        let stderr_ring = Arc::clone(&runtime.stderr_ring);
4633        let terminal_ring = Arc::clone(&runtime.terminal_ring);
4634        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
4635        let (tx, rx) = mpsc::channel(4);
4636        let module_id = spec.module_id.clone();
4637        let module = SupervisedModule {
4638            inner: Arc::new(SupervisedModuleInner {
4639                module_id: module_id.clone(),
4640                registry: Arc::clone(&self.registry),
4641                snapshot,
4642                configuration,
4643                stderr_ring,
4644                terminal_ring,
4645                commands: tx,
4646                monitor: Mutex::new(None),
4647                effective_drain_timeout,
4648                provenance_probe: self.provenance_probe.clone(),
4649                retained: runtime.child_roster.retained_view(),
4650            }),
4651        };
4652        // Identity gates and the configured mark must precede the roster entry,
4653        // so a client never sees a module without its admission policy.
4654        if let Some(supervisor_handle) = &self.supervisor_handle {
4655            supervisor_handle.insert(module.clone());
4656        }
4657        (module, rx)
4658    }
4659
4660    fn start_monitor(
4661        &self,
4662        spec: ModuleSpec,
4663        runtime: SupervisorRuntimeConfig,
4664        module: &SupervisedModule,
4665        child: Option<SupervisedChild>,
4666        commands: mpsc::Receiver<SupervisorCommand>,
4667    ) {
4668        let monitor = tokio::spawn(supervise_loop(
4669            spec,
4670            runtime,
4671            Arc::clone(&self.registry),
4672            Arc::clone(&self.process_liveness),
4673            Arc::clone(&module.inner.snapshot),
4674            child,
4675            commands,
4676        ));
4677        *module
4678            .inner
4679            .monitor
4680            .lock()
4681            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
4682    }
4683}
4684
4685impl Default for Supervisor {
4686    fn default() -> Self {
4687        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
4688    }
4689}
4690
4691/// Handle to one supervised child process.
4692#[derive(Clone)]
4693pub struct SupervisedModule {
4694    inner: Arc<SupervisedModuleInner>,
4695}
4696
4697struct SupervisedModuleInner {
4698    retained: crate::child_roster::RetainedView,
4699    module_id: String,
4700    registry: Arc<Registry>,
4701    snapshot: SharedSnapshot,
4702    configuration: Arc<Mutex<SupervisedConfiguration>>,
4703    stderr_ring: Arc<Mutex<StderrRing>>,
4704    terminal_ring: Arc<Mutex<TerminalRing>>,
4705    commands: mpsc::Sender<SupervisorCommand>,
4706    monitor: Mutex<Option<JoinHandle<()>>>,
4707    effective_drain_timeout: Arc<Mutex<Duration>>,
4708    provenance_probe: ExecutableIdentityProbe,
4709}
4710
4711impl fmt::Debug for SupervisedModule {
4712    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4713        f.debug_struct("SupervisedModule")
4714            .field("module_id", &self.inner.module_id)
4715            .field("status", &self.status())
4716            .finish_non_exhaustive()
4717    }
4718}
4719
4720impl SupervisedModule {
4721    pub(crate) fn check_approved_start_failure(
4722        &self,
4723        generation: &crate::config_gate::runtime::ApprovalIdentity,
4724    ) -> Result<bool, SuperviseError> {
4725        let state = lock_snapshot(&self.inner.snapshot)?;
4726        if let Some((failed_generation, detail)) = &state.approved_start_failure {
4727            if failed_generation == generation {
4728                return Err(SuperviseError::ReloadFailed {
4729                    module_id: self.inner.module_id.clone(),
4730                    reason: detail.clone(),
4731                });
4732            }
4733        }
4734        Ok(state.approved_start_failure.is_some())
4735    }
4736
4737    pub(crate) fn has_approved_start_failure(
4738        &self,
4739        identity: &crate::config_gate::runtime::ApprovalIdentity,
4740    ) -> bool {
4741        lock_snapshot(&self.inner.snapshot).is_ok_and(|state| {
4742            state
4743                .approved_start_failure
4744                .as_ref()
4745                .is_some_and(|(failed, _)| failed == identity)
4746        })
4747    }
4748
4749    pub(crate) fn needs_approved_gate_restart(
4750        &self,
4751        approval: &crate::config_gate::runtime::ApprovalIdentity,
4752    ) -> bool {
4753        lock_snapshot(&self.inner.snapshot)
4754            .is_ok_and(|state| state.needs_approved_gate_restart(approval))
4755    }
4756
4757    #[cfg(all(test, unix))]
4758    pub(crate) fn config_hold_origin_for_test(
4759        &self,
4760    ) -> (bool, Option<crate::config_gate::runtime::ApprovalIdentity>) {
4761        let state = lock_snapshot(&self.inner.snapshot).unwrap();
4762        (
4763            state.config_hold_daemon_initiated,
4764            state.config_hold_approval.clone(),
4765        )
4766    }
4767
4768    pub(crate) async fn config_policy_probe(
4769        &self,
4770        connection: crate::ConnectionId,
4771        generation: u64,
4772        attempt: u64,
4773    ) -> Result<(), SuperviseError> {
4774        self.inner
4775            .commands
4776            .send(SupervisorCommand::ConfigPolicyProbe {
4777                connection,
4778                generation,
4779                attempt,
4780            })
4781            .await
4782            .map_err(|_| SuperviseError::CommandClosed {
4783                module_id: self.inner.module_id.clone(),
4784            })
4785    }
4786    pub(crate) async fn config_exit_hold(&self) -> Result<(), SuperviseError> {
4787        let (reply, result) = oneshot::channel();
4788        self.inner
4789            .commands
4790            .send(SupervisorCommand::ConfigExitHold { reply })
4791            .await
4792            .map_err(|_| SuperviseError::CommandClosed {
4793                module_id: self.module_id().to_string(),
4794            })?;
4795        result.await.map_err(|_| SuperviseError::CommandClosed {
4796            module_id: self.module_id().to_string(),
4797        })?
4798    }
4799    pub(crate) async fn config_delivery_failed(
4800        &self,
4801        connection: crate::ConnectionId,
4802        hold: bool,
4803    ) -> Result<(), SuperviseError> {
4804        self.inner
4805            .commands
4806            .send(SupervisorCommand::ConfigDeliveryFailed { connection, hold })
4807            .await
4808            .map_err(|_| SuperviseError::CommandClosed {
4809                module_id: self.module_id().to_string(),
4810            })
4811    }
4812
4813    pub(crate) async fn config_apply_restart(&self, generation: u64) -> Result<(), SuperviseError> {
4814        self.inner
4815            .commands
4816            .send(SupervisorCommand::ConfigApplyRestart {
4817                generation,
4818                supersedes_swap: false,
4819            })
4820            .await
4821            .map_err(|_| SuperviseError::CommandClosed {
4822                module_id: self.inner.module_id.clone(),
4823            })
4824    }
4825    pub fn module_id(&self) -> &str {
4826        &self.inner.module_id
4827    }
4828
4829    /// Test-only: put one probe miss on the streak, the way
4830    /// `handle_health_probe_failure` does, so tests can assert what a later
4831    /// event does to the streak without driving the whole probe loop.
4832    #[cfg(test)]
4833    pub(crate) fn record_health_probe_failure_for_test(
4834        &self,
4835        detail: &str,
4836    ) -> Result<(), SuperviseError> {
4837        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
4838            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
4839            state.health.detail = Some(detail.to_string());
4840        })
4841    }
4842
4843    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
4844        Ok(lock_snapshot(&self.inner.snapshot)?.state)
4845    }
4846
4847    pub(crate) fn registration_end_reason(
4848        &self,
4849    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
4850        let snapshot = lock_snapshot(&self.inner.snapshot)?;
4851        Ok(match snapshot.state {
4852            ModuleState::Draining if snapshot.draining_to_replace => {
4853                Some(RegistrationEndReason::SupervisorRestart)
4854            }
4855            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
4856            ModuleState::Restarting if snapshot.draining_to_replace => {
4857                Some(RegistrationEndReason::SupervisorRestart)
4858            }
4859            _ => None,
4860        })
4861    }
4862
4863    /// The module's retained stderr, newest lines last.
4864    ///
4865    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
4866    /// module, `supervisor.list` renders every module, and putting it in the
4867    /// shared snapshot would make each status read carry a payload almost nobody
4868    /// asked for. Callers that want the text ask for it.
4869    pub fn stderr_tail(
4870        &self,
4871        max_lines: Option<usize>,
4872        max_bytes: Option<usize>,
4873    ) -> StderrTailSnapshot {
4874        self.inner
4875            .stderr_ring
4876            .lock()
4877            .unwrap_or_else(|poisoned| poisoned.into_inner())
4878            .snapshot(max_lines, max_bytes)
4879    }
4880
4881    /// The module's bounded terminal history, oldest retained exit first.
4882    ///
4883    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
4884    /// daemon whose in-memory history was necessarily reset.
4885    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
4886        self.inner
4887            .terminal_ring
4888            .lock()
4889            .unwrap_or_else(|poisoned| poisoned.into_inner())
4890            .snapshot()
4891    }
4892
4893    /// Retained observations from the current ring and all journal generations.
4894    ///
4895    /// Blocking: this reads the journal files. Async callers use
4896    /// `Self::read_durable_terminal_history`.
4897    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
4898        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
4899    }
4900
4901    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
4902    /// read (up to every retained generation) never occupies a runtime worker.
4903    /// Fails only if the blocking task could not finish (runtime shutdown or a
4904    /// panic in the read).
4905    pub(crate) async fn read_durable_terminal_history(
4906        &self,
4907    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
4908        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
4909        let module_id = self.inner.module_id.clone();
4910        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
4911            .await
4912    }
4913
4914    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
4915        self.status_with_snapshot_lock(&self.inner.snapshot, None)
4916            .map(|(status, _)| status)
4917    }
4918
4919    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
4920        let state = lock_snapshot(&self.inner.snapshot)?;
4921        Ok(state
4922            .failed_retry
4923            .pending
4924            .filter(|_| state.state == ModuleState::Failed && state.enabled)
4925            .map(|retry| {
4926                unix_ms_now().saturating_add(
4927                    u64::try_from(
4928                        retry
4929                            .deadline
4930                            .saturating_duration_since(Instant::now())
4931                            .as_millis(),
4932                    )
4933                    .unwrap_or(u64::MAX),
4934                )
4935            }))
4936    }
4937
4938    #[cfg(all(test, unix))]
4939    pub(crate) fn retry_deadline_for_test(&self) -> Option<Instant> {
4940        lock_snapshot(&self.inner.snapshot)
4941            .unwrap()
4942            .failed_retry
4943            .pending
4944            .map(|retry| retry.deadline)
4945    }
4946
4947    pub(crate) fn record_deliberate_severance(
4948        &self,
4949        identity: ProcessIdentity,
4950    ) -> Result<bool, SuperviseError> {
4951        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4952        if snapshot.pid != Some(identity.pid)
4953            || snapshot.process_start_time != Some(identity.start_time)
4954        {
4955            return Ok(false);
4956        }
4957        snapshot.deliberate_severance = Some(identity);
4958        Ok(true)
4959    }
4960
4961    /// Read status for a channel-0 renderer and report a contended snapshot lock.
4962    ///
4963    /// Internal supervision callers use [`Self::status`] so writer-side machinery
4964    /// does not produce reader-observability logs.
4965    pub(crate) fn status_for_control(
4966        &self,
4967        caller: &'static str,
4968    ) -> Result<ModuleStatus, SuperviseError> {
4969        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
4970            .map(|(status, _)| status)
4971    }
4972
4973    fn status_with_snapshot_lock(
4974        &self,
4975        snapshot: &SharedSnapshot,
4976        caller: Option<&'static str>,
4977    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
4978        let restart_policy = self
4979            .inner
4980            .configuration
4981            .lock()
4982            .map_err(|_| SuperviseError::StatePoisoned {
4983                module_id: Some(self.inner.module_id.clone()),
4984            })?
4985            .restart_policy;
4986        let mut guard = match caller {
4987            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
4988            None => lock_snapshot(snapshot)?,
4989        };
4990        // Read the budget through the pruning path so a reader sees the same
4991        // in-window count the restart decision would use, not a stale total.
4992        let restart_count = guard.crash_restarts_in_window(restart_policy.window, Instant::now());
4993        let snapshot = guard.clone();
4994        drop(guard);
4995        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
4996            SuperviseError::StatePoisoned {
4997                module_id: Some(self.inner.module_id.clone()),
4998            }
4999        })?;
5000        let registration_active = self
5001            .inner
5002            .registry
5003            .get_module(&self.inner.module_id)
5004            .map_err(SuperviseError::Registry)?
5005            .is_some();
5006        let protocol = snapshot
5007            .spawned_protocol
5008            .unwrap_or(self.declared_protocol()?);
5009        let running_process =
5010            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
5011        // Registration is the difference between the two protocols and the only
5012        // one: a subc module that has not registered cannot serve a request even
5013        // though its process is up, and a `none` module never registers at all,
5014        // so requiring it there would pin `live` to false for the whole life of
5015        // a perfectly healthy process.
5016        let live = match protocol {
5017            ModuleProtocol::Subc => running_process && registration_active,
5018            ModuleProtocol::None => running_process,
5019        };
5020        // Read only the retained cache: a status read must not resend signals,
5021        // rewrite the record, or keep the containment resource alive.
5022        let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
5023        let carry_configured = self
5024            .inner
5025            .configuration
5026            .lock()
5027            .map_err(|_| SuperviseError::StatePoisoned {
5028                module_id: Some(self.inner.module_id.clone()),
5029            })?
5030            .spec
5031            .carry_containment;
5032        let recovery_block = carries_containment(
5033            carry_configured,
5034            self.inner
5035                .retained
5036                .has_carried_container(&self.inner.module_id),
5037        )
5038        .then(|| self.inner.retained.recovery_block())
5039        .flatten();
5040        let launch_nonce_source = observed_launch_nonce_source(&snapshot);
5041        let pid = snapshot.reported_pid();
5042        let mut health = snapshot.health;
5043        if let Some(detail) = recovery_block
5044            .as_ref()
5045            .or_else(|| retained.as_ref().map(|(_, detail)| detail))
5046        {
5047            health.detail = Some(match health.detail {
5048                Some(previous) => format!("{previous}; {detail}"),
5049                None => detail.clone(),
5050            });
5051        }
5052
5053        Ok((
5054            ModuleStatus {
5055                admission_refused: snapshot.admission_refused,
5056                legacy_registration_spawn: snapshot.legacy_registration_spawn,
5057                registration_declaration: snapshot.registration_declaration.clone(),
5058                stop_reason: snapshot.stop_reason,
5059                waiting_for_previous_pid: retained
5060                    .as_ref()
5061                    .map(|(pid, _)| *pid)
5062                    .filter(|pid| *pid > 0)
5063                    .or(snapshot.waiting_for_previous_pid),
5064                waiting_for_previous_detail: recovery_block
5065                    .or_else(|| retained.map(|(_, detail)| detail))
5066                    .or(snapshot.waiting_for_previous_detail),
5067                module_id: self.inner.module_id.clone(),
5068                launch_nonce_source,
5069                state: snapshot.state,
5070                enabled: snapshot.enabled,
5071                process_alive: snapshot.process_alive,
5072                registration_active,
5073                protocol,
5074                live,
5075                restart_count,
5076                lifetime_restarts: snapshot.lifetime_restarts,
5077                spawn_generation: snapshot.spawn_generation,
5078                swap_outcome: snapshot.swap_outcome,
5079                max_restarts: restart_policy.max_restarts,
5080                restart_window: restart_policy.window,
5081                drain_timeout,
5082                restart_backoff: restart_policy.backoff,
5083                restart_max_backoff: restart_policy.max_backoff,
5084                pid,
5085                spawned_at_ms: snapshot.spawned_at_ms,
5086                spawned_from: snapshot.spawned_from,
5087                launch_spec_digest: snapshot.launch_spec_digest,
5088                process_start_time: snapshot.process_start_time,
5089                last_exit: snapshot.last_exit,
5090                health,
5091            },
5092            snapshot.spawned_file_identity.clone(),
5093        ))
5094    }
5095
5096    #[cfg(test)]
5097    pub(crate) fn hold_snapshot_for_test(
5098        &self,
5099        acquired: std::sync::mpsc::Sender<()>,
5100        hold: Duration,
5101    ) -> std::thread::JoinHandle<()> {
5102        let snapshot = Arc::clone(&self.inner.snapshot);
5103        std::thread::spawn(move || {
5104            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
5105            acquired
5106                .send(())
5107                .expect("test receiver waits for snapshot lock");
5108            std::thread::sleep(hold);
5109        })
5110    }
5111
5112    /// The status and the running-image check for `supervisor.provenance`,
5113    /// taken from one status read. The exec acknowledgement can land between
5114    /// two separate reads, and the reply would then pair "no pid yet" with an
5115    /// image observed after the module started, which describes no single
5116    /// moment.
5117    pub(crate) async fn status_and_running_image_agreement(
5118        &self,
5119    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
5120        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
5121        let image = self
5122            .inner
5123            .provenance_probe
5124            .observe(
5125                status.pid,
5126                status.spawned_from.as_deref(),
5127                identity,
5128                status.process_start_time,
5129            )
5130            .await;
5131        Ok((status, image))
5132    }
5133
5134    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
5135        let snapshot = match lock_snapshot(&self.inner.snapshot) {
5136            Ok(snapshot) => snapshot.clone(),
5137            Err(_) => {
5138                return subc_control::RunningImageAgreement::Unavailable {
5139                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
5140                };
5141            }
5142        };
5143        self.inner
5144            .provenance_probe
5145            .observe(
5146                snapshot.reported_pid(),
5147                snapshot.spawned_from.as_deref(),
5148                snapshot.spawned_file_identity.clone(),
5149                snapshot.process_start_time,
5150            )
5151            .await
5152    }
5153
5154    /// Memory and CPU time of the module's current process, read now. Only the
5155    /// process the supervisor spawned is read, not processes it has started.
5156    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
5157        #[cfg(windows)]
5158        {
5159            let image = lock_snapshot(&self.inner.snapshot)
5160                .ok()
5161                .and_then(|snapshot| {
5162                    snapshot
5163                        .spawned_file_identity
5164                        .as_ref()
5165                        .and_then(|identity| identity.windows_image.clone())
5166                });
5167            match image {
5168                Some(image) => {
5169                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
5170                }
5171                None => subc_control::ChildResourceUsage::Unavailable {
5172                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
5173                },
5174            }
5175        }
5176        #[cfg(not(windows))]
5177        {
5178            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
5179                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
5180                Err(_) => {
5181                    return subc_control::ChildResourceUsage::Unavailable {
5182                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
5183                    }
5184                }
5185            };
5186            crate::child_resources::read(pid, start_time)
5187        }
5188    }
5189
5190    #[cfg(all(test, any(target_os = "linux", windows)))]
5191    pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
5192        self.inner
5193            .snapshot
5194            .try_lock()
5195            .map(|snapshot| snapshot.subtree_target.clone())
5196            .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
5197    }
5198
5199    #[cfg(test)]
5200    pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
5201        self.inner.snapshot.lock().unwrap().subtree_target = target;
5202    }
5203
5204    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
5205        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
5206        Ok(match snapshot.state {
5207            ModuleState::Restarting => true,
5208            ModuleState::Failed | ModuleState::Disabled => false,
5209            _ if snapshot.enabled && snapshot.respawn_pending => true,
5210            _ => daemon_will_restart(
5211                &mut snapshot,
5212                &self
5213                    .inner
5214                    .configuration
5215                    .lock()
5216                    .map_err(|_| SuperviseError::StatePoisoned {
5217                        module_id: Some(self.inner.module_id.clone()),
5218                    })?
5219                    .restart_policy,
5220                Instant::now(),
5221            ),
5222        })
5223    }
5224
5225    #[cfg(test)]
5226    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
5227        self.is_warming_with_snapshot_lock(None)
5228    }
5229
5230    pub(crate) fn is_warming_for_control(
5231        &self,
5232        caller: &'static str,
5233    ) -> Result<bool, SuperviseError> {
5234        self.is_warming_with_snapshot_lock(Some(caller))
5235    }
5236
5237    fn is_warming_with_snapshot_lock(
5238        &self,
5239        caller: Option<&'static str>,
5240    ) -> Result<bool, SuperviseError> {
5241        let snapshot = match caller {
5242            Some(caller) => {
5243                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
5244            }
5245            None => lock_snapshot(&self.inner.snapshot)?,
5246        }
5247        .clone();
5248        Ok(matches!(
5249            snapshot.state,
5250            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
5251        ))
5252    }
5253
5254    /// Drain the module and stop monitoring it.
5255    pub async fn drain(&self) -> Result<(), SuperviseError> {
5256        self.stop().await
5257    }
5258
5259    pub(crate) async fn retire_requested(
5260        &self,
5261        requester: Option<Principal>,
5262    ) -> Result<(), SuperviseError> {
5263        match self.state()? {
5264            ModuleState::Stopped => return Ok(()),
5265            ModuleState::Failed
5266                if lock_snapshot(&self.inner.snapshot)?
5267                    .failed_retry
5268                    .pending
5269                    .is_none() =>
5270            {
5271                return Ok(())
5272            }
5273            ModuleState::Failed
5274            | ModuleState::Starting
5275            | ModuleState::Running
5276            | ModuleState::Unresponsive
5277            | ModuleState::Restarting
5278            | ModuleState::Draining
5279            | ModuleState::Disabled => {}
5280        }
5281
5282        let (reply_tx, reply_rx) = oneshot::channel();
5283        self.inner
5284            .commands
5285            .send(SupervisorCommand::Retire {
5286                requester,
5287                reply: reply_tx,
5288            })
5289            .await
5290            .map_err(|_| SuperviseError::CommandClosed {
5291                module_id: self.inner.module_id.clone(),
5292            })?;
5293        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5294            module_id: self.inner.module_id.clone(),
5295        })?
5296    }
5297
5298    pub async fn stop(&self) -> Result<(), SuperviseError> {
5299        self.stop_requested(None).await
5300    }
5301
5302    pub(crate) async fn stop_requested(
5303        &self,
5304        requester: Option<Principal>,
5305    ) -> Result<(), SuperviseError> {
5306        match self.state()? {
5307            ModuleState::Stopped => return Ok(()),
5308            ModuleState::Failed
5309                if lock_snapshot(&self.inner.snapshot)?
5310                    .failed_retry
5311                    .pending
5312                    .is_none() =>
5313            {
5314                return Ok(())
5315            }
5316            ModuleState::Failed
5317            | ModuleState::Starting
5318            | ModuleState::Running
5319            | ModuleState::Unresponsive
5320            | ModuleState::Restarting
5321            | ModuleState::Draining
5322            | ModuleState::Disabled => {}
5323        }
5324
5325        let (reply_tx, reply_rx) = oneshot::channel();
5326        self.inner
5327            .commands
5328            .send(SupervisorCommand::Drain {
5329                requester,
5330                reply: reply_tx,
5331            })
5332            .await
5333            .map_err(|_| SuperviseError::CommandClosed {
5334                module_id: self.inner.module_id.clone(),
5335            })?;
5336        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5337            module_id: self.inner.module_id.clone(),
5338        })?
5339    }
5340
5341    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
5342        self.restart_requested(drain_timeout_ms, None).await
5343    }
5344
5345    pub(crate) async fn restart_requested(
5346        &self,
5347        drain_timeout_ms: Option<u64>,
5348        requester: Option<Principal>,
5349    ) -> Result<(), SuperviseError> {
5350        let queued_at = Instant::now();
5351        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
5352        let (reply_tx, reply_rx) = oneshot::channel();
5353        self.inner
5354            .commands
5355            .send(SupervisorCommand::Restart {
5356                requester,
5357                drain_timeout_ms,
5358                received_at_generation,
5359                queued_at,
5360                reply: reply_tx,
5361            })
5362            .await
5363            .map_err(|_| SuperviseError::CommandClosed {
5364                module_id: self.inner.module_id.clone(),
5365            })?;
5366        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5367            module_id: self.inner.module_id.clone(),
5368        })?
5369    }
5370
5371    /// Blue/green restart: see `SupervisorCommand::Swap` and the
5372    /// `supervisor_swap` module. Returns once the swap has cut over (the old
5373    /// process then drains in the background of the supervise loop), has been
5374    /// superseded by an approved configuration restart, or has failed. Both
5375    /// supersession and failure leave the old process serving until replacement.
5376    pub async fn swap(
5377        &self,
5378        ready_timeout: Option<Duration>,
5379    ) -> Result<SwapOutcome, SuperviseError> {
5380        self.swap_requested(ready_timeout, None).await
5381    }
5382
5383    pub(crate) async fn swap_requested(
5384        &self,
5385        ready_timeout: Option<Duration>,
5386        requester: Option<Principal>,
5387    ) -> Result<SwapOutcome, SuperviseError> {
5388        let (reply_tx, reply_rx) = oneshot::channel();
5389        self.inner
5390            .commands
5391            .send(SupervisorCommand::Swap {
5392                requester,
5393                ready_timeout,
5394                reply: reply_tx,
5395            })
5396            .await
5397            .map_err(|_| SuperviseError::CommandClosed {
5398                module_id: self.inner.module_id.clone(),
5399            })?;
5400        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5401            module_id: self.inner.module_id.clone(),
5402        })?
5403    }
5404
5405    pub async fn reload(&self) -> Result<(), SuperviseError> {
5406        self.reload_requested(None).await
5407    }
5408
5409    pub(crate) async fn reload_requested(
5410        &self,
5411        requester: Option<Principal>,
5412    ) -> Result<(), SuperviseError> {
5413        let (reply_tx, reply_rx) = oneshot::channel();
5414        self.inner
5415            .commands
5416            .send(SupervisorCommand::Reload {
5417                requester,
5418                reply: reply_tx,
5419            })
5420            .await
5421            .map_err(|_| SuperviseError::CommandClosed {
5422                module_id: self.inner.module_id.clone(),
5423            })?;
5424        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5425            module_id: self.inner.module_id.clone(),
5426        })?
5427    }
5428
5429    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
5430        self.set_enabled_requested(enabled, None).await
5431    }
5432
5433    pub(crate) async fn set_enabled_requested(
5434        &self,
5435        enabled: bool,
5436        requester: Option<Principal>,
5437    ) -> Result<bool, SuperviseError> {
5438        self.set_enabled_with_budget_policy(enabled, requester, true)
5439            .await
5440    }
5441
5442    pub(crate) async fn release_config_hold(&self) -> Result<bool, SuperviseError> {
5443        self.set_enabled_with_budget_policy(true, None, false).await
5444    }
5445
5446    #[cfg(all(test, unix))]
5447    pub(crate) fn seed_crash_budget_for_test(&self, count: u32) {
5448        let policy = self.inner.configuration.lock().unwrap().restart_policy;
5449        let mut state = self.inner.snapshot.lock().unwrap();
5450        for _ in 0..count {
5451            state.record_crash_restart(&policy, Instant::now());
5452        }
5453    }
5454
5455    async fn set_enabled_with_budget_policy(
5456        &self,
5457        enabled: bool,
5458        requester: Option<Principal>,
5459        reset_budget: bool,
5460    ) -> Result<bool, SuperviseError> {
5461        let (reply_tx, reply_rx) = oneshot::channel();
5462        self.inner
5463            .commands
5464            .send(SupervisorCommand::SetEnabled {
5465                requester,
5466                enabled,
5467                reset_budget,
5468                reply: reply_tx,
5469            })
5470            .await
5471            .map_err(|_| SuperviseError::CommandClosed {
5472                module_id: self.inner.module_id.clone(),
5473            })?;
5474        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5475            module_id: self.inner.module_id.clone(),
5476        })?
5477    }
5478
5479    /// The current process's protocol, or the configured protocol when down.
5480    /// A rescan stores the next launch spec without changing how an existing
5481    /// process registers, serves routes, is probed, or exits.
5482    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
5483        let configured = self
5484            .inner
5485            .configuration
5486            .lock()
5487            .map_err(|_| SuperviseError::StatePoisoned {
5488                module_id: Some(self.inner.module_id.clone()),
5489            })?
5490            .spec
5491            .protocol;
5492        let state = lock_snapshot(&self.inner.snapshot)?;
5493        Ok(state.spawned_protocol.unwrap_or(configured))
5494    }
5495
5496    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
5497        let configuration =
5498            self.inner
5499                .configuration
5500                .lock()
5501                .map_err(|_| SuperviseError::StatePoisoned {
5502                    module_id: Some(self.inner.module_id.clone()),
5503                })?;
5504        Ok((configuration.spec.clone(), configuration.health.clone()))
5505    }
5506
5507    pub(crate) fn configured_register_timeout(&self) -> Result<Duration, SuperviseError> {
5508        self.inner
5509            .configuration
5510            .lock()
5511            .map(|configuration| configuration.register_timeout)
5512            .map_err(|_| SuperviseError::StatePoisoned {
5513                module_id: Some(self.inner.module_id.clone()),
5514            })
5515    }
5516
5517    pub(crate) fn configured_enabled(&self) -> Result<bool, SuperviseError> {
5518        self.inner
5519            .configuration
5520            .lock()
5521            .map(|configuration| configuration.enabled)
5522            .map_err(|_| SuperviseError::StatePoisoned {
5523                module_id: Some(self.inner.module_id.clone()),
5524            })
5525    }
5526
5527    /// Replace this module's launch spec, keeping its health and drain policy,
5528    /// the way a rescan does for a changed config entry. The running process is
5529    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
5530    #[cfg(any(test, feature = "test-support"))]
5531    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
5532        let (_, health) = self.configuration()?;
5533        let drain_timeout_ms = u64::try_from(
5534            self.inner
5535                .effective_drain_timeout
5536                .lock()
5537                .unwrap_or_else(|poisoned| poisoned.into_inner())
5538                .as_millis(),
5539        )
5540        .ok();
5541        self.update_configuration(spec, health, drain_timeout_ms)
5542            .await
5543    }
5544
5545    pub(crate) async fn reconcile_approved_configuration(
5546        &self,
5547        approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5548    ) -> Result<(), SuperviseError> {
5549        let entry = approved
5550            .module_entry(&self.inner.module_id)
5551            .map_err(|error| SuperviseError::InvalidSpec {
5552                reason: error.to_string(),
5553            })?;
5554        validate_spec(&entry.module_spec())?;
5555        let (reply, received) = oneshot::channel();
5556        self.inner
5557            .commands
5558            .send(SupervisorCommand::ApprovedConfiguration { approved, reply })
5559            .await
5560            .map_err(|_| SuperviseError::CommandClosed {
5561                module_id: self.inner.module_id.clone(),
5562            })?;
5563        received.await.map_err(|_| SuperviseError::CommandClosed {
5564            module_id: self.inner.module_id.clone(),
5565        })??;
5566        let mut configuration =
5567            self.inner
5568                .configuration
5569                .lock()
5570                .map_err(|_| SuperviseError::StatePoisoned {
5571                    module_id: Some(self.inner.module_id.clone()),
5572                })?;
5573        configuration.spec = entry.module_spec();
5574        configuration.enabled = entry.enabled;
5575        configuration.health = entry.health;
5576        configuration.register_timeout = entry.register_timeout;
5577        configuration.restart_policy = entry.restart;
5578        Ok(())
5579    }
5580
5581    pub(crate) async fn update_configuration(
5582        &self,
5583        spec: ModuleSpec,
5584        health: HealthConfig,
5585        drain_timeout_ms: Option<u64>,
5586    ) -> Result<(), SuperviseError> {
5587        if spec.module_id != self.inner.module_id {
5588            return Err(SuperviseError::InvalidSpec {
5589                reason: "a supervised module's module_id cannot be changed".to_string(),
5590            });
5591        }
5592        validate_spec(&spec)?;
5593        let (reply_tx, reply_rx) = oneshot::channel();
5594        self.inner
5595            .commands
5596            .send(SupervisorCommand::UpdateConfiguration {
5597                spec: spec.clone(),
5598                health: health.clone(),
5599                drain_timeout_ms,
5600                reply: reply_tx,
5601            })
5602            .await
5603            .map_err(|_| SuperviseError::CommandClosed {
5604                module_id: self.inner.module_id.clone(),
5605            })?;
5606        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5607            module_id: self.inner.module_id.clone(),
5608        })??;
5609        let mut configuration =
5610            self.inner
5611                .configuration
5612                .lock()
5613                .map_err(|_| SuperviseError::StatePoisoned {
5614                    module_id: Some(self.inner.module_id.clone()),
5615                })?;
5616        configuration.spec = spec;
5617        configuration.health = health;
5618        Ok(())
5619    }
5620}
5621
5622impl Drop for SupervisedModuleInner {
5623    fn drop(&mut self) {
5624        let Ok(mut monitor) = self.monitor.lock() else {
5625            return;
5626        };
5627        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
5628            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
5629                state.state = ModuleState::Stopped;
5630                clear_current_process_facts(state);
5631            });
5632            monitor.abort();
5633        }
5634        let _ = monitor.take();
5635    }
5636}
5637
5638#[derive(Debug)]
5639enum SupervisorCommand {
5640    ApprovedConfiguration {
5641        approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5642        reply: oneshot::Sender<Result<(), SuperviseError>>,
5643    },
5644    ConfigPolicyProbe {
5645        connection: crate::ConnectionId,
5646        generation: u64,
5647        attempt: u64,
5648    },
5649    ConfigApplyRestart {
5650        generation: u64,
5651        supersedes_swap: bool,
5652    },
5653    ConfigExitHold {
5654        reply: oneshot::Sender<Result<(), SuperviseError>>,
5655    },
5656    ConfigDeliveryFailed {
5657        connection: crate::ConnectionId,
5658        hold: bool,
5659    },
5660    Drain {
5661        requester: Option<Principal>,
5662        reply: oneshot::Sender<Result<(), SuperviseError>>,
5663    },
5664    Retire {
5665        requester: Option<Principal>,
5666        reply: oneshot::Sender<Result<(), SuperviseError>>,
5667    },
5668    Restart {
5669        requester: Option<Principal>,
5670        /// Operator override for this one restart's drain budget, in ms. `None`
5671        /// uses the module's configured/default budget; `Some(0)` cuts
5672        /// immediately (wedge bounce: a stuck request never settles, so
5673        /// waiting only delays recovery).
5674        drain_timeout_ms: Option<u64>,
5675        /// The module's `spawn_generation` when the request was received, before
5676        /// it waited in the command queue. A queued restart whose module has
5677        /// since spawned a newer process is already satisfied (see the handler).
5678        received_at_generation: u64,
5679        /// When the request entered the command queue, so the handler can log
5680        /// how long it waited behind the loop's other work.
5681        queued_at: Instant,
5682        reply: oneshot::Sender<Result<(), SuperviseError>>,
5683    },
5684    Reload {
5685        requester: Option<Principal>,
5686        reply: oneshot::Sender<Result<(), SuperviseError>>,
5687    },
5688    SetEnabled {
5689        reset_budget: bool,
5690        requester: Option<Principal>,
5691        enabled: bool,
5692        reply: oneshot::Sender<Result<bool, SuperviseError>>,
5693    },
5694    UpdateConfiguration {
5695        spec: ModuleSpec,
5696        health: HealthConfig,
5697        /// Per-module drain override from the new config; `None` re-resolves to
5698        /// the supervisor-wide default.
5699        drain_timeout_ms: Option<u64>,
5700        reply: oneshot::Sender<Result<(), SuperviseError>>,
5701    },
5702    Swap {
5703        requester: Option<Principal>,
5704        /// How long the candidate may take to register and declare itself
5705        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
5706        ready_timeout: Option<Duration>,
5707        /// Answered at cutover or failure; the incumbent's drain follows.
5708        reply: oneshot::Sender<Result<SwapOutcome, SuperviseError>>,
5709    },
5710}
5711
5712/// Successful daemon-local swap disposition. Supersession leaves the incumbent
5713/// serving until an approved replacement starts with the latest configured
5714/// command; the legacy control acknowledgment does not carry this disposition.
5715#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5716pub enum SwapOutcome {
5717    Cutover,
5718    SupersededByConfigRestart,
5719}
5720
5721impl SwapOutcome {
5722    pub fn as_str(self) -> &'static str {
5723        match self {
5724            Self::Cutover => "cutover",
5725            Self::SupersededByConfigRestart => "superseded_by_config_restart",
5726        }
5727    }
5728}
5729
5730#[derive(Debug)]
5731pub enum SuperviseError {
5732    ConfigHeld {
5733        module_id: String,
5734        code: String,
5735        detail: String,
5736    },
5737    ConfigDeliveryDeferred {
5738        module_id: String,
5739    },
5740    InvalidSpec {
5741        reason: String,
5742    },
5743    Spawn {
5744        program: PathBuf,
5745        source: io::Error,
5746        cgroup_path: Option<PathBuf>,
5747    },
5748    Cgroup {
5749        module_id: String,
5750        source: io::Error,
5751    },
5752    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
5753    /// than spawn a reserved module without its identity binding.
5754    LaunchNonce {
5755        reason: String,
5756    },
5757    Wait {
5758        module_id: String,
5759        source: io::Error,
5760    },
5761    Kill {
5762        module_id: String,
5763        source: io::Error,
5764    },
5765    Forwarding(ForwardingError),
5766    Registry(RegistryError),
5767    ReloadUnavailable {
5768        module_id: String,
5769        reason: String,
5770    },
5771    /// An operator restart/reload was requested for a module that is currently
5772    /// disabled. Restart/reload cycle a *running* module; a disabled module must
5773    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
5774    /// by a restart, so these commands are rejected instead of re-enabling it.
5775    Disabled {
5776        module_id: String,
5777    },
5778    ReloadFailed {
5779        module_id: String,
5780        reason: String,
5781    },
5782    RegistrationStillActive {
5783        module_id: String,
5784        waited: Duration,
5785    },
5786    StatePoisoned {
5787        module_id: Option<String>,
5788    },
5789    CommandClosed {
5790        module_id: String,
5791    },
5792    /// A restart or reload arrived while a swap's candidate was warming. The
5793    /// swap owns the module until it cuts over or fails; a stop or disable
5794    /// would have aborted it instead.
5795    SwapInProgress {
5796        module_id: String,
5797    },
5798    /// A swap was refused before anything was spawned.
5799    SwapRefused {
5800        module_id: String,
5801        reason: SwapRefusal,
5802    },
5803    /// A swap spawned a candidate and gave up on it. The candidate has been
5804    /// killed and its slot freed; the incumbent was left serving and was never
5805    /// drained, except in the one `CutoverLost` case described on that arm.
5806    SwapFailed {
5807        module_id: String,
5808        arm: SwapFailureArm,
5809        detail: String,
5810        /// How the candidate exited, when it exited on its own before the
5811        /// supervisor gave up on it.
5812        candidate_exit: Option<ExitReport>,
5813    },
5814}
5815
5816/// Why a swap was refused before a candidate was spawned.
5817#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5818pub enum SwapRefusal {
5819    /// The module's config does not declare `overlap: "safe"`.
5820    OverlapExclusive,
5821    /// The module is not registered, so there is no incumbent to keep serving
5822    /// and nothing a swap would improve on; a plain restart is the tool.
5823    NotRegistered,
5824    /// The module does not speak the subc wire, so a candidate could never
5825    /// register or declare itself ready.
5826    ProtocolNone,
5827    /// The supervisor lacks the forwarding table (to cut routes over) or the
5828    /// shared handle (to admit the candidate's HELLO) that a swap needs.
5829    NotConfigured,
5830    /// A swap is already open for this module.
5831    AlreadySwapping,
5832}
5833
5834impl SwapRefusal {
5835    pub fn as_str(self) -> &'static str {
5836        match self {
5837            Self::OverlapExclusive => "overlap_exclusive",
5838            Self::NotRegistered => "not_registered",
5839            Self::ProtocolNone => "protocol_none",
5840            Self::NotConfigured => "not_configured",
5841            Self::AlreadySwapping => "already_swapping",
5842        }
5843    }
5844}
5845
5846/// Which failure arm ended a swap. Every arm but one leaves the incumbent
5847/// serving and undrained; see `CutoverLost`.
5848#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5849pub enum SwapFailureArm {
5850    /// The candidate process could not be started.
5851    SpawnFailed,
5852    /// The candidate did not register within the readiness budget.
5853    NeverRegistered,
5854    /// The candidate registered but did not declare itself ready in time.
5855    NeverReady,
5856    /// The candidate exited before cutover.
5857    CandidateExited,
5858    /// The candidate declared itself ready but failed its health probe.
5859    CandidateUnhealthy,
5860    /// An operator stop, disable or retire arrived while the candidate warmed.
5861    /// The candidate was killed and the operator's command then carried out on
5862    /// the incumbent.
5863    Interrupted,
5864    /// The candidate's connection closed at the moment of cutover. If it
5865    /// closed before forwarding moved, the incumbent is untouched. If it closed
5866    /// between the forwarding and registry halves of cutover, forwarding can no
5867    /// longer route to the incumbent, so the module is restarted plainly.
5868    CutoverLost,
5869}
5870
5871impl SwapFailureArm {
5872    pub fn as_str(self) -> &'static str {
5873        match self {
5874            Self::SpawnFailed => "spawn_failed",
5875            Self::NeverRegistered => "never_registered",
5876            Self::NeverReady => "never_ready",
5877            Self::CandidateExited => "candidate_exited",
5878            Self::CandidateUnhealthy => "candidate_unhealthy",
5879            Self::Interrupted => "interrupted",
5880            Self::CutoverLost => "cutover_lost",
5881        }
5882    }
5883}
5884
5885impl SuperviseError {
5886    pub(crate) fn is_config_gate_error(&self) -> bool {
5887        match self {
5888            Self::ConfigHeld { .. } => true,
5889            Self::Spawn { source, .. } => source
5890                .get_ref()
5891                .is_some_and(|cause| cause.is::<crate::config_gate::GateError>()),
5892            _ => false,
5893        }
5894    }
5895}
5896
5897impl fmt::Display for SuperviseError {
5898    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5899        match self {
5900            Self::ConfigHeld { module_id, code, detail } => write!(f, "{code}: module '{module_id}' held: {detail}"),
5901            Self::ConfigDeliveryDeferred { module_id } => write!(f, "config_gated: module '{module_id}' is waiting for its exited delivery target's failure record"),
5902            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
5903            Self::Spawn {
5904                program,
5905                source,
5906                cgroup_path: Some(cgroup_path),
5907            } => write!(
5908                f,
5909                "failed to place module in cgroup '{}' while spawning '{}': {source}",
5910                cgroup_path.display(),
5911                program.display()
5912            ),
5913            Self::Spawn {
5914                program,
5915                source,
5916                cgroup_path: None,
5917            } => write!(
5918                f,
5919                "failed to spawn module '{}': {source}",
5920                program.display()
5921            ),
5922            Self::Cgroup { module_id, source } => {
5923                write!(
5924                    f,
5925                    "failed to prepare cgroup for module '{module_id}': {source}"
5926                )
5927            }
5928            Self::LaunchNonce { reason } => {
5929                write!(
5930                    f,
5931                    "failed to generate reserved-module launch nonce: {reason}"
5932                )
5933            }
5934            Self::Wait { module_id, source } => {
5935                write!(f, "failed to wait for module '{module_id}': {source}")
5936            }
5937            Self::Kill { module_id, source } => {
5938                write!(f, "failed to kill module '{module_id}': {source}")
5939            }
5940            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
5941            Self::Registry(err) => write!(f, "registry error: {err}"),
5942            Self::ReloadUnavailable { module_id, reason } => {
5943                write!(f, "reload unavailable for module '{module_id}': {reason}")
5944            }
5945            Self::Disabled { module_id } => {
5946                write!(
5947                    f,
5948                    "module '{module_id}' is disabled; enable it before restart or reload"
5949                )
5950            }
5951            Self::ReloadFailed { module_id, reason } => {
5952                write!(f, "reload failed for module '{module_id}': {reason}")
5953            }
5954            Self::RegistrationStillActive { module_id, waited } => write!(
5955                f,
5956                "module '{module_id}' registration remained active after waiting {waited:?}"
5957            ),
5958            Self::StatePoisoned { module_id } => match module_id {
5959                Some(module_id) => {
5960                    write!(f, "supervisor state for module '{module_id}' was poisoned")
5961                }
5962                None => write!(f, "supervisor state was poisoned"),
5963            },
5964            Self::CommandClosed { module_id } => {
5965                write!(
5966                    f,
5967                    "supervisor command channel for module '{module_id}' is closed"
5968                )
5969            }
5970            Self::SwapInProgress { module_id } => write!(
5971                f,
5972                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
5973            ),
5974            Self::SwapRefused { module_id, reason } => match reason {
5975                SwapRefusal::OverlapExclusive => write!(
5976                    f,
5977                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
5978                ),
5979                SwapRefusal::NotRegistered => write!(
5980                    f,
5981                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
5982                ),
5983                SwapRefusal::ProtocolNone => write!(
5984                    f,
5985                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
5986                ),
5987                SwapRefusal::NotConfigured => write!(
5988                    f,
5989                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
5990                ),
5991                SwapRefusal::AlreadySwapping => {
5992                    write!(f, "module '{module_id}' is already being swapped")
5993                }
5994            },
5995            Self::SwapFailed {
5996                module_id,
5997                arm,
5998                detail,
5999                ..
6000            } => write!(
6001                f,
6002                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
6003                arm.as_str()
6004            ),
6005        }
6006    }
6007}
6008
6009impl Error for SuperviseError {
6010    fn source(&self) -> Option<&(dyn Error + 'static)> {
6011        match self {
6012            Self::Spawn { source, .. }
6013            | Self::Cgroup { source, .. }
6014            | Self::Wait { source, .. }
6015            | Self::Kill { source, .. } => Some(source),
6016            Self::Forwarding(err) => Some(err),
6017            Self::Registry(err) => Some(err),
6018            Self::LaunchNonce { .. }
6019            | Self::ConfigDeliveryDeferred { .. }
6020            | Self::ConfigHeld { .. }
6021            | Self::InvalidSpec { .. }
6022            | Self::ReloadUnavailable { .. }
6023            | Self::Disabled { .. }
6024            | Self::ReloadFailed { .. }
6025            | Self::RegistrationStillActive { .. }
6026            | Self::StatePoisoned { .. }
6027            | Self::CommandClosed { .. }
6028            | Self::SwapInProgress { .. }
6029            | Self::SwapRefused { .. }
6030            | Self::SwapFailed { .. } => None,
6031        }
6032    }
6033}
6034
6035pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
6036    if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
6037        return Err(SuperviseError::InvalidSpec {
6038            reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
6039        });
6040    }
6041    if spec.module_id.trim().is_empty() {
6042        return Err(SuperviseError::InvalidSpec {
6043            reason: "module_id must not be empty".to_string(),
6044        });
6045    }
6046
6047    Ok(())
6048}
6049
6050#[derive(Debug, Default)]
6051struct HealthProbeRuntime {
6052    spawn_generation: Option<u64>,
6053    configured_health: Option<HealthConfig>,
6054    registered_connection: Option<crate::ConnectionId>,
6055    advertised: bool,
6056    next_probe_at: Option<Instant>,
6057    probe_index: u64,
6058}
6059
6060fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
6061    lock_snapshot(snapshot)
6062        .ok()
6063        .and_then(|state| state.spawned_protocol)
6064        .unwrap_or(spec.protocol)
6065}
6066
6067impl HealthProbeRuntime {
6068    fn refresh_registration(
6069        &mut self,
6070        spec: &ModuleSpec,
6071        runtime: &SupervisorRuntimeConfig,
6072        registry: &Registry,
6073        snapshot: &SharedSnapshot,
6074    ) {
6075        let generation = lock_snapshot(snapshot)
6076            .ok()
6077            .map(|state| state.spawn_generation);
6078        if self.spawn_generation != generation {
6079            // A replacement cannot reuse a deadline that is already due for its
6080            // predecessor. HTTP children get a full cadence to start; wire
6081            // children still wait for their own health-advertising HELLO.
6082            *self = Self {
6083                spawn_generation: generation,
6084                ..Self::default()
6085            };
6086        }
6087        if self.configured_health.as_ref() != Some(&runtime.health) {
6088            self.configured_health = Some(runtime.health.clone());
6089            self.next_probe_at = None;
6090            self.registered_connection = None;
6091            self.probe_index = 0;
6092        }
6093        // A non-wire process never registers. Only an explicitly configured
6094        // HTTP endpoint can arm its health probe; an absent HELLO is not a
6095        // health failure for that kind of process.
6096        if running_protocol(spec, snapshot) == ModuleProtocol::None {
6097            self.registered_connection = None;
6098            self.advertised = runtime.health.http.is_some();
6099            if !self.advertised {
6100                self.next_probe_at = None;
6101                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6102                    let unknown = ModuleHealthStatus::default();
6103                    if state.health != unknown {
6104                        state.health = unknown;
6105                        state.last_probe_evidence = None;
6106                    }
6107                });
6108            } else if self.next_probe_at.is_none() {
6109                self.next_probe_at = Some(
6110                    Instant::now()
6111                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6112                );
6113            }
6114            return;
6115        }
6116
6117        let registration = match registry.get_module(&spec.module_id) {
6118            Ok(registration) => registration,
6119            Err(err) => {
6120                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
6121                self.advertised = false;
6122                self.next_probe_at = None;
6123                return;
6124            }
6125        };
6126
6127        let Some(registration) = registration else {
6128            self.registered_connection = None;
6129            self.advertised = false;
6130            self.next_probe_at = None;
6131            return;
6132        };
6133
6134        let advertised = registration
6135            .control_ops
6136            .iter()
6137            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
6138        if !advertised {
6139            self.registered_connection = Some(registration.connection_id);
6140            self.advertised = false;
6141            self.next_probe_at = None;
6142            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6143                if state.health.status != SupervisorHealthStatus::Unknown
6144                    || state.health.consecutive_failures != 0
6145                    || state.health.last_probe_ms.is_some()
6146                    || state.health.detail.is_some()
6147                    || state.health.metrics.is_some()
6148                {
6149                    state.health.status = SupervisorHealthStatus::Unknown;
6150                    state.last_probe_evidence = None;
6151                    state.health.consecutive_failures = 0;
6152                    state.health.last_probe_ms = None;
6153                    state.health.detail = None;
6154                    state.health.metrics = None;
6155                }
6156            });
6157            return;
6158        }
6159
6160        let reregistered = self.registered_connection != Some(registration.connection_id);
6161        self.registered_connection = Some(registration.connection_id);
6162        self.advertised = true;
6163        if reregistered || self.next_probe_at.is_none() {
6164            self.probe_index = 0;
6165            self.next_probe_at = Some(
6166                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6167            );
6168            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6169                state.health.status = SupervisorHealthStatus::Unknown;
6170                state.last_probe_evidence = None;
6171                state.health.consecutive_failures = 0;
6172                state.health.detail = None;
6173                state.health.metrics = None;
6174            });
6175        }
6176    }
6177
6178    fn wake_at(&self) -> Option<Instant> {
6179        if !self.advertised {
6180            return None;
6181        }
6182        self.next_probe_at
6183    }
6184
6185    fn due(&self) -> bool {
6186        self.advertised
6187            && self
6188                .next_probe_at
6189                .is_some_and(|next| Instant::now() >= next)
6190    }
6191
6192    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
6193        self.probe_index = self.probe_index.wrapping_add(1);
6194        self.next_probe_at = Some(
6195            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
6196        );
6197    }
6198}
6199
6200/// Evidence is separate from diagnostic prose so host starvation can hold only
6201/// silence-based restarts. A missing lane is still actionable during overload;
6202/// a timeout can instead mean the host has not scheduled a healthy module.
6203#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6204enum HealthProbeEvidence {
6205    /// The module's control lane is gone. Proof of death.
6206    LaneDead,
6207    /// No reply within the deadline. Proves nothing about the module's state.
6208    NoAnswer,
6209    /// The module replied, but not with a usable health report. Proves it is alive.
6210    BadAnswer,
6211    /// The daemon could not ask. Says nothing about the module.
6212    Misconfigured,
6213}
6214
6215#[derive(Debug)]
6216struct HealthProbeError {
6217    evidence: HealthProbeEvidence,
6218    message: String,
6219}
6220
6221impl HealthProbeError {
6222    fn lane_dead(message: impl Into<String>) -> Self {
6223        Self::with(HealthProbeEvidence::LaneDead, message)
6224    }
6225
6226    fn no_answer(message: impl Into<String>) -> Self {
6227        Self::with(HealthProbeEvidence::NoAnswer, message)
6228    }
6229
6230    fn bad_answer(message: impl Into<String>) -> Self {
6231        Self::with(HealthProbeEvidence::BadAnswer, message)
6232    }
6233
6234    fn misconfigured(message: impl Into<String>) -> Self {
6235        Self::with(HealthProbeEvidence::Misconfigured, message)
6236    }
6237
6238    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
6239        Self {
6240            evidence,
6241            message: message.into(),
6242        }
6243    }
6244
6245    /// Whether this observation is proof the module cannot serve.
6246    ///
6247    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
6248    /// variant that fires under CPU starvation, and treating it as proof is the
6249    /// defect this enum exists to make impossible to reintroduce silently.
6250    ///
6251    /// Outside a host-overload hold, silence still reaches the normal failure
6252    /// threshold so an isolated, genuinely wedged module can recover.
6253    #[allow(dead_code)]
6254    fn is_proof_of_death(&self) -> bool {
6255        matches!(self.evidence, HealthProbeEvidence::LaneDead)
6256    }
6257
6258    /// Short stable label for logs and the health snapshot.
6259    ///
6260    /// An operator reading `ck health` currently cannot tell "the module is gone"
6261    /// from "the module did not answer in five seconds", because both render as
6262    /// prose in the same field. These labels are what make the two
6263    /// distinguishable at a glance, and they are what a later restart-policy
6264    /// change will be argued from.
6265    fn label(&self) -> &'static str {
6266        match self.evidence {
6267            HealthProbeEvidence::LaneDead => "lane-dead",
6268            HealthProbeEvidence::NoAnswer => "no-answer",
6269            HealthProbeEvidence::BadAnswer => "bad-answer",
6270            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
6271        }
6272    }
6273}
6274
6275impl fmt::Display for HealthProbeError {
6276    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
6277        f.write_str(&self.message)
6278    }
6279}
6280
6281async fn run_health_probe_cycle(
6282    spec: &ModuleSpec,
6283    runtime: &SupervisorRuntimeConfig,
6284    registry: &Registry,
6285    process_liveness: &SupervisorProcessLiveness,
6286    snapshot: &SharedSnapshot,
6287    child: &mut Option<SupervisedChild>,
6288) {
6289    let now_ms = unix_ms_now();
6290    let delivery_probe = match &runtime.config_delivery {
6291        Some(delivery) => {
6292            delivery
6293                .health_probe(&spec.module_id, runtime.forwarding.as_deref(), registry)
6294                .await
6295        }
6296        None => None,
6297    };
6298    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
6299        .then_some(runtime.health.http.as_deref())
6300        .flatten();
6301    let result = match http {
6302        Some(url) => probe_http_health(url, runtime.health.deadline)
6303            .await
6304            .map(|report| (report, None)),
6305        None => probe_module_health_from(&spec.module_id, runtime, None)
6306            .await
6307            .map(|(report, probed)| (report, Some(probed))),
6308    };
6309    match result {
6310        Ok((report, probed)) => {
6311            handle_health_report(
6312                spec,
6313                runtime,
6314                registry,
6315                process_liveness,
6316                snapshot,
6317                child,
6318                report,
6319                now_ms,
6320                delivery_probe,
6321                probed,
6322            )
6323            .await;
6324        }
6325        Err(err) => {
6326            if http.is_some() {
6327                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6328                    state.health.status = SupervisorHealthStatus::Failing;
6329                });
6330            }
6331            handle_health_probe_failure(
6332                spec,
6333                runtime,
6334                registry,
6335                process_liveness,
6336                snapshot,
6337                child,
6338                err,
6339                now_ms,
6340                delivery_probe,
6341            )
6342            .await;
6343        }
6344    }
6345}
6346
6347pub(crate) struct HttpProbeTarget<'a> {
6348    address: std::net::SocketAddr,
6349    localhost: bool,
6350    authority: &'a str,
6351    path: String,
6352}
6353
6354/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
6355/// or TLS. A URL cannot turn a local health check into an outbound connection.
6356pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
6357    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
6358        return Err("must not contain whitespace, controls, or a fragment".into());
6359    }
6360    let rest = url
6361        .strip_prefix("http://")
6362        .ok_or("must use plain http://")?;
6363    let split = rest.find(['/', '?']).unwrap_or(rest.len());
6364    let (authority, suffix) = rest.split_at(split);
6365    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
6366        ("::1", rest)
6367    } else {
6368        let split = authority.find(':').unwrap_or(authority.len());
6369        authority.split_at(split)
6370    };
6371    let ip: std::net::IpAddr = match host {
6372        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
6373        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
6374        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
6375    };
6376    let port = if port.is_empty() {
6377        80
6378    } else {
6379        port.strip_prefix(':')
6380            .and_then(|p| p.parse::<u16>().ok())
6381            .filter(|p| *p > 0)
6382            .ok_or("must have a valid nonzero TCP port")?
6383    };
6384    let path = if suffix.is_empty() {
6385        "/".into()
6386    } else if suffix.starts_with('?') {
6387        format!("/{suffix}")
6388    } else {
6389        suffix.into()
6390    };
6391    Ok(HttpProbeTarget {
6392        address: std::net::SocketAddr::new(ip, port),
6393        localhost: host == "localhost",
6394        authority,
6395        path,
6396    })
6397}
6398
6399async fn probe_http_health(
6400    url: &str,
6401    deadline: Duration,
6402) -> Result<HealthReport, HealthProbeError> {
6403    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
6404    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
6405    // Keep partial diagnostics outside the timed future so cancellation does
6406    // not discard a status line or body bytes already received.
6407    let mut response_status = String::new();
6408    let mut body = Vec::new();
6409    let probe = async {
6410        // Resolve localhost ourselves so a hosts-file override cannot turn
6411        // this into an outbound request, while IPv6-only local servers work.
6412        let connection = match tokio::net::TcpStream::connect(target.address).await {
6413            Err(_) if target.localhost => {
6414                tokio::net::TcpStream::connect((
6415                    std::net::Ipv6Addr::LOCALHOST,
6416                    target.address.port(),
6417                ))
6418                .await
6419            }
6420            result => result,
6421        };
6422        let mut stream = connection.map_err(|error| {
6423            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
6424        })?;
6425        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
6426            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
6427        let mut reader = BufReader::new(stream);
6428        let mut budget = 16 * 1024;
6429        let status = http_line(&mut reader, &mut budget).await?;
6430        let mut words = status.split_ascii_whitespace();
6431        let version = words.next();
6432        let code = words
6433            .next()
6434            .filter(|word| word.len() == 3)
6435            .and_then(|word| word.parse::<u16>().ok());
6436        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
6437            || !code.is_some_and(|code| (100..600).contains(&code))
6438        {
6439            return Err(HealthProbeError::bad_answer(format!(
6440                "invalid HTTP status: {status}"
6441            )));
6442        }
6443        let code = code.expect("validated status code");
6444        response_status = status.clone();
6445        let mut length = None;
6446        let mut chunked = false;
6447        loop {
6448            let line = http_line(&mut reader, &mut budget).await?;
6449            if line.is_empty() {
6450                break;
6451            }
6452            if let Some((name, value)) = line.split_once(':') {
6453                if name.eq_ignore_ascii_case("content-length") {
6454                    length = Some(value.trim().parse::<u64>().map_err(|_| {
6455                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
6456                    })?);
6457                } else if name.eq_ignore_ascii_case("transfer-encoding") {
6458                    chunked = value.trim().eq_ignore_ascii_case("chunked");
6459                }
6460            }
6461        }
6462        if chunked {
6463            while body.len() < 200 {
6464                let line = http_line(&mut reader, &mut budget).await?;
6465                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
6466                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
6467                if size == 0 {
6468                    break;
6469                }
6470                let count = size.min((200 - body.len()) as u64) as usize;
6471                let start = body.len();
6472                (&mut reader)
6473                    .take(count as u64)
6474                    .read_to_end(&mut body)
6475                    .await
6476                    .map_err(|error| {
6477                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6478                    })?;
6479                if body.len() - start != count {
6480                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
6481                }
6482                if size > count as u64 || body.len() == 200 {
6483                    break;
6484                }
6485                if !http_line(&mut reader, &mut budget).await?.is_empty() {
6486                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
6487                }
6488            }
6489        } else {
6490            reader
6491                .take(length.unwrap_or(200).min(200))
6492                .read_to_end(&mut body)
6493                .await
6494                .map_err(|error| {
6495                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6496                })?;
6497        }
6498        if (200..300).contains(&code) {
6499            Ok(HealthReport::ok())
6500        } else {
6501            Err(HealthProbeError::bad_answer(
6502                "HTTP health endpoint returned non-2xx",
6503            ))
6504        }
6505    };
6506    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
6507        Err(HealthProbeError::no_answer(format!(
6508            "HTTP probe timed out after {deadline:?}"
6509        )))
6510    });
6511    if let Err(error) = &mut result {
6512        if !response_status.is_empty() {
6513            error.message = format!(
6514                "{}; {response_status}: {}",
6515                error.message,
6516                String::from_utf8_lossy(&body)
6517            );
6518        }
6519    }
6520    result
6521}
6522
6523async fn http_line(
6524    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
6525    remaining: &mut usize,
6526) -> Result<String, HealthProbeError> {
6527    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
6528    let mut line = Vec::new();
6529    (&mut *reader)
6530        .take(*remaining as u64)
6531        .read_until(b'\n', &mut line)
6532        .await
6533        .map_err(|error| {
6534            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
6535        })?;
6536    *remaining -= line.len();
6537    if !line.ends_with(b"\r\n") {
6538        return Err(HealthProbeError::bad_answer(
6539            "HTTP headers are incomplete or exceed 16 KiB",
6540        ));
6541    }
6542    line.truncate(line.len() - 2);
6543    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
6544}
6545
6546async fn probe_module_health(
6547    module_id: &str,
6548    runtime: &SupervisorRuntimeConfig,
6549    drain_deadline: Option<Instant>,
6550) -> Result<HealthReport, HealthProbeError> {
6551    probe_module_health_from(module_id, runtime, drain_deadline)
6552        .await
6553        .map(|(report, _)| report)
6554}
6555
6556/// [`probe_module_health`], also returning the connection whose registration
6557/// answered, so its answer is credited to that registration and no other.
6558async fn probe_module_health_from(
6559    module_id: &str,
6560    runtime: &SupervisorRuntimeConfig,
6561    drain_deadline: Option<Instant>,
6562) -> Result<(HealthReport, ConnectionId), HealthProbeError> {
6563    let Some(forwarding) = runtime.forwarding.as_ref() else {
6564        return Err(HealthProbeError::misconfigured(
6565            "supervisor was not configured with a forwarding table",
6566        ));
6567    };
6568    let probe_started_at = Instant::now();
6569    let mut deadline = probe_started_at + runtime.health.deadline;
6570    if let Some(drain_deadline) = drain_deadline {
6571        deadline = deadline.min(drain_deadline);
6572    }
6573    let pending = if drain_deadline.is_some() {
6574        forwarding.begin_drain_health_probe_rpc_for(
6575            module_id,
6576            MODULE_CONTROL_OP_HEALTH_CHECK,
6577            probe_started_at,
6578            deadline,
6579        )
6580    } else {
6581        forwarding.begin_health_probe_rpc_for(
6582            module_id,
6583            MODULE_CONTROL_OP_HEALTH_CHECK,
6584            probe_started_at,
6585            deadline,
6586        )
6587    }
6588    .map_err(|err| {
6589        // The endpoint is not registered, so there is no live control lane to
6590        // ask. That is the module being absent, not slow.
6591        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6592    })?;
6593    let probed = pending.endpoint.connection_id;
6594    await_health_probe(forwarding, pending, deadline, runtime.health.deadline)
6595        .await
6596        .map(|report| (report, probed))
6597}
6598
6599/// [`probe_module_health`] for one endpoint rather than the id's active one.
6600///
6601/// A swap probes two processes that no by-id lookup reaches: its candidate
6602/// before cutover, and its superseded incumbent (for busy gauges) while the
6603/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
6604/// bounds the by-id drain probe.
6605async fn probe_endpoint_health(
6606    endpoint: crate::ModuleEndpointId,
6607    runtime: &SupervisorRuntimeConfig,
6608    deadline_cap: Option<Instant>,
6609) -> Result<HealthReport, HealthProbeError> {
6610    let Some(forwarding) = runtime.forwarding.as_ref() else {
6611        return Err(HealthProbeError::misconfigured(
6612            "supervisor was not configured with a forwarding table",
6613        ));
6614    };
6615    let probe_started_at = Instant::now();
6616    let mut deadline = probe_started_at + runtime.health.deadline;
6617    if let Some(cap) = deadline_cap {
6618        deadline = deadline.min(cap);
6619    }
6620    let pending = forwarding
6621        .begin_endpoint_health_probe_rpc_for(
6622            endpoint,
6623            MODULE_CONTROL_OP_HEALTH_CHECK,
6624            probe_started_at,
6625            deadline,
6626        )
6627        .map_err(|err| {
6628            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6629        })?;
6630    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
6631}
6632
6633/// Send a begun health probe and classify its answer.
6634async fn await_health_probe(
6635    forwarding: &ForwardingTable,
6636    pending: PendingModuleControlRpc,
6637    deadline: Instant,
6638    probe_budget: Duration,
6639) -> Result<HealthReport, HealthProbeError> {
6640    let PendingModuleControlRpc {
6641        endpoint,
6642        module_sink,
6643        negotiated_ver,
6644        corr,
6645        receiver,
6646    } = pending;
6647    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6648        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
6649    })?;
6650    let frame = Frame::build_with_version(
6651        negotiated_ver,
6652        FrameType::Request,
6653        control_flags(),
6654        0,
6655        0,
6656        corr,
6657        body,
6658    )
6659    .map_err(|err| {
6660        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
6661    })?;
6662
6663    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
6664    // blocks waiting for capacity when the module's egress queue is full, and an
6665    // unbounded await here freezes the whole supervision actor (it stops polling
6666    // Child::wait and supervisor commands), making the module unrecoverable
6667    // in-band. On timeout the probe fails like any transport failure.
6668    match timeout_at(deadline, module_sink.send(frame)).await {
6669        Ok(Ok(())) => {}
6670        Ok(Err(err)) => {
6671            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6672            // A closed sink means the module's egress channel is gone -- the
6673            // receiving half is dropped when its connection tears down. Proof.
6674            return Err(HealthProbeError::lane_dead(format!(
6675                "failed to send health.check: {err}"
6676            )));
6677        }
6678        Err(_elapsed) => {
6679            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6680            // A full egress queue means the module is not draining its socket, which
6681            // is consistent with a wedged module AND with one whose reader is merely
6682            // starved. Silence, not proof.
6683            return Err(HealthProbeError::no_answer(
6684                "health.check send timed out before enqueue (module egress full)",
6685            ));
6686        }
6687    }
6688
6689    match timeout_at(deadline, receiver).await {
6690        Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => Err(HealthProbeError::bad_answer(
6691            "health.check returned an unexpected config response",
6692        )),
6693        // Each arm records WHAT WAS OBSERVED. Four of them are the module
6694        // demonstrably answering -- rejected, non-health, malformed, wrong op --
6695        // and those prove it is alive even though the probe failed.
6696        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6697            response.health_report().ok_or_else(|| {
6698                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
6699            })
6700        }
6701        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
6702            format!("health.check rejected: {}", body.message),
6703        )),
6704        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6705            Err(HealthProbeError::lane_dead(message))
6706        }
6707        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6708            Err(HealthProbeError::bad_answer(message))
6709        }
6710        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6711            Err(HealthProbeError::bad_answer(format!(
6712                "expected module-control op '{expected}', got '{actual}'"
6713            )))
6714        }
6715        // A reply that crosses the deadline before this waiter observes it is
6716        // still proof of life. The forwarding path records its end-to-end latency
6717        // before delivering this classification.
6718        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
6719            "module answered health.check after its daemon deadline",
6720        )),
6721        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
6722            "health.check waiter was canceled before the module responded",
6723        )),
6724        Err(_) => {
6725            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
6726            Err(HealthProbeError::no_answer(format!(
6727                "module did not answer health.check within {probe_budget:?}"
6728            )))
6729        }
6730    }
6731}
6732
6733/// Act on a health probe's answer. `probed` is the connection whose
6734/// registration answered (`None` for an HTTP probe, which reaches no
6735/// registration); only that registration's spawn is credited with readiness.
6736#[allow(clippy::too_many_arguments)]
6737async fn handle_health_report(
6738    spec: &ModuleSpec,
6739    runtime: &SupervisorRuntimeConfig,
6740    registry: &Registry,
6741    process_liveness: &SupervisorProcessLiveness,
6742    snapshot: &SharedSnapshot,
6743    child: &mut Option<SupervisedChild>,
6744    report: HealthReport,
6745    now_ms: u64,
6746    delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6747    probed: Option<ConnectionId>,
6748) {
6749    let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6750        (Some(delivery), Some(probe)) => {
6751            delivery
6752                .health_result(
6753                    &spec.module_id,
6754                    runtime.forwarding.as_deref(),
6755                    registry,
6756                    probe,
6757                    report.status == HealthStatus::Ok,
6758                )
6759                .await
6760        }
6761        _ => false,
6762    };
6763    if let (Some(delivery), Some(spawn)) = (
6764        &runtime.config_delivery,
6765        child
6766            .as_ref()
6767            .and_then(|child| child.config_delivery.as_ref()),
6768    ) {
6769        delivery
6770            .mark_readiness(
6771                &spec.module_id,
6772                runtime.forwarding.as_deref(),
6773                probed,
6774                spawn,
6775                report.status == HealthStatus::Ok,
6776            )
6777            .await;
6778    }
6779    let status = supervisor_health_status(report.status);
6780    let detail = report.detail.clone();
6781    let metrics = truncate_health_metrics(report.metrics);
6782    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6783        state.health.status = status;
6784        state.last_probe_evidence = None;
6785        state.health.last_probe_ms = Some(now_ms);
6786        state.health.detail = detail.clone();
6787        state.health.metrics = metrics.clone();
6788        state.health.consecutive_failures = 0;
6789        state.failed_retry.observe_health(
6790            report.status == HealthStatus::Ok,
6791            runtime.restart_policy.window,
6792        );
6793    });
6794
6795    if delivery_failed {
6796        return;
6797    }
6798    let action = match report.status {
6799        HealthStatus::Ok => return,
6800        HealthStatus::Degraded => runtime.health.on_degraded,
6801        HealthStatus::Failing => runtime.health.on_failing,
6802    };
6803    apply_l3_health_action(
6804        spec,
6805        runtime,
6806        registry,
6807        process_liveness,
6808        snapshot,
6809        child,
6810        status,
6811        detail.as_deref(),
6812        action,
6813        now_ms,
6814    )
6815    .await;
6816}
6817
6818#[allow(clippy::too_many_arguments)]
6819async fn handle_health_probe_failure(
6820    spec: &ModuleSpec,
6821    runtime: &SupervisorRuntimeConfig,
6822    registry: &Registry,
6823    process_liveness: &SupervisorProcessLiveness,
6824    snapshot: &SharedSnapshot,
6825    child: &mut Option<SupervisedChild>,
6826    err: HealthProbeError,
6827    now_ms: u64,
6828    delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6829) {
6830    let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6831        (Some(delivery), Some(probe)) => {
6832            delivery
6833                .health_result(
6834                    &spec.module_id,
6835                    runtime.forwarding.as_deref(),
6836                    registry,
6837                    probe,
6838                    false,
6839                )
6840                .await
6841        }
6842        _ => false,
6843    };
6844    let threshold = runtime.health.failure_threshold.max(1);
6845    let held = if err.evidence == HealthProbeEvidence::NoAnswer {
6846        runtime.host_health.record_late_module(&spec.module_id)
6847    } else {
6848        false
6849    };
6850    let mut failures = 0;
6851    // Carry the evidence class into the operator-visible detail. Without it,
6852    // "module did not answer within 5s" and "the control lane is gone" are two
6853    // prose strings in the same field, and the reader has to know the codebase to
6854    // tell which one is proof of anything.
6855    let detail = if held {
6856        format!("[{}] {err}; host overloaded", err.label())
6857    } else {
6858        format!("[{}] {err}", err.label())
6859    };
6860    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6861        state.last_probe_evidence = Some(err.evidence);
6862        state
6863            .failed_retry
6864            .observe_health(false, runtime.restart_policy.window);
6865        // A failed wire probe invalidates the last report, even before the
6866        // restart threshold. HTTP probes already mark failures as Failing.
6867        if held {
6868            state.health.status = SupervisorHealthStatus::Degraded;
6869        } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
6870            state.health.status = SupervisorHealthStatus::Unknown;
6871        }
6872        state.health.last_probe_ms = Some(now_ms);
6873        // Held silence breaks the streak: after the host recovers, only new
6874        // failures count. Replaying pre-overload misses would restart immediately.
6875        state.health.consecutive_failures = if held {
6876            0
6877        } else {
6878            state.health.consecutive_failures.saturating_add(1)
6879        };
6880        state.health.detail = Some(detail.clone());
6881        state.health.metrics = None;
6882        failures = state.health.consecutive_failures;
6883    });
6884
6885    if delivery_failed {
6886        return;
6887    }
6888    if held || failures < threshold {
6889        warn!(
6890            module_id = %spec.module_id,
6891            consecutive_failures = failures,
6892            threshold,
6893            evidence = err.label(),
6894            detail = %detail,
6895            "health.check probe failed"
6896        );
6897        return;
6898    }
6899
6900    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6901        state.state = ModuleState::Unresponsive;
6902        state.health.status = SupervisorHealthStatus::Unresponsive;
6903    });
6904    // The evidence class is logged at the kill site because this is the line an
6905    // operator reads after an unexplained restart. A streak of `no-answer` under
6906    // machine load is the known false-positive shape; a `lane-dead` is not.
6907    if runtime.health.critical {
6908        error!(
6909            module_id = %spec.module_id,
6910            status = "unresponsive",
6911            evidence = err.label(),
6912            detail = %detail,
6913            "critical module health alert"
6914        );
6915    } else {
6916        warn!(
6917            module_id = %spec.module_id,
6918            status = "unresponsive",
6919            evidence = err.label(),
6920            detail = %detail,
6921            "module health threshold breached"
6922        );
6923    }
6924    if let Err(err) = health_restart_child(
6925        spec,
6926        runtime,
6927        registry,
6928        process_liveness,
6929        snapshot,
6930        child,
6931        SupervisorHealthStatus::Unresponsive,
6932        Some(&detail),
6933        now_ms,
6934    )
6935    .await
6936    {
6937        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6938    }
6939}
6940
6941#[allow(clippy::too_many_arguments)]
6942async fn apply_l3_health_action(
6943    spec: &ModuleSpec,
6944    runtime: &SupervisorRuntimeConfig,
6945    registry: &Registry,
6946    process_liveness: &SupervisorProcessLiveness,
6947    snapshot: &SharedSnapshot,
6948    child: &mut Option<SupervisedChild>,
6949    status: SupervisorHealthStatus,
6950    detail: Option<&str>,
6951    action: HealthAction,
6952    now_ms: u64,
6953) {
6954    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
6955    match action {
6956        HealthAction::Report => {
6957            info!(
6958                module_id = %spec.module_id,
6959                status = ?status,
6960                detail,
6961                "module reported non-ok health"
6962            );
6963        }
6964        HealthAction::Alert => {
6965            error!(
6966                module_id = %spec.module_id,
6967                status = ?status,
6968                detail,
6969                "module health alert"
6970            );
6971        }
6972        HealthAction::Restart => {
6973            if let Err(err) = health_restart_child(
6974                spec,
6975                runtime,
6976                registry,
6977                process_liveness,
6978                snapshot,
6979                child,
6980                status,
6981                detail,
6982                now_ms,
6983            )
6984            .await
6985            {
6986                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6987            }
6988        }
6989    }
6990}
6991
6992#[allow(clippy::too_many_arguments)]
6993async fn health_restart_child(
6994    spec: &ModuleSpec,
6995    runtime: &SupervisorRuntimeConfig,
6996    registry: &Registry,
6997    process_liveness: &SupervisorProcessLiveness,
6998    snapshot: &SharedSnapshot,
6999    child: &mut Option<SupervisedChild>,
7000    status: SupervisorHealthStatus,
7001    detail: Option<&str>,
7002    now_ms: u64,
7003) -> Result<(), SuperviseError> {
7004    budgeted_restart_child(
7005        spec,
7006        runtime,
7007        registry,
7008        process_liveness,
7009        snapshot,
7010        child,
7011        status,
7012        detail,
7013        now_ms,
7014        true,
7015    )
7016    .await
7017}
7018
7019#[allow(clippy::too_many_arguments)]
7020async fn budgeted_restart_child(
7021    spec: &ModuleSpec,
7022    runtime: &SupervisorRuntimeConfig,
7023    registry: &Registry,
7024    process_liveness: &SupervisorProcessLiveness,
7025    snapshot: &SharedSnapshot,
7026    child: &mut Option<SupervisedChild>,
7027    status: SupervisorHealthStatus,
7028    detail: Option<&str>,
7029    now_ms: u64,
7030    health_failure: bool,
7031) -> Result<(), SuperviseError> {
7032    if let Some(error) = runtime
7033        .config_delivery
7034        .as_ref()
7035        .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
7036        .filter(|error| error.code != "config_delivery_deferred")
7037    {
7038        return Err(config_delivery::gate_hold_error(
7039            spec,
7040            error,
7041            runtime.config_delivery.as_ref(),
7042        ));
7043    }
7044    let recovery_reason = if health_failure {
7045        "health_check"
7046    } else {
7047        "configuration_delivery"
7048    };
7049    let (enabled, schedule) = {
7050        let mut state = lock_snapshot(snapshot)?;
7051        let enabled = state.enabled;
7052        let schedule = if enabled {
7053            state.next_crash_restart(&runtime.restart_policy, Instant::now())
7054        } else {
7055            None
7056        };
7057        (enabled, schedule)
7058    };
7059
7060    if !enabled {
7061        return Err(SuperviseError::Disabled {
7062            module_id: spec.module_id.clone(),
7063        });
7064    }
7065
7066    if schedule.is_none() {
7067        if health_failure {
7068            record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
7069        }
7070        error!(
7071            module_id = %spec.module_id,
7072            status = ?status,
7073            detail,
7074            max_restarts = runtime.restart_policy.max_restarts,
7075            window_secs = runtime.restart_policy.window.as_secs(),
7076            reason = %runtime.restart_policy.budget_exhausted_detail(),
7077            recovery_reason,
7078            "automatic restart budget exhausted; marking module failed"
7079        );
7080        let stop_notice = begin_forwarding_drain_if_configured(
7081            spec,
7082            runtime,
7083            registry,
7084            snapshot,
7085            Some(true),
7086            RouteCloseReason::Disable,
7087        )
7088        .await?;
7089        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7090            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
7091        })?;
7092        let result = drain_optional_child(
7093            &spec.module_id,
7094            spec.protocol,
7095            stop_notice,
7096            registry,
7097            runtime.forwarding.as_deref(),
7098            snapshot,
7099            &runtime.terminal_ring,
7100            &runtime.spawn_events,
7101            child,
7102            runtime.drain_timeout,
7103            ModuleState::Failed,
7104            Some(true),
7105            &runtime.failure_context(true),
7106        )
7107        .await;
7108        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7109            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
7110                state.failed_retry.schedule(&spec.module_id);
7111            }
7112        })?;
7113        result?;
7114        process_liveness.untrack_if_current(&spec.module_id, snapshot);
7115        return Ok(());
7116    }
7117
7118    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
7119    let mut restart_count = 0;
7120    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7121        restart_count = state.crash_restarts.len();
7122        state.state = ModuleState::Unresponsive;
7123        if health_failure {
7124            state.health.status = status;
7125            state.health.last_action = Some(HealthAction::Restart.to_string());
7126            state.health.last_action_ms = Some(now_ms);
7127        }
7128    })?;
7129    warn!(
7130        module_id = %spec.module_id,
7131        status = ?status,
7132        detail,
7133        restart_count,
7134        restart_in_window = schedule.restart_in_window,
7135        delay_ms = schedule.delay.as_millis() as u64,
7136        recovery_reason,
7137        "automatic module restart"
7138    );
7139
7140    let stop_notice = begin_forwarding_drain_if_configured(
7141        spec,
7142        runtime,
7143        registry,
7144        snapshot,
7145        Some(true),
7146        RouteCloseReason::Restart,
7147    )
7148    .await?;
7149    drain_optional_child(
7150        &spec.module_id,
7151        spec.protocol,
7152        stop_notice,
7153        registry,
7154        runtime.forwarding.as_deref(),
7155        snapshot,
7156        &runtime.terminal_ring,
7157        &runtime.spawn_events,
7158        child,
7159        runtime.drain_timeout,
7160        ModuleState::Restarting,
7161        Some(true),
7162        &runtime.failure_context(true),
7163    )
7164    .await?;
7165    schedule_respawn(
7166        runtime,
7167        snapshot,
7168        &spec.module_id,
7169        schedule.delay,
7170        RespawnKind::Spawn,
7171        true,
7172    )
7173}
7174
7175fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
7176    if let Some(reply) = runtime
7177        .deferred_reload_reply
7178        .lock()
7179        .unwrap_or_else(|p| p.into_inner())
7180        .take()
7181    {
7182        let _ = reply.send(Err(SuperviseError::ReloadFailed {
7183            module_id: module_id.to_string(),
7184            reason: reason.to_string(),
7185        }));
7186    }
7187}
7188
7189fn park_config_spawn(
7190    spec: &ModuleSpec,
7191    runtime: &SupervisorRuntimeConfig,
7192    snapshot: &SharedSnapshot,
7193    failure: &FailureContext,
7194    admission_refused: Option<String>,
7195) -> Result<(), SuperviseError> {
7196    *runtime
7197        .scheduled_respawn
7198        .lock()
7199        .unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
7200    if let Some(reply) = runtime
7201        .deferred_reload_reply
7202        .lock()
7203        .unwrap_or_else(|poisoned| poisoned.into_inner())
7204        .take()
7205    {
7206        let _ = reply.send(Err(SuperviseError::ConfigHeld {
7207            module_id: spec.module_id.clone(),
7208            code: "config_gated".into(),
7209            detail: "module is held by configuration admission".into(),
7210        }));
7211    }
7212    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7213        state.admission_refused = admission_refused;
7214        state.config_hold_daemon_initiated = failure.daemon_initiated;
7215        state.config_hold_approval = failure.approval.clone();
7216        state.config_hold_parked = true;
7217        state.state = ModuleState::Stopped;
7218        state.respawn_pending = false;
7219        state.failed_retry.pending = None;
7220        // Parking ends this attempt and discards its failure timestamp. If a
7221        // later release fails, its retry delay starts at that new failure.
7222        state.failed_retry.failed_at = None;
7223        clear_current_process_facts(state);
7224    })
7225}
7226
7227fn try_park_config_attempt(
7228    spec: &ModuleSpec,
7229    runtime: &SupervisorRuntimeConfig,
7230    snapshot: &SharedSnapshot,
7231    failure: &FailureContext,
7232    program_base: Option<&subc_config::Digest>,
7233) -> Result<bool, SuperviseError> {
7234    let result = (|| {
7235        if !lock_snapshot(snapshot)?.enabled {
7236            return Ok(false);
7237        }
7238        #[cfg(test)]
7239        if let Some(delivery) = &runtime.config_delivery {
7240            delivery.0.pause_before_config_park_for_test();
7241        }
7242        let Some(delivery) = &runtime.config_delivery else {
7243            return Ok(false);
7244        };
7245        let parked = delivery
7246            .0
7247            .with_recorded_admission_hold(&spec.module_id, program_base, |refusal| {
7248                park_config_spawn(spec, runtime, snapshot, failure, refusal)
7249            })
7250            .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(delivery)))?;
7251        if let Some(result) = parked {
7252            result?;
7253            Ok(true)
7254        } else {
7255            Ok(false)
7256        }
7257    })();
7258    if let Err(error) = &result {
7259        // A failed park still belongs to the attempt that tried to park. Mark it
7260        // Failed here, before the monitor can use a daemon/operator origin saved
7261        // for a different attempt to decide whether to schedule a slow retry.
7262        fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7263        record_approved_attempt_failure(spec, snapshot, failure, error);
7264    }
7265    result
7266}
7267
7268fn record_approved_attempt_failure(
7269    spec: &ModuleSpec,
7270    snapshot: &SharedSnapshot,
7271    failure: &FailureContext,
7272    error: &SuperviseError,
7273) {
7274    let Some(approval) = &failure.approval else {
7275        return;
7276    };
7277    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7278        if state.enabled
7279            && state.state == ModuleState::Failed
7280            && state
7281                .approved_start_failure
7282                .as_ref()
7283                .is_none_or(|(identity, _)| identity != approval)
7284        {
7285            state.approved_start_failure = Some((approval.clone(), error.to_string()));
7286        }
7287    });
7288}
7289
7290fn hold_config_spawn(
7291    spec: &ModuleSpec,
7292    runtime: &SupervisorRuntimeConfig,
7293    snapshot: &SharedSnapshot,
7294    failure: &FailureContext,
7295) -> Result<(), SuperviseError> {
7296    if !lock_snapshot(snapshot)?.enabled {
7297        return Ok(());
7298    }
7299    if !try_park_config_attempt(spec, runtime, snapshot, failure, None)? {
7300        // This caller consumed an actual start attempt. If its hold ended before
7301        // parking, retain that attempt instead of waiting for a lost release.
7302        defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
7303    }
7304    Ok(())
7305}
7306
7307fn schedule_respawn(
7308    runtime: &SupervisorRuntimeConfig,
7309    snapshot: &SharedSnapshot,
7310    module_id: &str,
7311    delay: Duration,
7312    kind: RespawnKind,
7313    daemon_initiated: bool,
7314) -> Result<(), SuperviseError> {
7315    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
7316    update_snapshot(snapshot, Some(module_id), |state| {
7317        state.respawn_pending = true
7318    })?;
7319    *runtime
7320        .scheduled_respawn
7321        .lock()
7322        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7323        deadline: Instant::now() + delay,
7324        kind,
7325        daemon_initiated,
7326        approval: None,
7327    });
7328    Ok(())
7329}
7330
7331const CONFIG_ADMISSION_RETRY_DELAY: Duration = Duration::from_millis(100);
7332
7333fn defer_config_spawn(
7334    spec: &ModuleSpec,
7335    runtime: &SupervisorRuntimeConfig,
7336    snapshot: &SharedSnapshot,
7337    kind: RespawnKind,
7338    failure: &FailureContext,
7339) -> Result<(), SuperviseError> {
7340    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7341        state.state = ModuleState::Restarting;
7342        state.respawn_pending = true;
7343        clear_current_process_facts(state);
7344    })?;
7345    *runtime
7346        .scheduled_respawn
7347        .lock()
7348        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7349        deadline: Instant::now() + CONFIG_ADMISSION_RETRY_DELAY,
7350        kind,
7351        daemon_initiated: failure.daemon_initiated,
7352        approval: failure.approval.clone(),
7353    });
7354    Ok(())
7355}
7356
7357fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
7358    let _ = update_snapshot(snapshot, Some(module_id), |state| {
7359        state.health.last_action = Some(action);
7360        state.health.last_action_ms = Some(now_ms);
7361    });
7362}
7363
7364fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
7365    match status {
7366        HealthStatus::Ok => SupervisorHealthStatus::Ok,
7367        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
7368        HealthStatus::Failing => SupervisorHealthStatus::Failing,
7369    }
7370}
7371
7372/// Caps the metrics blob stored in the cached supervisor snapshot, which is
7373/// returned to every `supervisor.list` and `supervisor.health` caller.
7374///
7375/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
7376/// path: that request exists to return a module's complete metrics object, and
7377/// `ck health <module-id>` documents it as the way to see what the cached view
7378/// truncates. The asymmetry is the feature.
7379///
7380/// So a new caller must decide which side it is on rather than assume the cap is
7381/// universal. Reaching for it on a fresh-probe path would silently reintroduce
7382/// the truncation that path exists to avoid.
7383fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
7384    let metrics = metrics?;
7385    match serde_json::to_vec(&metrics) {
7386        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
7387            "truncated": true,
7388            "original_bytes": encoded.len(),
7389        })),
7390        Ok(_) | Err(_) => Some(metrics),
7391    }
7392}
7393
7394/// Spread health probes so a fleet-wide restart does not converge them.
7395///
7396/// The delay is derived from the module id and probe index rather than a random
7397/// source, so it is deterministic per module: a module keeps its own offset
7398/// across daemon restarts instead of re-rolling into a collision.
7399fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
7400    if cadence.is_zero() {
7401        return Duration::ZERO;
7402    }
7403    let cadence_ms = cadence.as_millis() as u64;
7404    // This early return is REDUNDANT, deliberately, and a mutation run will show
7405    // it surviving removal. Recording why here so the next person to notice does
7406    // not have to re-derive it:
7407    //
7408    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
7409    //   a zero cadence and builds the Duration from whole milliseconds, so a
7410    //   sub-millisecond cadence cannot come from config.
7411    // - Even if reached it changes no answer. The `.max(1)` below makes the span
7412    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
7413    //   -- exactly what this returns.
7414    //
7415    // Kept as a guard against a future widening of the config parser (accepting
7416    // microseconds, say), which would make the sub-millisecond case reachable.
7417    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
7418    // divides by zero. Remove this and nothing changes.
7419    if cadence_ms == 0 {
7420        return cadence;
7421    }
7422    // Note that this never returns less than one cadence, including for the FIRST
7423    // probe. So a freshly registered module reports health `unknown` for a full
7424    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
7425    // ready to answer.
7426    //
7427    // That is a property of the supervisor's schedule, not of any module: an
7428    // operator watching a restart sees `unknown` and cannot tell it from a module
7429    // that is slow to warm. Measured on two unrelated modules, both flipping to
7430    // `ok` between 22s and 32s after restart.
7431    //
7432    // Left as-is because spreading the first probe is what keeps a fleet-wide
7433    // restart from firing fourteen simultaneous probes into a cold machine. The
7434    // alternative -- probe at t+0 and jitter only from the second onward -- trades
7435    // that thundering herd for a faster first reading.
7436    let jitter_span = (cadence_ms / 10).max(1);
7437    let hash = module_id.as_bytes().iter().fold(
7438        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
7439        |acc, byte| {
7440            acc.wrapping_mul(1099511628211)
7441                .wrapping_add(u64::from(*byte))
7442        },
7443    );
7444    cadence + Duration::from_millis(hash % jitter_span)
7445}
7446
7447#[cfg(test)]
7448mod tests {
7449    use super::*;
7450
7451    #[test]
7452    fn registration_retirement_self_prefixed_owner_is_not_a_prefix_submodule() {
7453        let handle = SupervisorHandle::new();
7454        handle.set_spawn_nonce("prefix:owner", "owner-nonce".into());
7455        handle.set_reserved_prefixes("prefix:owner", &["prefix:".into()]);
7456        let owner = handle
7457            .reserved_hello_verification("prefix:owner", Some("owner-nonce"))
7458            .unwrap();
7459        assert!(owner.nonce_verified);
7460        assert!(
7461            owner.prefix_owner.is_none(),
7462            "an unreserved programmatic owner is not its own prefix child"
7463        );
7464        let child = handle
7465            .reserved_hello_verification("prefix:child", Some("owner-nonce"))
7466            .unwrap();
7467        assert!(child.nonce_verified);
7468        assert_eq!(child.prefix_owner.as_deref(), Some("prefix:owner"));
7469    }
7470
7471    #[test]
7472    fn readding_a_module_clears_its_rescan_removal_tombstone() {
7473        let handle = SupervisorHandle::new();
7474        let module_id = "readded-tombstone";
7475        handle.record_rescan_removal(module_id);
7476        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
7477
7478        handle.apply_identity_configuration(&ModuleSpec {
7479            module_id: module_id.to_string(),
7480            program: PathBuf::from("/test/module"),
7481            args: Vec::new(),
7482            env: Vec::new(),
7483            reserved: false,
7484            reserved_prefixes: Vec::new(),
7485            protocol: ModuleProtocol::Subc,
7486            overlap: Default::default(),
7487            carry_containment: false,
7488        });
7489
7490        assert!(
7491            handle.removal_tombstone_age_ms(module_id).is_none(),
7492            "a re-added module must not retain a stale removal tombstone"
7493        );
7494    }
7495
7496    /// What one module's owner looked like from the control plane at the
7497    /// instant after its first process was spawned.
7498    #[derive(Debug, PartialEq, Eq)]
7499    struct OwnerInSpawnWindow {
7500        module_id: String,
7501        configured: bool,
7502        on_roster: bool,
7503        admission_refusal: Option<&'static str>,
7504    }
7505
7506    /// A supervised module's process can connect, register, sync its scopes
7507    /// and describe them as soon as it is spawned. A plain `spawn` does not put
7508    /// the module on the roster until afterward; configured supervision does
7509    /// so before spawning. In both cases the owner must
7510    /// already read as configured, so a scoped `route.open` against it is
7511    /// refused as retryable `scope_not_synced` and not as terminal
7512    /// `scope_not_live` ("will never sync").
7513    ///
7514    /// The hook runs just after each initial spawn attempt, before returning to
7515    /// the caller, so no race with a real child's registration is needed.
7516    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7517    async fn a_new_module_is_configured_before_its_first_process_can_register() {
7518        use crate::scopes::ScopeTable;
7519        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
7520
7521        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
7522        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
7523            module_id: module_id.to_string(),
7524            program,
7525            args: Vec::new(),
7526            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7527                .into_iter()
7528                .map(|key| (key.to_string(), dir.path().display().to_string()))
7529                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7530                .collect(),
7531            reserved: false,
7532            reserved_prefixes: Vec::new(),
7533            protocol: ModuleProtocol::Subc,
7534            overlap: Default::default(),
7535            carry_containment: false,
7536        };
7537        let live = super::terminal_history_tests::fake_aft_stub_path();
7538        let missing = dir.path().join("definitely-missing-module");
7539
7540        let handle = SupervisorHandle::new();
7541        let mut supervisor =
7542            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7543                .with_handle(handle.clone());
7544        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
7545        let hook_handle = handle.clone();
7546        let hook_observed = Arc::clone(&observed);
7547        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7548            // Exactly what the control plane computes for a scoped route.open
7549            // naming this module as the owner of a scope it has not synced.
7550            let configured = hook_handle.is_configured(module_id);
7551            let selector = ScopeSelector::new(
7552                Principal::Reserved {
7553                    module_id: module_id.to_string(),
7554                },
7555                "s".to_string(),
7556                Some(1),
7557            );
7558            let carrier = Principal::Reserved {
7559                module_id: "carrier".to_string(),
7560            };
7561            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
7562                .admit(&carrier, module_id, &selector, configured)
7563            {
7564                Ok(_) => None,
7565                Err(refusal) => Some(refusal.code),
7566            };
7567            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
7568                module_id: module_id.to_string(),
7569                configured,
7570                on_roster: hook_handle.get(module_id).is_some(),
7571                admission_refusal,
7572            });
7573        })));
7574
7575        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
7576        let configured = supervisor
7577            .supervise_configured(stub("configured", live.clone()), true)
7578            .unwrap();
7579        let with_health = supervisor
7580            .supervise_configured_with_health(
7581                stub("with-health", live.clone()),
7582                true,
7583                HealthConfig::default(),
7584                None,
7585                RestartPolicy::default(),
7586            )
7587            .unwrap();
7588        // The failed-spawn path still puts the module on the roster (as
7589        // failed), so it is configured throughout.
7590        let failed = supervisor
7591            .supervise_configured_with_health(
7592                stub("failed-spawn", missing.clone()),
7593                true,
7594                HealthConfig::default(),
7595                None,
7596                RestartPolicy::default(),
7597            )
7598            .unwrap();
7599        // A failed plain `spawn` puts nothing on the roster, so its mark is
7600        // taken back once the spawn has failed.
7601        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
7602
7603        let expected = [
7604            "plain",
7605            "configured",
7606            "with-health",
7607            "failed-spawn",
7608            "spawn-error",
7609        ]
7610        .into_iter()
7611        .map(|module_id| OwnerInSpawnWindow {
7612            module_id: module_id.to_string(),
7613            configured: true,
7614            on_roster: !matches!(module_id, "plain" | "spawn-error"),
7615            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
7616        })
7617        .collect::<Vec<_>>();
7618        assert_eq!(*observed.lock().unwrap(), expected);
7619
7620        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
7621            assert!(
7622                handle.get(module_id).is_some(),
7623                "{module_id} is on the roster"
7624            );
7625            assert!(
7626                handle.is_configured(module_id),
7627                "{module_id} stays configured"
7628            );
7629        }
7630        assert!(handle.get("spawn-error").is_none());
7631        assert!(
7632            !handle.is_configured("spawn-error"),
7633            "a plain spawn that failed must not leave its module marked configured"
7634        );
7635
7636        // Leaving the roster clears the mark with it.
7637        handle.retire("failed-spawn");
7638        assert!(!handle.is_configured("failed-spawn"));
7639
7640        for module in [plain, configured, with_health] {
7641            module.stop().await.unwrap();
7642        }
7643        drop(failed);
7644    }
7645
7646    /// A configured module reads as starting until the supervisor has admitted
7647    /// its first process (exec confirmed, identity durably recorded), even
7648    /// though that process is already running by then and can register over
7649    /// HELLO, which is what lists it in catalog.list. So a client that needs
7650    /// the supervisor state must wait for that state; seeing the module in
7651    /// the catalog does not mean supervisor.list already says running.
7652    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7653    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
7654        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
7655        let spec = ModuleSpec {
7656            module_id: "spawned-not-admitted".to_string(),
7657            program: super::terminal_history_tests::fake_aft_stub_path(),
7658            args: Vec::new(),
7659            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7660                .into_iter()
7661                .map(|key| (key.to_string(), dir.path().display().to_string()))
7662                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7663                .collect(),
7664            reserved: false,
7665            reserved_prefixes: Vec::new(),
7666            protocol: ModuleProtocol::Subc,
7667            overlap: Default::default(),
7668            carry_containment: false,
7669        };
7670        let handle = SupervisorHandle::new();
7671        let mut supervisor =
7672            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7673                .with_handle(handle.clone());
7674        let observed = Arc::new(Mutex::new(None));
7675        let hook_handle = handle.clone();
7676        let hook_observed = Arc::clone(&observed);
7677        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7678            let state = hook_handle
7679                .get(module_id)
7680                .expect("a configured module is on the roster before it is spawned")
7681                .state()
7682                .unwrap();
7683            *hook_observed.lock().unwrap() = Some(state);
7684        })));
7685
7686        let module = supervisor.supervise_configured(spec, true).unwrap();
7687        assert_eq!(
7688            *observed.lock().unwrap(),
7689            Some(ModuleState::Starting),
7690            "between the spawn and its admission the supervisor still reports starting"
7691        );
7692        assert_eq!(module.state().unwrap(), ModuleState::Running);
7693        module.stop().await.unwrap();
7694    }
7695
7696    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
7697        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
7698        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
7699            snapshot.process_alive = true;
7700            snapshot.pid = Some(41);
7701            snapshot.spawned_at_ms = Some(42);
7702            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
7703            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
7704            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
7705                device: 43,
7706                inode: 44,
7707                #[cfg(windows)]
7708                windows_image: None,
7709            });
7710        })
7711        .unwrap();
7712        snapshot
7713    }
7714
7715    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
7716        let snapshot = lock_snapshot(snapshot).unwrap();
7717        assert!(!snapshot.process_alive);
7718        assert_eq!(snapshot.pid, None);
7719        assert_eq!(snapshot.spawned_at_ms, None);
7720        assert_eq!(snapshot.spawned_from, None);
7721        assert_eq!(snapshot.launch_spec_digest, None);
7722        assert_eq!(snapshot.spawned_file_identity, None);
7723    }
7724
7725    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7726    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
7727        let supervisor =
7728            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7729        let mut runtime = supervisor.runtime_config();
7730        runtime.test_seed_stale_facts_before_enable_spawn = true;
7731        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
7732        let mut child = None;
7733        let spec = ModuleSpec {
7734            module_id: "failed-enable-clears-facts".to_string(),
7735            program: PathBuf::from("/definitely/missing/failed-enable-module"),
7736            args: Vec::new(),
7737            env: Vec::new(),
7738            reserved: false,
7739            reserved_prefixes: Vec::new(),
7740            protocol: ModuleProtocol::Subc,
7741            overlap: Default::default(),
7742            carry_containment: false,
7743        };
7744
7745        let result = set_child_enabled(
7746            &spec,
7747            &runtime,
7748            &supervisor.registry,
7749            &supervisor.process_liveness,
7750            &snapshot,
7751            &mut child,
7752            true,
7753            &runtime.failure_context(false),
7754        )
7755        .await;
7756
7757        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
7758        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7759        assert_snapshot_process_facts_cleared(&snapshot);
7760    }
7761
7762    #[tokio::test]
7763    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
7764        let supervisor =
7765            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7766        let runtime = supervisor.runtime_config();
7767        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
7768            ModuleState::Restarting,
7769            true,
7770        )));
7771        let spec = ModuleSpec {
7772            module_id: "start-stranded-restarting".to_string(),
7773            program: super::terminal_history_tests::fake_aft_stub_path(),
7774            args: Vec::new(),
7775            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
7776            reserved: false,
7777            reserved_prefixes: Vec::new(),
7778            protocol: ModuleProtocol::None,
7779            overlap: Default::default(),
7780            carry_containment: false,
7781        };
7782        let mut child = None;
7783        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
7784        assert!(!super::set_child_enabled(
7785            &spec,
7786            &runtime,
7787            &Registry::default(),
7788            &supervisor.process_liveness,
7789            &snapshot,
7790            &mut child,
7791            true,
7792            &runtime.failure_context(false),
7793        )
7794        .await
7795        .unwrap());
7796        assert!(child.is_none());
7797        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
7798        assert!(super::set_child_enabled(
7799            &spec,
7800            &runtime,
7801            &Registry::default(),
7802            &supervisor.process_liveness,
7803            &snapshot,
7804            &mut child,
7805            true,
7806            &runtime.failure_context(false),
7807        )
7808        .await
7809        .unwrap());
7810        assert_eq!(
7811            lock_snapshot(&snapshot).unwrap().state,
7812            ModuleState::Running
7813        );
7814        let mut child = child.unwrap();
7815        child.start_kill().unwrap();
7816        child.wait().await.unwrap();
7817    }
7818
7819    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7820    async fn failed_reload_spawn_clears_current_process_facts() {
7821        let supervisor =
7822            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7823        let mut runtime = supervisor.runtime_config();
7824        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
7825        let snapshot = stale_process_snapshot(ModuleState::Running, true);
7826        let mut child = None;
7827        let spec = ModuleSpec {
7828            module_id: "failed-reload-clears-facts".to_string(),
7829            program: PathBuf::from("/unused/failed-reload-module"),
7830            args: Vec::new(),
7831            env: Vec::new(),
7832            reserved: false,
7833            reserved_prefixes: Vec::new(),
7834            protocol: ModuleProtocol::Subc,
7835            overlap: Default::default(),
7836            carry_containment: false,
7837        };
7838
7839        let result = handle_reload_spawn_failure(
7840            &spec,
7841            &runtime,
7842            &supervisor.process_liveness,
7843            &snapshot,
7844            &mut child,
7845            "forced reload spawn failure".to_string(),
7846        )
7847        .await;
7848
7849        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
7850        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7851        assert_snapshot_process_facts_cleared(&snapshot);
7852    }
7853
7854    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7855    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
7856        let supervisor =
7857            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7858        let snapshot = stale_process_snapshot(ModuleState::Running, true);
7859        let module = supervisor.supervised_module(
7860            ModuleSpec {
7861                module_id: "drop-clears-facts".to_string(),
7862                program: PathBuf::from("/unused/drop-module"),
7863                args: Vec::new(),
7864                env: Vec::new(),
7865                reserved: false,
7866                reserved_prefixes: Vec::new(),
7867                protocol: ModuleProtocol::Subc,
7868                overlap: Default::default(),
7869                carry_containment: false,
7870            },
7871            supervisor.runtime_config(),
7872            Arc::clone(&snapshot),
7873            None,
7874        );
7875        assert!(!module
7876            .inner
7877            .monitor
7878            .lock()
7879            .unwrap()
7880            .as_ref()
7881            .unwrap()
7882            .is_finished());
7883
7884        drop(module);
7885
7886        assert_eq!(
7887            lock_snapshot(&snapshot).unwrap().state,
7888            ModuleState::Stopped
7889        );
7890        assert_snapshot_process_facts_cleared(&snapshot);
7891    }
7892
7893    #[cfg(unix)]
7894    #[tokio::test]
7895    async fn rescan_preserves_running_protocol_until_respawn() {
7896        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
7897        let initial = ModuleSpec {
7898            module_id: "rescan-protocol".into(),
7899            program: PathBuf::from("/bin/sleep"),
7900            args: vec!["60".into()],
7901            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7902                .into_iter()
7903                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
7904                .collect(),
7905            reserved: false,
7906            reserved_prefixes: vec![],
7907            protocol: ModuleProtocol::None,
7908            overlap: Default::default(),
7909            carry_containment: false,
7910        };
7911        let supervisor =
7912            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7913        let module = supervisor.spawn(initial.clone()).unwrap();
7914        assert!(module.status().unwrap().live);
7915        let mut next = initial;
7916        next.protocol = ModuleProtocol::Subc;
7917        module
7918            .update_configuration(next.clone(), HealthConfig::default(), None)
7919            .await
7920            .unwrap();
7921        assert!(
7922            module.status().unwrap().live,
7923            "rescan must not require HELLO from the old non-wire process"
7924        );
7925        let runtime = supervisor.runtime_config();
7926        let action = on_child_exit(
7927            &next,
7928            RestartPolicy::default(),
7929            &supervisor.registry,
7930            &module.inner.snapshot,
7931            &runtime.terminal_ring,
7932            &runtime.spawn_events,
7933            &runtime.child_roster,
7934            ExitReport {
7935                kind: ExitKind::Clean,
7936                code: Some(0),
7937                signal: None,
7938                at_ms: unix_ms_now(),
7939            },
7940            true,
7941        )
7942        .await;
7943        assert!(
7944            matches!(action, NextAction::Restart { .. }),
7945            "the old non-wire process's clean exit must restart"
7946        );
7947        module.drain().await.unwrap();
7948    }
7949
7950    #[cfg(unix)]
7951    #[tokio::test]
7952    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
7953        use std::os::unix::fs::PermissionsExt;
7954        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
7955        let script = dir.join("module.sh");
7956        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
7957        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
7958        let record_path = dir.join("live-children.json");
7959        let supervisor =
7960            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7961                .with_live_children_record(&record_path);
7962        for (program, args) in [
7963            (PathBuf::from("sleep"), vec!["60".into()]),
7964            (script, vec![]),
7965        ] {
7966            let spec = ModuleSpec {
7967                module_id: "image-identity".into(),
7968                program: program.clone(),
7969                args,
7970                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7971                    .into_iter()
7972                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
7973                    .collect(),
7974                reserved: false,
7975                reserved_prefixes: vec![],
7976                protocol: ModuleProtocol::None,
7977                overlap: Default::default(),
7978                carry_containment: false,
7979            };
7980            let module = supervisor.spawn(spec).unwrap();
7981            #[cfg(target_os = "macos")]
7982            {
7983                // SETEXEC confirmation is asynchronous; the orphan record must
7984                // identify the final image, never the intermediate trampoline.
7985                let deadline = Instant::now() + Duration::from_secs(5);
7986                while crate::live_children::read_record(&record_path)
7987                    .unwrap()
7988                    .iter()
7989                    .all(|entry| entry.executable.is_none())
7990                {
7991                    assert!(Instant::now() < deadline, "module image was not confirmed");
7992                    tokio::time::sleep(Duration::from_millis(5)).await;
7993                }
7994            }
7995            let entry = crate::live_children::read_record(&record_path)
7996                .unwrap()
7997                .pop()
7998                .unwrap();
7999            let observed = subc_os::Process::open(entry.pid)
8000                .unwrap()
8001                .unwrap()
8002                .observe()
8003                .unwrap();
8004            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
8005            module.drain().await.unwrap();
8006            assert_eq!(
8007                verdict,
8008                crate::live_children::IdentityVerdict::Matches,
8009                "program {program:?}: recorded {entry:?}, observed {observed:?}"
8010            );
8011        }
8012    }
8013
8014    #[cfg(unix)]
8015    fn http_fixture(
8016        dir: &std::path::Path,
8017        url: &str,
8018        threshold: u32,
8019    ) -> crate::daemon_config::ConfiguredModule {
8020        let path = dir.join("subc.jsonc");
8021        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
8022            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
8023            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
8024            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
8025            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
8026        }}}).to_string()).unwrap();
8027        crate::daemon_config::load(&path)
8028            .unwrap()
8029            .unwrap()
8030            .modules
8031            .pop()
8032            .unwrap()
8033    }
8034
8035    #[cfg(unix)]
8036    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
8037        timeout(Duration::from_secs(5), async {
8038            loop {
8039                if module.status().unwrap().health.status == status {
8040                    break;
8041                }
8042                sleep(Duration::from_millis(5)).await;
8043            }
8044        })
8045        .await
8046        .unwrap_or_else(|_| {
8047            panic!(
8048                "expected {status:?}, got {:?}",
8049                module.status().unwrap().health
8050            )
8051        });
8052    }
8053
8054    #[cfg(unix)]
8055    #[tokio::test]
8056    async fn http_health_status_flips_ok_failing_ok() {
8057        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8058        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
8059        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8060        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
8061        let serving_status = status.clone();
8062        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8063        let server = tokio::spawn(async move {
8064            loop {
8065                let (mut stream, _) = listener.accept().await.unwrap();
8066                let mut request = [0u8; 2048];
8067                let count = stream.read(&mut request).await.unwrap();
8068                assert!(count > 0, "a probe must send an HTTP request");
8069                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
8070                let body = if code == 200 {
8071                    "ready"
8072                } else {
8073                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
8074                };
8075                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
8076                let _ = stream.write_all(response.as_bytes()).await;
8077            }
8078        });
8079        let configured = http_fixture(&dir, &url, 1000);
8080        let module =
8081            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8082                .supervise_configured_with_health(
8083                    configured.module_spec(),
8084                    true,
8085                    configured.health,
8086                    configured.drain_timeout_ms,
8087                    configured.restart,
8088                )
8089                .unwrap();
8090        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8091        status.store(503, std::sync::atomic::Ordering::SeqCst);
8092        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
8093        assert!(module
8094            .status()
8095            .unwrap()
8096            .health
8097            .detail
8098            .unwrap()
8099            .contains("scratch failure"));
8100        status.store(200, std::sync::atomic::Ordering::SeqCst);
8101        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8102        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
8103        let before = module.status().unwrap();
8104        let (spec, mut health) = module.configuration().unwrap();
8105        health.http = None;
8106        module
8107            .update_configuration(spec.clone(), health.clone(), Some(10))
8108            .await
8109            .unwrap();
8110        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
8111        health.http = Some(url);
8112        module
8113            .update_configuration(spec, health, Some(10))
8114            .await
8115            .unwrap();
8116        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8117        assert_eq!(
8118            module.status().unwrap().pid,
8119            before.pid,
8120            "changing a probe must apply live, not restart its process"
8121        );
8122        let (spec, mut health) = module.configuration().unwrap();
8123        health.failure_threshold = 2;
8124        module
8125            .update_configuration(spec, health, Some(10))
8126            .await
8127            .unwrap();
8128        status.store(503, std::sync::atomic::Ordering::SeqCst);
8129        timeout(Duration::from_secs(5), async {
8130            while module.status().unwrap().spawn_generation == before.spawn_generation {
8131                sleep(Duration::from_millis(5)).await;
8132            }
8133        })
8134        .await
8135        .expect("sustained HTTP 503 responses must trigger the health restart policy");
8136        module.drain().await.unwrap();
8137        server.abort();
8138    }
8139
8140    #[cfg(unix)]
8141    #[tokio::test]
8142    async fn http_health_replacement_recovers_after_one_restart() {
8143        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8144        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
8145        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8146        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8147        let mut configured = http_fixture(&dir, &url, 2);
8148        configured.restart.max_restarts = 2;
8149        let module =
8150            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8151                .supervise_configured_with_health(
8152                    configured.module_spec(),
8153                    true,
8154                    configured.health,
8155                    configured.drain_timeout_ms,
8156                    configured.restart,
8157                )
8158                .unwrap();
8159        let first_generation = module.status().unwrap().spawn_generation;
8160        let observed = module.clone();
8161        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
8162        let server = tokio::spawn(async move {
8163            let mut generation = first_generation;
8164            let mut probes = 0;
8165            loop {
8166                let (mut stream, _) = listener.accept().await.unwrap();
8167                let mut request = [0; 2048];
8168                assert!(stream.read(&mut request).await.unwrap() > 0);
8169                let current = observed.status().unwrap().spawn_generation;
8170                if current != generation {
8171                    generation = current;
8172                    probes = 0;
8173                }
8174                probes += 1;
8175                // The original process never recovers. Each replacement has one
8176                // transient miss, then answers successfully for its remaining life.
8177                let healthy = generation != first_generation && probes > 1;
8178                let code = if healthy { 200 } else { 503 };
8179                let response = format!(
8180                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
8181                );
8182                let _ = stream.write_all(response.as_bytes()).await;
8183                if healthy {
8184                    healthy_tx.send(()).await.unwrap();
8185                }
8186            }
8187        });
8188        timeout(Duration::from_secs(5), async {
8189            let mut healthy_probes = 0;
8190            loop {
8191                if module.state().unwrap() == ModuleState::Failed {
8192                    break;
8193                }
8194                while healthy_rx.try_recv().is_ok() {
8195                    healthy_probes += 1;
8196                }
8197                if healthy_probes >= 3
8198                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
8199                {
8200                    break;
8201                }
8202                sleep(Duration::from_millis(5)).await;
8203            }
8204        })
8205        .await
8206        .expect("the replacement must settle healthy without exhausting its restart budget");
8207        let status = module.status().unwrap();
8208        module.drain().await.unwrap();
8209        server.abort();
8210        assert_eq!(status.state, ModuleState::Running);
8211        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
8212        assert_eq!(status.health.consecutive_failures, 0);
8213        assert_eq!(status.lifetime_restarts, 1);
8214        assert_eq!(status.restart_count, 1);
8215    }
8216
8217    #[cfg(unix)]
8218    #[tokio::test]
8219    async fn http_health_sustained_failure_exhausts_restart_budget() {
8220        use std::collections::BTreeMap;
8221        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8222        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
8223        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8224        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8225        let mut configured = http_fixture(&dir, &url, 2);
8226        configured.restart.max_restarts = 2;
8227        let module =
8228            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8229                .supervise_configured_with_health(
8230                    configured.module_spec(),
8231                    true,
8232                    configured.health,
8233                    configured.drain_timeout_ms,
8234                    configured.restart,
8235                )
8236                .unwrap();
8237        let observed = module.clone();
8238        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
8239        let served = probes.clone();
8240        let server = tokio::spawn(async move {
8241            loop {
8242                let (mut stream, _) = listener.accept().await.unwrap();
8243                let mut request = [0; 2048];
8244                assert!(stream.read(&mut request).await.unwrap() > 0);
8245                *served
8246                    .lock()
8247                    .unwrap()
8248                    .entry(observed.status().unwrap().spawn_generation)
8249                    .or_default() += 1;
8250                let _ = stream
8251                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
8252                    .await;
8253            }
8254        });
8255        timeout(Duration::from_secs(5), async {
8256            while module.state().unwrap() != ModuleState::Failed {
8257                sleep(Duration::from_millis(5)).await;
8258            }
8259        })
8260        .await
8261        .expect("sustained failures must still exhaust the health restart budget");
8262        let status = module.status().unwrap();
8263        server.abort();
8264        assert_eq!(status.restart_count, 2);
8265        assert_eq!(status.lifetime_restarts, 2);
8266        assert!(!status.process_alive);
8267        assert_eq!(
8268            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
8269            vec![2, 2, 2]
8270        );
8271    }
8272
8273    #[cfg(unix)]
8274    #[tokio::test]
8275    async fn spawn_clears_the_previous_process_health() {
8276        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
8277        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
8278        configured.health.cadence = Duration::from_secs(3600);
8279        let module =
8280            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8281                .supervise_configured_with_health(
8282                    configured.module_spec(),
8283                    false,
8284                    configured.health,
8285                    configured.drain_timeout_ms,
8286                    configured.restart,
8287                )
8288                .unwrap();
8289        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
8290            state.health.status = SupervisorHealthStatus::Unresponsive;
8291            state.health.consecutive_failures = 3;
8292            state.health.detail = Some("old process refused connections".into());
8293            state.health.metrics = Some(serde_json::json!({"old": true}));
8294            state.health.last_probe_ms = Some(123);
8295            state.health.last_action = Some("restart".into());
8296            state.health.last_action_ms = Some(124);
8297        })
8298        .unwrap();
8299        module.set_enabled(true).await.unwrap();
8300        let health = module.status().unwrap().health;
8301        module.drain().await.unwrap();
8302        assert_eq!(health.consecutive_failures, 0);
8303        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
8304        assert_eq!(health.detail, None);
8305        assert_eq!(health.metrics, None);
8306        assert_eq!(health.last_probe_ms, None);
8307        assert_eq!(health.last_action.as_deref(), Some("restart"));
8308        assert_eq!(health.last_action_ms, Some(124));
8309    }
8310
8311    #[cfg(unix)]
8312    #[tokio::test]
8313    async fn http_health_no_listener_restarts_after_consecutive_failures() {
8314        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
8315        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8316        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
8317        drop(unused);
8318        let configured = http_fixture(&dir, &url, 2);
8319        let module =
8320            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8321                .supervise_configured_with_health(
8322                    configured.module_spec(),
8323                    true,
8324                    configured.health,
8325                    configured.drain_timeout_ms,
8326                    configured.restart,
8327                )
8328                .unwrap();
8329        let before = module.status().unwrap().spawn_generation;
8330        timeout(Duration::from_secs(5), async {
8331            loop {
8332                let status = module.status().unwrap();
8333                if status.spawn_generation > before {
8334                    assert!(status.lifetime_restarts > 0);
8335                    break;
8336                }
8337                sleep(Duration::from_millis(5)).await;
8338            }
8339        })
8340        .await
8341        .expect("sustained HTTP refusal must trigger the health restart policy");
8342        module.drain().await.unwrap();
8343    }
8344
8345    #[cfg(unix)]
8346    #[tokio::test]
8347    async fn http_health_timeout_honours_deadline() {
8348        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8349        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8350        let server = tokio::spawn(async move {
8351            let _held = listener.accept().await.unwrap();
8352            std::future::pending::<()>().await;
8353        });
8354        let error = timeout(
8355            Duration::from_secs(1),
8356            probe_http_health(&url, Duration::from_millis(10)),
8357        )
8358        .await
8359        .expect("the probe must enforce its own deadline")
8360        .unwrap_err();
8361        server.abort();
8362        assert!(error.to_string().contains("timed out"));
8363    }
8364
8365    #[cfg(unix)]
8366    #[tokio::test]
8367    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
8368        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8369        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
8370        let url = format!(
8371            "http://localhost:{}/healthz",
8372            listener.local_addr().unwrap().port()
8373        );
8374        let server = tokio::spawn(async move {
8375            let (mut stream, _) = listener.accept().await.unwrap();
8376            let mut request = [0u8; 2048];
8377            assert!(stream.read(&mut request).await.unwrap() > 0);
8378            stream
8379                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
8380                .await
8381                .unwrap();
8382        });
8383        // The deadline only bounds a hang. A probe that never tried the IPv6
8384        // address would be refused on 127.0.0.1 and fail at once, so a longer
8385        // deadline does not weaken the assertion; one second timed out under a
8386        // loaded parallel test run.
8387        assert_eq!(
8388            probe_http_health(&url, Duration::from_secs(10))
8389                .await
8390                .unwrap()
8391                .status,
8392            HealthStatus::Ok
8393        );
8394        server.await.unwrap();
8395    }
8396
8397    #[cfg(unix)]
8398    #[tokio::test]
8399    async fn http_health_timeout_keeps_partial_status_and_body() {
8400        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8401        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8402        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8403        let server = tokio::spawn(async move {
8404            let (mut stream, _) = listener.accept().await.unwrap();
8405            let mut request = [0u8; 2048];
8406            assert!(stream.read(&mut request).await.unwrap() > 0);
8407            stream
8408                .write_all(
8409                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
8410                )
8411                .await
8412                .unwrap();
8413            std::future::pending::<()>().await;
8414        });
8415        let error = probe_http_health(&url, Duration::from_secs(1))
8416            .await
8417            .unwrap_err()
8418            .to_string();
8419        server.abort();
8420        assert!(
8421            error.contains("timed out")
8422                && error.contains("503 Unavailable")
8423                && error.contains("partial diagnostic"),
8424            "{error}"
8425        );
8426    }
8427
8428    #[cfg(unix)]
8429    #[tokio::test]
8430    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
8431        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8432        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8433        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8434        let server = tokio::spawn(async move {
8435            let (mut stream, _) = listener.accept().await.unwrap();
8436            let mut request = [0u8; 2048];
8437            assert!(stream.read(&mut request).await.unwrap() > 0);
8438            let body = format!("{}not-in-diagnostic", "x".repeat(400));
8439            let response = format!(
8440                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
8441                body.len()
8442            );
8443            stream.write_all(response.as_bytes()).await.unwrap();
8444        });
8445        let error = probe_http_health(&url, Duration::from_secs(1))
8446            .await
8447            .unwrap_err()
8448            .to_string();
8449        server.await.unwrap();
8450        assert!(error.ends_with(&"x".repeat(200)), "{error}");
8451        assert_eq!(error.split(": ").last().unwrap().len(), 200);
8452        assert!(!error.contains("not-in-diagnostic"));
8453    }
8454
8455    #[cfg(unix)]
8456    #[tokio::test]
8457    async fn http_health_real_nats_server_monitoring() {
8458        if std::process::Command::new("nats-server")
8459            .arg("--version")
8460            .env("XDG_DATA_HOME", std::env::temp_dir())
8461            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
8462            .env("XDG_CONFIG_HOME", std::env::temp_dir())
8463            .output()
8464            .is_err()
8465        {
8466            eprintln!(
8467                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
8468            );
8469            return;
8470        }
8471        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
8472        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8473        let port = monitor.local_addr().unwrap().port();
8474        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8475        let client_port = client.local_addr().unwrap().port();
8476        let config = dir.join("server.conf");
8477        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
8478        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
8479        configured.program = PathBuf::from("nats-server");
8480        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
8481        drop(monitor);
8482        drop(client);
8483        let module =
8484            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8485                .supervise_configured_with_health(
8486                    configured.module_spec(),
8487                    true,
8488                    configured.health,
8489                    configured.drain_timeout_ms,
8490                    configured.restart,
8491                )
8492                .unwrap();
8493        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8494        module.drain().await.unwrap();
8495    }
8496
8497    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8498    async fn configuration_update_does_not_replace_captured_running_process_facts() {
8499        let supervisor =
8500            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8501        let snapshot = stale_process_snapshot(ModuleState::Running, true);
8502        let initial = ModuleSpec {
8503            module_id: "rescan-preserves-spawn-facts".to_string(),
8504            program: PathBuf::from("/spawned/module"),
8505            args: Vec::new(),
8506            env: Vec::new(),
8507            reserved: false,
8508            reserved_prefixes: Vec::new(),
8509            protocol: ModuleProtocol::Subc,
8510            overlap: Default::default(),
8511            carry_containment: false,
8512        };
8513        let module = supervisor.supervised_module(
8514            initial.clone(),
8515            supervisor.runtime_config(),
8516            snapshot,
8517            None,
8518        );
8519        let before = module.status().unwrap();
8520        let mut replacement = initial;
8521        replacement.program = PathBuf::from("/rescanned/replacement-module");
8522
8523        module
8524            .update_configuration(replacement, HealthConfig::default(), None)
8525            .await
8526            .unwrap();
8527
8528        let after = module.status().unwrap();
8529        assert_eq!(after.pid, before.pid);
8530        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
8531        assert_eq!(after.spawned_from, before.spawned_from);
8532        drop(module);
8533    }
8534}
8535
8536fn unix_ms_now() -> u64 {
8537    SystemTime::now()
8538        .duration_since(UNIX_EPOCH)
8539        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
8540        .unwrap_or(0)
8541}
8542
8543async fn supervise_loop(
8544    mut spec: ModuleSpec,
8545    mut runtime: SupervisorRuntimeConfig,
8546    registry: Arc<Registry>,
8547    process_liveness: Arc<SupervisorProcessLiveness>,
8548    snapshot: SharedSnapshot,
8549    mut child: Option<SupervisedChild>,
8550    mut commands: mpsc::Receiver<SupervisorCommand>,
8551) {
8552    let mut health_probe = HealthProbeRuntime::default();
8553    // Registry writes (including embedded callers) notify this module only.
8554    // Subscribe before the first refresh; watch retains changes that arrive
8555    // while commands or probes are running. No polling fallback is needed.
8556    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
8557        Ok(changes) => Some(changes),
8558        Err(err) => {
8559            // A poisoned registry cannot accept further writes, so it cannot
8560            // recover via a timer. Keep exit and command handling alive.
8561            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
8562            None
8563        }
8564    };
8565    // All restart backoffs run here, including health and operator requests.
8566    // While one is pending the loop serves commands, so disable or drain can
8567    // cancel the replacement without spawning a process just to stop it.
8568    let mut pending_respawn: Option<PendingRespawn> = None;
8569    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
8570    // before anything else so a stop that interrupted a swap runs at once.
8571    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
8572    let mut previous_instance_recheck = tokio::time::interval_at(
8573        Instant::now() + Duration::from_secs(5),
8574        Duration::from_secs(5),
8575    );
8576    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
8577    'supervision: loop {
8578        #[cfg(test)]
8579        {
8580            lock_snapshot(&snapshot).unwrap().actor_select = None;
8581        }
8582        #[cfg(target_os = "macos")]
8583        if let Some(active) = child.as_mut() {
8584            active.confirm_privacy_exec().await;
8585        }
8586        if let Some(scheduled) = runtime
8587            .scheduled_respawn
8588            .lock()
8589            .unwrap_or_else(|p| p.into_inner())
8590            .take()
8591        {
8592            runtime.recovery_is_daemon = scheduled.daemon_initiated;
8593            pending_respawn = Some(scheduled);
8594        }
8595        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
8596            pending_respawn = None;
8597            cancel_deferred_reload(
8598                &runtime,
8599                &spec.module_id,
8600                "respawn cancelled by a supervisor command",
8601            );
8602        }
8603        if child.is_none()
8604            && pending_respawn.is_none()
8605            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
8606        {
8607            cancel_deferred_reload(
8608                &runtime,
8609                &spec.module_id,
8610                "respawn cancelled before a replacement was spawned",
8611            );
8612            let mut stranded = false;
8613            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8614                state.respawn_pending = false;
8615                state.coalesced_restart_pending = false;
8616                if matches!(
8617                    state.state,
8618                    ModuleState::Restarting
8619                        | ModuleState::Starting
8620                        | ModuleState::Draining
8621                        | ModuleState::Unresponsive
8622                ) {
8623                    error!(module_id = %spec.module_id, state = ?state.state,
8624                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
8625                    stranded = true;
8626                }
8627            });
8628            if stranded {
8629                fail_snapshot(
8630                    &snapshot,
8631                    Some(&spec.module_id),
8632                    None,
8633                    &runtime.failure_context(runtime.recovery_is_daemon),
8634                );
8635            }
8636        }
8637        if child.is_none()
8638            && (runtime.child_roster.is_closed()
8639                || pending_respawn.is_none()
8640                || !lock_snapshot(&snapshot)
8641                    .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
8642        {
8643            containment::terminate(&runtime.child_roster.containment).await;
8644        }
8645        if let Some(command) = requeued.pop_front() {
8646            if !handle_supervisor_command(
8647                command,
8648                &mut spec,
8649                &mut runtime,
8650                &registry,
8651                &process_liveness,
8652                &snapshot,
8653                &mut child,
8654                &mut commands,
8655                &mut requeued,
8656            )
8657            .await
8658            {
8659                containment::terminate(&runtime.child_roster.containment).await;
8660                return;
8661            }
8662            if child.is_some() || !respawn_still_pending(&snapshot) {
8663                pending_respawn = None;
8664                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8665                    state.respawn_pending = false
8666                });
8667            }
8668            continue;
8669        }
8670        if child.is_some() {
8671            runtime.recovery_is_daemon = true;
8672            #[cfg(test)]
8673            {
8674                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
8675            }
8676            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
8677            let active = child.as_mut().expect("child checked above");
8678            if active.registered_since_spawn() {
8679                if let Some(delivery) = &runtime.config_delivery {
8680                    delivery.queue_registered(
8681                        &spec.module_id,
8682                        runtime.forwarding.as_ref(),
8683                        &registry,
8684                        active,
8685                    );
8686                }
8687            }
8688            if runtime.child_roster.is_closed() || !active.awaiting_registration() {
8689                active.registration_deadline = None;
8690            }
8691            let registration_deadline = active.registration_deadline;
8692            let no_checks = runtime.health.http.is_none()
8693                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
8694                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
8695            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
8696                state
8697                    .failed_retry
8698                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
8699            });
8700            let wake_at = match (health_probe.wake_at(), uptime_reset) {
8701                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
8702                (probe, None) => probe,
8703            };
8704            let wake_at = match (wake_at, registration_deadline) {
8705                (Some(wake), Some(register)) => Some(wake.min(register)),
8706                (wake, register) => wake.or(register),
8707            };
8708            #[cfg(test)]
8709            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
8710            let probe_sleep = supervisor_sleep(wake_at);
8711            tokio::pin!(probe_sleep);
8712            // A `protocol: "none"` child (a plain process that never registers
8713            // over the subc wire, such as nats-server) with no HTTP health
8714            // check has nothing to probe, so it waits only for its exit or a
8715            // supervisor command. A rescan that adds an HTTP check is a command.
8716            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
8717            #[cfg(test)]
8718            {
8719                let registration_pending = wire_child
8720                    && registration_changes
8721                        .as_ref()
8722                        .is_some_and(|changes| changes.has_changed().unwrap());
8723                let mut state = lock_snapshot(&snapshot).unwrap();
8724                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
8725                    generation: state.spawn_generation,
8726                    turn: state.actor_turns,
8727                    registered_connection: health_probe.registered_connection,
8728                    next_probe_at: health_probe.next_probe_at,
8729                    wake_after,
8730                });
8731            }
8732            let registration = active
8733                .awaiting_registration()
8734                .then(|| active.registration_spawn.clone())
8735                .flatten();
8736            #[cfg(all(test, target_os = "linux"))]
8737            let registration_before_exit = !runtime.child_roster.admission_event_for_test(
8738                &spec.module_id,
8739                "observe_registration_before_exit",
8740                Some(active.pid),
8741            ) && registration.is_some();
8742            #[cfg(not(all(test, target_os = "linux")))]
8743            let registration_before_exit = false;
8744            #[cfg(all(test, target_os = "linux"))]
8745            let registration_child_is_running = |child: &mut SupervisedChild| {
8746                // Model the interval after SIGKILL is accepted but before the
8747                // child becomes waitable, without delaying the actual kill.
8748                registration_before_exit || registration_child_is_running(child)
8749            };
8750            let registration_failed = async {
8751                match &registration {
8752                    Some(spawn) => spawn.wait_registration_failure().await,
8753                    None => std::future::pending().await,
8754                }
8755            };
8756            let active_child = child.as_mut().expect("child checked above");
8757            tokio::select! {
8758                _ = registration_failed => {
8759                    if !active_child.registered_since_spawn() && registration_child_is_running(active_child) {
8760                        active_child.registration_deadline = None;
8761                        active_child.spawn_failure = Some("registration_channel_closed".into());
8762                        if let Err(error) = stop_unregistered_child(active_child).await {
8763                            fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8764                                &runtime.spawn_events, format!("registration_channel_closed: {error}"),
8765                                &runtime.failure_context(true));
8766                            child = None;
8767                        }
8768                    }
8769                    continue;
8770                }
8771                wait_result = active_child.wait(), if !registration_before_exit => {
8772                    if let Ok(mut state) = lock_snapshot(&snapshot) {
8773                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
8774                    }
8775                    #[cfg(test)]
8776                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8777                    // Every arm below that gives up on the CHILD must keep the
8778                    // supervision task itself alive (child = None, loop
8779                    // continues into command-serving mode). Returning here
8780                    // closes the command channel, which makes the module
8781                    // permanently unrestartable in-band: a clean child exit
8782                    // of an enabled module once wedged the fleet this way
8783                    // ('supervisor command channel is closed') and required a
8784                    // full daemon restart to recover.
8785                    let exit_report = match wait_result {
8786                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
8787                        Err(err) => {
8788                            active_child.drain_stderr(&spec.module_id).await;
8789                            fail_after_wait_error(
8790                                &spec.module_id,
8791                                &snapshot,
8792                                &runtime.terminal_ring,
8793                                &runtime.spawn_events,
8794                                format!("failed to wait for supervised module: {err}"),
8795                            &runtime.failure_context(true),
8796                            );
8797                            untrack_if_registration_released(
8798                                &process_liveness,
8799                                &registry,
8800                                &spec.module_id,
8801                                &snapshot,
8802                            );
8803                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
8804                            child = None;
8805                            continue;
8806                        }
8807                    };
8808                    active_child.drain_stderr(&spec.module_id).await;
8809
8810                    let delivery=runtime.config_delivery.clone();
8811                    let exited_spawn=active_child.spawned_at;
8812                    let spawn=active_child.config_delivery.clone();
8813                    let refusal=active_child.spawn_failure.as_deref().filter(|detail|detail.starts_with("program_refused:") && !active_child.live_program_binding.as_ref().is_some_and(|binding|binding.is_refused())).zip(active_child.admission_base.clone()).map(|(detail,base)|(base,subc_config::raw_digest(detail.as_bytes())));
8814                    let module_id=spec.module_id.clone();
8815                    // Publication may own the store mutex for local durable I/O.
8816                    // Keep commands selectable until the child's exit is recorded;
8817                    // no respawn disposition is chosen before that observation.
8818                    let observed=async move {
8819                        if let Some(delivery)=delivery {
8820                            delivery.0.on_blocking(move |owner| {
8821                                if let Some(spawn)=spawn { owner.delivery_child_exited(&spawn)?; }
8822                                if let Some((base,identity))=refusal {owner.record_program_refusal(&module_id,&base,identity)?;}
8823                                Ok(())
8824                            }).await
8825                        } else {Ok(())}
8826                    };
8827                    tokio::pin!(observed);
8828                    loop {
8829                        tokio::select! {
8830                            result=&mut observed=>{
8831                                if let Err(error)=result {warn!(module_id=%spec.module_id,%error,"configuration exit observation failed");}
8832                                break;
8833                            }
8834                            command=commands.recv()=>{
8835                                let Some(command)=command else {containment::terminate(&runtime.child_roster.containment).await;return;};
8836                                let deferred=||SuperviseError::ConfigDeliveryDeferred {module_id:spec.module_id.clone()};
8837                                let command=match command {
8838                                    SupervisorCommand::Restart {reply,..}|SupervisorCommand::Reload {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8839                                    SupervisorCommand::Swap {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8840                                    command @ (SupervisorCommand::ConfigApplyRestart {..}|SupervisorCommand::ConfigPolicyProbe {..}|SupervisorCommand::ConfigDeliveryFailed {..})=>{requeued.push_back(command);continue;}
8841                                    command=>command,
8842                                };
8843                                if !handle_supervisor_command(command,&mut spec,&mut runtime,&registry,&process_liveness,&snapshot,&mut child,&mut commands,&mut requeued).await {
8844                                    containment::terminate(&runtime.child_roster.containment).await;return;
8845                                }
8846                                if child.as_ref().is_none_or(|current|current.spawned_at!=exited_spawn) {continue 'supervision;}
8847                            }
8848                        }
8849                    }
8850                    let active_child=child.as_mut().expect("exit observation retained its child");
8851                    // An unavailable image can race the program's own exit.
8852                    // Only the admission kill is budget-free; another exit must
8853                    // retain its ordinary crash/clean-exit accounting.
8854                    let admission_kill = admission_exit::is_admission_kill(active_child.admission_kill_sent, &exit_report);
8855                    let deferred_spawn=admission_kill && active_child.spawn_failure.as_deref().is_some_and(|detail|detail.starts_with("config_delivery_deferred:"));
8856                    let refused_program = active_child.live_program_binding.as_ref().is_some_and(|binding| binding.is_refused());
8857                    let exit_held = !deferred_spawn && (refused_program || runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.exit_is_held(&spec.module_id)));
8858                    let parked = exit_held && (!refused_program || active_child.admission_base.is_some())
8859                        && try_park_config_attempt(&spec, &runtime, &snapshot, &runtime.failure_context(true),
8860                            if refused_program { active_child.admission_base.as_ref() } else { None })
8861                            .unwrap_or_else(|error| { warn!(module_id=%spec.module_id, %error, "exit hold could not be recorded; using ordinary exit accounting"); false });
8862                    let next = if deferred_spawn {
8863                        record_terminal_with_detail(&spec.module_id,&runtime.terminal_ring,&runtime.spawn_events,&exit_report,TerminalDisposition::Restarting,active_child.spawn_failure.clone());
8864                        let _=update_snapshot(&snapshot,Some(&spec.module_id),|state| {
8865                            state.state=ModuleState::Restarting;
8866                            clear_current_process_facts(state);
8867                            state.last_exit=Some(exit_report.clone());
8868                        });
8869                        NextAction::Restart {schedule:None}
8870                    } else if parked {
8871                        record_terminal_with_detail(&spec.module_id, &runtime.terminal_ring, &runtime.spawn_events, &exit_report, TerminalDisposition::Stopped, active_child.spawn_failure.clone().or_else(|| Some("config_gated".into())));
8872                        NextAction::Stop { registration_released: registry.get_module(&spec.module_id).ok().flatten().is_none() }
8873                    } else { on_child_exit(
8874                        &spec,
8875                        runtime.restart_policy,
8876                        &registry,
8877                        &snapshot,
8878                        &runtime.terminal_ring,
8879                        &runtime.spawn_events,
8880                        &runtime.child_roster,
8881                        exit_report,
8882                        true,
8883                    ).await };
8884                    runtime.start_requester = None;
8885                    runtime.pending_activity_restart = false;
8886                    // The exit is recorded, so a daemon shutdown may stop
8887                    // waiting for this child (see `SupervisedChild::wait`).
8888                    let carry = matches!(next, NextAction::Restart { .. })
8889                        && active_child.spawn_failure.is_none()
8890                        && !runtime.child_roster.is_closed();
8891                    if let Err(error) = active_child.finish_exit(carry).await {
8892                        containment::terminate(&runtime.child_roster.containment).await;
8893                        fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8894                            &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
8895                            &runtime.failure_context(true));
8896                        child = None;
8897                        continue;
8898                    }
8899                    match next {
8900                        NextAction::Stop { registration_released } => {
8901                            if registration_released {
8902                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
8903                            }
8904                            child = None;
8905                        }
8906                        NextAction::Restart { schedule } => {
8907                            let delay = if deferred_spawn {CONFIG_ADMISSION_RETRY_DELAY} else {schedule.map_or(
8908                                runtime.restart_policy.delay_for_restart(0),
8909                                |schedule| schedule.delay,
8910                            )};
8911                            if let Some(schedule) = schedule {
8912                                log_crash_respawn(&spec.module_id, schedule);
8913                            }
8914                            // The exited child is fully recorded at this point,
8915                            // so release it and count the backoff down in the
8916                            // command-serving branch below rather than sleeping
8917                            // here: commands cannot be received from inside this
8918                            // select arm, and an operator disable or drain that
8919                            // arrives during the backoff must cancel the pending
8920                            // respawn instead of waiting for it to spawn first.
8921                            child = None;
8922                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true, approval: None });
8923                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
8924                        }
8925                    }
8926                }
8927                command = commands.recv() => {
8928                    #[cfg(test)]
8929                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8930                    let Some(command) = command else {
8931                        containment::terminate(&runtime.child_roster.containment).await;
8932                        return;
8933                    };
8934                    if !handle_supervisor_command(
8935                        command,
8936                        &mut spec,
8937                        &mut runtime,
8938                        &registry,
8939                        &process_liveness,
8940                        &snapshot,
8941                        &mut child,
8942                        &mut commands,
8943                        &mut requeued,
8944                    ).await {
8945                        containment::terminate(&runtime.child_roster.containment).await;
8946                        return;
8947                    }
8948                }
8949                _ = async {
8950                    match registration_changes.as_mut() {
8951                        Some(changes) => { let _ = changes.changed().await; }
8952                        None => std::future::pending().await,
8953                    }
8954                }, if wire_child => {
8955                    #[cfg(test)]
8956                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8957                }
8958                _ = &mut probe_sleep => {
8959                    #[cfg(test)]
8960                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8961                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
8962                        // Recheck after the wake: HELLO, exit or shutdown may
8963                        // have become ready in the same executor tick.
8964                        active_child.registration_deadline = None;
8965                        if !runtime.child_roster.is_closed()
8966                            && !active_child.registered_since_spawn()
8967                            && registration_child_is_running(active_child)
8968                        {
8969                            warn!(module_id = %spec.module_id, pid = active_child.pid,
8970                                wait_secs = runtime.register_timeout.as_secs(),
8971                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
8972                            active_child.spawn_failure = Some("registration_timeout".into());
8973                            if let Err(error) = stop_unregistered_child(active_child).await {
8974                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
8975                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8976                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
8977                                    &runtime.failure_context(true));
8978                                child = None;
8979                            }
8980                            // The ordinary exit handler records the child's exit,
8981                            // spends crash budget, and schedules recovery.
8982                            continue;
8983                        }
8984                    }
8985                    if health_probe.due() {
8986                        run_health_probe_cycle(
8987                            &spec,
8988                            &runtime,
8989                            &registry,
8990                            &process_liveness,
8991                            &snapshot,
8992                            &mut child,
8993                        ).await;
8994                        if child.is_none() {
8995                            runtime.start_requester = None;
8996                            runtime.pending_activity_restart = false;
8997                        }
8998                        if child.is_some() {
8999                            health_probe.schedule_next(&spec, runtime.health.cadence);
9000                        }
9001                    }
9002                }
9003            }
9004        } else if let Some(pending) = pending_respawn.clone() {
9005            tokio::select! {
9006                _ = runtime.child_roster.closed() => {
9007                    pending_respawn = None;
9008                    let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9009                        state.state = ModuleState::Stopped;
9010                        state.respawn_pending = false;
9011                    });
9012                    containment::terminate(&runtime.child_roster.containment).await;
9013                }
9014                _ = async {
9015                    sleep_until(pending.deadline).await;
9016                    if let Some(delivery) = &runtime.config_delivery {
9017                        delivery.unfenced(&spec.module_id).await;
9018                    }
9019                } => {
9020                    pending_respawn = None;
9021                    let failure = pending.failure_context(&runtime);
9022                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9023                    // A command handled below while the backoff elapsed may
9024                    // have stopped the module; never respawn past an operator's
9025                    // disable or drain.
9026                    if !respawn_still_pending(&snapshot) {
9027                        continue;
9028                    }
9029                    // The daemon began shutting down during the backoff: the
9030                    // spawn would be refused anyway, and refusing it here
9031                    // leaves the module stopped instead of reporting a
9032                    // failed restart.
9033                    if runtime.child_roster.is_closed() {
9034                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9035                            state.state = ModuleState::Stopped;
9036                        });
9037                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
9038                        continue;
9039                    }
9040                    if let Err(err) = release_dead_registration(
9041                        &registry,
9042                        runtime.forwarding.as_deref(),
9043                        &snapshot,
9044                        &spec.module_id,
9045                        &failure,
9046                    ).await {
9047                        record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9048                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
9049                        continue;
9050                    }
9051
9052                    if matches!(pending.kind, RespawnKind::Reload) {
9053                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
9054                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
9055                        if matches!(result, Err(SuperviseError::ConfigDeliveryDeferred { .. })) {
9056                            *runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()) = reply;
9057                            continue;
9058                        }
9059                        if result.is_ok() && runtime.pending_activity_restart {
9060                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9061                        }
9062                        runtime.pending_activity_restart = false;
9063                        if let Some(reply) = reply { let _ = reply.send(result); }
9064                        continue;
9065                    }
9066                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9067                    match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9068                        Ok(next_child) => {
9069                            child = Some(next_child);
9070                            if runtime.pending_activity_restart {
9071                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9072                                runtime.pending_activity_restart = false;
9073                            }
9074                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
9075                        }
9076                        Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9077                            pending_respawn = Some(PendingRespawn { deadline:Instant::now()+CONFIG_ADMISSION_RETRY_DELAY, ..pending });
9078                            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
9079                        }
9080                        Err(SuperviseError::ConfigHeld { .. }) => {
9081                            let _ = hold_config_spawn(&spec, &runtime, &snapshot, &failure);
9082                            runtime.pending_activity_restart = false;
9083                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9084                        }
9085                        Err(err) => {
9086                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &failure);
9087                            record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9088                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9089                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
9090                        }
9091                    }
9092                }
9093                command = commands.recv() => {
9094                    let Some(command) = command else {
9095                        containment::terminate(&runtime.child_roster.containment).await;
9096                        return;
9097                    };
9098                    if !handle_supervisor_command(
9099                        command,
9100                        &mut spec,
9101                        &mut runtime,
9102                        &registry,
9103                        &process_liveness,
9104                        &snapshot,
9105                        &mut child,
9106                        &mut commands,
9107                        &mut requeued,
9108                    ).await {
9109                        containment::terminate(&runtime.child_roster.containment).await;
9110                        return;
9111                    }
9112                    // Reconcile the pending respawn with what the command did:
9113                    // a start may already have spawned a fresh child,
9114                    // while a disable or drain moved the snapshot out of the
9115                    // state the respawn was counting down from.
9116                    if child.is_some() || !respawn_still_pending(&snapshot) {
9117                        pending_respawn = None;
9118                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9119                    }
9120                }
9121            }
9122        } else {
9123            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
9124                (state.enabled && state.state == ModuleState::Failed)
9125                    .then_some(state.failed_retry.pending)
9126                    .flatten()
9127            });
9128            let command = if lock_snapshot(&snapshot)
9129                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
9130            {
9131                tokio::select! {
9132                    command = commands.recv() => command,
9133                    _ = previous_instance_recheck.tick() => {
9134                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
9135                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
9136                            if enabled && !runtime.child_roster.is_closed() {
9137                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9138                                match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9139                                    Ok(next_child) => child = Some(next_child),
9140                                    Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9141                                        let _ = defer_config_spawn(&spec, &runtime, &snapshot, RespawnKind::Spawn, &runtime.failure_context(true));
9142                                    }
9143                                    Err(SuperviseError::ConfigHeld { .. }) => {
9144                                        let _ = hold_config_spawn(&spec, &runtime, &snapshot, &runtime.failure_context(true));
9145                                    }
9146                                    Err(error) => {
9147                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
9148                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
9149                                    }
9150                                }
9151                            } else {
9152                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9153                                    state.waiting_for_previous_pid = None;
9154                                    state.waiting_for_previous_detail = None;
9155                                });
9156                            }
9157                        }
9158                        continue;
9159                    }
9160                }
9161            } else {
9162                tokio::select! {
9163                    biased;
9164                    command = commands.recv() => command,
9165                    _ = async {
9166                        match slow_retry {
9167                            Some(retry) => sleep_until(retry.deadline).await,
9168                            None => std::future::pending().await,
9169                        }
9170                    } => {
9171                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
9172                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9173                            state.failed_retry.pending = None;
9174                        });
9175                        if runtime.child_roster.is_closed() {
9176                            continue;
9177                        }
9178                        info!(module_id = %spec.module_id, attempt = retry.attempt,
9179                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
9180                        if let Err(error) = set_child_enabled(
9181                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
9182                            &runtime.failure_context(true),
9183                        ).await {
9184                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
9185                        }
9186                        continue;
9187                    }
9188                }
9189            };
9190            let Some(command) = command else {
9191                containment::terminate(&runtime.child_roster.containment).await;
9192                return;
9193            };
9194            if !handle_supervisor_command(
9195                command,
9196                &mut spec,
9197                &mut runtime,
9198                &registry,
9199                &process_liveness,
9200                &snapshot,
9201                &mut child,
9202                &mut commands,
9203                &mut requeued,
9204            )
9205            .await
9206            {
9207                containment::terminate(&runtime.child_roster.containment).await;
9208                return;
9209            }
9210        }
9211    }
9212}
9213
9214// Preserve the deadline until the future is polled. Starting a relative sleep
9215// after the executor yields would restart part of the countdown.
9216async fn supervisor_sleep(wake_at: Option<Instant>) {
9217    match wake_at {
9218        Some(deadline) => sleep_until(deadline).await,
9219        None => std::future::pending().await,
9220    }
9221}
9222
9223fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
9224    info!(
9225        module_id,
9226        restart_in_window = schedule.restart_in_window,
9227        delay_ms = schedule.delay.as_millis() as u64,
9228        "respawning after crash"
9229    );
9230}
9231
9232/// Whether the respawn a backoff was counting down to is still wanted. A
9233/// disable or drain handled while the backoff elapsed moves the snapshot out
9234/// of `Restarting`, and the operator's stop must win over the pending respawn,
9235/// so every sleep-then-spawn path re-validates against the live snapshot
9236/// instead of assuming the state it left behind still holds.
9237fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
9238    matches!(
9239        lock_snapshot(snapshot),
9240        Ok(state) if state.enabled && state.state == ModuleState::Restarting
9241    )
9242}
9243
9244enum NextAction {
9245    Stop {
9246        registration_released: bool,
9247    },
9248    Restart {
9249        schedule: Option<CrashRestartSchedule>,
9250    },
9251}
9252
9253#[allow(clippy::too_many_arguments)]
9254async fn handle_supervisor_command(
9255    command: SupervisorCommand,
9256    spec: &mut ModuleSpec,
9257    runtime: &mut SupervisorRuntimeConfig,
9258    registry: &Arc<Registry>,
9259    process_liveness: &SupervisorProcessLiveness,
9260    snapshot: &SharedSnapshot,
9261    child: &mut Option<SupervisedChild>,
9262    commands: &mut mpsc::Receiver<SupervisorCommand>,
9263    requeued: &mut VecDeque<SupervisorCommand>,
9264) -> bool {
9265    if let SupervisorCommand::SetEnabled {
9266        enabled: true,
9267        reset_budget: false,
9268        reply,
9269        ..
9270    } = command
9271    {
9272        let parked = lock_snapshot(snapshot).ok().and_then(|state| {
9273            (state.config_hold_parked
9274                && state.enabled
9275                && !state.process_alive
9276                && child.is_none()
9277                && state.state == ModuleState::Stopped)
9278                .then(|| {
9279                    (
9280                        state.config_hold_daemon_initiated,
9281                        state.config_hold_approval.clone(),
9282                    )
9283                })
9284        });
9285        // Clearing a configuration-delivery block must not restart a child that
9286        // exited on its own or exhausted its crash budget. Resume only an actor
9287        // stopped by a configuration hold; otherwise leave its slow retry alone.
9288        let result = if let Some((daemon_initiated, approval)) = parked {
9289            runtime.recovery_is_daemon = daemon_initiated;
9290            let failure = FailureContext {
9291                approval,
9292                ..runtime.failure_context(daemon_initiated)
9293            };
9294            let result = set_child_enabled_with_budget_policy(
9295                spec,
9296                runtime,
9297                registry,
9298                process_liveness,
9299                snapshot,
9300                child,
9301                true,
9302                &failure,
9303                false,
9304            )
9305            .await;
9306            if let Err(error) = &result {
9307                record_approved_attempt_failure(spec, snapshot, &failure, error);
9308            }
9309            result
9310        } else {
9311            Ok(false)
9312        };
9313        let _ = reply.send(result);
9314        return true;
9315    }
9316    let blocked = runtime
9317        .config_delivery
9318        .as_ref()
9319        .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
9320        .filter(|error| error.code != "config_delivery_deferred")
9321        .map(|error| {
9322            config_delivery::gate_hold_error(spec, error, runtime.config_delivery.as_ref())
9323        })
9324        .or_else(|| recovery_block_error(spec, &runtime.child_roster))
9325        .or_else(|| {
9326            runtime
9327                .child_roster
9328                .previous_instance(&spec.module_id)
9329                .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
9330        });
9331    let command = if let Some(error) = blocked {
9332        match command {
9333            SupervisorCommand::Restart { reply, .. } | SupervisorCommand::Reload { reply, .. } => {
9334                let _ = reply.send(Err(error));
9335                return true;
9336            }
9337            SupervisorCommand::Swap { reply, .. } => {
9338                let _ = reply.send(Err(error));
9339                return true;
9340            }
9341            SupervisorCommand::ConfigApplyRestart { .. } => return true,
9342            SupervisorCommand::SetEnabled {
9343                enabled: true,
9344                requester,
9345                reply,
9346                ..
9347            } => {
9348                if matches!(error, SuperviseError::ConfigHeld { .. })
9349                    && lock_snapshot(snapshot).is_ok_and(|state| !state.enabled)
9350                    && runtime
9351                        .config_delivery
9352                        .as_ref()
9353                        .is_some_and(|delivery| delivery.0.can_stage_unstamp_enable())
9354                {
9355                    // Stage only the pending unstamp's desired state. A stamped
9356                    // unapproved actor stays refused; admission release starts
9357                    // an unstamped actor only after reconciliation completes.
9358                    let mut changed = false;
9359                    let result = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9360                        changed = !state.enabled;
9361                        state.enabled = true;
9362                        if child.is_none() && state.waiting_for_previous_pid.is_none() {
9363                            state.state = ModuleState::Starting;
9364                            state.failed_retry.cancel();
9365                        }
9366                    })
9367                    .and_then(|_| {
9368                        if child.is_none()
9369                            && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none()
9370                        {
9371                            hold_config_spawn(
9372                                spec,
9373                                runtime,
9374                                snapshot,
9375                                &runtime.failure_context(false),
9376                            )?;
9377                        }
9378                        Ok(changed)
9379                    });
9380                    if result.is_ok() && changed {
9381                        runtime.start_requester = requester;
9382                    }
9383                    let _ = reply.send(result);
9384                    return true;
9385                }
9386                let _ = reply.send(Err(error));
9387                return true;
9388            }
9389            other => other,
9390        }
9391    } else {
9392        command
9393    };
9394    // A refused start or restart leaves the Failed actor's existing recovery
9395    // deadline intact. Cancel only after the command passes admission.
9396    if matches!(
9397        command,
9398        SupervisorCommand::Drain { .. }
9399            | SupervisorCommand::Retire { .. }
9400            | SupervisorCommand::SetEnabled { .. }
9401            | SupervisorCommand::Restart { .. }
9402    ) {
9403        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9404            state.failed_retry.cancel();
9405        });
9406    }
9407    match command {
9408        SupervisorCommand::ApprovedConfiguration { approved, reply } => {
9409            let result = async {
9410                let next = approved.module_entry(&spec.module_id).map_err(|error| {
9411                    SuperviseError::InvalidSpec { reason: error.to_string() }
9412                })?;
9413                let retry_approved_failure = {
9414                    let mut state = lock_snapshot(snapshot)?;
9415                    if let Some((generation, detail)) = &state.approved_start_failure {
9416                        if *generation == approved.identity() && next.enabled {
9417                            return Err(SuperviseError::Spawn {
9418                                program: spec.program.clone(),
9419                                source: std::io::Error::other(detail.clone()),
9420                                cgroup_path: None,
9421                            });
9422                        }
9423                    }
9424                    let retry = state.approved_start_failure.is_some();
9425                    state.approved_start_failure = None;
9426                    retry
9427                };
9428                stage_supervisor_configuration(
9429                    spec, runtime, snapshot, next.module_spec(), next.health,
9430                    next.drain_timeout_ms,
9431                );
9432                runtime.restart_policy = next.restart;
9433                runtime.register_timeout = next.register_timeout;
9434                if let Some(active) = child.as_mut()
9435                    .filter(|child| child.registration_deadline.is_some())
9436                {
9437                    active.registration_deadline = Some(active.spawned_at + next.register_timeout);
9438                }
9439                let enabled = lock_snapshot(snapshot)?.enabled;
9440                // An approval retries a changed failed entry or its own earlier
9441                // failed start, not an unrelated crash-budget failure. For an
9442                // unchanged entry, reconciled means the approved configuration
9443                // is effective; it does not claim the process has recovered.
9444                // A terminal gate stop is different: a new approval owns its
9445                // recovery even when the daemon entry itself did not change.
9446                let retry_gate_stopped = lock_snapshot(snapshot)?
9447                    .needs_approved_gate_restart(&approved.identity());
9448                let retry_failed = enabled && child.is_none()
9449                    && ((lock_snapshot(snapshot)?.state == ModuleState::Failed
9450                        && (retry_approved_failure || approved.changes_entry(&spec.module_id)))
9451                        || retry_gate_stopped);
9452                if retry_failed {
9453                    // This approval owns a new start, so the failed attempt's
9454                    // daemon retry must not survive either result of that start.
9455                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
9456                        state.failed_retry.cancel();
9457                        if retry_gate_stopped {
9458                            state.stop_reason = None;
9459                            state.config_exit_failed_approval = None;
9460                        }
9461                    })?;
9462                }
9463                if enabled != next.enabled || retry_failed {
9464                    let failure = FailureContext {
9465                        approval: next.enabled.then(|| approved.identity()),
9466                        ..runtime.failure_context(false)
9467                    };
9468                    let held = next.enabled && runtime.config_delivery.as_ref()
9469                        .is_some_and(|delivery| delivery.0.has_recorded_admission_hold(&spec.module_id)
9470                            || delivery.0.check_module_hold(&spec.module_id).is_err_and(|error| error.code == "config_delivery_deferred"));
9471                    if held {
9472                        // Preserve module-delivery failure/recovery holds. The
9473                        // admission release starts the actor when its own hold ends.
9474                        reset_restart_count(snapshot, &spec.module_id)?;
9475                        update_snapshot(snapshot, Some(&spec.module_id), |state| {
9476                            state.enabled = true;
9477                            if child.is_none() && state.waiting_for_previous_pid.is_none() {
9478                                state.state = ModuleState::Starting;
9479                            }
9480                        })?;
9481                        if child.is_none() && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none() {
9482                            hold_config_spawn(spec, runtime, snapshot, &failure)?;
9483                        }
9484                    } else {
9485                        match set_child_enabled_with_budget_policy(
9486                            spec, runtime, registry, process_liveness, snapshot, child,
9487                            next.enabled, &failure, true,
9488                        ).await {
9489                            Ok(_) => {},
9490                            Err(SuperviseError::ConfigHeld { .. }) if next.enabled => {},
9491                            Err(error) => return Err(error),
9492                        }
9493                        if !next.enabled {
9494                            containment::terminate(&runtime.child_roster.containment).await;
9495                        }
9496                    }
9497                }
9498                if (child.as_ref().is_some_and(|child| child.registration_spawn.is_some() && child.admission_base.is_none())
9499                        || (child.is_none() && lock_snapshot(snapshot)?.state == ModuleState::Restarting
9500                            && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.stamp_channel_cutover_pending())))
9501                    && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.module_is_gated(&spec.module_id))
9502                {
9503                    let failure = FailureContext {
9504                        approval: Some(approved.identity()),
9505                        ..runtime.failure_context(false)
9506                    };
9507                    // A stamp cannot adopt an unverified legacy process, even
9508                    // when its approved replacement no longer speaks subc.
9509                    // Stopping that process spends no crash restart budget.
9510                    if child.is_some() {
9511                        restart_child_with_budget_policy(spec, runtime, registry,
9512                            process_liveness, snapshot, child, runtime.drain_timeout, false).await?;
9513                    }
9514                    // The stamp is already committed. A deferred or held launch
9515                    // belongs to the normal retry/hold loop, not a failed apply.
9516                    update_snapshot(snapshot, Some(&spec.module_id), |state| state.respawn_pending = false)?;
9517                    match spawn_and_mark_running_async(spec, runtime, snapshot).await {
9518                        Ok(replacement) => *child = Some(replacement),
9519                        Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9520                            defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, &failure)?;
9521                        }
9522                        Err(SuperviseError::ConfigHeld { .. }) => hold_config_spawn(spec, runtime, snapshot, &failure)?,
9523                        Err(error) => {
9524                            fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
9525                            return Err(error);
9526                        }
9527                    }
9528                }
9529                info!(module_id=%spec.module_id, config_generation=approved.generation(), "approved daemon entry reconciled");
9530                Ok(())
9531            }
9532            .await;
9533            if let Err(error) = &result {
9534                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9535                    if state.enabled
9536                        && state.state == ModuleState::Failed
9537                        && state
9538                            .approved_start_failure
9539                            .as_ref()
9540                            .is_none_or(|(generation, _)| *generation != approved.identity())
9541                    {
9542                        state.approved_start_failure =
9543                            Some((approved.identity(), error.to_string()));
9544                    }
9545                });
9546            }
9547            let _ = reply.send(result);
9548            true
9549        }
9550        SupervisorCommand::ConfigPolicyProbe {
9551            connection,
9552            generation,
9553            attempt,
9554        } => {
9555            let Some(delivery) = &runtime.config_delivery else {
9556                return true;
9557            };
9558            let Ok(gate) = delivery.0.module_gate(&spec.module_id) else {
9559                return true;
9560            };
9561            let guard = gate.lock().await;
9562            let pending = delivery
9563                .0
9564                .on_blocking(move |owner| {
9565                    owner.policy_probe_pending(connection, generation, attempt)
9566                })
9567                .await
9568                .unwrap_or(false)
9569                && child.is_some()
9570                && registry
9571                    .get_module(&spec.module_id)
9572                    .ok()
9573                    .flatten()
9574                    .is_some_and(|registration| registration.connection_id == connection);
9575            drop(guard);
9576            if pending {
9577                run_health_probe_cycle(spec, runtime, registry, process_liveness, snapshot, child)
9578                    .await;
9579            }
9580            true
9581        }
9582        SupervisorCommand::ConfigApplyRestart {
9583            generation,
9584            supersedes_swap,
9585        } => {
9586            let Some(owner) = runtime
9587                .config_delivery
9588                .as_ref()
9589                .map(|delivery| delivery.0.clone())
9590            else {
9591                return true;
9592            };
9593            let Ok(gate) = owner.module_gate(&spec.module_id) else {
9594                return true;
9595            };
9596            let _guard = gate.lock().await;
9597            let spawn = child
9598                .as_ref()
9599                .and_then(|child| child.config_delivery.clone());
9600            let module = spec.module_id.clone();
9601            let current = owner
9602                .on_blocking(move |owner| {
9603                    Ok((supersedes_swap
9604                        || owner
9605                            .pending_config_restarts()?
9606                            .contains(&(module.clone(), generation)))
9607                        && owner.requires_fresher_spawn(&module, spawn.as_ref())?
9608                            == Some(generation))
9609                })
9610                .await
9611                .unwrap_or(false);
9612            if !current
9613                || !lock_snapshot(snapshot).is_ok_and(|state| {
9614                    state.enabled && !state.respawn_pending && state.process_alive
9615                })
9616            {
9617                return true;
9618            }
9619            drop(_guard);
9620            info!(module_id=%spec.module_id, config_generation=generation, "approved configuration restart preserves the crash budget");
9621            if let Err(error) = restart_child_with_budget_policy(
9622                spec,
9623                runtime,
9624                registry,
9625                process_liveness,
9626                snapshot,
9627                child,
9628                runtime.drain_timeout,
9629                false,
9630            )
9631            .await
9632            {
9633                warn!(module_id=%spec.module_id, %error, "approved configuration restart could not be scheduled");
9634            }
9635            true
9636        }
9637        SupervisorCommand::ConfigExitHold { reply } => {
9638            // A notification owns no start, backoff, reload reply or previous-
9639            // instance wait. Only an idle, registered initial start has no work
9640            // to preserve; its initialized startup context can be parked here.
9641            let idle_start = child.is_none()
9642                && lock_snapshot(snapshot).is_ok_and(|state| {
9643                    state.enabled
9644                        && state.state == ModuleState::Starting
9645                        && !state.process_alive
9646                        && !state.respawn_pending
9647                        && state.waiting_for_previous_pid.is_none()
9648                        && state.waiting_for_previous_detail.is_none()
9649                        && state.failed_retry.pending.is_none()
9650                })
9651                && runtime
9652                    .scheduled_respawn
9653                    .lock()
9654                    .unwrap_or_else(|p| p.into_inner())
9655                    .is_none()
9656                && runtime
9657                    .deferred_reload_reply
9658                    .lock()
9659                    .unwrap_or_else(|p| p.into_inner())
9660                    .is_none();
9661            let result = if idle_start {
9662                try_park_config_attempt(
9663                    spec,
9664                    runtime,
9665                    snapshot,
9666                    &runtime.failure_context(runtime.recovery_is_daemon),
9667                    None,
9668                )
9669                .map(|_| ())
9670            } else {
9671                Ok(())
9672            };
9673            let _ = reply.send(result);
9674            true
9675        }
9676        SupervisorCommand::ConfigDeliveryFailed { connection, hold } => {
9677            if registry
9678                .get_module(&spec.module_id)
9679                .ok()
9680                .flatten()
9681                .is_none_or(|registration| registration.connection_id != connection)
9682            {
9683                return true;
9684            }
9685            runtime.recovery_is_daemon = true;
9686            // With a recoverable generation, delivery-failure recovery spends
9687            // one crash-budget entry: the module rejected its configuration.
9688            // Approved configuration restarts spend none.
9689            if hold {
9690                let notice = begin_forwarding_drain_if_configured(
9691                    spec,
9692                    runtime,
9693                    registry,
9694                    snapshot,
9695                    Some(true),
9696                    RouteCloseReason::Disable,
9697                )
9698                .await;
9699                if let Ok(notice) = notice {
9700                    let drained = drain_optional_child(
9701                        &spec.module_id,
9702                        spec.protocol,
9703                        notice,
9704                        registry,
9705                        runtime.forwarding.as_deref(),
9706                        snapshot,
9707                        &runtime.terminal_ring,
9708                        &runtime.spawn_events,
9709                        child,
9710                        runtime.drain_timeout,
9711                        ModuleState::Failed,
9712                        Some(true),
9713                        &runtime.failure_context(true),
9714                    )
9715                    .await;
9716                    if drained.is_ok() {
9717                        match try_park_config_attempt(
9718                            spec,
9719                            runtime,
9720                            snapshot,
9721                            &runtime.failure_context(true),
9722                            None,
9723                        ) {
9724                            Ok(true) => {}
9725                            Ok(false) => {
9726                                if let Err(error) = budgeted_restart_child(
9727                                    spec,
9728                                    runtime,
9729                                    registry,
9730                                    process_liveness,
9731                                    snapshot,
9732                                    child,
9733                                    SupervisorHealthStatus::Unknown,
9734                                    Some("configuration delivery failed after its hold ended"),
9735                                    unix_ms_now(),
9736                                    false,
9737                                )
9738                                .await
9739                                {
9740                                    warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9741                                }
9742                            }
9743                            Err(error) => {
9744                                warn!(module_id=%spec.module_id, %error, "configuration failure could not be parked")
9745                            }
9746                        }
9747                    }
9748                }
9749            } else if let Err(error) = budgeted_restart_child(
9750                spec,
9751                runtime,
9752                registry,
9753                process_liveness,
9754                snapshot,
9755                child,
9756                SupervisorHealthStatus::Unknown,
9757                Some("configuration delivery failed"),
9758                unix_ms_now(),
9759                false,
9760            )
9761            .await
9762            {
9763                warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9764            }
9765            true
9766        }
9767        SupervisorCommand::Drain { requester, reply } => {
9768            // A plain stop runs no forwarding drain, so nothing reaches the
9769            // module over its connection before the wait: ask by signal.
9770            let result = drain_optional_child(
9771                &spec.module_id,
9772                spec.protocol,
9773                StopNotice::NotSent,
9774                registry,
9775                runtime.forwarding.as_deref(),
9776                snapshot,
9777                &runtime.terminal_ring,
9778                &runtime.spawn_events,
9779                child,
9780                runtime.drain_timeout,
9781                ModuleState::Stopped,
9782                None,
9783                &FailureContext::operator(),
9784            )
9785            .await;
9786            containment::terminate(&runtime.child_roster.containment).await;
9787            let registration_released = result.is_ok();
9788            if registration_released {
9789                runtime.record_activity_transition(
9790                    "subc.module_stopped",
9791                    &spec.module_id,
9792                    requester,
9793                );
9794            }
9795            let _ = reply.send(result);
9796            if registration_released {
9797                process_liveness.untrack_if_current(&spec.module_id, snapshot);
9798            }
9799            false
9800        }
9801        SupervisorCommand::Retire { requester, reply } => {
9802            let result = async {
9803                let stop_notice = begin_forwarding_drain_if_configured(
9804                    spec,
9805                    runtime,
9806                    registry,
9807                    snapshot,
9808                    None,
9809                    RouteCloseReason::Disable,
9810                )
9811                .await?;
9812                drain_optional_child(
9813                    &spec.module_id,
9814                    spec.protocol,
9815                    stop_notice,
9816                    registry,
9817                    runtime.forwarding.as_deref(),
9818                    snapshot,
9819                    &runtime.terminal_ring,
9820                    &runtime.spawn_events,
9821                    child,
9822                    runtime.drain_timeout,
9823                    ModuleState::Stopped,
9824                    None,
9825                    &FailureContext::operator(),
9826                )
9827                .await
9828            }
9829            .await;
9830            containment::terminate(&runtime.child_roster.containment).await;
9831            let registration_released = result.is_ok();
9832            if registration_released {
9833                runtime.record_activity_transition(
9834                    "subc.module_stopped",
9835                    &spec.module_id,
9836                    requester,
9837                );
9838            }
9839            let _ = reply.send(result);
9840            if registration_released {
9841                process_liveness.untrack_if_current(&spec.module_id, snapshot);
9842            }
9843            false
9844        }
9845        SupervisorCommand::Restart {
9846            requester,
9847            drain_timeout_ms,
9848            received_at_generation,
9849            queued_at,
9850            reply,
9851        } => {
9852            // Without this line a restart that waited in the queue (behind a
9853            // health probe cycle or another command) was invisible: the log
9854            // showed only the drain timing out, minutes after the operator's call.
9855            info!(
9856                module_id = %spec.module_id,
9857                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
9858                "restart command dequeued"
9859            );
9860            // ACK AT INITIATION, not completion. The blocking form deadlocked any
9861            // caller whose own request lane rides the module being restarted: the
9862            // caller's in-flight request keeps the drain from quiescing, the drain
9863            // keeps the restart from completing, and the completion keeps the reply
9864            // from releasing the caller — so the drain always timed out and cut the
9865            // initiator with a GOODBYE, even on a healthy module. Replying once the
9866            // restart is validated lets a self-lane caller settle, which is exactly
9867            // what makes the drain succeed. Completion is observable via
9868            // supervisor.list / module status; a post-ack failure lands the module
9869            // in a visible terminal state below rather than in a reply nobody can
9870            // receive.
9871            let validation = match lock_snapshot(snapshot) {
9872                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
9873                    module_id: spec.module_id.clone(),
9874                }),
9875                Ok(_) => Ok(()),
9876                Err(err) => Err(err),
9877            };
9878            let initiated = validation.is_ok();
9879            let _ = reply.send(validation);
9880            // A restart asks for a fresh process. Commands run one at a time,
9881            // so a restart queued behind another restart (two operator calls
9882            // in quick succession) is dequeued the moment the first one has
9883            // spawned its replacement -- before that process has sent HELLO.
9884            // Running it would drain and kill the process the first restart
9885            // just produced, which is the opposite of what both callers asked
9886            // for. If a process spawned after this request was received is
9887            // still supervised, the request is already satisfied. Not when the
9888            // configuration changed since that spawn: then the newer process
9889            // predates the spec this restart may exist to apply.
9890            // Generation publication can lag a child that is already serving
9891            // while its identity is recorded. Only process creation after request
9892            // receipt satisfies a restart, not publication after receipt.
9893            let satisfied_by_generation = if initiated
9894                && child
9895                    .as_ref()
9896                    .is_some_and(|child| child.spawned_at > queued_at)
9897            {
9898                lock_snapshot(snapshot).ok().and_then(|state| {
9899                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
9900                })
9901            } else {
9902                None
9903            };
9904            let satisfied_by_pending = initiated
9905                && child.is_none()
9906                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
9907                    // No child exists yet, so this replacement will necessarily
9908                    // be created after the queued request, unlike a published
9909                    // generation whose process may already have been serving.
9910                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
9911                    if pending {
9912                        state.coalesced_restart_pending = true;
9913                    }
9914                    pending
9915                });
9916            if satisfied_by_pending {
9917                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
9918            } else if let Some(generation) = satisfied_by_generation {
9919                info!(
9920                    module_id = %spec.module_id,
9921                    received_at_generation,
9922                    "restart already satisfied by generation {generation}; not restarting again"
9923                );
9924            } else if initiated {
9925                // Precedence: this restart's operator override, else the module's
9926                // configured budget (already resolved into the runtime).
9927                let drain_timeout = drain_timeout_ms
9928                    .map(Duration::from_millis)
9929                    .unwrap_or(runtime.drain_timeout);
9930                if let Err(err) = restart_child(
9931                    spec,
9932                    runtime,
9933                    registry,
9934                    process_liveness,
9935                    snapshot,
9936                    child,
9937                    drain_timeout,
9938                )
9939                .await
9940                {
9941                    warn!(
9942                        module_id = %spec.module_id,
9943                        error = %err,
9944                        "operator restart failed after initiation ack; module state carries the outcome"
9945                    );
9946                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9947                        state.state = ModuleState::Failed;
9948                        clear_current_process_facts(state);
9949                    });
9950                } else {
9951                    runtime.start_requester = requester;
9952                    runtime.pending_activity_restart = true;
9953                }
9954            }
9955            true
9956        }
9957        SupervisorCommand::Reload { requester, reply } => {
9958            let result =
9959                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
9960            if result.is_ok() {
9961                runtime.start_requester = requester;
9962                runtime.pending_activity_restart = true;
9963            }
9964            if result.is_ok()
9965                && runtime
9966                    .scheduled_respawn
9967                    .lock()
9968                    .unwrap_or_else(|p| p.into_inner())
9969                    .as_ref()
9970                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
9971            {
9972                *runtime
9973                    .deferred_reload_reply
9974                    .lock()
9975                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
9976            } else {
9977                let _ = reply.send(result);
9978            }
9979            true
9980        }
9981        SupervisorCommand::SetEnabled {
9982            enabled,
9983            requester,
9984            reply,
9985            reset_budget,
9986        } => {
9987            let previous_requester = runtime.start_requester.clone();
9988            if enabled {
9989                runtime.start_requester = requester.clone();
9990            }
9991            let result = set_child_enabled_with_budget_policy(
9992                spec,
9993                runtime,
9994                registry,
9995                process_liveness,
9996                snapshot,
9997                child,
9998                enabled,
9999                &runtime.failure_context(false),
10000                reset_budget,
10001            )
10002            .await;
10003            if !enabled {
10004                containment::terminate(&runtime.child_roster.containment).await;
10005            }
10006            if matches!(result, Ok(true)) {
10007                runtime.pending_activity_restart = false;
10008                if enabled {
10009                    runtime.start_requester = requester;
10010                } else {
10011                    runtime.start_requester = None;
10012                    runtime.record_activity_transition(
10013                        "subc.module_disabled",
10014                        &spec.module_id,
10015                        requester,
10016                    );
10017                }
10018            } else {
10019                runtime.start_requester = previous_requester;
10020            }
10021            let _ = reply.send(result);
10022            true
10023        }
10024        SupervisorCommand::UpdateConfiguration {
10025            spec: next_spec,
10026            health,
10027            drain_timeout_ms,
10028            reply,
10029        } => {
10030            if runtime
10031                .config_delivery
10032                .as_ref()
10033                .is_some_and(|delivery| delivery.0.file_configuration_is_gated(&spec.module_id))
10034            {
10035                let _ = reply.send(Err(SuperviseError::ConfigHeld {
10036                    module_id: spec.module_id.clone(),
10037                    code: "config_gated".into(),
10038                    detail: "live file reconfiguration cannot replace stamped authority".into(),
10039                }));
10040                return true;
10041            }
10042            stage_supervisor_configuration(
10043                spec,
10044                runtime,
10045                snapshot,
10046                next_spec,
10047                health,
10048                drain_timeout_ms,
10049            );
10050            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10051                state.configuration_updated_since_spawn = true;
10052            });
10053            let _ = reply.send(Ok(()));
10054            true
10055        }
10056        SupervisorCommand::Swap {
10057            requester,
10058            ready_timeout,
10059            reply,
10060        } => {
10061            if spec.carry_containment
10062                || containment::current(&runtime.child_roster.containment).is_some()
10063            {
10064                let _ = reply.send(Err(SuperviseError::InvalidSpec {
10065                    reason: "a carried container cannot overlap a swap candidate".to_owned(),
10066                }));
10067                return true;
10068            }
10069            let mut swap_runtime = runtime.clone();
10070            swap_runtime.start_requester = requester;
10071            let before = lock_snapshot(snapshot)
10072                .ok()
10073                .map(|state| state.spawn_generation);
10074            let end = swap::run_swap(
10075                spec,
10076                &swap_runtime,
10077                registry,
10078                process_liveness,
10079                snapshot,
10080                child,
10081                commands,
10082                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
10083                reply,
10084            )
10085            .await;
10086            if before
10087                != lock_snapshot(snapshot)
10088                    .ok()
10089                    .map(|state| state.spawn_generation)
10090            {
10091                runtime.start_requester = swap_runtime.start_requester;
10092            }
10093            requeued.extend(end.requeue);
10094            true
10095        }
10096    }
10097}
10098
10099fn stage_supervisor_configuration(
10100    spec: &mut ModuleSpec,
10101    runtime: &mut SupervisorRuntimeConfig,
10102    snapshot: &SharedSnapshot,
10103    next_spec: ModuleSpec,
10104    health: HealthConfig,
10105    drain_timeout_ms: Option<u64>,
10106) {
10107    if let Some(handle) = &runtime.supervisor_handle {
10108        handle.apply_identity_configuration(&next_spec);
10109    }
10110    let launch_changed = *spec != next_spec;
10111    *spec = next_spec;
10112    if launch_changed {
10113        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10114            state.configuration_updated_since_spawn = true
10115        });
10116    }
10117    let health_changed = runtime.health != health;
10118    runtime.health = health;
10119    // Reset the cadence and old endpoint's failure streak on a live
10120    // health-policy change rather than waiting for its old deadline.
10121    if health_changed {
10122        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10123            state.health = ModuleHealthStatus::default();
10124            state.last_probe_evidence = None;
10125        });
10126    }
10127    runtime.drain_timeout = drain_timeout_ms
10128        .map(Duration::from_millis)
10129        .unwrap_or(runtime.default_drain_timeout);
10130    *runtime
10131        .effective_drain_timeout
10132        .lock()
10133        .unwrap_or_else(|p| p.into_inner()) = runtime.drain_timeout;
10134}
10135
10136async fn restart_child(
10137    spec: &ModuleSpec,
10138    runtime: &SupervisorRuntimeConfig,
10139    registry: &Registry,
10140    process_liveness: &SupervisorProcessLiveness,
10141    snapshot: &SharedSnapshot,
10142    child: &mut Option<SupervisedChild>,
10143    drain_timeout: Duration,
10144) -> Result<(), SuperviseError> {
10145    restart_child_with_budget_policy(
10146        spec,
10147        runtime,
10148        registry,
10149        process_liveness,
10150        snapshot,
10151        child,
10152        drain_timeout,
10153        true,
10154    )
10155    .await
10156}
10157
10158#[allow(clippy::too_many_arguments)]
10159async fn restart_child_with_budget_policy(
10160    spec: &ModuleSpec,
10161    runtime: &SupervisorRuntimeConfig,
10162    registry: &Registry,
10163    process_liveness: &SupervisorProcessLiveness,
10164    snapshot: &SharedSnapshot,
10165    child: &mut Option<SupervisedChild>,
10166    drain_timeout: Duration,
10167    reset_budget: bool,
10168) -> Result<(), SuperviseError> {
10169    // Restart cycles a running module; it must not silently start a disabled one.
10170    if !lock_snapshot(snapshot)?.enabled {
10171        return Err(SuperviseError::Disabled {
10172            module_id: spec.module_id.clone(),
10173        });
10174    }
10175    let stop_notice = begin_forwarding_drain_with_timeout(
10176        spec,
10177        runtime,
10178        registry,
10179        snapshot,
10180        None,
10181        RouteCloseReason::Restart,
10182        drain_timeout,
10183    )
10184    .await?;
10185
10186    if child.is_some() {
10187        drain_optional_child(
10188            &spec.module_id,
10189            spec.protocol,
10190            stop_notice,
10191            registry,
10192            runtime.forwarding.as_deref(),
10193            snapshot,
10194            &runtime.terminal_ring,
10195            &runtime.spawn_events,
10196            child,
10197            drain_timeout,
10198            ModuleState::Restarting,
10199            Some(true),
10200            &FailureContext::operator(),
10201        )
10202        .await?;
10203    } else {
10204        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10205            state.enabled = true;
10206            state.state = ModuleState::Restarting;
10207            clear_current_process_facts(state);
10208        })?;
10209        release_dead_registration(
10210            registry,
10211            runtime.forwarding.as_deref(),
10212            snapshot,
10213            &spec.module_id,
10214            &runtime.failure_context(false),
10215        )
10216        .await?;
10217    }
10218
10219    if reset_budget {
10220        reset_restart_count(snapshot, &spec.module_id)?;
10221    }
10222    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10223    schedule_respawn(
10224        runtime,
10225        snapshot,
10226        &spec.module_id,
10227        runtime.restart_policy.backoff,
10228        RespawnKind::Spawn,
10229        false,
10230    )
10231}
10232
10233async fn reload_child(
10234    spec: &ModuleSpec,
10235    runtime: &SupervisorRuntimeConfig,
10236    registry: &Registry,
10237    process_liveness: &SupervisorProcessLiveness,
10238    snapshot: &SharedSnapshot,
10239    child: &mut Option<SupervisedChild>,
10240) -> Result<(), SuperviseError> {
10241    // Reload cycles a running module; it must not silently start a disabled one.
10242    if !lock_snapshot(snapshot)?.enabled {
10243        return Err(SuperviseError::Disabled {
10244            module_id: spec.module_id.clone(),
10245        });
10246    }
10247    let stop_notice = begin_forwarding_drain(
10248        spec,
10249        runtime,
10250        registry,
10251        snapshot,
10252        Some(true),
10253        RouteCloseReason::Reload,
10254    )
10255    .await?;
10256
10257    if child.is_some() {
10258        drain_optional_child(
10259            &spec.module_id,
10260            spec.protocol,
10261            stop_notice,
10262            registry,
10263            runtime.forwarding.as_deref(),
10264            snapshot,
10265            &runtime.terminal_ring,
10266            &runtime.spawn_events,
10267            child,
10268            runtime.drain_timeout,
10269            ModuleState::Restarting,
10270            Some(true),
10271            &FailureContext::operator(),
10272        )
10273        .await?;
10274    } else {
10275        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10276            state.enabled = true;
10277            state.state = ModuleState::Restarting;
10278            clear_current_process_facts(state);
10279        })?;
10280        release_dead_registration(
10281            registry,
10282            runtime.forwarding.as_deref(),
10283            snapshot,
10284            &spec.module_id,
10285            &runtime.failure_context(false),
10286        )
10287        .await?;
10288    }
10289
10290    reset_restart_count(snapshot, &spec.module_id)?;
10291    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10292    schedule_respawn(
10293        runtime,
10294        snapshot,
10295        &spec.module_id,
10296        runtime.restart_policy.backoff,
10297        RespawnKind::Reload,
10298        false,
10299    )
10300}
10301
10302async fn finish_reload_child(
10303    spec: &ModuleSpec,
10304    runtime: &SupervisorRuntimeConfig,
10305    registry: &Registry,
10306    process_liveness: &SupervisorProcessLiveness,
10307    snapshot: &SharedSnapshot,
10308    child: &mut Option<SupervisedChild>,
10309) -> Result<(), SuperviseError> {
10310    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10311    let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10312        Ok(next_child) => next_child,
10313        Err(error @ SuperviseError::ConfigDeliveryDeferred { .. }) => {
10314            defer_config_spawn(
10315                spec,
10316                runtime,
10317                snapshot,
10318                RespawnKind::Reload,
10319                &runtime.failure_context(runtime.recovery_is_daemon),
10320            )?;
10321            return Err(error);
10322        }
10323        Err(error @ SuperviseError::ConfigHeld { .. }) => {
10324            let failure = runtime.failure_context(runtime.recovery_is_daemon);
10325            if try_park_config_attempt(spec, runtime, snapshot, &failure, None)? {
10326                return Err(error);
10327            }
10328            defer_config_spawn(spec, runtime, snapshot, RespawnKind::Reload, &failure)?;
10329            return Err(SuperviseError::ConfigDeliveryDeferred {
10330                module_id: spec.module_id.clone(),
10331            });
10332        }
10333        Err(err) => {
10334            return handle_reload_spawn_failure(
10335                spec,
10336                runtime,
10337                process_liveness,
10338                snapshot,
10339                child,
10340                format!("new child failed to spawn: {err}"),
10341            )
10342            .await;
10343        }
10344    };
10345    *child = Some(next_child);
10346
10347    let wait_outcome = {
10348        let active_child = child.as_mut().expect("new reload child was just stored");
10349        wait_for_registration_after_reload(
10350            registry,
10351            &spec.module_id,
10352            snapshot,
10353            active_child,
10354            REGISTRY_RELEASE_TIMEOUT,
10355        )
10356        .await?
10357    };
10358
10359    match wait_outcome {
10360        RegistrationWaitOutcome::Registered => {
10361            child
10362                .as_mut()
10363                .expect("registered reload child")
10364                .registration_deadline = None;
10365            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
10366            Ok(())
10367        }
10368        RegistrationWaitOutcome::Exited(exit_report) => {
10369            if let Some(active_child) = child.as_mut() {
10370                active_child.drain_stderr(&spec.module_id).await;
10371            }
10372            // Keep the exited child's roster guard until its terminal is written.
10373            // Shutdown waits on that guard, not on the child Option used for respawn.
10374            let mut exited_child = child.take().expect("exited reload child is still stored");
10375            #[cfg(test)]
10376            if let Some(gate) = &runtime.test_reload_exit_record_gate {
10377                gate.reached.notify_one();
10378                gate.resume.notified().await;
10379            }
10380            let result = handle_reload_child_registration_failure(
10381                spec,
10382                runtime,
10383                registry,
10384                process_liveness,
10385                snapshot,
10386                child,
10387                ReloadRegistrationFailure {
10388                    exit_report: registration_failure_exit_report(exit_report),
10389                    reason: exited_child
10390                        .spawn_failure
10391                        .clone()
10392                        .unwrap_or_else(|| "new child exited before registering".to_string()),
10393                },
10394            )
10395            .await;
10396            exited_child
10397                .finish_exit(false)
10398                .await
10399                .map_err(|source| SuperviseError::Wait {
10400                    module_id: spec.module_id.clone(),
10401                    source,
10402                })?;
10403            result
10404        }
10405        RegistrationWaitOutcome::TimedOut => {
10406            let mut timed_out_child = child
10407                .take()
10408                .expect("timed-out reload child is still running");
10409            timed_out_child
10410                .start_kill()
10411                .map_err(|source| SuperviseError::Kill {
10412                    module_id: spec.module_id.clone(),
10413                    source,
10414                })?;
10415            let status = timed_out_child
10416                .wait()
10417                .await
10418                .map_err(|source| SuperviseError::Wait {
10419                    module_id: spec.module_id.clone(),
10420                    source,
10421                })?;
10422            timed_out_child.drain_stderr(&spec.module_id).await;
10423            let result = handle_reload_child_registration_failure(
10424                spec,
10425                runtime,
10426                registry,
10427                process_liveness,
10428                snapshot,
10429                child,
10430                ReloadRegistrationFailure {
10431                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
10432                        snapshot,
10433                        &timed_out_child,
10434                        &status,
10435                    )),
10436                    reason: format!(
10437                        "new child did not register within {:?}",
10438                        REGISTRY_RELEASE_TIMEOUT
10439                    ),
10440                },
10441            )
10442            .await;
10443            timed_out_child
10444                .finish_exit(false)
10445                .await
10446                .map_err(|source| SuperviseError::Wait {
10447                    module_id: spec.module_id.clone(),
10448                    source,
10449                })?;
10450            result
10451        }
10452    }
10453}
10454
10455#[allow(clippy::too_many_arguments)]
10456async fn set_child_enabled(
10457    spec: &ModuleSpec,
10458    runtime: &SupervisorRuntimeConfig,
10459    registry: &Registry,
10460    process_liveness: &SupervisorProcessLiveness,
10461    snapshot: &SharedSnapshot,
10462    child: &mut Option<SupervisedChild>,
10463    enabled: bool,
10464    failure: &FailureContext,
10465) -> Result<bool, SuperviseError> {
10466    set_child_enabled_with_budget_policy(
10467        spec,
10468        runtime,
10469        registry,
10470        process_liveness,
10471        snapshot,
10472        child,
10473        enabled,
10474        failure,
10475        true,
10476    )
10477    .await
10478}
10479
10480#[allow(clippy::too_many_arguments)]
10481async fn set_child_enabled_with_budget_policy(
10482    spec: &ModuleSpec,
10483    runtime: &SupervisorRuntimeConfig,
10484    registry: &Registry,
10485    process_liveness: &SupervisorProcessLiveness,
10486    snapshot: &SharedSnapshot,
10487    child: &mut Option<SupervisedChild>,
10488    enabled: bool,
10489    failure: &FailureContext,
10490    reset_budget: bool,
10491) -> Result<bool, SuperviseError> {
10492    let (current_enabled, current_state, respawn_pending) = {
10493        let state = lock_snapshot(snapshot)?;
10494        (state.enabled, state.state, state.respawn_pending)
10495    };
10496    // `start` (enable on an already-enabled module) heals TERMINAL states instead
10497    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
10498    // clean (Stopped) has no live process and no other in-band recovery — the
10499    // operator's start is the explicit recovery act and resets the budget. Without
10500    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
10501    // which the 2026-07-14 aft outage proved is a trap when the failed module is
10502    // the one providing every agent's shell.
10503    let revive_terminal = enabled
10504        && current_enabled
10505        && child.is_none()
10506        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
10507            || (current_state == ModuleState::Restarting && !respawn_pending));
10508    if current_enabled == enabled && !revive_terminal {
10509        return Ok(false);
10510    }
10511
10512    if enabled {
10513        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10514            state.enabled = true;
10515            state.state = ModuleState::Starting;
10516            clear_current_process_facts(state);
10517        })?;
10518        #[cfg(test)]
10519        if runtime.test_seed_stale_facts_before_enable_spawn {
10520            update_snapshot(snapshot, Some(&spec.module_id), |state| {
10521                state.process_alive = true;
10522                state.pid = Some(41);
10523                state.spawned_at_ms = Some(42);
10524                state.spawned_from = Some(PathBuf::from("/spawned/module"));
10525                state.spawned_file_identity = Some(SpawnedFileIdentity {
10526                    device: 43,
10527                    inode: 44,
10528                    #[cfg(windows)]
10529                    windows_image: None,
10530                });
10531            })?;
10532        }
10533        release_dead_registration(
10534            registry,
10535            runtime.forwarding.as_deref(),
10536            snapshot,
10537            &spec.module_id,
10538            failure,
10539        )
10540        .await?;
10541        if reset_budget {
10542            reset_restart_count(snapshot, &spec.module_id)?;
10543        }
10544        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10545        let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10546            Ok(next_child) => next_child,
10547            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
10548                defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
10549                return Ok(true);
10550            }
10551            Err(error @ SuperviseError::ConfigHeld { .. }) => {
10552                if let Err(parking_error) = hold_config_spawn(spec, runtime, snapshot, failure) {
10553                    process_liveness.untrack_if_current(&spec.module_id, snapshot);
10554                    return Err(parking_error);
10555                }
10556                return Err(error);
10557            }
10558            Err(err) => {
10559                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
10560                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10561                return Err(err);
10562            }
10563        };
10564        *child = Some(next_child);
10565        debug!(module_id = %spec.module_id, "supervised module enabled");
10566        Ok(true)
10567    } else {
10568        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10569            state.config_hold_parked = false;
10570            state.config_hold_daemon_initiated = false;
10571            state.config_hold_approval = None;
10572        })?;
10573        let stop_notice = begin_forwarding_drain_if_configured(
10574            spec,
10575            runtime,
10576            registry,
10577            snapshot,
10578            Some(false),
10579            RouteCloseReason::Disable,
10580        )
10581        .await?;
10582        drain_optional_child(
10583            &spec.module_id,
10584            spec.protocol,
10585            stop_notice,
10586            registry,
10587            runtime.forwarding.as_deref(),
10588            snapshot,
10589            &runtime.terminal_ring,
10590            &runtime.spawn_events,
10591            child,
10592            runtime.drain_timeout,
10593            ModuleState::Disabled,
10594            Some(false),
10595            failure,
10596        )
10597        .await?;
10598        debug!(module_id = %spec.module_id, "supervised module disabled");
10599        Ok(true)
10600    }
10601}
10602
10603#[allow(clippy::too_many_arguments)]
10604async fn on_child_exit(
10605    spec: &ModuleSpec,
10606    policy: RestartPolicy,
10607    registry: &Registry,
10608    snapshot: &SharedSnapshot,
10609    terminal_ring: &Arc<Mutex<TerminalRing>>,
10610    spawn_events: &SpawnEventFeed,
10611    roster: &ChildRoster,
10612    exit_report: ExitReport,
10613    daemon_initiated: bool,
10614) -> NextAction {
10615    // Once the daemon has begun shutting down, no exit is a crash to recover
10616    // from: the module is exiting because the daemon is going away (EOF on its
10617    // connection, or a service manager signalling the whole cgroup). Record it
10618    // as such and never schedule a respawn, which would only start a process
10619    // for the shutdown to end again.
10620    if roster.is_closed() {
10621        return on_child_exit_during_daemon_shutdown(
10622            spec,
10623            registry,
10624            snapshot,
10625            terminal_ring,
10626            spawn_events,
10627            exit_report,
10628        )
10629        .await;
10630    }
10631    // Deliberate stops the supervisor asks for (operator stop, disable,
10632    // restart, reload, swap, a health restart, a drain that runs out of budget)
10633    // takes the child out of the supervise loop and reaps it in
10634    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
10635    // that reaches this point is either unrequested or a first-HELLO timeout,
10636    // which is deliberately classified as a crash even after a clean TERM exit.
10637    //
10638    // For a subc-wire module a clean exit is still a stop: those modules are
10639    // written to re-raise SIGTERM, so a stray outside signal already reads as a
10640    // crash, and exiting 0 is a deliberate choice the module made. A
10641    // `protocol: "none"` module is a stock program we cannot change, and many
10642    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
10643    // stop would leave the module down for good after any stray signal, so it
10644    // goes through the crash path instead: it spends restart budget, respawns
10645    // with the crash backoff, and ends `failed` when the budget runs out.
10646    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
10647        && running_protocol(spec, snapshot) == ModuleProtocol::None;
10648    match exit_report.kind {
10649        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
10650            info!(
10651                module_id = %spec.module_id,
10652                exit_code = ?exit_report.code,
10653                exit_signal = ?exit_report.signal,
10654                "supervised module exited cleanly"
10655            );
10656            if let Err(err) = publish_after_terminal_record(
10657                &spec.module_id,
10658                || {
10659                    record_terminal(
10660                        &spec.module_id,
10661                        terminal_ring,
10662                        spawn_events,
10663                        &exit_report,
10664                        TerminalDisposition::Stopped,
10665                    )
10666                },
10667                || {
10668                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10669                        state.state = ModuleState::Stopped;
10670                        clear_current_process_facts(state);
10671                        state.last_exit = Some(exit_report.clone());
10672                    })
10673                },
10674            ) {
10675                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
10676            }
10677            let registration_released = match wait_for_registration_release(
10678                registry,
10679                &spec.module_id,
10680                REGISTRY_RELEASE_TIMEOUT,
10681            )
10682            .await
10683            {
10684                Ok(()) => true,
10685                Err(err) => {
10686                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
10687                    false
10688                }
10689            };
10690            NextAction::Stop {
10691                registration_released,
10692            }
10693        }
10694        ExitKind::Clean | ExitKind::Crash => {
10695            if unrequested_clean_exit_of_protocol_none {
10696                warn!(
10697                    module_id = %spec.module_id,
10698                    exit_code = ?exit_report.code,
10699                    exit_signal = ?exit_report.signal,
10700                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
10701                );
10702            } else {
10703                warn!(
10704                    module_id = %spec.module_id,
10705                    exit_code = ?exit_report.code,
10706                    exit_signal = ?exit_report.signal,
10707                    "supervised module exited abnormally (crash)"
10708                );
10709            }
10710            let mut restart_schedule = None;
10711            let mut disposition = TerminalDisposition::Disabled;
10712            let mut final_state = ModuleState::Disabled;
10713            // Set only when the budget is what stopped the module, so the
10714            // terminal record says which limit was hit rather than leaving
10715            // `failed` to be read as "crashed once, badly".
10716            let mut disposition_detail = lock_snapshot(snapshot)
10717                .ok()
10718                .and_then(|mut state| state.spawn_failure.take());
10719            let now = Instant::now();
10720            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10721                if state.enabled {
10722                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
10723                        // Reserving the last slot exhausts the budget for the
10724                        // replacement, not for this exit. Publish its scheduled
10725                        // recovery atomically with the reservation so connection
10726                        // cleanup sees it while terminal recording still holds
10727                        // the visible state at Running.
10728                        state.respawn_pending = true;
10729                        final_state = ModuleState::Restarting;
10730                        restart_schedule = Some(schedule);
10731                        disposition = TerminalDisposition::Restarting;
10732                    } else {
10733                        final_state = ModuleState::Failed;
10734                        disposition = TerminalDisposition::Failed;
10735                        let budget = policy.budget_exhausted_detail();
10736                        disposition_detail =
10737                            Some(disposition_detail.take().map_or_else(
10738                                || budget.clone(),
10739                                |cause| format!("{cause}; {budget}"),
10740                            ));
10741                    }
10742                } else {
10743                    disposition = TerminalDisposition::Disabled;
10744                }
10745            }) {
10746                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
10747                return NextAction::Stop {
10748                    registration_released: false,
10749                };
10750            }
10751            if disposition == TerminalDisposition::Failed {
10752                // The window is in the message, not only in the fields: this line
10753                // is read in a scrollback where a bare `max_restarts=3` reads as a
10754                // lifetime cap and sends the operator looking for three crashes
10755                // that never happened together.
10756                error!(
10757                    module_id = %spec.module_id,
10758                    max_restarts = policy.max_restarts,
10759                    window_secs = policy.window.as_secs(),
10760                    "module stopped: {}",
10761                    policy.budget_exhausted_detail()
10762                );
10763            }
10764            if let Err(err) = publish_after_terminal_record(
10765                &spec.module_id,
10766                || {
10767                    record_terminal_with_detail(
10768                        &spec.module_id,
10769                        terminal_ring,
10770                        spawn_events,
10771                        &exit_report,
10772                        disposition,
10773                        disposition_detail,
10774                    )
10775                },
10776                || {
10777                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10778                        state.state = final_state;
10779                        clear_current_process_facts(state);
10780                        if final_state == ModuleState::Failed
10781                            && daemon_initiated
10782                            && !roster.is_closed()
10783                        {
10784                            state.failed_retry.schedule(&spec.module_id);
10785                        }
10786                        state.last_exit = Some(exit_report.clone());
10787                    })
10788                },
10789            ) {
10790                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
10791            }
10792
10793            if let Some(schedule) = restart_schedule {
10794                NextAction::Restart {
10795                    schedule: Some(schedule),
10796                }
10797            } else {
10798                let registration_released = match wait_for_registration_release(
10799                    registry,
10800                    &spec.module_id,
10801                    REGISTRY_RELEASE_TIMEOUT,
10802                )
10803                .await
10804                {
10805                    Ok(()) => true,
10806                    Err(err) => {
10807                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
10808                        false
10809                    }
10810                };
10811                NextAction::Stop {
10812                    registration_released,
10813                }
10814            }
10815        }
10816        ExitKind::DeliberateSeverance => {
10817            warn!(
10818                module_id = %spec.module_id,
10819                exit_code = ?exit_report.code,
10820                exit_signal = ?exit_report.signal,
10821                "supervised module exited after deliberate connection severance"
10822            );
10823            let should_restart = match lock_snapshot(snapshot) {
10824                Ok(state) => state.enabled,
10825                Err(err) => {
10826                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
10827                    return NextAction::Stop {
10828                        registration_released: false,
10829                    };
10830                }
10831            };
10832            let final_state = if should_restart {
10833                ModuleState::Restarting
10834            } else {
10835                ModuleState::Disabled
10836            };
10837            if let Err(err) = publish_after_terminal_record(
10838                &spec.module_id,
10839                || {
10840                    record_terminal(
10841                        &spec.module_id,
10842                        terminal_ring,
10843                        spawn_events,
10844                        &exit_report,
10845                        terminal_disposition(final_state),
10846                    )
10847                },
10848                || {
10849                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10850                        state.state = final_state;
10851                        clear_current_process_facts(state);
10852                        state.last_exit = Some(exit_report.clone());
10853                        state.lifetime_restarts += 1;
10854                    })
10855                },
10856            ) {
10857                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
10858                return NextAction::Stop {
10859                    registration_released: false,
10860                };
10861            }
10862
10863            if should_restart {
10864                NextAction::Restart { schedule: None }
10865            } else {
10866                let registration_released = match wait_for_registration_release(
10867                    registry,
10868                    &spec.module_id,
10869                    REGISTRY_RELEASE_TIMEOUT,
10870                )
10871                .await
10872                {
10873                    Ok(()) => true,
10874                    Err(err) => {
10875                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
10876                        false
10877                    }
10878                };
10879                NextAction::Stop {
10880                    registration_released,
10881                }
10882            }
10883        }
10884    }
10885}
10886
10887async fn on_child_exit_during_daemon_shutdown(
10888    spec: &ModuleSpec,
10889    registry: &Registry,
10890    snapshot: &SharedSnapshot,
10891    terminal_ring: &Arc<Mutex<TerminalRing>>,
10892    spawn_events: &SpawnEventFeed,
10893    exit_report: ExitReport,
10894) -> NextAction {
10895    info!(
10896        module_id = %spec.module_id,
10897        exit_code = ?exit_report.code,
10898        exit_signal = ?exit_report.signal,
10899        exit_kind = ?exit_report.kind,
10900        "supervised module exited during daemon shutdown; not restarting it"
10901    );
10902    if let Err(err) = publish_after_terminal_record(
10903        &spec.module_id,
10904        || {
10905            record_terminal(
10906                &spec.module_id,
10907                terminal_ring,
10908                spawn_events,
10909                &exit_report,
10910                TerminalDisposition::DaemonShutdown,
10911            )
10912        },
10913        || {
10914            update_snapshot(snapshot, Some(&spec.module_id), |state| {
10915                state.state = ModuleState::Stopped;
10916                clear_current_process_facts(state);
10917                state.last_exit = Some(exit_report.clone());
10918            })
10919        },
10920    ) {
10921        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
10922    }
10923    let registration_released =
10924        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
10925            .await
10926            .is_ok();
10927    NextAction::Stop {
10928        registration_released,
10929    }
10930}
10931
10932// Keep the status lock out of terminal I/O. Readers may observe the previous
10933// state while recording is in progress, but never a completed exit without its
10934// history. Journal errors are logged by the journal and retained in the ring;
10935// a recording panic still publishes the final state before resuming the unwind.
10936fn publish_after_terminal_record<T>(
10937    module_id: &str,
10938    record: impl FnOnce(),
10939    publish: impl FnOnce() -> T,
10940) -> T {
10941    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
10942    let published = publish();
10943    if let Err(panic) = recorded {
10944        error!(
10945            module_id,
10946            "terminal recording panicked; published final state before resuming unwind"
10947        );
10948        std::panic::resume_unwind(panic);
10949    }
10950    published
10951}
10952
10953fn fail_after_wait_error(
10954    module_id: &str,
10955    snapshot: &SharedSnapshot,
10956    terminal_ring: &Arc<Mutex<TerminalRing>>,
10957    spawn_events: &SpawnEventFeed,
10958    detail: String,
10959    failure: &FailureContext,
10960) {
10961    let report = wait_error_exit_report();
10962    publish_after_terminal_record(
10963        module_id,
10964        || {
10965            record_terminal_with_detail(
10966                module_id,
10967                terminal_ring,
10968                spawn_events,
10969                &report,
10970                TerminalDisposition::Failed,
10971                Some(detail),
10972            )
10973        },
10974        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
10975    );
10976}
10977
10978#[cfg(test)]
10979fn record_wait_error_terminal(
10980    module_id: &str,
10981    terminal_ring: &Arc<Mutex<TerminalRing>>,
10982    spawn_events: &SpawnEventFeed,
10983) {
10984    record_terminal(
10985        module_id,
10986        terminal_ring,
10987        spawn_events,
10988        &wait_error_exit_report(),
10989        TerminalDisposition::Failed,
10990    );
10991}
10992
10993fn record_terminal(
10994    module_id: &str,
10995    terminal_ring: &Arc<Mutex<TerminalRing>>,
10996    spawn_events: &SpawnEventFeed,
10997    exit_report: &ExitReport,
10998    disposition: TerminalDisposition,
10999) {
11000    record_terminal_with_detail(
11001        module_id,
11002        terminal_ring,
11003        spawn_events,
11004        exit_report,
11005        disposition,
11006        None,
11007    );
11008}
11009
11010/// The ring lock is held only to capture the read (see
11011/// `TerminalJournal::capture_read`), so this module's exits keep recording
11012/// while the journal files are read. Blocking: it reads files.
11013fn durable_terminal_history_of(
11014    terminal_ring: &Mutex<TerminalRing>,
11015    module_id: &str,
11016) -> subc_control::TerminalHistory {
11017    let read = terminal_ring
11018        .lock()
11019        .unwrap_or_else(|p| p.into_inner())
11020        .capture_durable_history();
11021    read.read(module_id)
11022}
11023
11024fn record_terminal_with_detail(
11025    module_id: &str,
11026    terminal_ring: &Arc<Mutex<TerminalRing>>,
11027    spawn_events: &SpawnEventFeed,
11028    exit_report: &ExitReport,
11029    disposition: TerminalDisposition,
11030    disposition_detail: Option<String>,
11031) {
11032    #[cfg(test)]
11033    terminal_history_tests::before_terminal_record(spawn_events);
11034    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
11035    let record = TerminalRecord {
11036        exit_code: exit_report.code,
11037        exit_signal: exit_report.signal,
11038        at_ms: exit_report.at_ms,
11039        disposition,
11040        exit_kind: exit_report.kind.into(),
11041        disposition_detail,
11042    };
11043    terminal_ring
11044        .lock()
11045        .unwrap_or_else(|poisoned| poisoned.into_inner())
11046        .record_exit(module_id, record);
11047}
11048
11049fn untrack_if_registration_released(
11050    process_liveness: &SupervisorProcessLiveness,
11051    registry: &Registry,
11052    module_id: &str,
11053    snapshot: &SharedSnapshot,
11054) {
11055    match registry.get_module(module_id) {
11056        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
11057        Ok(Some(_)) => {}
11058        Err(err) => {
11059            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
11060        }
11061    }
11062}
11063
11064/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
11065/// then apply the module's configured entries minus daemon-private capture keys.
11066///
11067/// Separated from `spawn_child` only so it can be asserted without spawning a
11068/// process — a duplicate of this logic in a test would pass while the real one
11069/// drifted, which is the defect class this function exists to avoid.
11070/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
11071/// nonce. A `protocol: "none"` module gets neither, because it cannot use
11072/// either and the argument would stop a stock binary from starting at all.
11073/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
11074///
11075/// The plain-spawn form, kept for the tests that assert its plan; spawns go
11076/// through [`apply_wire_spawn_args_for_admission`].
11077#[cfg(test)]
11078fn apply_wire_spawn_args(
11079    command: &mut Command,
11080    spec: &ModuleSpec,
11081    connection_file_path: Option<&std::path::Path>,
11082    handle: Option<&SupervisorHandle>,
11083) -> Result<Option<NonceHandoff>, SuperviseError> {
11084    apply_wire_spawn_args_for_role(
11085        command,
11086        spec,
11087        connection_file_path,
11088        handle,
11089        SpawnRole::Plain,
11090    )
11091}
11092
11093/// The read end of a spawn's launch-nonce pipe, prepared by
11094/// [`apply_wire_spawn_args_for_admission`] and installed as the child's descriptor 3
11095/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
11096/// server before spawn and authenticates its client after the child exists.
11097#[cfg(unix)]
11098type NonceHandoff = subc_os::LaunchNonceHandoff;
11099#[cfg(windows)]
11100type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
11101#[cfg(not(any(unix, windows)))]
11102type NonceHandoff = std::convert::Infallible;
11103
11104/// Create the launch nonce that a new module process must present when it
11105/// registers or opens a consumer connection, including a replacement process
11106/// started alongside a still-serving module.
11107///
11108/// A plain spawn replaces the module's recorded nonce. A swap candidate records
11109/// a separate candidate token so the still-serving incumbent and its consumers
11110/// keep their nonce. Both records are installed before the process exists, so
11111/// the child's initial HELLO registration cannot arrive ahead of its nonce.
11112///
11113/// On Unix the nonce is delivered only through a pipe. It is written into
11114/// a pipe whose read end the child gets as descriptor 3, named by
11115/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
11116/// process of the same user cannot read it with `ps eww`. That handoff is
11117/// returned rather than installed here. The combined descriptor placement runs
11118/// after the cgroup callback and placement barrier, so replacing slots 3-5
11119/// cannot close a descriptor either callback still needs.
11120/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
11121#[cfg(test)]
11122fn apply_wire_spawn_args_for_role(
11123    command: &mut Command,
11124    spec: &ModuleSpec,
11125    connection_file_path: Option<&std::path::Path>,
11126    handle: Option<&SupervisorHandle>,
11127    role: SpawnRole,
11128) -> Result<Option<NonceHandoff>, SuperviseError> {
11129    apply_wire_spawn_args_for_admission(command, spec, connection_file_path, handle, role, None)
11130}
11131
11132fn apply_wire_spawn_args_for_admission(
11133    command: &mut Command,
11134    spec: &ModuleSpec,
11135    connection_file_path: Option<&std::path::Path>,
11136    handle: Option<&SupervisorHandle>,
11137    role: SpawnRole,
11138    admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11139) -> Result<Option<NonceHandoff>, SuperviseError> {
11140    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
11141    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
11142    // included: a daemon started from a module's process tree inherits it,
11143    // and passing it on would point the child at a descriptor it does not
11144    // have.
11145    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
11146    #[cfg(windows)]
11147    {
11148        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
11149        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
11150    }
11151    // Remove inherited or configured copies too: withholding must mean absent.
11152    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
11153    command.env_remove(subc_os::registration_channel::REGISTRATION_FD_ENV);
11154    command.env_remove(subc_os::registration_channel::REGISTRATION_PIPE_ENV);
11155    command.env_remove(subc_os::registration_channel::DAEMON_PID_ENV);
11156    if role == SpawnRole::Plain {
11157        if let Some(handle) = handle {
11158            #[cfg(all(test, unix))]
11159            {
11160                let pause = {
11161                    let mut slot = handle.plain_identity_policy_pause.lock().unwrap();
11162                    if slot
11163                        .as_ref()
11164                        .is_some_and(|(module, _)| module == &spec.module_id)
11165                    {
11166                        slot.take().map(|(_, pause)| pause)
11167                    } else {
11168                        None
11169                    }
11170                };
11171                if let Some(pause) = pause {
11172                    pause.reached.send(()).unwrap();
11173                    pause.release.recv().unwrap();
11174                }
11175            }
11176            // A rescan preserves the running child's nonce and reservation.
11177            // Replacement spawn preparation applies the new protocol now,
11178            // before protocol:none can return without minting a nonce.
11179            handle.apply_identity_policy(spec, spec.protocol);
11180        }
11181    }
11182    if spec.protocol == ModuleProtocol::None {
11183        return Ok(None);
11184    }
11185    if let Some(connection_file_path) = connection_file_path {
11186        command.arg(SUBC_ARG).arg(connection_file_path);
11187    }
11188
11189    if admission.is_some_and(|admission| admission.launch.is_some()) {
11190        if let Some(handle) = handle {
11191            match role {
11192                SpawnRole::Plain => {
11193                    handle
11194                        .spawn_nonces
11195                        .lock()
11196                        .unwrap_or_else(|p| p.into_inner())
11197                        .remove(&spec.module_id);
11198                    if spec.reserved {
11199                        handle
11200                            .reserved_nonces
11201                            .lock()
11202                            .unwrap_or_else(|p| p.into_inner())
11203                            .insert(spec.module_id.clone(), None);
11204                    }
11205                }
11206                // Swaps still need their lifecycle record, but the empty value
11207                // is not a bearer token and is never accepted by nonce HELLO.
11208                SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, String::new()),
11209            }
11210        }
11211        return Ok(None);
11212    }
11213    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
11214    // route.open attestation. Reserved modules additionally use the same nonce
11215    // for HELLO id-squatting protection. A respawn rotates both records.
11216    let nonce = generate_launch_nonce()?;
11217    if let Some(handle) = handle {
11218        match role {
11219            SpawnRole::Plain => {
11220                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
11221                if spec.reserved {
11222                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
11223                }
11224            }
11225            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
11226        }
11227    }
11228    #[cfg(unix)]
11229    let handoff = {
11230        let handoff =
11231            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
11232                program: spec.program.clone(),
11233                source,
11234                cgroup_path: None,
11235            })?;
11236        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
11237        Some(handoff)
11238    };
11239    #[cfg(windows)]
11240    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
11241    #[cfg(not(any(unix, windows)))]
11242    let handoff = None;
11243    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
11244    // until the daemon's per-spawn delivery records show that every running
11245    // Windows module has read its secret from the named pipe.
11246    #[cfg(not(unix))]
11247    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
11248    Ok(handoff)
11249}
11250
11251pub(crate) fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
11252    command.env_remove(CK_LOG_ENV);
11253    // The spawn role is the supervisor's to set, and only on a swap candidate
11254    // (see `apply_spawn_role`). Removing it here, rather than just not setting
11255    // it, is what makes it absent on a plain spawn: the daemon's own
11256    // environment could carry it, and so could a spec built outside daemon
11257    // config (config refuses it as an `env` key). A module reading it on a
11258    // plain restart would pick the long swap budget and leave callers waiting.
11259    command.env_remove(SUBC_SPAWN_ROLE_ENV);
11260    for (key, value) in &spec.env {
11261        // cortexkit-log currently exposes retention only as a Rust struct, not
11262        // environment names. These values are daemon-private sink metadata and
11263        // must never become a public child-process contract by being inherited.
11264        if matches!(
11265            key.as_str(),
11266            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
11267        ) || key == SUBC_SPAWN_ROLE_ENV
11268        {
11269            continue;
11270        }
11271        command.env(key, value);
11272    }
11273}
11274
11275#[cfg(windows)]
11276fn prepare_windows_registration_pipe(
11277    command: &mut Command,
11278    spec: &ModuleSpec,
11279) -> Result<Option<SpawnRegistrationChannel>, SuperviseError> {
11280    if spec.protocol == ModuleProtocol::None {
11281        return Ok(None);
11282    }
11283    let pipe = subc_os::registration_channel::RegistrationPipe::new().map_err(|source| {
11284        SuperviseError::Spawn {
11285            program: spec.program.clone(),
11286            source,
11287            cgroup_path: None,
11288        }
11289    })?;
11290    command
11291        .env(
11292            subc_os::registration_channel::REGISTRATION_PIPE_ENV,
11293            pipe.name(),
11294        )
11295        .env(
11296            subc_os::registration_channel::DAEMON_PID_ENV,
11297            std::process::id().to_string(),
11298        );
11299    Ok(Some(pipe))
11300}
11301
11302#[cfg(windows)]
11303fn prepare_windows_nonce_pipe(
11304    command: &mut Command,
11305    spec: &ModuleSpec,
11306    nonce: &str,
11307) -> Option<NonceHandoff> {
11308    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
11309        Ok(handoff) => {
11310            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
11311            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
11312            // still register until every module reads the pipe. A reader without
11313            // this variable set to env returns the pipe error instead of using
11314            // an inherited or configured environment copy.
11315            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
11316            Some(handoff)
11317        }
11318        Err(error) => {
11319            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
11320            None
11321        }
11322    }
11323}
11324
11325#[cfg(windows)]
11326fn start_windows_nonce_pipe(
11327    handoff: Option<NonceHandoff>,
11328    pid: u32,
11329    spec: &ModuleSpec,
11330) -> Option<subc_os::LaunchNoncePipeDelivery> {
11331    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
11332    // same maximum wait used for registration during a reload. Older modules
11333    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
11334    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
11335        Ok(delivery) => Some(delivery),
11336        Err(error) => {
11337            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
11338            None
11339        }
11340    })
11341}
11342
11343fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
11344    #[cfg(windows)]
11345    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
11346        return Some(
11347            if snapshot
11348                .pipe_consumed
11349                .as_ref()
11350                .is_some_and(|consumed| consumed.get().is_some())
11351            {
11352                "pipe"
11353            } else {
11354                "env"
11355            }
11356            .to_owned(),
11357        );
11358    }
11359    let _ = snapshot;
11360    None
11361}
11362
11363/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
11364/// of a blue/green swap.
11365#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11366enum SpawnRole {
11367    Plain,
11368    SwapCandidate,
11369}
11370
11371/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
11372/// `apply_child_env` has already removed the variable for every spawn.
11373fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
11374    if role == SpawnRole::SwapCandidate {
11375        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
11376    }
11377}
11378
11379fn previous_instance_error(
11380    spec: &ModuleSpec,
11381    pid: u32,
11382    reason: crate::live_children::RetainedReason,
11383) -> SuperviseError {
11384    SuperviseError::Spawn {
11385        program: spec.program.clone(),
11386        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
11387        cgroup_path: None,
11388    }
11389}
11390
11391fn carries_containment(configured: bool, has_container: bool) -> bool {
11392    configured || has_container
11393}
11394
11395fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
11396    if !carries_containment(
11397        spec.carry_containment,
11398        containment::current(&roster.containment).is_some(),
11399    ) {
11400        return None;
11401    }
11402    Some(SuperviseError::Spawn {
11403        program: spec.program.clone(),
11404        source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
11405        cgroup_path: None,
11406    })
11407}
11408
11409#[allow(clippy::too_many_arguments)]
11410fn spawn_child(
11411    spec: &ModuleSpec,
11412    connection_file_path: Option<&std::path::Path>,
11413    handle: Option<&SupervisorHandle>,
11414    ring: &Arc<Mutex<StderrRing>>,
11415    capture_logs_dir: Option<&std::path::Path>,
11416    roster: &ChildRoster,
11417    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11418    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11419    register_timeout: Duration,
11420) -> Result<SupervisedChild, SuperviseError> {
11421    #[cfg(all(test, unix))]
11422    if let Some(source) = containment_tests::join_failure(spec, roster) {
11423        containment::terminate_blocking(&roster.containment);
11424        return Err(SuperviseError::Spawn {
11425            program: spec.program.clone(),
11426            source,
11427            cgroup_path: None,
11428        });
11429    }
11430    let result = spawn_child_in_slot(
11431        spec,
11432        connection_file_path,
11433        handle,
11434        ring,
11435        capture_logs_dir,
11436        roster,
11437        config_delivery,
11438        #[cfg(target_os = "linux")]
11439        cgroup_placement,
11440        SpawnRole::Plain,
11441        false,
11442        register_timeout,
11443    );
11444    if result.is_err() {
11445        if roster.admission_pending(&spec.module_id) {
11446            containment::terminate_deferred(&roster.containment);
11447        } else {
11448            containment::terminate_blocking(&roster.containment);
11449        }
11450    }
11451    result
11452}
11453
11454/// Spawn one process of `spec` into a slot.
11455///
11456/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
11457/// A swap candidate needs a different cgroup from the process it is replacing,
11458/// which is still alive: in the same cgroup the two would be one kill domain,
11459/// and killing a failed candidate could take the incumbent with it.
11460///
11461/// The stderr capture file is `<module_id>.stderr.log` for every process of
11462/// the module, whichever slot it is in, because that is the one file
11463/// `ck module logs` reads. During a swap's overlap both processes append to it;
11464/// the daemon writes whole lines, so the two interleave by line, which is also
11465/// the merged view an operator wants while a swap runs.
11466#[allow(clippy::too_many_arguments)]
11467fn spawn_child_in_slot(
11468    spec: &ModuleSpec,
11469    connection_file_path: Option<&std::path::Path>,
11470    handle: Option<&SupervisorHandle>,
11471    ring: &Arc<Mutex<StderrRing>>,
11472    capture_logs_dir: Option<&std::path::Path>,
11473    roster: &ChildRoster,
11474    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11475    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11476    role: SpawnRole,
11477    alternate_slot: bool,
11478    register_timeout: Duration,
11479) -> Result<SupervisedChild, SuperviseError> {
11480    let prepared = prepare_child_in_slot(
11481        spec,
11482        connection_file_path,
11483        handle,
11484        capture_logs_dir,
11485        roster,
11486        config_delivery,
11487        #[cfg(target_os = "linux")]
11488        cgroup_placement,
11489        role,
11490        alternate_slot,
11491        false,
11492    )?;
11493    let gate = config_delivery
11494        .map(|delivery| delivery.module_gate(spec))
11495        .transpose()?;
11496    let _gate = gate
11497        .as_ref()
11498        .map(|gate| {
11499            gate.try_lock()
11500                .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11501                    module_id: spec.module_id.clone(),
11502                })
11503        })
11504        .transpose()?;
11505    if let (Some(delivery), Some(admission)) = (config_delivery, &prepared.spawn_admission) {
11506        delivery.check(spec, admission)?;
11507    }
11508    let checked_base = prepared
11509        .spawn_admission
11510        .as_ref()
11511        .and_then(|admission| admission.approved_base())
11512        .cloned();
11513    let mut result = launch_child_in_slot(spec, ring, roster, prepared, register_timeout);
11514    drop(_gate);
11515    if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11516        (config_delivery, &result)
11517    {
11518        if let Some(base) = checked_base.as_ref().filter(|_| code == "program_refused") {
11519            if !delivery.record_refusal(spec, base, detail)? {
11520                result = Err(spawn_plan_stale(spec));
11521            }
11522        }
11523    }
11524    result
11525}
11526
11527struct AsyncInitialIdentity<'a> {
11528    supervisor: &'a Supervisor,
11529    module: String,
11530    committed: bool,
11531}
11532impl Drop for AsyncInitialIdentity<'_> {
11533    fn drop(&mut self) {
11534        if !self.committed {
11535            self.supervisor.abandon_unrostered(&self.module);
11536        }
11537    }
11538}
11539
11540/// Child-free preparation can be abandoned without orphaning a process. After
11541/// this value returns, the async caller validates its launch witness and creates
11542/// and publishes the child without another await.
11543struct PreparedChild {
11544    registration_spawn: Option<Arc<RegistrationSpawn>>,
11545    registration_handle: Option<SupervisorHandle>,
11546    #[cfg(any(unix, windows))]
11547    registration_channel: Option<SpawnRegistrationChannel>,
11548    config_context: Option<config_delivery::ConfigDelivery>,
11549    binding_publication: Option<ProgramBindingPublication>,
11550    launch_spec: ModuleSpec,
11551    spawn_admission: Option<crate::config_gate::runtime::SpawnAdmission>,
11552    command: Command,
11553    carrying: bool,
11554    carried: Option<Arc<containment::CarriedContainment>>,
11555    #[cfg(unix)]
11556    launch_identity: Option<containment::LaunchIdentity>,
11557    #[cfg(unix)]
11558    group_target: containment::GroupTarget,
11559    #[cfg(target_os = "linux")]
11560    cgroup_placement: Option<subc_cgroup::Placement>,
11561    #[cfg(target_os = "linux")]
11562    cgroup_name: String,
11563    #[cfg(target_os = "linux")]
11564    cgroup_cleanup: PreparedCgroup,
11565    cgroup_path: Option<PathBuf>,
11566    #[cfg(unix)]
11567    admission_gate: Option<admission::Gate>,
11568    output_sink: ChildOutputSink,
11569    #[cfg(target_os = "macos")]
11570    privacy_exec: Option<PrivacyExec>,
11571    #[cfg(windows)]
11572    image_capture: subc_os::ExecutableCapture,
11573    #[cfg(windows)]
11574    nonce_handoff: Option<NonceHandoff>,
11575    spawned_at: Instant,
11576    config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
11577    delivery_registration: Option<config_delivery::RegistrationGuard>,
11578    program_binding: Option<SpawnProgramBinding>,
11579}
11580
11581#[cfg(target_os = "linux")]
11582struct PreparedCgroup(Option<(subc_cgroup::Placement, String)>);
11583#[cfg(target_os = "linux")]
11584impl Drop for PreparedCgroup {
11585    fn drop(&mut self) {
11586        if let Some((placement, name)) = &self.0 {
11587            remove_module_cgroup(placement, name);
11588        }
11589    }
11590}
11591
11592pub(crate) struct SpawnProgramBinding {
11593    pub(crate) path: PathBuf,
11594    // Keep the inode alive if the configured name is replaced while preparation
11595    // is suspended, so replacement cannot pass through inode reuse.
11596    _file: std::fs::File,
11597    pub(crate) identity: subc_os::RetainedFileIdentity,
11598}
11599
11600impl SpawnProgramBinding {
11601    pub(crate) fn capture(spec: &ModuleSpec) -> Result<Self, SuperviseError> {
11602        Self::capture_after_resolution(spec, |_| {})
11603    }
11604
11605    fn capture_after_resolution(
11606        spec: &ModuleSpec,
11607        before_open: impl FnOnce(&std::path::Path),
11608    ) -> Result<Self, SuperviseError> {
11609        require_program_search_path(spec)?;
11610        #[cfg(windows)]
11611        let windows_capture =
11612            windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
11613                program: spec.program.clone(),
11614                source,
11615                cgroup_path: None,
11616            })?;
11617        #[cfg(windows)]
11618        let path = windows_capture.path().to_path_buf();
11619        #[cfg(not(windows))]
11620        let path = if spec.program.is_absolute() || spec.program.components().count() > 1 {
11621            spec.program.clone()
11622        } else {
11623            let path = spec
11624                .env
11625                .iter()
11626                .rev()
11627                .find(|(key, _)| key == "PATH")
11628                .map(|(_, path)| std::ffi::OsString::from(path))
11629                .or_else(|| std::env::var_os("PATH"))
11630                .unwrap_or_else(|| {
11631                    if cfg!(target_os = "macos") {
11632                        // Match Darwin's spawn search when PATH is absent, not
11633                        // the current-directory search of an explicitly empty PATH.
11634                        "/usr/bin:/bin".into()
11635                    } else {
11636                        Default::default()
11637                    }
11638                });
11639            std::env::split_paths(&path)
11640                .map(|dir| dir.join(&spec.program))
11641                .find(|path| {
11642                    if !path.is_file() {
11643                        return false;
11644                    }
11645                    #[cfg(unix)]
11646                    {
11647                        rustix::fs::accessat(
11648                            rustix::fs::CWD,
11649                            path,
11650                            rustix::fs::Access::EXEC_OK,
11651                            rustix::fs::AtFlags::EACCESS,
11652                        )
11653                        .is_ok()
11654                    }
11655                    #[cfg(not(unix))]
11656                    {
11657                        true
11658                    }
11659                })
11660                .ok_or_else(|| SuperviseError::Spawn {
11661                    program: spec.program.clone(),
11662                    source: io::Error::new(io::ErrorKind::NotFound, "executable not found in PATH"),
11663                    cgroup_path: None,
11664                })?
11665        };
11666        let resolved = std::fs::canonicalize(&path).map_err(|source| SuperviseError::Spawn {
11667            program: spec.program.clone(),
11668            source,
11669            cgroup_path: None,
11670        })?;
11671        before_open(&resolved);
11672        // Resolving links does not open their target. The descriptor open must
11673        // remain nonblocking and no-follow if that resolved name is substituted.
11674        let file =
11675            subc_os::open_regular_nofollow(&resolved).map_err(|source| SuperviseError::Spawn {
11676                program: spec.program.clone(),
11677                source: io::Error::other(source),
11678                cgroup_path: None,
11679            })?;
11680        let identity =
11681            subc_os::retained_file_identity(&file).map_err(|source| SuperviseError::Spawn {
11682                program: spec.program.clone(),
11683                source,
11684                cgroup_path: None,
11685            })?;
11686        Ok(Self {
11687            path,
11688            _file: file,
11689            identity,
11690        })
11691    }
11692    fn current(&self) -> bool {
11693        #[cfg(unix)]
11694        {
11695            subc_os::file_identity(&self.path) == Some(self.identity)
11696        }
11697        #[cfg(windows)]
11698        {
11699            std::fs::canonicalize(&self.path)
11700                .ok()
11701                .and_then(|path| subc_os::open_regular_nofollow(&path).ok())
11702                .and_then(|file| subc_os::retained_file_identity(&file).ok())
11703                == Some(self.identity)
11704        }
11705    }
11706}
11707
11708fn spawn_plan_stale(spec: &ModuleSpec) -> SuperviseError {
11709    SuperviseError::ConfigDeliveryDeferred {
11710        module_id: spec.module_id.clone(),
11711    }
11712}
11713
11714fn require_program_search_path(spec: &ModuleSpec) -> Result<(), SuperviseError> {
11715    if !cfg!(target_os = "macos")
11716        && !spec.program.is_absolute()
11717        && spec.program.components().count() == 1
11718        && !spec.env.iter().any(|(key, _)| {
11719            if cfg!(windows) {
11720                key.eq_ignore_ascii_case("PATH")
11721            } else {
11722                key == "PATH"
11723            }
11724        })
11725        && std::env::var_os("PATH").is_none()
11726    {
11727        // The OS default search is not a current-directory search. Refusing an
11728        // unspecified search path prevents approving a decoy in the current directory.
11729        return Err(SuperviseError::Spawn {
11730            program: spec.program.clone(),
11731            source: io::Error::new(io::ErrorKind::InvalidInput, "bare program requires PATH"),
11732            cgroup_path: None,
11733        });
11734    }
11735    Ok(())
11736}
11737
11738async fn spawn_child_async(
11739    spec: &ModuleSpec,
11740    runtime: &SupervisorRuntimeConfig,
11741) -> Result<SupervisedChild, SuperviseError> {
11742    #[cfg(all(test, unix))]
11743    if let Some(source) = containment_tests::join_failure(spec, &runtime.child_roster) {
11744        containment::terminate_blocking(&runtime.child_roster.containment);
11745        return Err(SuperviseError::Spawn {
11746            program: spec.program.clone(),
11747            source,
11748            cgroup_path: None,
11749        });
11750    }
11751    let result = spawn_child_in_slot_async(spec, runtime, SpawnRole::Plain, false).await;
11752    if result.is_err() {
11753        if runtime.child_roster.admission_pending(&spec.module_id) {
11754            containment::terminate_deferred(&runtime.child_roster.containment);
11755        } else {
11756            containment::terminate_blocking(&runtime.child_roster.containment);
11757        }
11758    }
11759    result
11760}
11761
11762async fn spawn_child_in_slot_async(
11763    spec: &ModuleSpec,
11764    runtime: &SupervisorRuntimeConfig,
11765    role: SpawnRole,
11766    alternate_slot: bool,
11767) -> Result<SupervisedChild, SuperviseError> {
11768    let gate = runtime
11769        .config_delivery
11770        .as_ref()
11771        .map(|delivery| delivery.0.module_gate(&spec.module_id))
11772        .transpose()
11773        .map_err(|error| SuperviseError::InvalidSpec {
11774            reason: error.to_string(),
11775        })?;
11776    if gate.as_ref().is_some_and(|gate| gate.try_lock().is_err()) {
11777        return Err(SuperviseError::ConfigDeliveryDeferred {
11778            module_id: spec.module_id.clone(),
11779        });
11780    }
11781    let prepared_spec = spec.clone();
11782    let prepared_runtime = runtime.clone();
11783    let prepared = tokio::task::spawn_blocking(move || {
11784        let prepared = prepare_child_in_slot(
11785            &prepared_spec,
11786            prepared_runtime.connection_file_path.as_deref(),
11787            prepared_runtime.supervisor_handle.as_ref(),
11788            prepared_runtime.capture_logs_dir.as_deref(),
11789            &prepared_runtime.child_roster,
11790            prepared_runtime.spawn_delivery(),
11791            #[cfg(target_os = "linux")]
11792            prepared_runtime.cgroup_placement.as_ref(),
11793            role,
11794            alternate_slot,
11795            true,
11796        );
11797        prepared
11798    })
11799    .await
11800    .map_err(|_| SuperviseError::InvalidSpec {
11801        reason: "configuration launch preparation stopped".into(),
11802    })??;
11803    if let Some(registration) = &prepared.delivery_registration {
11804        if !registration.launch_current()? {
11805            return Err(spawn_plan_stale(spec));
11806        }
11807    }
11808    let _gate = match gate {
11809        Some(gate) => {
11810            Some(
11811                gate.try_lock_owned()
11812                    .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11813                        module_id: spec.module_id.clone(),
11814                    })?,
11815            )
11816        }
11817        None => None,
11818    };
11819    if let (Some(delivery), Some(admission)) = (runtime.spawn_delivery(), &prepared.spawn_admission)
11820    {
11821        delivery.check(spec, admission)?;
11822    }
11823    if prepared
11824        .spawn_admission
11825        .as_ref()
11826        .is_none_or(|admission| admission.launch.is_none())
11827        && prepared
11828            .program_binding
11829            .as_ref()
11830            .is_some_and(|binding| !binding.current())
11831    {
11832        return Err(spawn_plan_stale(spec));
11833    }
11834    // No await may separate this last identity check from ownership publication.
11835    let checked_base = prepared
11836        .spawn_admission
11837        .as_ref()
11838        .and_then(|admission| admission.approved_base())
11839        .cloned();
11840    let mut result = launch_child_in_slot(
11841        spec,
11842        &runtime.stderr_ring,
11843        &runtime.child_roster,
11844        prepared,
11845        runtime.register_timeout,
11846    );
11847    drop(_gate);
11848    if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11849        (runtime.config_delivery.as_ref(), &result)
11850    {
11851        if let Some(base) = checked_base.filter(|_| code == "program_refused") {
11852            let module = spec.module_id.clone();
11853            let identity = subc_config::raw_digest(detail.as_bytes());
11854            let recorded = delivery
11855                .0
11856                .on_blocking(move |owner| owner.record_program_refusal(&module, &base, identity))
11857                .await;
11858            match recorded {
11859                Ok(true) => {}
11860                Ok(false) => result = Err(spawn_plan_stale(spec)),
11861                Err(error) => {
11862                    result = Err(config_delivery::gate_hold_error(
11863                        spec,
11864                        error,
11865                        Some(delivery),
11866                    ))
11867                }
11868            }
11869        }
11870    }
11871    result
11872}
11873
11874fn validate_spawn_program(
11875    spec: &ModuleSpec,
11876    admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11877    delivery: Option<config_delivery::SpawnDelivery<'_>>,
11878) -> Result<(), SuperviseError> {
11879    if let Some(admission) = admission {
11880        if let Some(program) = &admission.launch {
11881            if let Err(error) = program.validate_for_spawn() {
11882                // A captured unsigned image is a definite refusal. Uncertain
11883                // reads are deferred by validate_for_spawn, not attributed to
11884                // the operator as a changed or unapprovable program.
11885                if let (Some(context), Some(base)) = (delivery, admission.approved_base()) {
11886                    if error.code == "program_refused"
11887                        && !context.record_refusal(spec, base, &error.detail)?
11888                    {
11889                        return Err(spawn_plan_stale(spec));
11890                    }
11891                }
11892                return Err(config_delivery::gate_hold_error(
11893                    spec,
11894                    error,
11895                    delivery.map(|context| context.runtime()).as_ref(),
11896                ));
11897            }
11898        }
11899    }
11900    Ok(())
11901}
11902
11903#[allow(clippy::too_many_arguments)]
11904fn prepare_child_in_slot(
11905    spec: &ModuleSpec,
11906    connection_file_path: Option<&std::path::Path>,
11907    handle: Option<&SupervisorHandle>,
11908    capture_logs_dir: Option<&std::path::Path>,
11909    roster: &ChildRoster,
11910    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11911    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11912    role: SpawnRole,
11913    alternate_slot: bool,
11914    asynchronous: bool,
11915) -> Result<PreparedChild, SuperviseError> {
11916    require_program_search_path(spec)?;
11917    let spawn_admission = config_delivery
11918        .map(|delivery| delivery.prepare(spec))
11919        .transpose()?;
11920    #[cfg(target_os = "macos")]
11921    let gated = spawn_admission
11922        .as_ref()
11923        .is_some_and(|admission| admission.approved_base().is_some());
11924    let launch_spec = spawn_admission
11925        .as_ref()
11926        .and_then(|admission| admission.launch.as_ref())
11927        .map(|launch| launch.spec.clone())
11928        .unwrap_or_else(|| spec.clone());
11929    let spec = &launch_spec;
11930    if let Some(error) = recovery_block_error(spec, roster) {
11931        return Err(error);
11932    }
11933    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
11934        return Err(previous_instance_error(spec, pid, reason));
11935    }
11936    if roster.is_closed() {
11937        return Err(SuperviseError::Spawn {
11938            program: spec.program.clone(),
11939            source: io::Error::other("the daemon is shutting down; not starting a new process"),
11940            cgroup_path: None,
11941        });
11942    }
11943    let carried = containment::current(&roster.containment);
11944    let carrying = carries_containment(spec.carry_containment, carried.is_some());
11945    if role != SpawnRole::Plain && carrying {
11946        return Err(SuperviseError::InvalidSpec {
11947            reason: "a carried container cannot overlap a swap candidate".to_owned(),
11948        });
11949    }
11950    #[cfg(unix)]
11951    let launch_identity = if carrying && carried.is_none() {
11952        Some(
11953            containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
11954                program: spec.program.clone(),
11955                source,
11956                cgroup_path: None,
11957            })?,
11958        )
11959    } else {
11960        None
11961    };
11962    #[cfg(target_os = "linux")]
11963    let cgroup_placement = match &carried {
11964        Some(container) => container.cgroup().map(|(placement, _)| placement),
11965        None => cgroup_placement,
11966    };
11967    #[cfg(unix)]
11968    let group_target = if let Some(container) = carried
11969        .as_ref()
11970        .filter(|container| container.group().is_some())
11971    {
11972        containment::GroupTarget::Pinned(Arc::clone(container))
11973    } else {
11974        #[cfg(target_os = "linux")]
11975        let cgroup_container = carrying && cgroup_placement.is_some();
11976        #[cfg(not(target_os = "linux"))]
11977        let cgroup_container = false;
11978        if cgroup_container {
11979            containment::GroupTarget::NoGroup
11980        } else {
11981            containment::GroupTarget::OwnLeader
11982        }
11983    };
11984    #[cfg(target_os = "linux")]
11985    let cgroup_name = {
11986        // Slot names alone are not kill domains: a retired incumbent may still
11987        // be draining when a later enable/restart spawns into the same slot.
11988        // Decimal entropy keeps the suffix unambiguous; Placement performs
11989        // the module-id escaping and constructs the filesystem path.
11990        if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
11991            name.to_owned()
11992        } else if cgroup_placement.is_none() {
11993            swap::cgroup_name(&spec.module_id, alternate_slot)
11994        } else {
11995            let nonce = generate_launch_nonce()?;
11996            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
11997            // Leave room for byte escaping and the suffix under NAME_MAX. The
11998            // label is only for humans; the nonce identifies the kill domain.
11999            let mut end = spec.module_id.len().min(64);
12000            while !spec.module_id.is_char_boundary(end) {
12001                end -= 1;
12002            }
12003            format!(
12004                "{}_{suffix}",
12005                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
12006            )
12007        }
12008    };
12009    #[cfg(not(target_os = "linux"))]
12010    let _ = alternate_slot;
12011    #[cfg(windows)]
12012    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
12013        program: spec.program.clone(),
12014        source,
12015        cgroup_path: None,
12016    })?;
12017    #[cfg(windows)]
12018    let program_binding = if asynchronous {
12019        let mut resolved = spec.clone();
12020        resolved.program = image_capture.path().to_path_buf();
12021        Some(SpawnProgramBinding::capture(&resolved)?)
12022    } else {
12023        None
12024    };
12025    #[cfg(not(windows))]
12026    let program_binding = (asynchronous || cfg!(target_os = "macos"))
12027        .then(|| SpawnProgramBinding::capture(spec))
12028        .transpose()?;
12029    #[cfg(target_os = "macos")]
12030    validate_spawn_program(spec, spawn_admission.as_ref(), config_delivery)?;
12031    #[cfg(target_os = "macos")]
12032    let (mut command, privacy_exec, exec_ack) = privacy_command(
12033        spec,
12034        program_binding
12035            .as_ref()
12036            .expect("macOS captures every launch"),
12037        roster,
12038        // Ungated launches retain legacy unsuspended exec, including scripts. Suspended
12039        // verification admits approved programs only. prepare() above fails
12040        // closed if the gate cannot decide; an image sample is not authority.
12041        gated,
12042    )?;
12043    #[cfg(windows)]
12044    let mut command = Command::new(image_capture.path());
12045    #[cfg(not(any(target_os = "macos", windows)))]
12046    let mut command = Command::new(
12047        program_binding
12048            .as_ref()
12049            .map_or(spec.program.as_path(), |binding| binding.path.as_path()),
12050    );
12051    command.args(&spec.args);
12052    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
12053    // that is the whole of the intent, so remove that one key rather than the
12054    // environment.
12055    //
12056    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
12057    // and took the POSIX environment with it. Modules spawned that way had no
12058    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
12059    // logging:
12060    //
12061    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
12062    //     both unset it fell back to the temp dir alone and `ck` could not find
12063    //     a daemon running on the same machine from inside any module's process
12064    //     tree — reporting a path the file has never lived at, which reads as
12065    //     "the daemon did not write its file".
12066    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
12067    //     the RELATIVE `.local/share`, so a module deriving its own store path
12068    //     resolved it against its own CWD. That is the store-fragmentation
12069    //     defect the daemon already refuses in config (`parse_doc` rejects a
12070    //     relative `storage.data_home`) arriving by derivation instead.
12071    //   * anything a module spawns inherited it: git without ~/.gitconfig,
12072    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
12073    //     quietly rather than erroring.
12074    //
12075    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
12076    // offered one candidate under /tmp while the file sat in /run/user/1000.
12077    //
12078    // A configured module is unaffected either way: `module_spec()` puts the
12079    // resolved CK_LOG into `spec.env`, which is applied below and therefore
12080    // wins over anything ambient.
12081    apply_child_env(&mut command, spec);
12082    let origin_witness = crate::program_origin::LaunchWitness::capture_command(spec, &command)
12083        .map_err(|source| SuperviseError::Spawn {
12084            program: spec.program.clone(),
12085            source,
12086            cgroup_path: None,
12087        })?;
12088    command.current_dir(&origin_witness.working_directory);
12089    apply_spawn_role(&mut command, role);
12090    let nonce_handoff = apply_wire_spawn_args_for_admission(
12091        &mut command,
12092        spec,
12093        connection_file_path,
12094        handle,
12095        role,
12096        spawn_admission.as_ref(),
12097    )?;
12098    #[cfg(unix)]
12099    let mut slots = subc_os::registration_channel::ChildDescriptorSlots::default();
12100    #[cfg(unix)]
12101    let registration_channel = if spec.protocol == ModuleProtocol::Subc {
12102        let (parent, child, metadata) = subc_os::registration_channel::registration_socketpair()
12103            .map_err(|source| SuperviseError::Spawn {
12104                program: spec.program.clone(),
12105                source,
12106                cgroup_path: None,
12107            })?;
12108        slots
12109            .channel(child)
12110            .map_err(|source| SuperviseError::Spawn {
12111                program: spec.program.clone(),
12112                source,
12113                cgroup_path: None,
12114            })?;
12115        command
12116            .env(subc_os::registration_channel::REGISTRATION_FD_ENV, metadata)
12117            .env(
12118                subc_os::registration_channel::DAEMON_PID_ENV,
12119                std::process::id().to_string(),
12120            );
12121        Some(parent)
12122    } else {
12123        None
12124    };
12125    #[cfg(windows)]
12126    let registration_channel = prepare_windows_registration_pipe(&mut command, spec)?;
12127    let binding_publication = if spawn_admission
12128        .as_ref()
12129        .is_some_and(|admission| admission.launch.is_some())
12130    {
12131        handle.and_then(|handle| handle.begin_program_binding(&spec.module_id, role))
12132    } else {
12133        None
12134    };
12135    let registration_spawn = (spec.protocol == ModuleProtocol::Subc)
12136        .then(|| {
12137            handle.map(|handle| {
12138                let spawn = RegistrationSpawn::new_with_binding_requirement(
12139                    handle,
12140                    &spec.module_id,
12141                    role,
12142                    binding_publication.clone(),
12143                    spawn_admission
12144                        .as_ref()
12145                        .is_some_and(|admission| admission.launch.is_some()),
12146                );
12147                if let Some(admission) = &spawn_admission {
12148                    admission.initialize_origin(&spawn.authority, &origin_witness);
12149                }
12150                handle.record_nonce_launch(&spawn);
12151                spawn
12152            })
12153        })
12154        .flatten();
12155
12156    #[cfg(target_os = "linux")]
12157    let cgroup_path = cgroup_placement
12158        .map(|placement| {
12159            if let Some(container) = &carried {
12160                let path = container
12161                    .cgroup_path()
12162                    .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
12163                placement.existing_module_path(path)?.ok_or_else(|| {
12164                    io::Error::new(
12165                        io::ErrorKind::NotFound,
12166                        "carried cgroup disappeared before replacement",
12167                    )
12168                })
12169            } else {
12170                if carrying {
12171                    containment::prepare_cgroup_path(placement, &cgroup_name)
12172                } else {
12173                    placement.module_path(&cgroup_name)
12174                }
12175            }
12176        })
12177        .transpose()
12178        .map_err(|source| SuperviseError::Cgroup {
12179            module_id: spec.module_id.clone(),
12180            source,
12181        })?;
12182    #[cfg(not(target_os = "linux"))]
12183    let cgroup_path: Option<PathBuf> = None;
12184    // Own a newly created directory before any fallible admission or command
12185    // preparation. Existing carried containment must never be removed here.
12186    #[cfg(target_os = "linux")]
12187    let cgroup_cleanup = PreparedCgroup(
12188        cgroup_placement
12189            .filter(|_| carried.is_none())
12190            .map(|placement| (placement.clone(), cgroup_name.clone())),
12191    );
12192    #[cfg(unix)]
12193    let mut admission_gate = if carrying {
12194        Some(
12195            admission::Gate::install(&mut command, spec, roster).map_err(|source| {
12196                SuperviseError::Spawn {
12197                    program: spec.program.clone(),
12198                    source,
12199                    cgroup_path: cgroup_path.clone(),
12200                }
12201            })?,
12202        )
12203    } else {
12204        None
12205    };
12206    #[cfg(target_os = "linux")]
12207    if let Some(path) = &cgroup_path {
12208        apply_cgroup_placement(&mut command, spec, path)?;
12209    }
12210
12211    let output_sink = if let Some(logs_dir) = capture_logs_dir {
12212        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
12213        match ChildOutputSink::open(&path, capture_retention(spec)) {
12214            Ok(sink) => sink,
12215            Err(error) => {
12216                warn!(
12217                    module_id = %spec.module_id,
12218                    path = %path.display(),
12219                    error = %error,
12220                    "could not open child output capture file; forwarding to stderr"
12221                );
12222                ChildOutputSink::Stderr
12223            }
12224        }
12225    } else {
12226        ChildOutputSink::Stderr
12227    };
12228
12229    command.stdout(Stdio::piped());
12230    command.stderr(Stdio::piped());
12231    command.kill_on_drop(!carrying);
12232    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
12233    // before exec). In the daemon's group, a service manager that kills the
12234    // job's process group when the daemon exits (launchd's default) killed
12235    // every module at the same moment its control connection closed, so no
12236    // module ever ran its EOF teardown on a daemon stop. Outside that group a
12237    // module is reached only by the daemon: the EOF it sees when its
12238    // connection closes, and the bounded stop in `child_roster` for anything
12239    // still running after that. On Linux this composes with the cgroup
12240    // placement above: that is a pre_exec write to cgroup.procs, std performs
12241    // setpgid in the child before running pre_exec callbacks, and the two
12242    // change independent process attributes.
12243    //
12244    // stdin is /dev/null because a process outside the terminal's foreground
12245    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
12246    // by hand would otherwise hand down. Under a service manager stdin is
12247    // already /dev/null.
12248    #[cfg(unix)]
12249    command.process_group(
12250        carried
12251            .as_ref()
12252            .map(|container| container.verified_join_group())
12253            .transpose()
12254            .map_err(|source| SuperviseError::Spawn {
12255                program: spec.program.clone(),
12256                source,
12257                cgroup_path: cgroup_path.clone(),
12258            })?
12259            .flatten()
12260            .unwrap_or(0) as i32,
12261    );
12262    command.stdin(Stdio::null());
12263    #[cfg(unix)]
12264    if let Some(gate) = &mut admission_gate {
12265        gate.install_placement(&mut command);
12266    }
12267    // Fill all reserved slots in one allocation-free step, after placement has
12268    // stopped using descriptors that the fixed slots may replace.
12269    #[cfg(unix)]
12270    {
12271        if let Some(handoff) = nonce_handoff {
12272            slots.nonce(handoff);
12273        }
12274        #[cfg(target_os = "macos")]
12275        slots
12276            .acknowledgement(exec_ack.into_descriptor())
12277            .map_err(|source| SuperviseError::Spawn {
12278                program: spec.program.clone(),
12279                source,
12280                cgroup_path: cgroup_path.clone(),
12281            })?;
12282        slots.install(command.as_std_mut());
12283    }
12284    #[cfg(not(any(unix, windows)))]
12285    let _ = nonce_handoff;
12286
12287    // Containment, step 1 of 3 (issue #109): create the child suspended so it
12288    // cannot run a single instruction -- and therefore cannot spawn a
12289    // grandchild -- before it is in the job. See `contain_spawned_child` for the
12290    // other two steps and why the window matters.
12291    #[cfg(windows)]
12292    {
12293        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
12294            program: spec.program.clone(),
12295            source,
12296            cgroup_path: None,
12297        })?;
12298        subc_jobobject::suspend_on_create_async(&mut command);
12299    }
12300    // Timestamp before the spawn call: on Unix the child can serve before that
12301    // call even returns. This conservative bound never labels an older child
12302    // as a replacement for a request received during process creation.
12303    let spawned_at = Instant::now();
12304    let delivery_context = config_delivery;
12305    let config_delivery = config_delivery
12306        .map(|delivery| {
12307            delivery.spawned(
12308                spec,
12309                spawn_admission
12310                    .as_ref()
12311                    .expect("delivery prepares spawn admission"),
12312                spawned_at,
12313                role,
12314                registration_spawn
12315                    .as_ref()
12316                    .map(|spawn| spawn.authority.clone()),
12317            )
12318        })
12319        .transpose()?
12320        .flatten();
12321    let mut delivery_registration =
12322        delivery_context.map(|context| context.registration_guard(config_delivery.as_ref(), role));
12323    if let Some(registration) = &mut delivery_registration {
12324        registration.defer_cleanup();
12325    }
12326    if asynchronous {
12327        #[cfg(test)]
12328        if let Some(context) = delivery_context {
12329            context.pause_preflight_for_test();
12330        }
12331    }
12332    validate_spawn_program(spec, spawn_admission.as_ref(), delivery_context)?;
12333    Ok(PreparedChild {
12334        registration_spawn,
12335        registration_handle: handle.cloned(),
12336        #[cfg(any(unix, windows))]
12337        registration_channel,
12338        config_context: delivery_context.map(|delivery| delivery.runtime()),
12339        binding_publication,
12340        launch_spec,
12341        spawn_admission,
12342        command,
12343        carrying,
12344        carried: carried.clone(),
12345        #[cfg(unix)]
12346        launch_identity,
12347        #[cfg(unix)]
12348        group_target,
12349        #[cfg(target_os = "linux")]
12350        cgroup_placement: cgroup_placement.cloned(),
12351        #[cfg(target_os = "linux")]
12352        cgroup_cleanup,
12353        #[cfg(target_os = "linux")]
12354        cgroup_name,
12355        cgroup_path,
12356        #[cfg(unix)]
12357        admission_gate,
12358        output_sink,
12359        #[cfg(target_os = "macos")]
12360        privacy_exec,
12361        #[cfg(windows)]
12362        image_capture,
12363        #[cfg(windows)]
12364        nonce_handoff,
12365        spawned_at,
12366        config_delivery,
12367        delivery_registration,
12368        program_binding,
12369    })
12370}
12371
12372fn launch_child_in_slot(
12373    _spec: &ModuleSpec,
12374    ring: &Arc<Mutex<StderrRing>>,
12375    roster: &ChildRoster,
12376    prepared: PreparedChild,
12377    register_timeout: Duration,
12378) -> Result<SupervisedChild, SuperviseError> {
12379    let launch_spec = prepared.launch_spec.clone();
12380    let registration_declaration = prepared
12381        .spawn_admission
12382        .as_ref()
12383        .and_then(|admission| admission.registration_declaration(&launch_spec.module_id));
12384    let admission_base = prepared
12385        .spawn_admission
12386        .as_ref()
12387        .and_then(|admission| admission.approved_base())
12388        .cloned();
12389    let spec = &launch_spec;
12390    let expected_image = prepared
12391        .spawn_admission
12392        .as_ref()
12393        .and_then(|admission| admission.launch.as_ref())
12394        .map(|launch| launch.image);
12395    let program_launch = prepared
12396        .spawn_admission
12397        .and_then(|admission| admission.launch);
12398    let config_owner = prepared
12399        .config_context
12400        .as_ref()
12401        .map(|context| context.0.clone());
12402    let binding_publication = prepared.binding_publication;
12403    #[cfg(test)]
12404    if let Some(context) = &prepared.config_context {
12405        context.0.before_spawn_exec_for_test();
12406    }
12407    #[cfg(feature = "test-support")]
12408    if let Some(context) = &prepared.config_context {
12409        context
12410            .0
12411            .pause_spawn_before_exec()
12412            .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(context)))?;
12413    }
12414    let mut command = prepared.command;
12415    let carrying = prepared.carrying;
12416    let carried = prepared.carried;
12417    #[cfg(unix)]
12418    let launch_identity = prepared.launch_identity;
12419    #[cfg(unix)]
12420    let group_target = prepared.group_target;
12421    #[cfg(target_os = "linux")]
12422    let cgroup_placement = prepared.cgroup_placement.as_ref();
12423    #[cfg(target_os = "linux")]
12424    let cgroup_name = prepared.cgroup_name;
12425    #[cfg(target_os = "linux")]
12426    let mut cgroup_cleanup = prepared.cgroup_cleanup;
12427    #[cfg(target_os = "linux")]
12428    let _ = cgroup_cleanup.0.take();
12429    let cgroup_path = prepared.cgroup_path;
12430    #[cfg(unix)]
12431    let admission_gate = prepared.admission_gate;
12432    let output_sink = prepared.output_sink;
12433    #[cfg(target_os = "macos")]
12434    let mut privacy_exec = prepared.privacy_exec;
12435    #[cfg(target_os = "macos")]
12436    if let Some(pending) = &mut privacy_exec {
12437        pending.admission_binding = binding_publication.clone();
12438        if expected_image.is_some() {
12439            pending.expected = expected_image;
12440        }
12441    }
12442    #[cfg(any(unix, windows))]
12443    let registration_channel = prepared.registration_channel;
12444    let registration_spawn = prepared.registration_spawn;
12445    let registration_handle = prepared.registration_handle;
12446    #[cfg(windows)]
12447    let image_capture = prepared.image_capture;
12448    #[cfg(windows)]
12449    let nonce_handoff = prepared.nonce_handoff;
12450    let spawned_at = prepared.spawned_at;
12451    let config_delivery = prepared.config_delivery;
12452    let delivery_registration = prepared.delivery_registration;
12453    let _program_binding = prepared.program_binding;
12454    #[cfg(all(test, unix))]
12455    containment_tests::record_spawn_attempt(spec, &mut command);
12456    #[cfg(unix)]
12457    let spawned = match admission_gate {
12458        Some(gate) => gate
12459            .spawn(
12460                command,
12461                spec,
12462                roster,
12463                launch_identity,
12464                #[cfg(target_os = "linux")]
12465                cgroup_placement
12466                    .zip(cgroup_path.as_ref())
12467                    .map(|(placement, path)| {
12468                        (placement.clone(), cgroup_name.clone(), path.clone())
12469                    }),
12470            )
12471            .map(|(child, admitted)| (child, Some(admitted))),
12472        None => {
12473            let spawned =
12474                subc_os::child_spawn::tokio_spawn(&mut command).map(|child| (child, None));
12475            // The callback owns all child-only sources. Closing them here is
12476            // necessary for channel EOF and macOS exec acknowledgement EOF.
12477            drop(command);
12478            spawned
12479        }
12480    };
12481    #[cfg(not(unix))]
12482    let spawned = subc_os::child_spawn::tokio_spawn(&mut command);
12483    let spawned = match spawned {
12484        Ok(child) => child,
12485        Err(source) => {
12486            if carried
12487                .as_ref()
12488                .and_then(|container| container.group())
12489                .is_some()
12490            {
12491                containment::terminate_blocking(&roster.containment);
12492            }
12493            #[cfg(target_os = "linux")]
12494            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12495                remove_module_cgroup(placement, &cgroup_name);
12496            }
12497            return Err(SuperviseError::Spawn {
12498                program: spec.program.clone(),
12499                source,
12500                cgroup_path,
12501            });
12502        }
12503    };
12504    #[cfg(unix)]
12505    let (mut child, mut admitted) = spawned;
12506    #[cfg(unix)]
12507    let group_target = admitted
12508        .as_ref()
12509        .map(|attempt| attempt.group_target())
12510        .unwrap_or(group_target);
12511    #[cfg(not(unix))]
12512    let mut child = spawned;
12513    #[cfg(test)]
12514    spawn_record_tests::after_create(&child, spec);
12515
12516    // Bind while the child is still suspended, before job assignment resumes it.
12517    // Keeping the executable handle open across CreateProcessW denies rename
12518    // and deletion, so the file cannot be exchanged during spawn. Bind that
12519    // file ID to the process handle returned by spawn, never a reopened PID.
12520    #[cfg(windows)]
12521    let windows_image = match image_capture.bind_tokio(&child) {
12522        Ok(image) => Arc::new(image),
12523        Err(source) => {
12524            kill_and_reap_rejected_child(&mut child);
12525            return Err(SuperviseError::Spawn {
12526                program: spec.program.clone(),
12527                source,
12528                cgroup_path: None,
12529            });
12530        }
12531    };
12532    // Assign the suspended child to its module job; resume only after its
12533    // process identity has been durably recorded for orphan recovery.
12534    #[cfg(windows)]
12535    let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
12536    let spawned_at_ms = unix_ms_now();
12537    #[cfg(not(windows))]
12538    let spawned_from = spec.program.clone();
12539    #[cfg(windows)]
12540    let spawned_from = windows_image.path().to_path_buf();
12541    #[cfg(not(windows))]
12542    let spawned_file_identity = spawned_file_identity(&spawned_from);
12543    #[cfg(windows)]
12544    let spawned_file_identity = Some(SpawnedFileIdentity {
12545        device: 0,
12546        inode: 0,
12547        windows_image: Some(Arc::clone(&windows_image)),
12548    });
12549    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
12550        program: spec.program.clone(),
12551        source: io::Error::other("spawned child exposed no live pid"),
12552        cgroup_path: cgroup_path.clone(),
12553    })?;
12554    #[cfg(windows)]
12555    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
12556    #[cfg(not(windows))]
12557    let process_start_time = admitted
12558        .as_ref()
12559        .map(|admitted| admitted.birth)
12560        .or_else(|| crate::provenance::process_start_time(pid));
12561    #[cfg(windows)]
12562    let process_start_time = Some(windows_image.start_time());
12563    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
12564    #[cfg(all(test, target_os = "macos"))]
12565    privacy_exec_boundary_tests::before_image_sample(spec, pid);
12566    #[cfg(all(test, target_os = "linux"))]
12567    assert!(roster.admission_event_for_test(&spec.module_id, "before_image_binding", Some(pid)));
12568    // Record the committed kernel image, not the configured pathname: PATH
12569    // lookup and shebang interpretation can select a different executable.
12570    // Linux's posix_spawn can return before that image is installed, so the
12571    // observer also waits for the kernel's first-exec flag to clear.
12572    #[cfg(not(windows))]
12573    let recorded_image = observe_spawned_image(pid);
12574    #[cfg(all(test, target_os = "linux"))]
12575    let recorded_image = if roster.admission_event_for_test(
12576        &spec.module_id,
12577        "after_spawn_image_sample",
12578        Some(pid),
12579    ) {
12580        recorded_image
12581    } else {
12582        None
12583    };
12584    #[cfg(target_os = "linux")]
12585    // An exited child cannot receive HELLO service and has no readable /proc/exe.
12586    // Keep its real exit for the supervisor instead of turning a crash into a hold.
12587    let exited_before_binding = expected_image.is_some()
12588        && recorded_image.and_then(|image| image.executable) != expected_image
12589        && child.try_wait().ok().flatten().is_some();
12590    #[cfg(target_os = "linux")]
12591    let unavailable_initial_image = expected_image.is_some()
12592        && recorded_image.and_then(|image| image.executable).is_none()
12593        && !exited_before_binding;
12594    #[cfg(not(target_os = "linux"))]
12595    let unavailable_initial_image = false;
12596    #[cfg(target_os = "linux")]
12597    if expected_image
12598        .is_some_and(|expected| recorded_image.and_then(|image| image.executable) != Some(expected))
12599        && !exited_before_binding
12600        && !unavailable_initial_image
12601    {
12602        kill_module_cgroup(cgroup_placement, &cgroup_name);
12603        kill_and_reap_rejected_child(&mut child, group_target.clone());
12604        return Err(SuperviseError::ConfigHeld {
12605            module_id: spec.module_id.clone(),
12606            code: "program_refused".into(),
12607            detail: format!(
12608                "spawned executable {:?} does not match hashed program {:?}",
12609                recorded_image.and_then(|image| image.executable),
12610                expected_image
12611            ),
12612        });
12613    }
12614    #[cfg(not(target_os = "linux"))]
12615    let _ = expected_image;
12616    #[cfg(target_os = "macos")]
12617    let binding_ready = privacy_exec.is_none();
12618    #[cfg(target_os = "linux")]
12619    let binding_ready = !exited_before_binding;
12620    #[cfg(not(any(target_os = "macos", target_os = "linux")))]
12621    let binding_ready = cfg!(windows);
12622    // spawn() confirms only the first exec, into the trampoline. Never persist
12623    // the trampoline image; the asynchronous acknowledgement publishes the
12624    // module image once the trampoline has replaced itself with the module.
12625    #[cfg(target_os = "macos")]
12626    let recorded_image = if privacy_exec.is_some() {
12627        None
12628    } else {
12629        recorded_image
12630    };
12631    #[cfg(target_os = "linux")]
12632    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
12633    #[cfg(not(target_os = "linux"))]
12634    let recorded_cgroup_name = None;
12635    #[cfg(unix)]
12636    let preadmitted = admitted
12637        .as_mut()
12638        .and_then(|admitted| admitted.leader.take());
12639    #[cfg(not(unix))]
12640    let preadmitted: Option<crate::child_roster::RosterGuard> = None;
12641    let roster_guard = if let Some(guard) = preadmitted {
12642        Ok(guard)
12643    } else {
12644        roster.admit(
12645            spec.module_id.clone(),
12646            pid,
12647            spec.protocol,
12648            process_start_time,
12649            crate::child_roster::RecordedIdentity {
12650                container: None,
12651                #[cfg(unix)]
12652                carried: None,
12653                #[cfg(not(windows))]
12654                start_time: recorded_image.map(|image| image.start_time),
12655                #[cfg(windows)]
12656                start_time: process_start_time,
12657                #[cfg(not(windows))]
12658                executable: recorded_image
12659                    .and_then(|image| image.executable)
12660                    .map(crate::live_children::ExecutableIdentity::from),
12661                #[cfg(windows)]
12662                executable: Some(crate::live_children::ExecutableIdentity::from(
12663                    windows_image.identity(),
12664                )),
12665                cgroup_name: recorded_cgroup_name,
12666                #[cfg(target_os = "linux")]
12667                cgroup_placement: cgroup_placement.cloned(),
12668            },
12669        )
12670    };
12671    let roster_guard = match roster_guard {
12672        Ok(guard) => guard,
12673        Err(failure) => {
12674            return failure.with_cleanup(|source| {
12675                #[cfg(windows)]
12676                if let Some(job) = &job {
12677                    let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12678                }
12679                #[cfg(target_os = "linux")]
12680                kill_module_cgroup(cgroup_placement, &cgroup_name);
12681                #[cfg(unix)]
12682                let rejected_group = group_target.clone();
12683                kill_and_reap_rejected_child(
12684                    &mut child,
12685                    #[cfg(unix)]
12686                    rejected_group,
12687                );
12688                #[cfg(target_os = "linux")]
12689                if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12690                    remove_module_cgroup(placement, &cgroup_name);
12691                }
12692                Err(SuperviseError::Spawn {
12693                    program: spec.program.clone(),
12694                    source: io::Error::other(format!(
12695                        "could not durably record module child: {source}"
12696                    )),
12697                    cgroup_path,
12698                })
12699            });
12700        }
12701    };
12702    #[cfg(unix)]
12703    if admitted.is_some() {
12704        if let Some(image) = recorded_image {
12705            if let Err(source) = roster_guard.confirm_executable(image) {
12706                kill_and_reap_rejected_child(&mut child, group_target.clone());
12707                return Err(SuperviseError::Spawn {
12708                    program: spec.program.clone(),
12709                    source,
12710                    cgroup_path,
12711                });
12712            }
12713        }
12714    }
12715    // The pre-spawn check can pass just before shutdown begins. Ownership is
12716    // published at the start of admission, even while a record write is queued.
12717    // If shutdown already saw an empty roster before that publication, this
12718    // check sees the roster closed and ends the child before starting a module.
12719    if roster.is_closed() {
12720        if carrying {
12721            #[cfg(unix)]
12722            let rejected_group = admitted
12723                .as_ref()
12724                .map(|admitted| admitted.rejection_target())
12725                .unwrap_or_else(|| group_target.clone());
12726            #[cfg(windows)]
12727            if let Some(job) = &job {
12728                let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12729            }
12730            kill_and_reap_rejected_child(
12731                &mut child,
12732                #[cfg(unix)]
12733                rejected_group,
12734            );
12735            #[cfg(all(test, unix))]
12736            let _ =
12737                roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12738            drop(roster_guard);
12739            return Err(SuperviseError::Spawn {
12740                program: spec.program.clone(),
12741                source: io::Error::other(
12742                    "the daemon began shutting down during carried admission; ended the container",
12743                ),
12744                cgroup_path,
12745            });
12746        }
12747        // This child was never admitted, so there is no module protocol shutdown to wait for.
12748        #[cfg(target_os = "linux")]
12749        kill_module_cgroup(cgroup_placement, &cgroup_name);
12750        if let Err(error) = child.start_kill() {
12751            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
12752        }
12753        #[cfg(target_os = "linux")]
12754        if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12755            // This spawn was never admitted, so shutdown has no roster entry
12756            // to await. Do not detach its cleanup: the runtime could exit
12757            // before that task reaps the rejected child and removes its group.
12758            while matches!(child.try_wait(), Ok(None)) {
12759                std::thread::yield_now();
12760            }
12761            if matches!(
12762                subc_cgroup::kill_module(Some(placement), &cgroup_name),
12763                subc_cgroup::KillOutcome::Killed
12764            ) {
12765                if let Ok(path) = placement.module_path(&cgroup_name) {
12766                    while std::fs::read_to_string(path.join("cgroup.events"))
12767                        .ok()
12768                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
12769                    {
12770                        std::thread::yield_now();
12771                    }
12772                }
12773            }
12774            remove_module_cgroup(placement, &cgroup_name);
12775        }
12776        drop(roster_guard);
12777        return Err(SuperviseError::Spawn {
12778            program: spec.program.clone(),
12779            source: io::Error::other(
12780                "the daemon began shutting down while this process was starting; ended it",
12781            ),
12782            cgroup_path,
12783        });
12784    }
12785
12786    let stdout_pump = match child.stdout.take() {
12787        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
12788        None => {
12789            warn!(
12790                module_id = %spec.module_id,
12791                "spawned child exposed no stdout pipe; file capture will be incomplete"
12792            );
12793            None
12794        }
12795    };
12796    let stderr_pump = match child.stderr.take() {
12797        Some(stderr) => {
12798            let generation = ring
12799                .lock()
12800                .unwrap_or_else(|poisoned| poisoned.into_inner())
12801                .begin_process();
12802            Some(StderrPump {
12803                task: tokio::spawn(pump_stderr_to(
12804                    stderr,
12805                    Arc::clone(ring),
12806                    generation,
12807                    output_sink,
12808                )),
12809                generation,
12810            })
12811        }
12812        None => {
12813            // Spawning succeeded but the pipe did not materialise. Recording it as
12814            // uncaptured keeps the tail honest: the alternative is an empty tail
12815            // that reads as a module which printed nothing.
12816            ring.lock()
12817                .unwrap_or_else(|poisoned| poisoned.into_inner())
12818                .mark_not_captured("stderr pipe was not available on spawn");
12819            warn!(
12820                module_id = %spec.module_id,
12821                "spawned child exposed no stderr pipe; tail will be unavailable"
12822            );
12823            None
12824        }
12825    };
12826
12827    let exec = Arc::new(crate::activity::ExecOutcome::new());
12828    #[cfg(target_os = "macos")]
12829    let pending_exec = privacy_exec.is_some();
12830    #[cfg(not(target_os = "macos"))]
12831    let pending_exec = false;
12832    if !pending_exec {
12833        exec.finish(crate::activity::ExecState::CommittedExec);
12834    }
12835    #[cfg(windows)]
12836    let subtree_target = job
12837        .as_ref()
12838        .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
12839        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
12840    #[cfg(target_os = "linux")]
12841    let subtree_target = cgroup_path
12842        .map(crate::subtree_resources::Target::Cgroup)
12843        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
12844    #[cfg(not(any(target_os = "linux", windows)))]
12845    let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
12846    let mut supervised = SupervisedChild {
12847        registration_spawn,
12848        #[cfg(any(unix, windows))]
12849        registration_channel,
12850        live_program_binding: None,
12851        registered_config_connection: None,
12852        admission_base,
12853        registration_declaration,
12854        _program_binding: binding_publication,
12855        subtree_target,
12856        child: containment::ProcessChild::plain(child, pid),
12857        exec: crate::activity::ExecOwner(exec),
12858        #[cfg(windows)]
12859        nonce_delivery,
12860        protocol: spec.protocol,
12861        #[cfg(target_os = "linux")]
12862        module_id: cgroup_name,
12863        #[cfg(target_os = "linux")]
12864        cgroup_placement: cgroup_placement.cloned(),
12865        #[cfg(windows)]
12866        job,
12867        stdout_pump,
12868        stderr_pump,
12869        stderr_ring: Arc::clone(ring),
12870        spawned_at,
12871        config_delivery,
12872        registration_deadline: None,
12873        registration_watch: None,
12874        spawned_at_ms,
12875        spawned_from,
12876        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
12877        spawned_file_identity,
12878        process_start_time,
12879        process_identity,
12880        pid,
12881        roster_guard: Some(roster_guard),
12882        #[cfg(target_os = "macos")]
12883        privacy_exec,
12884        #[cfg(target_os = "macos")]
12885        report_ready: Arc::new(OnceLock::new()),
12886        spawn_failure: None,
12887        admission_kill_sent: false,
12888    };
12889    if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
12890        supervised.child.reject(
12891            #[cfg(unix)]
12892            group_target,
12893        );
12894        #[cfg(all(test, unix))]
12895        let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12896        return Err(error);
12897    }
12898    if let (Some(launch), Some(owner), Some(base)) = (
12899        program_launch,
12900        config_owner,
12901        supervised.admission_base.clone(),
12902    ) {
12903        supervised.live_program_binding =
12904            content_binding::LiveProgramBinding::new(launch, &supervised, owner, base);
12905        if let (Some(publication), Some(live)) = (
12906            &supervised._program_binding,
12907            &supervised.live_program_binding,
12908        ) {
12909            publication.attach(live);
12910        }
12911    }
12912    #[cfg(unix)]
12913    if let Some(admitted) = admitted.take() {
12914        admitted.disarm();
12915    }
12916    #[cfg(windows)]
12917    if let Err(error) = resume_suspended_child(pid, spec) {
12918        containment::terminate_blocking(&roster.containment);
12919        let _ = supervised.child.start_kill();
12920        return Err(error);
12921    }
12922    if binding_ready {
12923        // An unreadable initial image is not a definite mismatch. Keep the
12924        // owned child and delivery registration until its reaped status decides
12925        // whether it crashed on its own or was stopped by admission's kill.
12926        let checked = if unavailable_initial_image {
12927            Err(crate::config_gate::GateError::new(
12928                "config_delivery_deferred",
12929                "initial process image is unavailable",
12930            ))
12931        } else {
12932            match (
12933                &supervised._program_binding,
12934                &supervised.live_program_binding,
12935            ) {
12936                (Some(binding), _) => binding.verify(),
12937                (None, Some(live)) => live.check(),
12938                _ if supervised.admission_base.is_some() => {
12939                    Err(crate::config_gate::GateError::new(
12940                        "config_delivery_deferred",
12941                        "live program witness is unavailable",
12942                    ))
12943                }
12944                _ => Ok(()),
12945            }
12946        };
12947        #[cfg(all(test, target_os = "linux"))]
12948        let checked = if roster.admission_event_for_test(
12949            &spec.module_id,
12950            "after_content_witness_check",
12951            Some(pid),
12952        ) {
12953            checked
12954        } else {
12955            Err(crate::config_gate::GateError::new(
12956                "config_delivery_deferred",
12957                "current process image is unavailable",
12958            ))
12959        };
12960        if let Err(error) = checked {
12961            if let Some(binding) = &mut supervised._program_binding {
12962                binding.reject();
12963            }
12964            // Reap through the normal child-reaping path, retaining the real exit record and
12965            // its roster ownership. A refusal must also end any helpers.
12966            if error.code == "program_refused" || supervised.try_wait().ok().flatten().is_none() {
12967                supervised.spawn_failure = Some(format!("{}: {}", error.code, error.detail));
12968                #[cfg(all(test, target_os = "linux"))]
12969                assert!(roster.admission_event_for_test(
12970                    &spec.module_id,
12971                    if unavailable_initial_image {
12972                        "before_initial_image_kill"
12973                    } else {
12974                        "before_content_witness_kill"
12975                    },
12976                    Some(pid),
12977                ));
12978                let _ = supervised.start_admission_kill();
12979                #[cfg(all(test, target_os = "linux"))]
12980                if supervised.admission_kill_sent {
12981                    assert!(roster.admission_event_for_test(
12982                        &spec.module_id,
12983                        "after_admission_kill_sent",
12984                        Some(pid),
12985                    ));
12986                }
12987            }
12988        }
12989    }
12990    if let Some(registration) = delivery_registration {
12991        registration.commit();
12992    }
12993    if let Some(spawn) = &supervised.registration_spawn {
12994        if let Some(identity) = supervised.process_identity {
12995            let _ = spawn.identity.set(identity);
12996        }
12997        if supervised.spawn_failure.is_some() {
12998            spawn.refuse();
12999        } else if binding_ready {
13000            spawn.ready();
13001        }
13002        #[cfg(any(unix, windows))]
13003        if let (Some(handle), Some(channel)) =
13004            (registration_handle, supervised.registration_channel.take())
13005        {
13006            handle.serve_registration(channel, spawn.clone(), pid, spawned_at + register_timeout);
13007        }
13008    }
13009    Ok(supervised)
13010}
13011
13012#[cfg(test)]
13013mod spawn_record_tests {
13014    use super::*;
13015
13016    thread_local! {
13017        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
13018    }
13019
13020    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
13021        LAST_PID.set(child.id());
13022        #[cfg(windows)]
13023        if let Some((_, address)) = _spec
13024            .env
13025            .iter()
13026            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
13027        {
13028            use std::io::{Read, Write};
13029            let mut stream = std::net::TcpStream::connect(address).unwrap();
13030            stream
13031                .write_all(&child.id().unwrap().to_ne_bytes())
13032                .unwrap();
13033            let mut release = [0];
13034            stream.read_exact(&mut release).unwrap();
13035        }
13036    }
13037
13038    #[cfg(windows)]
13039    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
13040        if spec
13041            .env
13042            .iter()
13043            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
13044        {
13045            Err(io::Error::other("injected module job assignment failure"))
13046        } else {
13047            Ok(())
13048        }
13049    }
13050
13051    fn spec() -> ModuleSpec {
13052        ModuleSpec {
13053            module_id: "spawn-record-worker".into(),
13054            #[cfg(unix)]
13055            program: "/bin/sleep".into(),
13056            #[cfg(windows)]
13057            program: std::env::current_exe().unwrap(),
13058            #[cfg(unix)]
13059            args: vec!["60".into()],
13060            #[cfg(windows)]
13061            args: vec![
13062                "--exact".into(),
13063                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
13064                "--nocapture".into(),
13065            ],
13066            env: vec![],
13067            reserved: false,
13068            reserved_prefixes: vec![],
13069            protocol: ModuleProtocol::None,
13070            overlap: Default::default(),
13071            carry_containment: false,
13072        }
13073    }
13074
13075    #[tokio::test]
13076    async fn record_sync_failure_fails_spawn_and_reaps_child() {
13077        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
13078        let supervisor =
13079            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
13080                .with_live_children_record(home.join("live-children.json"));
13081        crate::live_children::fail_next_file_sync();
13082        let result = supervisor.spawn(spec());
13083        let failed = result.is_err();
13084        if let Ok(module) = result {
13085            module.stop().await.unwrap();
13086        }
13087        assert!(failed, "record failure must fail spawn");
13088        let pid = LAST_PID.get().expect("the real OS child was created");
13089        assert!(
13090            subc_os::Process::open(pid).unwrap().is_none(),
13091            "child {pid} survived rejected admission or was not reaped"
13092        );
13093    }
13094
13095    #[cfg(target_os = "macos")]
13096    #[test]
13097    #[ignore = "manual on-device latency measurement"]
13098    fn durable_record_write_latency_50() {
13099        let home = cortexkit_test_support::ScratchDir::new("record-latency");
13100        let path = home.join("live-children.json");
13101        let children = vec![crate::live_children::LiveChild {
13102            module_id: "latency-worker".into(),
13103            pid: std::process::id(),
13104            protocol: ModuleProtocol::None,
13105            start_time: Some(1),
13106            executable: None,
13107            cgroup_name: None,
13108            container: None,
13109        }];
13110        let mut samples = Vec::new();
13111        for _ in 0..50 {
13112            let start = std::time::Instant::now();
13113            crate::live_children::write_record(&path, &children).unwrap();
13114            samples.push(start.elapsed());
13115        }
13116        samples.sort();
13117        eprintln!(
13118            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
13119            (samples[24] + samples[25]) / 2,
13120            samples[47]
13121        );
13122    }
13123
13124    #[cfg(windows)]
13125    #[test]
13126    fn windows_store_touch_fixture() {
13127        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
13128            std::fs::write(path, b"module ran").unwrap();
13129        }
13130    }
13131
13132    #[cfg(windows)]
13133    #[tokio::test]
13134    async fn windows_daemon_fixture() {
13135        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
13136            return;
13137        };
13138        let mut spec = spec();
13139        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
13140        let supervisor =
13141            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13142        let _module = supervisor.spawn(spec).unwrap();
13143    }
13144
13145    #[cfg(windows)]
13146    #[test]
13147    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
13148        use std::io::Read;
13149        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13150        // An outer job models a scheduler/service host. The daemon's root and
13151        // root job must nest without requiring breakaway from this job.
13152        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13153        command
13154            .args([
13155                "--exact",
13156                "supervise::spawn_record_tests::windows_daemon_fixture",
13157                "--nocapture",
13158            ])
13159            .env(
13160                "SUBC_TEST_DAEMON_BARRIER",
13161                listener.local_addr().unwrap().to_string(),
13162            );
13163        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13164        listener.set_nonblocking(true).unwrap();
13165        let deadline = std::time::Instant::now() + Duration::from_secs(30);
13166        let mut stream = loop {
13167            if let Ok((stream, _)) = listener.accept() {
13168                break stream;
13169            }
13170            assert!(
13171                daemon.child.try_wait().unwrap().is_none(),
13172                "daemon failed before rendezvous"
13173            );
13174            assert!(
13175                std::time::Instant::now() < deadline,
13176                "daemon did not reach CreateProcess failpoint"
13177            );
13178            std::thread::yield_now();
13179        };
13180        stream
13181            .set_read_timeout(Some(Duration::from_secs(30)))
13182            .unwrap();
13183        let mut pid = [0; 4];
13184        stream.read_exact(&mut pid).unwrap();
13185        let pid = u32::from_ne_bytes(pid);
13186        assert!(subc_jobobject::process_exists(pid));
13187        // Terminate only the daemon, not its outer job: otherwise the host job
13188        // would also kill a child whose daemon never joined its root job.
13189        daemon.child.kill().unwrap();
13190        daemon.child.wait().unwrap();
13191        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
13192        if !exited {
13193            daemon.job.terminate().unwrap();
13194        }
13195        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
13196    }
13197
13198    #[cfg(windows)]
13199    #[tokio::test]
13200    async fn windows_containment_failure_never_resumes() {
13201        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
13202        let sentinel = home.join("store-touched");
13203        let mut spec = spec();
13204        spec.env
13205            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
13206        spec.env.push((
13207            "SUBC_TEST_STORE_TOUCH".into(),
13208            sentinel.display().to_string(),
13209        ));
13210        let supervisor =
13211            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13212        let result = supervisor.spawn(spec);
13213        // A version that resumes despite assignment failure must have time to
13214        // execute its sentinel before dropping the returned module kills it.
13215        if let Ok(module) = result {
13216            let _ = tokio::time::timeout(Duration::from_secs(10), async {
13217                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
13218                    tokio::task::yield_now().await;
13219                }
13220            })
13221            .await;
13222            let _ = module.stop().await;
13223            panic!(
13224                "assignment failure resumed a module; store touched={}",
13225                sentinel.exists()
13226            );
13227        }
13228        assert!(
13229            !sentinel.exists(),
13230            "refused module executed store-touch sentinel"
13231        );
13232        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
13233    }
13234
13235    #[cfg(windows)]
13236    #[tokio::test]
13237    async fn windows_nested_daemon_fixture() {
13238        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
13239            return;
13240        };
13241        let mut spec = spec();
13242        spec.env.push((
13243            "SUBC_TEST_STORE_TOUCH".into(),
13244            sentinel.to_str().unwrap().into(),
13245        ));
13246        let supervisor =
13247            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13248        let module = supervisor
13249            .spawn(spec)
13250            .expect("per-module job must nest below daemon and host jobs");
13251        tokio::time::timeout(Duration::from_secs(10), async {
13252            while !PathBuf::from(&sentinel).exists() {
13253                tokio::task::yield_now().await;
13254            }
13255        })
13256        .await
13257        .unwrap();
13258        module.stop().await.unwrap();
13259    }
13260
13261    #[cfg(windows)]
13262    #[test]
13263    fn windows_module_job_nests_under_host_job() {
13264        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
13265        let sentinel = home.join("nested-store-touched");
13266        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13267        command
13268            .args([
13269                "--exact",
13270                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
13271                "--nocapture",
13272            ])
13273            .env("SUBC_TEST_NESTED_STORE", &sentinel);
13274        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13275        assert!(daemon.child.wait().unwrap().success());
13276        assert!(sentinel.exists(), "nested protocol:none module never ran");
13277    }
13278}
13279
13280#[cfg(target_os = "linux")]
13281pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
13282    use subc_cgroup::KillOutcome;
13283    match subc_cgroup::kill_module(placement, module_id) {
13284        KillOutcome::Killed => {}
13285        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
13286            debug!(
13287                module_id,
13288                "cgroup tree kill unavailable; using direct-child kill"
13289            );
13290        }
13291        KillOutcome::IoError { path, error } => {
13292            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
13293        }
13294    }
13295}
13296
13297/// Assign a freshly spawned Windows child without ever resuming on failure.
13298///
13299/// Steps 2 and 3 of the suspended-create contract: the job is created and the
13300/// child assigned **while it is still suspended** (step 1 is
13301/// `suspend_on_create_async` at the spawn site). The caller resumes only after
13302/// the child's roster admission is durable. Failed containment kills and reaps
13303/// the suspended child instead of running an unprotected module.
13304#[cfg(windows)]
13305fn contain_spawned_child(
13306    child: &mut Child,
13307    spec: &ModuleSpec,
13308    carried: Option<&Arc<containment::CarriedContainment>>,
13309) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
13310    let assignment = || {
13311        let job = match carried {
13312            Some(container) => container.job(),
13313            None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
13314        };
13315        #[cfg(test)]
13316        spawn_record_tests::assignment_failpoint(spec)?;
13317        job.lock()
13318            .unwrap_or_else(|p| p.into_inner())
13319            .assign(child)?;
13320        Ok::<_, io::Error>(job)
13321    };
13322    match assignment() {
13323        Ok(job) => Ok(Some(job)),
13324        Err(source) => {
13325            kill_and_reap_rejected_child(child);
13326            Err(SuperviseError::Spawn {
13327                program: spec.program.clone(),
13328                source: io::Error::other(format!(
13329                    "Windows module job containment failed: {source}"
13330                )),
13331                cgroup_path: None,
13332            })
13333        }
13334    }
13335}
13336
13337/// Rejection is synchronous: do not return a spawn error while the child can
13338/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
13339fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
13340    #[cfg(unix)]
13341    group.signal_rejected(child);
13342    let _ = child.start_kill();
13343    while matches!(child.try_wait(), Ok(None)) {
13344        std::thread::yield_now();
13345    }
13346}
13347
13348/// Resume a suspended child, killing it if it cannot be started.
13349///
13350/// A suspended process holds a pid and does nothing, so there is no useful
13351/// state to return: the caller gets an error and the spawn fails.
13352#[cfg(windows)]
13353fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
13354    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
13355        // Kill it here rather than leaving a suspended process for the caller
13356        // to notice; `kill_on_drop` would eventually do this, but the module
13357        // would have been reported as running in between.
13358        let _ = subc_os::child_spawn::status(
13359            std::process::Command::new("taskkill.exe")
13360                .args(["/PID", &pid.to_string(), "/T", "/F"])
13361                .stdin(Stdio::null())
13362                .stdout(Stdio::null())
13363                .stderr(Stdio::null()),
13364        );
13365        return Err(SuperviseError::Spawn {
13366            program: spec.program.clone(),
13367            source,
13368            cgroup_path: None,
13369        });
13370    }
13371    Ok(())
13372}
13373
13374#[cfg(target_os = "linux")]
13375fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13376    match placement.remove_module(module_id) {
13377        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
13378        Err(error) => warn!(
13379            module_id,
13380            error = %error,
13381            "could not remove module cgroup after process exit; continuing teardown"
13382        ),
13383    }
13384}
13385
13386#[cfg(target_os = "linux")]
13387const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
13388#[cfg(target_os = "linux")]
13389const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
13390
13391#[cfg(target_os = "linux")]
13392async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13393    let path = placement.module_path(module_id).ok();
13394    if path.as_ref().is_some_and(|path| {
13395        std::fs::read_to_string(path.join("cgroup.events"))
13396            .ok()
13397            .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
13398    }) {
13399        remove_module_cgroup(placement, module_id);
13400        return;
13401    }
13402
13403    // Freeze before enumerating: a frozen task cannot exit and release its PID
13404    // for reuse by a process outside this module while SIGTERM is being sent.
13405    if terminate_reaped_cgroup(placement, module_id).await {
13406        if let Some(path) = &path {
13407            let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
13408            while cgroup_is_populated(path) && Instant::now() < deadline {
13409                sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
13410            }
13411        }
13412    }
13413
13414    // Reaping the direct child is not proof its descendants exited. Kill any
13415    // survivors; when `cgroup.kill` succeeds, wait for the kernel to report the
13416    // cgroup empty before rmdir.
13417    if matches!(
13418        subc_cgroup::kill_module(Some(placement), module_id),
13419        subc_cgroup::KillOutcome::Killed
13420    ) {
13421        if let Some(path) = &path {
13422            while cgroup_is_populated(path) {
13423                sleep(Duration::from_millis(1)).await;
13424            }
13425        }
13426    }
13427    remove_module_cgroup(placement, module_id);
13428}
13429
13430#[cfg(target_os = "linux")]
13431fn cgroup_is_populated(path: &std::path::Path) -> bool {
13432    std::fs::read_to_string(path.join("cgroup.events"))
13433        .ok()
13434        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
13435}
13436
13437#[cfg(target_os = "linux")]
13438async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
13439    let frozen =
13440        match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
13441            .await
13442        {
13443            subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
13444            outcome => {
13445                tracing::debug!(
13446                    module_id,
13447                    ?outcome,
13448                    "cgroup freeze failed; skipping SIGTERM grace"
13449                );
13450                return false;
13451            }
13452        };
13453    let listed = match frozen.processes() {
13454        subc_cgroup::ProcsOutcome::Listed(pids) => {
13455            for pid in pids.into_iter().filter(|pid| *pid > 1) {
13456                if let Some(pid) = rustix::process::Pid::from_raw(pid) {
13457                    if let Err(error) =
13458                        rustix::process::kill_process(pid, rustix::process::Signal::TERM)
13459                    {
13460                        tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
13461                    }
13462                }
13463            }
13464            true
13465        }
13466        outcome => {
13467            tracing::debug!(
13468                module_id,
13469                ?outcome,
13470                "cgroup process listing failed; skipping SIGTERM grace"
13471            );
13472            false
13473        }
13474    };
13475    match frozen.thaw() {
13476        subc_cgroup::ThawOutcome::Thawed => listed,
13477        outcome => {
13478            tracing::debug!(
13479                module_id,
13480                ?outcome,
13481                "cgroup thaw failed; proceeding to cgroup.kill"
13482            );
13483            false
13484        }
13485    }
13486}
13487
13488#[cfg(target_os = "linux")]
13489fn apply_cgroup_placement(
13490    command: &mut Command,
13491    spec: &ModuleSpec,
13492    path: &std::path::Path,
13493) -> Result<(), SuperviseError> {
13494    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
13495        module_id: spec.module_id.clone(),
13496        source,
13497    })
13498}
13499
13500fn capture_retention(spec: &ModuleSpec) -> Retention {
13501    let defaults = Retention::default();
13502    let value = |name: &str| {
13503        spec.env
13504            .iter()
13505            .rev()
13506            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
13507    };
13508    Retention {
13509        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
13510            .and_then(|value| value.parse().ok())
13511            .unwrap_or(defaults.max_file_mb),
13512        keep: value(CAPTURE_KEEP_ENV)
13513            .and_then(|value| value.parse().ok())
13514            .unwrap_or(defaults.keep),
13515        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
13516            .and_then(|value| value.parse().ok())
13517            .unwrap_or(defaults.max_age_days),
13518    }
13519}
13520
13521/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
13522/// module's registration to the exact process the supervisor spawned.
13523fn generate_launch_nonce() -> Result<String, SuperviseError> {
13524    let mut bytes = [0u8; 32];
13525    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
13526        reason: source.to_string(),
13527    })?;
13528    let mut hex = String::with_capacity(64);
13529    for b in bytes {
13530        use std::fmt::Write;
13531        let _ = write!(hex, "{b:02x}");
13532    }
13533    Ok(hex)
13534}
13535
13536/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
13537/// signal about how many leading bytes matched.
13538fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
13539    if a.len() != b.len() {
13540        return false;
13541    }
13542    let mut diff = 0u8;
13543    for (x, y) in a.iter().zip(b.iter()) {
13544        diff |= x ^ y;
13545    }
13546    diff == 0
13547}
13548
13549/// The kernel's image after an acknowledged exec, shared by ordinary launches
13550/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
13551/// not identities inferred from a configured pathname.
13552#[cfg(not(windows))]
13553fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
13554    let process = subc_os::Process::open(pid).ok().flatten()?;
13555    #[cfg(target_os = "linux")]
13556    return sample_after_linux_exec(
13557        || {
13558            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
13559            let committed = linux_exec_committed(&stat);
13560            #[cfg(test)]
13561            spawn_image_tests::report_exec_state(committed);
13562            committed
13563        },
13564        || process.observe(),
13565    );
13566    #[cfg(not(target_os = "linux"))]
13567    process.observe()
13568}
13569
13570/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
13571/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
13572/// installation. Until it clears, /proc/pid/exe can still name the daemon.
13573#[cfg(target_os = "linux")]
13574fn linux_exec_committed(stat: &str) -> Option<bool> {
13575    const PF_FORKNOEXEC: u64 = 0x40;
13576    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
13577    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
13578}
13579
13580#[cfg(target_os = "linux")]
13581fn sample_after_linux_exec(
13582    mut exec_committed: impl FnMut() -> Option<bool>,
13583    observe: impl FnOnce() -> Option<subc_os::Observation>,
13584) -> Option<subc_os::Observation> {
13585    let deadline = std::time::Instant::now() + Duration::from_secs(1);
13586    while !exec_committed()? {
13587        // A failed or indefinitely stalled exec must not publish the inherited
13588        // image as authority to signal a future orphan. Missing identity keeps
13589        // the existing conservative orphan-sweep behavior.
13590        if std::time::Instant::now() >= deadline {
13591            return None;
13592        }
13593        std::thread::sleep(Duration::from_micros(50));
13594    }
13595    observe()
13596}
13597
13598#[cfg(all(test, target_os = "linux"))]
13599mod spawn_image_tests {
13600    use super::*;
13601    use std::cell::{Cell, RefCell};
13602
13603    enum ExecEvent {
13604        Uncommitted,
13605        Image(Option<subc_os::Observation>),
13606    }
13607
13608    thread_local! {
13609        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
13610    }
13611
13612    pub(super) fn report_exec_state(committed: Option<bool>) {
13613        if committed == Some(false) {
13614            EXEC_EVENTS.with(|events| {
13615                if let Some(events) = events.borrow_mut().take() {
13616                    let _ = events.send(ExecEvent::Uncommitted);
13617                }
13618            });
13619        }
13620    }
13621
13622    #[test]
13623    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
13624        use std::io::{Read, Write};
13625        use std::os::fd::AsRawFd;
13626
13627        let descriptor = std::fs::File::open("/dev/null").unwrap();
13628        let mut command = std::process::Command::new("/bin/sleep");
13629        command.arg("30");
13630        let (mut ready, mut release) =
13631            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
13632                .unwrap();
13633        let spawn = std::thread::spawn(move || command.spawn());
13634        let mut message = [0u8; 8];
13635        ready.read_exact(&mut message).unwrap();
13636        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
13637        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
13638        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
13639        let (events, received) = std::sync::mpsc::channel();
13640        let observer = std::thread::spawn(move || {
13641            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
13642            let image = observe_spawned_image(pid);
13643            let _ = events.send(ExecEvent::Image(image));
13644        });
13645        // Release only once the production observer has seen the uncommitted
13646        // exec, or has incorrectly returned the inherited image. Either way,
13647        // release and reap before asserting so a regression cannot leak a child.
13648        let first = received.recv_timeout(Duration::from_secs(5));
13649        release.write_all(&[1]).unwrap();
13650        let mut child = spawn.join().unwrap().unwrap();
13651        let after = match first {
13652            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
13653                Ok(ExecEvent::Image(image)) => image,
13654                _ => None,
13655            },
13656            Ok(ExecEvent::Image(image)) => image,
13657            Err(_) => None,
13658        };
13659        observer.join().unwrap();
13660        child.kill().unwrap();
13661        child.wait().unwrap();
13662
13663        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
13664        assert_eq!(
13665            inherited.unwrap().executable,
13666            subc_os::file_identity(&std::env::current_exe().unwrap())
13667        );
13668        assert_eq!(
13669            after.unwrap().executable,
13670            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
13671        );
13672    }
13673
13674    #[test]
13675    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
13676        let committed = Cell::new(false);
13677        let inherited = subc_os::Observation {
13678            start_time: 23,
13679            executable: Some(subc_os::FileIdentity {
13680                device: 29,
13681                inode: 31,
13682            }),
13683        };
13684        let module = subc_os::Observation {
13685            start_time: 23,
13686            executable: Some(subc_os::FileIdentity {
13687                device: 29,
13688                inode: 37,
13689            }),
13690        };
13691        let polls = Cell::new(0);
13692        let image = sample_after_linux_exec(
13693            || {
13694                polls.set(polls.get() + 1);
13695                // Hold the inherited image through the first poll, as in the
13696                // posix_spawn parent-wake window, then commit the module image.
13697                committed.set(polls.get() == 2);
13698                Some(committed.get())
13699            },
13700            || Some(if committed.get() { module } else { inherited }),
13701        );
13702        assert_eq!(image, Some(module));
13703        assert_eq!(polls.get(), 2);
13704    }
13705
13706    #[test]
13707    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
13708        assert_eq!(
13709            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
13710            Some(false)
13711        );
13712        assert_eq!(
13713            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
13714            Some(true)
13715        );
13716        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
13717        assert_eq!(linux_exec_committed("17 (a) R"), None);
13718    }
13719
13720    #[test]
13721    fn unreadable_exec_state_never_samples_an_inherited_image() {
13722        assert_eq!(
13723            sample_after_linux_exec(
13724                || None,
13725                || panic!("must not sample before exec confirmation")
13726            ),
13727            None
13728        );
13729    }
13730
13731    #[test]
13732    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
13733        assert_eq!(
13734            sample_after_linux_exec(
13735                || Some(false),
13736                || panic!("must not sample after an exec timeout")
13737            ),
13738            None
13739        );
13740    }
13741}
13742
13743/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
13744/// different executable after the no-delete capture was taken.
13745#[cfg(windows)]
13746pub(crate) fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
13747    if let Err(error) = require_program_search_path(spec) {
13748        return Err(match error {
13749            SuperviseError::Spawn { source, .. } => source,
13750            error => io::Error::other(error),
13751        });
13752    }
13753    let mut program = spec.program.clone();
13754    if program.is_absolute() || program.components().count() > 1 {
13755        // An explicit path can name a PE image with no extension. Try that
13756        // exact file first, as Command does, before falling back to .exe.
13757        match subc_os::ExecutableCapture::open(&program) {
13758            Ok(capture) => return Ok(capture),
13759            Err(error)
13760                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
13761            Err(error) => return Err(error),
13762        }
13763        program.set_extension("exe");
13764        return subc_os::ExecutableCapture::open(&program);
13765    }
13766    if program.extension().is_none() {
13767        program.set_extension("exe");
13768    }
13769    let custom_path = spec
13770        .env
13771        .iter()
13772        .rev()
13773        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
13774    let mut directories = Vec::new();
13775    // These are Windows' default executable search locations. A configured
13776    // PATH still overrides the inherited PATH, not the system directories.
13777    if let Some(parent) = std::env::current_exe()?.parent() {
13778        directories.push(parent.to_path_buf());
13779    }
13780    directories.push(std::env::current_dir()?);
13781    if let Some(windows) = std::env::var_os("SystemRoot") {
13782        let windows = PathBuf::from(windows);
13783        directories.push(windows.join("System32"));
13784        directories.push(windows.clone());
13785    }
13786    let path = custom_path
13787        .map(|(_, value)| std::ffi::OsString::from(value))
13788        .or_else(|| std::env::var_os("PATH"));
13789    if let Some(path) = path {
13790        directories.extend(std::env::split_paths(&path));
13791    }
13792    for directory in directories {
13793        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
13794            Ok(capture) => return Ok(capture),
13795            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
13796            Err(error) => return Err(error),
13797        }
13798    }
13799    Err(io::Error::new(
13800        io::ErrorKind::NotFound,
13801        "resolved executable not found",
13802    ))
13803}
13804
13805async fn spawn_and_mark_running_async(
13806    spec: &ModuleSpec,
13807    runtime: &SupervisorRuntimeConfig,
13808    snapshot: &SharedSnapshot,
13809) -> Result<SupervisedChild, SuperviseError> {
13810    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13811    let registration_watch = watch_registration(&runtime.registry, spec);
13812    #[cfg(unix)]
13813    let joining_group = containment::current(&runtime.child_roster.containment)
13814        .is_some_and(|container| container.group().is_some());
13815    let result = spawn_child_async(spec, runtime).await;
13816    #[cfg(unix)]
13817    let result = if result.is_err()
13818        && joining_group
13819        && !runtime.child_roster.is_closed()
13820        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13821        && runtime
13822            .child_roster
13823            .previous_instance(&spec.module_id)
13824            .is_none()
13825    {
13826        // A failed group join has already ended the old container. Its single
13827        // fresh-container retry belongs to the same restart budget entry.
13828        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13829        spawn_child_async(spec, runtime).await
13830    } else {
13831        result
13832    };
13833    let mut child = result?;
13834    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13835    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13836        containment::terminate_blocking(&runtime.child_roster.containment);
13837        return Err(error);
13838    }
13839    if let Some(spawn) = activity_spawn {
13840        spawn.complete(snapshot);
13841    }
13842    Ok(child)
13843}
13844
13845#[cfg(test)]
13846fn spawn_and_mark_running(
13847    spec: &ModuleSpec,
13848    runtime: &SupervisorRuntimeConfig,
13849    snapshot: &SharedSnapshot,
13850) -> Result<SupervisedChild, SuperviseError> {
13851    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13852    let registration_watch = watch_registration(&runtime.registry, spec);
13853    #[cfg(unix)]
13854    let joining_group = containment::current(&runtime.child_roster.containment)
13855        .is_some_and(|container| container.group().is_some());
13856    let spawn = || {
13857        spawn_child(
13858            spec,
13859            runtime.connection_file_path.as_deref(),
13860            runtime.supervisor_handle.as_ref(),
13861            &runtime.stderr_ring,
13862            runtime.capture_logs_dir.as_deref(),
13863            &runtime.child_roster,
13864            runtime.spawn_delivery(),
13865            #[cfg(target_os = "linux")]
13866            runtime.cgroup_placement.as_ref(),
13867            runtime.register_timeout,
13868        )
13869    };
13870    let result = spawn();
13871    #[cfg(unix)]
13872    let result = if result.is_err()
13873        && joining_group
13874        && !runtime.child_roster.is_closed()
13875        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13876        && runtime
13877            .child_roster
13878            .previous_instance(&spec.module_id)
13879            .is_none()
13880    {
13881        // The failed join already destroyed and verified the old container.
13882        // Try one fresh group as part of this same restart, without spending
13883        // another crash-budget entry. Cgroups and Windows jobs do not retry.
13884        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13885        spawn()
13886    } else {
13887        result
13888    };
13889    let child = result?;
13890    let mut child = child;
13891    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13892    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13893        containment::terminate_blocking(&runtime.child_roster.containment);
13894        return Err(error);
13895    }
13896    if let Some(spawn) = activity_spawn {
13897        spawn.complete(snapshot);
13898    }
13899    Ok(child)
13900}
13901
13902struct RegistrationWatch {
13903    changes: watch::Receiver<u64>,
13904    baseline: u64,
13905}
13906
13907// Capture before spawning: a fast child's accepted HELLO can disappear from the
13908// active registry before the monitor first runs. A pre-existing registration
13909// must not satisfy a new process's deadline either.
13910fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
13911    if spec.protocol != ModuleProtocol::Subc {
13912        return None;
13913    }
13914    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
13915    let baseline = *changes.borrow();
13916    Some(RegistrationWatch { changes, baseline })
13917}
13918
13919fn arm_registration_deadline(
13920    child: &mut SupervisedChild,
13921    budget: Duration,
13922    registration_watch: Option<RegistrationWatch>,
13923) {
13924    child.registration_watch = registration_watch;
13925    child.registration_deadline = (child.protocol == ModuleProtocol::Subc
13926        && child.awaiting_registration())
13927    .then_some(child.spawned_at + budget);
13928}
13929
13930async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
13931    child.retire_registration();
13932    if let Some(container) = child.child.container() {
13933        container.stop_members().await;
13934        child.kill_leader()?;
13935        child.wait().await?;
13936        return Ok(());
13937    }
13938    #[cfg(unix)]
13939    {
13940        // Children lead their own process groups. Signal the whole group so a
13941        // helper stuck before HELLO cannot survive its supervised parent.
13942        let group = unreaped_registration_group(child)?;
13943        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
13944        // Process-group safety rule: only signal a group while its leader is
13945        // our unreaped child. Even a zombie leader reserves its pid/group id;
13946        // reaping it before KILL would let that id name an unrelated group.
13947        // Sleep without waiting so TERM-ignoring helpers get the full grace.
13948        sleep(crate::child_roster::TERM_TO_KILL).await;
13949        let group = unreaped_registration_group(child)?;
13950        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
13951    }
13952    // Windows terminates the containing Job Object as well as the direct
13953    // child. Linux also kills the child's cgroup when placement is enabled.
13954    child.start_kill()?;
13955    child.wait().await?;
13956    Ok(())
13957}
13958
13959#[cfg(unix)]
13960fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
13961    if let Some(container) = child.child.container() {
13962        return container
13963            .group()
13964            .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
13965            .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
13966    }
13967    // The group id comes only from the pid recorded at spawn, when the child
13968    // created its own group. Never admit init's group or a negative group id.
13969    let group = i32::try_from(child.pid)
13970        .ok()
13971        .filter(|pid| *pid > 1)
13972        .and_then(rustix::process::Pid::from_raw)
13973        .ok_or_else(|| {
13974            io::Error::new(
13975                io::ErrorKind::InvalidInput,
13976                "invalid supervised process group",
13977            )
13978        })?;
13979    if child.child.id() != Some(child.pid) {
13980        return Err(io::Error::new(
13981            io::ErrorKind::InvalidInput,
13982            "refusing process-group signal after supervised child was reaped",
13983        ));
13984    }
13985    Ok(group)
13986}
13987
13988enum RegistrationWaitOutcome {
13989    Registered,
13990    Exited(ExitReport),
13991    TimedOut,
13992}
13993
13994struct ReloadRegistrationFailure {
13995    exit_report: ExitReport,
13996    reason: String,
13997}
13998
13999#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14000enum BusyGaugeObservation {
14001    Quiescent,
14002    Busy,
14003    Omitted,
14004}
14005
14006/// What one health answer said about one declared busy gauge.
14007#[derive(Debug, Clone, PartialEq, Eq)]
14008enum GaugeVerdict {
14009    /// The gauge read 0: this gauge is not what holds the drain.
14010    Zero,
14011    /// The gauge read a positive count.
14012    NonZero(u64),
14013    /// The gauge was present but not a non-negative integer (`0.0`, a string,
14014    /// a negative number). It counts as busy, because the daemon cannot tell
14015    /// it is idle. Carries the value as JSON text, cut to a short prefix.
14016    NotACount(String),
14017    /// The health answer had no metrics object, or one without this gauge.
14018    Missing,
14019    /// The probe produced no health answer to read; carries the probe error's
14020    /// evidence label (`no-answer` for a probe that timed out).
14021    ProbeFailed(&'static str),
14022}
14023
14024impl GaugeVerdict {
14025    fn reads_quiet(&self) -> bool {
14026        matches!(self, Self::Zero)
14027    }
14028}
14029
14030impl fmt::Display for GaugeVerdict {
14031    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14032        match self {
14033            Self::Zero => f.write_str("zero"),
14034            Self::NonZero(value) => write!(f, "non_zero({value})"),
14035            Self::NotACount(value) => write!(f, "not_a_count({value})"),
14036            Self::Missing => f.write_str("missing_from_health_body"),
14037            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
14038        }
14039    }
14040}
14041
14042/// The longest JSON text kept for a gauge value that is not a count, so a
14043/// module reporting a large object as a gauge cannot flood the drain log.
14044const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
14045
14046/// Read each declared gauge out of one health answer's metrics.
14047fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
14048    let metrics = metrics.and_then(Value::as_object);
14049    gauges
14050        .iter()
14051        .map(
14052            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
14053                None => GaugeVerdict::Missing,
14054                Some(value) => match value.as_u64() {
14055                    Some(0) => GaugeVerdict::Zero,
14056                    Some(count) => GaugeVerdict::NonZero(count),
14057                    None => {
14058                        let mut text = value.to_string();
14059                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
14060                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
14061                            while !text.is_char_boundary(end) {
14062                                end -= 1;
14063                            }
14064                            text.truncate(end);
14065                            text.push_str("...");
14066                        }
14067                        GaugeVerdict::NotACount(text)
14068                    }
14069                },
14070            },
14071        )
14072        .collect()
14073}
14074
14075/// The drain's overall reading of one health answer: omitted when any
14076/// declared gauge is missing, quiescent only when every gauge reads 0.
14077fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
14078    if verdicts
14079        .iter()
14080        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
14081    {
14082        BusyGaugeObservation::Omitted
14083    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
14084        BusyGaugeObservation::Quiescent
14085    } else {
14086        BusyGaugeObservation::Busy
14087    }
14088}
14089
14090/// One declared gauge's history over a drain: the latest verdict on it, and
14091/// the latest value actually read for it (which a later failed probe or
14092/// missing gauge does not erase). Times are offsets from the start of the
14093/// drain's wait.
14094#[derive(Debug, Clone, Default, PartialEq, Eq)]
14095struct GaugeTrace {
14096    latest: Option<(GaugeVerdict, Duration)>,
14097    last_value: Option<(GaugeVerdict, Duration)>,
14098}
14099
14100/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
14101/// its ceiling can say in its timeout line which declared gauge held it and
14102/// why. Without it, a gauge that read busy, a gauge the module never
14103/// reported, and a probe that never got an answer all looked the same: a
14104/// drain that ended at its ceiling with nothing in flight.
14105#[derive(Debug, Clone, Default, PartialEq, Eq)]
14106struct DrainGaugeTrace {
14107    /// Health probes the drain sent.
14108    probes: u32,
14109    /// Each declared gauge, in declaration order.
14110    gauges: Vec<(String, GaugeTrace)>,
14111}
14112
14113impl DrainGaugeTrace {
14114    fn new(gauges: &[String]) -> Self {
14115        Self {
14116            probes: 0,
14117            gauges: gauges
14118                .iter()
14119                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
14120                .collect(),
14121        }
14122    }
14123
14124    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
14125    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
14126        self.probes = self.probes.saturating_add(1);
14127        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
14128            trace.latest = Some((verdict.clone(), at));
14129            if matches!(
14130                verdict,
14131                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
14132            ) {
14133                trace.last_value = Some((verdict.clone(), at));
14134            }
14135        }
14136    }
14137}
14138
14139/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
14140/// declared gauge, separated by ` | `. A value is shown as its verdict
14141/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
14142/// and whether that counted as busy.
14143impl fmt::Display for DrainGaugeTrace {
14144    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14145        if self.gauges.is_empty() {
14146            return f.write_str("none declared");
14147        }
14148        for (index, (name, trace)) in self.gauges.iter().enumerate() {
14149            if index > 0 {
14150                f.write_str(" | ")?;
14151            }
14152            write!(f, "{name}: ")?;
14153            match &trace.latest {
14154                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
14155                None => f.write_str("never probed")?,
14156            }
14157            match &trace.last_value {
14158                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
14159                None => f.write_str("; no value ever read")?,
14160            }
14161        }
14162        Ok(())
14163    }
14164}
14165
14166pub(crate) fn declared_busy_gauges(
14167    registry: &Registry,
14168    module_id: &str,
14169) -> Result<Vec<String>, SuperviseError> {
14170    busy_gauges_of(
14171        registry
14172            .get_module(module_id)
14173            .map_err(SuperviseError::Registry)?,
14174    )
14175}
14176
14177/// [`declared_busy_gauges`] for the registration a connection holds, in any
14178/// slot: after cutover the incumbent is no longer the id's active
14179/// registration, and its own manifest is the one that names its gauges.
14180fn declared_busy_gauges_for_connection(
14181    registry: &Registry,
14182    connection_id: ConnectionId,
14183) -> Result<Vec<String>, SuperviseError> {
14184    busy_gauges_of(
14185        registry
14186            .get_module_by_connection(connection_id)
14187            .map_err(SuperviseError::Registry)?,
14188    )
14189}
14190
14191fn busy_gauges_of(
14192    registration: Option<crate::registry::ModuleRegistration>,
14193) -> Result<Vec<String>, SuperviseError> {
14194    let Some(registration) = registration else {
14195        return Ok(Vec::new());
14196    };
14197    let Some(self_signals) = registration.manifest.self_signals else {
14198        return Ok(Vec::new());
14199    };
14200
14201    let mut gauges = Vec::new();
14202    for declaration in self_signals {
14203        if declaration.kind != SelfSignalKind::Busy {
14204            continue;
14205        }
14206        match declaration.anchored_to {
14207            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
14208                gauges.extend(declared)
14209            }
14210            _ => {
14211                // An invalid Busy anchor is fail-safe: the empty name cannot be
14212                // present in a conforming health report, so this drain stays busy.
14213                gauges.push(String::new());
14214            }
14215        }
14216    }
14217    Ok(gauges)
14218}
14219
14220/// Wait for `endpoint` to have nothing in flight and, when the module declares
14221/// busy gauges, for a health probe to report them quiet. The probe is addressed
14222/// by `scope`: a swap's superseded incumbent must be asked about its own
14223/// gauges, and by module id the probe would reach the promoted candidate.
14224/// While the gauges read busy, the probe repeats every
14225/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
14226/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
14227/// timeout line.
14228#[allow(clippy::too_many_arguments)]
14229async fn wait_for_forwarding_quiescence(
14230    forwarding: &ForwardingTable,
14231    module_id: &str,
14232    runtime: &SupervisorRuntimeConfig,
14233    endpoint: crate::ModuleEndpointId,
14234    deadline: Instant,
14235    busy_gauges: &[String],
14236    scope: DrainScope,
14237    trace: &mut DrainGaugeTrace,
14238) -> Result<bool, SuperviseError> {
14239    let wait_started = Instant::now();
14240    let mut gauges_quiescent = busy_gauges.is_empty();
14241    let mut next_probe_at = wait_started;
14242    let mut omission_reported = false;
14243
14244    loop {
14245        let now = Instant::now();
14246        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
14247            let report = match scope {
14248                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
14249                DrainScope::Endpoint(endpoint) => {
14250                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
14251                }
14252            };
14253            let verdicts = match &report {
14254                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
14255                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
14256            };
14257            trace.record(&verdicts, wait_started.elapsed());
14258            gauges_quiescent = match report {
14259                Ok(_) => match busy_gauge_observation(&verdicts) {
14260                    BusyGaugeObservation::Quiescent => true,
14261                    BusyGaugeObservation::Busy => false,
14262                    BusyGaugeObservation::Omitted => {
14263                        // Once per drain: a module that never reports a gauge
14264                        // it declared would otherwise say so on every probe.
14265                        if !omission_reported {
14266                            forwarding
14267                                .counters()
14268                                .increment_drains_with_undeclared_gauge();
14269                            let missing = busy_gauges
14270                                .iter()
14271                                .zip(&verdicts)
14272                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
14273                                .map(|(gauge, _)| gauge.as_str())
14274                                .collect::<Vec<_>>()
14275                                .join(",");
14276                            warn!(
14277                                module_id,
14278                                missing_gauges = %missing,
14279                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
14280                            );
14281                            omission_reported = true;
14282                        }
14283                        false
14284                    }
14285                },
14286                Err(err) => {
14287                    warn!(
14288                        module_id,
14289                        error = %err,
14290                        "drain health.check did not produce declared busy gauges; treating module as busy"
14291                    );
14292                    false
14293                }
14294            };
14295            // Counted from the probe's completion: the probe above was awaited,
14296            // so no second probe can start while one is still outstanding.
14297            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
14298        }
14299
14300        let in_flight = forwarding
14301            .endpoint_in_flight_count(endpoint)
14302            .map_err(SuperviseError::Forwarding)?;
14303        if in_flight == 0 && gauges_quiescent {
14304            return Ok(true);
14305        }
14306
14307        let now = Instant::now();
14308        if now >= deadline {
14309            return Ok(false);
14310        }
14311        let mut wait = deadline
14312            .saturating_duration_since(now)
14313            .min(REGISTRY_RELEASE_POLL);
14314        if !busy_gauges.is_empty() {
14315            wait = wait.min(next_probe_at.saturating_duration_since(now));
14316        }
14317        sleep(wait).await;
14318    }
14319}
14320
14321/// The wait between drain busy-gauge probes: the larger of
14322/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
14323/// drain's own in-flight poll interval.
14324fn drain_busy_reprobe_interval() -> Duration {
14325    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
14326}
14327
14328/// Logs why a drain gave up: the drain waited its full timeout without the
14329/// module going quiet, and teardown is being forced.
14330///
14331/// The line names two kinds of evidence, because a bare "timed out" hides
14332/// causes that need different fixes:
14333/// - `holdouts`: the requests and routes still open when the timeout hit,
14334///   and on which connections. One stuck call is a module bug; a held stream
14335///   on every session is a module that should end its streams when it
14336///   receives `module.draining`.
14337/// - `gauges`: for each busy gauge the module declares, its latest reading
14338///   and how many probes ran. This separates a module that kept reporting
14339///   itself busy, one whose health answer never contained the gauge, and one
14340///   that never answered the probes, which otherwise all look like a timeout
14341///   with nothing in flight.
14342fn warn_drain_timed_out(
14343    module_id: &str,
14344    drain_timeout: Duration,
14345    reason: RouteCloseReason,
14346    holdouts: &crate::forwarding::DrainHoldouts,
14347    gauges: &DrainGaugeTrace,
14348) {
14349    warn!(
14350        module_id = %module_id,
14351        waited = ?drain_timeout,
14352        ?reason,
14353        held_requests = holdouts.requests,
14354        held_routes = holdouts.routes,
14355        total_routes = holdouts.total_routes,
14356        top_connections = ?holdouts.top_connections,
14357        // `module_channel:corr`, so the module can find each held request
14358        // in its own log; capped, so `held_requests` is the full count.
14359        held = %holdouts
14360            .held
14361            .iter()
14362            .map(|(channel, corr)| format!("{channel}:{corr}"))
14363            .collect::<Vec<_>>()
14364            .join(","),
14365        gauge_probes = gauges.probes,
14366        busy_gauges = %gauges,
14367        "route drain timed out before request quiescence; forcing teardown"
14368    );
14369}
14370
14371/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
14372///
14373/// `Ok` is always honest and passed straight through -- the wait actually measured
14374/// in-flight state. `Err` means the wait produced no measurement at all (the
14375/// forwarding table's lock was poisoned), so `false` is reported as the one honest
14376/// constant: the drain did not complete. Never recomputed from route state, never a
14377/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
14378fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
14379    match wait_result {
14380        Ok(drained) => *drained,
14381        Err(_) => false,
14382    }
14383}
14384
14385fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
14386    for released in released_routes {
14387        let frame = match Frame::build_with_version(
14388            released.negotiated_ver,
14389            FrameType::Goodbye,
14390            control_flags(),
14391            released.channel,
14392            released.epoch,
14393            0,
14394            Vec::new(),
14395        ) {
14396            Ok(frame) => frame,
14397            Err(err) => {
14398                warn!(
14399                    route_channel = released.channel,
14400                    error = %err,
14401                    "failed to build supervisor drain route GOODBYE frame"
14402                );
14403                continue;
14404            }
14405        };
14406        if !released.close_on_delivery_failure() {
14407            crate::forwarding::send_module_route_goodbye(
14408                &forwarding.counters(),
14409                &released.sink,
14410                frame,
14411                released.module_id.as_deref(),
14412                "supervisor drain",
14413            );
14414            continue;
14415        }
14416        if let Err(err) = released.sink.try_send(frame) {
14417            warn!(
14418                target_connection_id = released.connection_id.get(),
14419                route_channel = released.channel,
14420                error = %err,
14421                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
14422            );
14423            let _ = forwarding.escalate_client_delivery_failure(
14424                released.connection_id,
14425                released.channel,
14426                released.epoch,
14427                CloseReason::new(
14428                    "route_goodbye_delivery_failed",
14429                    format!(
14430                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
14431                        released.channel
14432                    ),
14433                ),
14434                crate::forwarding::UndeliveredFrame {
14435                    module_id: released.module_id.as_deref(),
14436                    sink: &released.sink,
14437                },
14438            );
14439        }
14440    }
14441}
14442
14443async fn send_module_draining(
14444    module_id: &str,
14445    reason: RouteCloseReason,
14446    deadline_ms: u64,
14447    target: &ModuleDrainTarget,
14448    _wait_for_flush: bool,
14449) -> bool {
14450    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
14451        reason,
14452        deadline_ms,
14453    }) {
14454        Ok(body) => body,
14455        Err(err) => {
14456            warn!(
14457                module_id,
14458                error = %err,
14459                "failed to encode module draining command"
14460            );
14461            return false;
14462        }
14463    };
14464    let frame = match Frame::build_with_version(
14465        target.negotiated_ver,
14466        FrameType::Push,
14467        control_flags(),
14468        0,
14469        0,
14470        0,
14471        body,
14472    ) {
14473        Ok(frame) => frame,
14474        Err(err) => {
14475            warn!(
14476                module_id,
14477                error = %err,
14478                "failed to build module draining command frame"
14479            );
14480            return false;
14481        }
14482    };
14483    #[cfg(unix)]
14484    if _wait_for_flush {
14485        return target.sink.send_flushed(frame).await.is_ok();
14486    }
14487    if let Err(err) = target.sink.try_send(frame) {
14488        warn!(
14489            module_id,
14490            target_connection_id = target.endpoint.connection_id.get(),
14491            error = %err,
14492            "module draining command was not delivered to peer"
14493        );
14494        return false;
14495    }
14496    true
14497}
14498
14499/// The channel-0 GOODBYE that tells a module its stop is planned.
14500fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
14501    match Frame::build_with_version(
14502        negotiated_ver,
14503        FrameType::Goodbye,
14504        control_flags(),
14505        0,
14506        0,
14507        0,
14508        Vec::new(),
14509    ) {
14510        Ok(frame) => Some(frame),
14511        Err(err) => {
14512            warn!(
14513                module_id,
14514                error = %err,
14515                "failed to build module GOODBYE frame"
14516            );
14517            None
14518        }
14519    }
14520}
14521
14522/// Send every registered module connection its module GOODBYE at daemon
14523/// shutdown, then request that connection's close.
14524///
14525/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
14526/// before EOF, so the GOODBYE must reach the socket before the close. A close
14527/// request does not wait for the connection's queued frames: its writer gets a
14528/// bounded grace after the close, is aborted if it overruns it, and the daemon
14529/// process may exit before that grace ends. So with `wait_for_flush`, each
14530/// connection is closed only after its writer has acknowledged writing the
14531/// GOODBYE, or once a short shared budget runs out, so one module that is not
14532/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
14533/// are only queued, for a shutdown the operator has told to stop waiting.
14534/// A connection that is already gone is skipped.
14535#[cfg(unix)]
14536async fn send_module_goodbyes_for_daemon_shutdown(
14537    forwarding: &Arc<ForwardingTable>,
14538    reason: &CloseReason,
14539    wait_for_flush: bool,
14540) {
14541    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
14542    let targets = match forwarding.module_connections() {
14543        Ok(targets) => targets,
14544        Err(err) => {
14545            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
14546            return;
14547        }
14548    };
14549    let deadline = Instant::now() + GOODBYE_BUDGET;
14550    let mut sends = tokio::task::JoinSet::new();
14551    for target in targets {
14552        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
14553            continue;
14554        };
14555        if !wait_for_flush {
14556            if let Err(err) = target.sink.try_send(frame) {
14557                debug!(
14558                    module_id = %target.module_id,
14559                    error = %err,
14560                    "shutdown module GOODBYE was not queued"
14561                );
14562            }
14563            continue;
14564        }
14565        let forwarding = Arc::clone(forwarding);
14566        let reason = reason.clone();
14567        sends.spawn(async move {
14568            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
14569                Ok(Ok(())) => {}
14570                Ok(Err(err)) => debug!(
14571                    module_id = %target.module_id,
14572                    error = %err,
14573                    "module connection closed before its shutdown GOODBYE was written"
14574                ),
14575                Err(_) => warn!(
14576                    module_id = %target.module_id,
14577                    budget = ?GOODBYE_BUDGET,
14578                    "shutdown module GOODBYE was not written within its budget; closing anyway"
14579                ),
14580            }
14581            forwarding.request_connection_close(target.endpoint.connection_id, reason);
14582        });
14583    }
14584    // Every task ends by the shared deadline, so this wait is bounded too.
14585    while sends.join_next().await.is_some() {}
14586}
14587
14588async fn send_module_goodbye(
14589    module_id: &str,
14590    forwarding: &ForwardingTable,
14591    target: &ModuleDrainTarget,
14592    wait_for_flush: bool,
14593) {
14594    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
14595        return;
14596    };
14597    // A channel is shut down synchronously after its drain. Its final frame
14598    // must reach the socket writer first; queue admission alone can lose it.
14599    let sent = if wait_for_flush {
14600        timeout(Duration::from_millis(500), target.sink.send_flushed(frame))
14601            .await
14602            .map_err(|_| "module GOODBYE flush exceeded 500ms".to_owned())
14603            .and_then(|result| result.map_err(|err| err.to_string()))
14604    } else {
14605        target.sink.try_send(frame).map_err(|err| err.to_string())
14606    };
14607    if let Err(err) = sent {
14608        warn!(
14609            module_id,
14610            target_connection_id = target.endpoint.connection_id.get(),
14611            error = %err,
14612            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
14613        );
14614        forwarding.request_connection_close(
14615            target.endpoint.connection_id,
14616            CloseReason::new(
14617                "module_goodbye_delivery_failed",
14618                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
14619            ),
14620        );
14621    }
14622}
14623
14624#[derive(Clone, Copy)]
14625struct ForwardingDrainContext<'a> {
14626    spec: &'a ModuleSpec,
14627    runtime: &'a SupervisorRuntimeConfig,
14628    registry: &'a Registry,
14629    scope: DrainScope,
14630}
14631
14632/// Which process a forwarding drain addresses.
14633#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14634enum DrainScope {
14635    /// Whatever endpoint is active for the module id: every plain stop,
14636    /// restart and reload. Also moves the module's state to `Draining`.
14637    Active,
14638    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
14639    /// module id would resolve to the promoted candidate and leave neither
14640    /// process routable. The module's state is left alone, since the promoted
14641    /// candidate is what it describes and that process is running.
14642    Endpoint(crate::ModuleEndpointId),
14643}
14644
14645/// Whether a child being drained has already been asked to stop by the time
14646/// its drain wait starts.
14647///
14648/// The drain wait is the same budget whatever this says. What it decides is
14649/// whether the supervisor must ask by signal before that wait begins: a child
14650/// that nobody asked will sit out the whole budget and then be SIGKILLed,
14651/// healthy or not.
14652#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14653enum StopNotice {
14654    /// The module was sent `module.draining` and a module GOODBYE over its own
14655    /// registered connection, and stops itself.
14656    SentOverConnection,
14657    /// The forwarding drain found no registered connection for the module: a
14658    /// subc child spawned moments ago that has not sent HELLO yet, or a
14659    /// `protocol: "none"` child, which never registers.
14660    NoConnection,
14661    /// This path sends nothing over the module's connection: the supervisor has
14662    /// no forwarding table, or the caller stops the child without a forwarding
14663    /// drain.
14664    NotSent,
14665}
14666
14667async fn begin_forwarding_drain(
14668    spec: &ModuleSpec,
14669    runtime: &SupervisorRuntimeConfig,
14670    registry: &Registry,
14671    snapshot: &SharedSnapshot,
14672    enabled: Option<bool>,
14673    reason: RouteCloseReason,
14674) -> Result<StopNotice, SuperviseError> {
14675    let Some(forwarding) = runtime.forwarding.as_ref() else {
14676        return Err(SuperviseError::ReloadUnavailable {
14677            module_id: spec.module_id.clone(),
14678            reason: "supervisor was not configured with a forwarding table".to_string(),
14679        });
14680    };
14681
14682    begin_forwarding_drain_with(
14683        forwarding,
14684        ForwardingDrainContext {
14685            spec,
14686            runtime,
14687            registry,
14688            scope: DrainScope::Active,
14689        },
14690        snapshot,
14691        enabled,
14692        reason,
14693        runtime.drain_timeout,
14694    )
14695    .await
14696}
14697
14698async fn begin_forwarding_drain_if_configured(
14699    spec: &ModuleSpec,
14700    runtime: &SupervisorRuntimeConfig,
14701    registry: &Registry,
14702    snapshot: &SharedSnapshot,
14703    enabled: Option<bool>,
14704    reason: RouteCloseReason,
14705) -> Result<StopNotice, SuperviseError> {
14706    begin_forwarding_drain_with_timeout(
14707        spec,
14708        runtime,
14709        registry,
14710        snapshot,
14711        enabled,
14712        reason,
14713        runtime.drain_timeout,
14714    )
14715    .await
14716}
14717
14718/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
14719/// budget, for paths where the operator overrides the module's configured one
14720/// (`supervisor.restart{drain_timeout_ms}`).
14721async fn begin_forwarding_drain_with_timeout(
14722    spec: &ModuleSpec,
14723    runtime: &SupervisorRuntimeConfig,
14724    registry: &Registry,
14725    snapshot: &SharedSnapshot,
14726    enabled: Option<bool>,
14727    reason: RouteCloseReason,
14728    drain_timeout: Duration,
14729) -> Result<StopNotice, SuperviseError> {
14730    let Some(forwarding) = runtime.forwarding.as_ref() else {
14731        return Ok(StopNotice::NotSent);
14732    };
14733
14734    begin_forwarding_drain_with(
14735        forwarding,
14736        ForwardingDrainContext {
14737            spec,
14738            runtime,
14739            registry,
14740            scope: DrainScope::Active,
14741        },
14742        snapshot,
14743        enabled,
14744        reason,
14745        drain_timeout,
14746    )
14747    .await
14748}
14749
14750async fn begin_forwarding_drain_with(
14751    forwarding: &ForwardingTable,
14752    context: ForwardingDrainContext<'_>,
14753    snapshot: &SharedSnapshot,
14754    enabled: Option<bool>,
14755    reason: RouteCloseReason,
14756    drain_timeout: Duration,
14757) -> Result<StopNotice, SuperviseError> {
14758    let ForwardingDrainContext {
14759        spec,
14760        runtime,
14761        registry,
14762        scope,
14763    } = context;
14764    debug_assert_ne!(reason, RouteCloseReason::Crash);
14765    let terminal = matches!(reason, RouteCloseReason::Disable);
14766    let drain_started_at = Instant::now();
14767    let drain_deadline = drain_started_at + drain_timeout;
14768    let deadline_ms =
14769        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
14770    let busy_gauges = match scope {
14771        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
14772        DrainScope::Endpoint(endpoint) => {
14773            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
14774        }
14775    };
14776
14777    // Admission gate first: route.open/commit and route REQUEST admission are closed
14778    // before the first quiescence check, so the outstanding count can only fall.
14779    let gate_started = Instant::now();
14780    let drain_target = match scope {
14781        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
14782        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
14783    }
14784    .map_err(SuperviseError::Forwarding)?;
14785    // The instant admission closed, and how long taking the forwarding write
14786    // lock to close it took. The timeout line reports only the quiescence
14787    // wait, so without this a drain that started late looked like one that
14788    // started on time.
14789    info!(
14790        module_id = %spec.module_id,
14791        ?reason,
14792        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
14793        connected = drain_target.is_some(),
14794        "module drain began; route admission closed"
14795    );
14796    if scope == DrainScope::Active {
14797        update_snapshot(snapshot, Some(&spec.module_id), |state| {
14798            state.state = ModuleState::Draining;
14799            state.draining_to_replace =
14800                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
14801            if let Some(enabled) = enabled {
14802                state.enabled = enabled;
14803            }
14804        })?;
14805    }
14806
14807    let Some(target) = drain_target.as_ref() else {
14808        // Nothing was sent: the module has no registered connection to carry
14809        // `module.draining` or a GOODBYE. The caller must not assume the child
14810        // was asked to stop.
14811        return Ok(StopNotice::NoConnection);
14812    };
14813    let flush_goodbye = runtime.supervisor_handle.as_ref().is_some_and(|handle| {
14814        handle
14815            .registration_spawn(target.endpoint.connection_id)
14816            .is_some()
14817    });
14818    {
14819        send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
14820        #[cfg(feature = "test-support")]
14821        forwarding.wait_before_route_closing_for_test().await;
14822        let routes = forwarding
14823            .endpoint_routes(target.endpoint)
14824            .map_err(SuperviseError::Forwarding)?;
14825        let routes_notified = routes.len();
14826        crate::control::send_route_control_pushes(
14827            forwarding,
14828            routes.clone(),
14829            ClientControlPush::RouteClosing {
14830                module_id: spec.module_id.clone(),
14831                channels: Vec::new(),
14832                reason,
14833            },
14834        );
14835        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
14836
14837        // `route.closing` was just sent above: from here on every return path,
14838        // including an early one, MUST send `route.closed` before propagating
14839        // anything else. A client holds `closing` as a promise that a verdict is
14840        // coming; leaving early without `closed` strands it waiting forever, since
14841        // `closing` carries no timeout of its own.
14842        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
14843        let wait_result = wait_for_forwarding_quiescence(
14844            forwarding,
14845            &spec.module_id,
14846            runtime,
14847            target.endpoint,
14848            drain_deadline,
14849            &busy_gauges,
14850            scope,
14851            &mut gauge_trace,
14852        )
14853        .await;
14854        let drained = drained_after_quiescence_wait(&wait_result);
14855        if let Err(err) = &wait_result {
14856            error!(
14857                module_id = %spec.module_id,
14858                ?reason,
14859                error = %err,
14860                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
14861            );
14862        } else if !drained {
14863            // Read before teardown releases the routes.
14864            let holdouts = forwarding
14865                .endpoint_drain_holdouts(target.endpoint)
14866                .unwrap_or_default();
14867            warn_drain_timed_out(
14868                &spec.module_id,
14869                drain_timeout,
14870                reason,
14871                &holdouts,
14872                &gauge_trace,
14873            );
14874        }
14875        crate::control::send_route_control_pushes(
14876            forwarding,
14877            routes,
14878            ClientControlPush::RouteClosed {
14879                module_id: spec.module_id.clone(),
14880                channels: Vec::new(),
14881                reason,
14882                drained,
14883                abandoned: target.abandoned_bindings.len() as u32,
14884                excluded_subscriptions: target.excluded_subscriptions,
14885                terminal: Some(terminal),
14886            },
14887        );
14888        wait_result?;
14889
14890        // `route.closed` has now been sent unconditionally above. From here the
14891        // remaining steps are cleanup (route + module GOODBYE) rather than a
14892        // promise the client is waiting on, but a lock-poisoned
14893        // `release_module_endpoint_routes` would otherwise skip the module
14894        // GOODBYE silently too -- send it before propagating the error.
14895        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
14896            Ok(routes) => routes,
14897            Err(err) => {
14898                warn!(
14899                    module_id = %spec.module_id,
14900                    ?reason,
14901                    error = %err,
14902                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
14903                );
14904                send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
14905                return Err(SuperviseError::Forwarding(err));
14906            }
14907        };
14908        let route_goodbye_count = released_routes.len();
14909        send_route_goodbyes(forwarding, released_routes);
14910        send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
14911
14912        // The drain's happy path was previously silent: every emission above is
14913        // best-effort with only its failure arm logged, so "were consumers told"
14914        // was unprovable from the daemon log (surfaced by a 30-minute consumer
14915        // hang where the open question was exactly whether teardown notice went
14916        // out). One summary line makes that class decidable in one grep.
14917        info!(
14918            module_id = %spec.module_id,
14919            ?reason,
14920            routes_notified,
14921            route_goodbyes = route_goodbye_count,
14922            abandoned_reservations = target.abandoned_bindings.len(),
14923            excluded_subscriptions = target.excluded_subscriptions,
14924            drained,
14925            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
14926        );
14927    }
14928
14929    Ok(StopNotice::SentOverConnection)
14930}
14931
14932/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
14933/// the only slot a plain (non-swap) spawn can register into.
14934async fn wait_for_registration_after_reload(
14935    registry: &Registry,
14936    module_id: &str,
14937    snapshot: &SharedSnapshot,
14938    child: &mut SupervisedChild,
14939    wait: Duration,
14940) -> Result<RegistrationWaitOutcome, SuperviseError> {
14941    wait_for_slot_registration(
14942        registry,
14943        crate::registry::RegistrationSlot::Active(module_id),
14944        module_id,
14945        snapshot,
14946        child,
14947        wait,
14948    )
14949    .await
14950}
14951
14952/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
14953///
14954/// Keyed on the slot rather than the bare module id because during a swap the
14955/// id's active slot is already held by the incumbent: an id-keyed wait would
14956/// report the incumbent's registration as the candidate's and a candidate that
14957/// never registers would look registered. A swap candidate waits on
14958/// `crate::registry::RegistrationSlot::Candidate`.
14959async fn wait_for_slot_registration(
14960    registry: &Registry,
14961    slot: crate::registry::RegistrationSlot<'_>,
14962    module_id: &str,
14963    snapshot: &SharedSnapshot,
14964    child: &mut SupervisedChild,
14965    wait: Duration,
14966) -> Result<RegistrationWaitOutcome, SuperviseError> {
14967    let deadline = Instant::now() + wait;
14968    loop {
14969        if registry
14970            .registration(slot)
14971            .map_err(SuperviseError::Registry)?
14972            .is_some()
14973        {
14974            return Ok(RegistrationWaitOutcome::Registered);
14975        }
14976
14977        let now = Instant::now();
14978        if now >= deadline {
14979            return Ok(RegistrationWaitOutcome::TimedOut);
14980        }
14981        let remaining = deadline.saturating_duration_since(now);
14982        let poll = remaining.min(REGISTRY_RELEASE_POLL);
14983
14984        tokio::select! {
14985            wait_result = child.wait() => {
14986                let status = wait_result.map_err(|source| SuperviseError::Wait {
14987                    module_id: module_id.to_string(),
14988                    source,
14989                })?;
14990                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
14991                    snapshot,
14992                    child,
14993                    &status,
14994                )));
14995            }
14996            _ = sleep(poll) => {}
14997        }
14998    }
14999}
15000
15001fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
15002    // A process stopped for missing its first HELLO did not provide service,
15003    // even if its TERM handler exits zero. Reload also uses this classification
15004    // when a replacement exits before registering.
15005    if exit_report.kind != ExitKind::DeliberateSeverance {
15006        exit_report.kind = ExitKind::Crash;
15007    }
15008    exit_report
15009}
15010
15011async fn handle_reload_child_registration_failure(
15012    spec: &ModuleSpec,
15013    runtime: &SupervisorRuntimeConfig,
15014    registry: &Registry,
15015    process_liveness: &SupervisorProcessLiveness,
15016    snapshot: &SharedSnapshot,
15017    _child: &mut Option<SupervisedChild>,
15018    failure: ReloadRegistrationFailure,
15019) -> Result<(), SuperviseError> {
15020    let ReloadRegistrationFailure {
15021        exit_report,
15022        reason,
15023    } = failure;
15024    match on_child_exit(
15025        spec,
15026        runtime.restart_policy,
15027        registry,
15028        snapshot,
15029        &runtime.terminal_ring,
15030        &runtime.spawn_events,
15031        &runtime.child_roster,
15032        exit_report,
15033        runtime.recovery_is_daemon,
15034    )
15035    .await
15036    {
15037        NextAction::Stop {
15038            registration_released,
15039        } => {
15040            if registration_released {
15041                process_liveness.untrack_if_current(&spec.module_id, snapshot);
15042            }
15043        }
15044        NextAction::Restart { schedule } => {
15045            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
15046                schedule.delay
15047            });
15048            if let Some(schedule) = schedule {
15049                log_crash_respawn(&spec.module_id, schedule);
15050            }
15051            schedule_respawn(
15052                runtime,
15053                snapshot,
15054                &spec.module_id,
15055                delay,
15056                RespawnKind::Spawn,
15057                true,
15058            )?;
15059        }
15060    }
15061    Err(SuperviseError::ReloadFailed {
15062        module_id: spec.module_id.clone(),
15063        reason,
15064    })
15065}
15066
15067async fn handle_reload_spawn_failure(
15068    spec: &ModuleSpec,
15069    runtime: &SupervisorRuntimeConfig,
15070    process_liveness: &SupervisorProcessLiveness,
15071    snapshot: &SharedSnapshot,
15072    _child: &mut Option<SupervisedChild>,
15073    reason: String,
15074) -> Result<(), SuperviseError> {
15075    let now = Instant::now();
15076    let mut schedule = None;
15077    update_snapshot(snapshot, Some(&spec.module_id), |state| {
15078        clear_current_process_facts(state);
15079        if state.enabled {
15080            schedule = state.next_crash_restart(&runtime.restart_policy, now);
15081            state.state = if schedule.is_some() {
15082                ModuleState::Restarting
15083            } else {
15084                if runtime
15085                    .failure_context(runtime.recovery_is_daemon)
15086                    .may_retry()
15087                {
15088                    state.failed_retry.schedule(&spec.module_id);
15089                }
15090                ModuleState::Failed
15091            };
15092        } else {
15093            state.state = ModuleState::Disabled;
15094        }
15095    })?;
15096    if let Some(schedule) = schedule {
15097        schedule_respawn(
15098            runtime,
15099            snapshot,
15100            &spec.module_id,
15101            schedule.delay,
15102            RespawnKind::Spawn,
15103            true,
15104        )?;
15105    } else {
15106        process_liveness.untrack_if_current(&spec.module_id, snapshot);
15107    }
15108    Err(SuperviseError::ReloadFailed {
15109        module_id: spec.module_id.clone(),
15110        reason,
15111    })
15112}
15113
15114fn control_flags() -> Flags {
15115    Flags::new(false, Priority::Passive, false)
15116}
15117
15118#[allow(clippy::too_many_arguments)]
15119async fn drain_optional_child(
15120    module_id: &str,
15121    protocol: ModuleProtocol,
15122    stop_notice: StopNotice,
15123    registry: &Registry,
15124    forwarding: Option<&ForwardingTable>,
15125    snapshot: &SharedSnapshot,
15126    terminal_ring: &Arc<Mutex<TerminalRing>>,
15127    spawn_events: &SpawnEventFeed,
15128    child: &mut Option<SupervisedChild>,
15129    drain_timeout: Duration,
15130    final_state: ModuleState,
15131    enabled: Option<bool>,
15132    failure: &FailureContext,
15133) -> Result<(), SuperviseError> {
15134    if let Some(child) = child.take() {
15135        drain_child_to_state(
15136            module_id,
15137            protocol,
15138            stop_notice,
15139            registry,
15140            forwarding,
15141            snapshot,
15142            terminal_ring,
15143            spawn_events,
15144            child,
15145            drain_timeout,
15146            final_state,
15147            enabled,
15148            failure,
15149        )
15150        .await
15151    } else {
15152        update_snapshot(snapshot, Some(module_id), |state| {
15153            state.state = final_state;
15154            if final_state == ModuleState::Failed {
15155                state.failed_retry.failed_at = Some(Instant::now());
15156            }
15157            if let Some(enabled) = enabled {
15158                state.enabled = enabled;
15159            }
15160            clear_current_process_facts(state);
15161        })?;
15162        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
15163    }
15164}
15165
15166#[allow(clippy::too_many_arguments)]
15167async fn drain_child_to_state(
15168    module_id: &str,
15169    _protocol: ModuleProtocol,
15170    stop_notice: StopNotice,
15171    registry: &Registry,
15172    forwarding: Option<&ForwardingTable>,
15173    snapshot: &SharedSnapshot,
15174    terminal_ring: &Arc<Mutex<TerminalRing>>,
15175    spawn_events: &SpawnEventFeed,
15176    mut child: SupervisedChild,
15177    drain_timeout: Duration,
15178    final_state: ModuleState,
15179    enabled: Option<bool>,
15180    failure: &FailureContext,
15181) -> Result<(), SuperviseError> {
15182    let protocol = child.protocol;
15183    child.retire_registration();
15184    update_snapshot(snapshot, Some(module_id), |state| {
15185        state.state = ModuleState::Draining;
15186        state.draining_to_replace = final_state == ModuleState::Restarting;
15187        if let Some(enabled) = enabled {
15188            state.enabled = enabled;
15189        }
15190    })?;
15191
15192    // The wait below is the same budget in every case; what differs is
15193    // whether anything has ASKED the child to stop before it starts. Only a
15194    // forwarding drain that reached the module's registered connection has
15195    // (`module.draining`, then a module GOODBYE). Every other child was told
15196    // nothing: a `protocol: "none"` module, which never registers; a subc
15197    // module spawned moments ago that has not sent HELLO yet; or a stop that
15198    // runs no forwarding drain. Without a signal the budget is only a delay
15199    // in front of SIGKILL -- and the not-yet-registered child is the worst
15200    // case, because it registers into a module that is already draining,
15201    // is never told, and is killed while healthy.
15202    if stop_notice != StopNotice::SentOverConnection {
15203        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
15204            info!(
15205                module_id,
15206                pid = child.pid,
15207                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15208                "module has no connection yet; requesting stop by signal"
15209            );
15210        }
15211        request_graceful_stop(module_id, &child);
15212    }
15213
15214    let exit_report = match timeout(drain_timeout, child.wait()).await {
15215        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
15216        Ok(Err(source)) => {
15217            fail_after_wait_error(
15218                module_id,
15219                snapshot,
15220                terminal_ring,
15221                spawn_events,
15222                format!("failed to wait during drain: {source}"),
15223                failure,
15224            );
15225            return Err(SuperviseError::Wait {
15226                module_id: module_id.to_string(),
15227                source,
15228            });
15229        }
15230        Err(_) => {
15231            // Mirror the sibling arm above: state is already `Draining`, and an
15232            // error propagated from here would strand it there -- a state
15233            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
15234            // `Failed | Stopped`), leaving an operator Restart as the only exit.
15235            // `Failed` before `?` keeps the module operator-visible and
15236            // revivable. Trigger is an ESRCH race (process exits between the
15237            // drain timeout firing and the kill) or a post-kill wait failure
15238            // (issue #34).
15239            //
15240            // Logged because the kill is otherwise visible only as signal 9 in
15241            // the terminal ring, and the budget it follows can be long enough
15242            // that consumers see a stretch of refusals with no stated cause.
15243            warn!(
15244                module_id,
15245                pid = child.pid,
15246                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15247                reason = ?final_state,
15248                ?stop_notice,
15249                "drain budget expired before the module exited; killing it"
15250            );
15251            let kill =
15252                if final_state == ModuleState::Restarting && child.child.container().is_some() {
15253                    child.kill_leader()
15254                } else {
15255                    child.start_kill()
15256                };
15257            kill.map_err(|source| {
15258                fail_after_wait_error(
15259                    module_id,
15260                    snapshot,
15261                    terminal_ring,
15262                    spawn_events,
15263                    format!("failed to kill during drain: {source}"),
15264                    failure,
15265                );
15266                SuperviseError::Kill {
15267                    module_id: module_id.to_string(),
15268                    source,
15269                }
15270            })?;
15271            let status = child.wait().await.map_err(|source| {
15272                fail_after_wait_error(
15273                    module_id,
15274                    snapshot,
15275                    terminal_ring,
15276                    spawn_events,
15277                    format!("failed to wait after drain kill: {source}"),
15278                    failure,
15279                );
15280                SuperviseError::Wait {
15281                    module_id: module_id.to_string(),
15282                    source,
15283                }
15284            })?;
15285            classify_reaped_child_exit(snapshot, &child, &status)
15286        }
15287    };
15288
15289    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
15290    publish_after_terminal_record(
15291        module_id,
15292        || {
15293            record_terminal_with_detail(
15294                module_id,
15295                terminal_ring,
15296                spawn_events,
15297                &exit_report,
15298                terminal_disposition(final_state),
15299                detail,
15300            )
15301        },
15302        || {
15303            update_snapshot(snapshot, Some(module_id), |state| {
15304                state.state = final_state;
15305                if final_state == ModuleState::Failed {
15306                    state.failed_retry.failed_at = Some(Instant::now());
15307                }
15308                if let Some(enabled) = enabled {
15309                    state.enabled = enabled;
15310                }
15311                clear_current_process_facts(state);
15312                state.last_exit = Some(exit_report.clone());
15313                if exit_report.kind == ExitKind::DeliberateSeverance {
15314                    state.lifetime_restarts += 1;
15315                }
15316            })
15317        },
15318    )?;
15319    child.drain_stderr(module_id).await;
15320
15321    let released =
15322        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
15323    child
15324        .finish_exit(
15325            released.is_ok()
15326                && final_state == ModuleState::Restarting
15327                && child.spawn_failure.is_none(),
15328        )
15329        .await
15330        .map_err(|source| SuperviseError::Wait {
15331            module_id: module_id.to_owned(),
15332            source,
15333        })?;
15334    released
15335}
15336
15337/// Ask a child that nothing else has asked to stop, by signal.
15338///
15339/// A registered subc module is asked over its own connection: the drain sends
15340/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
15341/// module GOODBYE, and the module stops itself. A module that speaks no subc
15342/// wire receives none of that, and neither does a subc module that has not
15343/// registered yet, so for them the drain budget would be pure delay in front of
15344/// a SIGKILL -- and for a process with a store to flush (JetStream is the
15345/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
15346/// into a recovery on the next start.
15347///
15348/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
15349/// rule rather than an optimisation: that module's graceful stop is already
15350/// running by the time its child is drained, and a signal would race it.
15351///
15352/// Best-effort by construction. A child that has already exited is the ordinary
15353/// case rather than an error (the kill lands on a reaped or exiting pid), so a
15354/// failure is logged at debug and the wait-then-kill below still decides the
15355/// outcome.
15356#[cfg(unix)]
15357fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
15358    let Some(pid) = child
15359        .id()
15360        .and_then(|pid| i32::try_from(pid).ok())
15361        .and_then(rustix::process::Pid::from_raw)
15362    else {
15363        debug!(
15364            module_id,
15365            "no pid to signal for teardown; falling through to the drain wait"
15366        );
15367        return;
15368    };
15369    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
15370        Ok(()) => debug!(
15371            module_id,
15372            "sent SIGTERM to a module nothing else asked to stop"
15373        ),
15374        Err(err) => debug!(
15375            module_id,
15376            error = %err,
15377            "SIGTERM to module failed; the drain wait and kill still apply"
15378        ),
15379    }
15380}
15381
15382/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
15383/// Windows does offer need cooperation this supervisor cannot assume: a console
15384/// control event requires sharing a console with the child, and `WM_CLOSE`
15385/// requires the child to pump a message loop. A supervised server process does
15386/// neither, so there is nothing to send and teardown is the wait followed by the
15387/// kill. Emulating a signal here would mean inventing a stop protocol, which is
15388/// the thing `protocol: "none"` exists to avoid.
15389#[cfg(not(unix))]
15390fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
15391    debug!(
15392        module_id,
15393        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
15394    );
15395}
15396
15397fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
15398    match final_state {
15399        ModuleState::Stopped => TerminalDisposition::Stopped,
15400        ModuleState::Disabled => TerminalDisposition::Disabled,
15401        ModuleState::Restarting => TerminalDisposition::Restarting,
15402        ModuleState::Failed => TerminalDisposition::Failed,
15403        ModuleState::Starting
15404        | ModuleState::Running
15405        | ModuleState::Unresponsive
15406        | ModuleState::Draining => {
15407            unreachable!("terminal exits only finish in terminal or restarting states")
15408        }
15409    }
15410}
15411
15412/// Release a reaped child's registration before allowing another spawn.
15413///
15414/// EOF is not a process-lifetime signal: an inherited socket can stay open
15415/// indefinitely, and serial frame dispatch can be waiting on egress instead of
15416/// reading EOF. After the normal release grace, request connection close (which
15417/// cancels both reads and dispatch), then allow one more release grace for the
15418/// connection guard's forwarding cleanup. Never evict a different connection.
15419async fn release_dead_registration(
15420    registry: &Registry,
15421    forwarding: Option<&ForwardingTable>,
15422    snapshot: &SharedSnapshot,
15423    module_id: &str,
15424    failure: &FailureContext,
15425) -> Result<(), SuperviseError> {
15426    let result = async {
15427        let registration = registry
15428            .get_module(module_id)
15429            .map_err(SuperviseError::Registry)?;
15430        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
15431            Ok(()) => return Ok(()),
15432            Err(SuperviseError::RegistrationStillActive { .. }) => {}
15433            Err(err) => return Err(err),
15434        }
15435        let pid = lock_snapshot(snapshot)?.reaped_pid;
15436        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
15437            warn!(
15438                module_id,
15439                pid,
15440                connection_id = registration.connection_id.get(),
15441                "reaped module registration outlived release grace; closing dead connection"
15442            );
15443            forwarding.request_connection_close(
15444                registration.connection_id,
15445                CloseReason::new(
15446                    "supervised_process_reaped",
15447                    format!("module '{module_id}' pid {pid} exited"),
15448                ),
15449            );
15450            wait_for_slot_registration_release(
15451                registry,
15452                crate::registry::RegistrationSlot::Connection(registration.connection_id),
15453                REGISTRY_RELEASE_TIMEOUT,
15454            )
15455            .await?;
15456        }
15457        wait_for_registration_release(registry, module_id, Duration::ZERO).await
15458    }
15459    .await;
15460    if let Err(err) = &result {
15461        fail_snapshot(snapshot, Some(module_id), None, failure);
15462        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
15463    }
15464    result
15465}
15466
15467/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
15468/// plain stop or restart waits for before it spawns a replacement.
15469async fn wait_for_registration_release(
15470    registry: &Registry,
15471    module_id: &str,
15472    wait: Duration,
15473) -> Result<(), SuperviseError> {
15474    wait_for_slot_registration_release(
15475        registry,
15476        crate::registry::RegistrationSlot::Active(module_id),
15477        wait,
15478    )
15479    .await
15480}
15481
15482/// Wait for the registration in `slot` to go away.
15483///
15484/// Keyed on the slot rather than the bare module id because a successful swap
15485/// never empties the id's active slot (the promoted candidate is in it), so an
15486/// id-keyed wait for the incumbent's release would always time out. Draining a
15487/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
15488/// incumbent's connection instead.
15489async fn wait_for_slot_registration_release(
15490    registry: &Registry,
15491    slot: crate::registry::RegistrationSlot<'_>,
15492    wait: Duration,
15493) -> Result<(), SuperviseError> {
15494    let deadline = Instant::now() + wait;
15495    let mut release_events = registration_release_events().subscribe();
15496    let still_active = |registration: &crate::registry::ModuleRegistration| {
15497        SuperviseError::RegistrationStillActive {
15498            module_id: registration.manifest.module_id.clone(),
15499            waited: wait,
15500        }
15501    };
15502    loop {
15503        let _observed_generation = *release_events.borrow_and_update();
15504        let Some(registration) = registry
15505            .registration(slot)
15506            .map_err(SuperviseError::Registry)?
15507        else {
15508            return Ok(());
15509        };
15510
15511        let now = Instant::now();
15512        if now >= deadline {
15513            return Err(still_active(&registration));
15514        }
15515
15516        let remaining = deadline.saturating_duration_since(now);
15517        match timeout(remaining, release_events.changed()).await {
15518            Ok(Ok(())) | Ok(Err(_)) => {}
15519            Err(_) => return Err(still_active(&registration)),
15520        }
15521    }
15522}
15523
15524#[cfg(test)]
15525mod slot_registration_wait_tests {
15526    use super::*;
15527    use crate::registry::{ConnectionId, RegistrationSlot};
15528    use subc_protocol::manifest::ModuleManifest;
15529
15530    #[tokio::test]
15531    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
15532        let registry = Arc::new(Registry::default());
15533        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
15534        let runtime = supervisor.runtime_config();
15535        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
15536        let spec = ModuleSpec {
15537            module_id: "enable-stale-registration".to_string(),
15538            program: PathBuf::from("/missing/enable-retry-test"),
15539            args: Vec::new(),
15540            env: Vec::new(),
15541            reserved: false,
15542            reserved_prefixes: Vec::new(),
15543            protocol: ModuleProtocol::Subc,
15544            overlap: Default::default(),
15545            carry_containment: false,
15546        };
15547        let connection = ConnectionId::new(90);
15548        registry
15549            .register_with_control_ops(
15550                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
15551                1,
15552                connection,
15553                Vec::new(),
15554            )
15555            .unwrap();
15556        let mut child = None;
15557        let err = set_child_enabled(
15558            &spec,
15559            &runtime,
15560            &registry,
15561            &supervisor.process_liveness,
15562            &snapshot,
15563            &mut child,
15564            true,
15565            &runtime.failure_context(false),
15566        )
15567        .await
15568        .unwrap_err();
15569        assert!(matches!(
15570            err,
15571            SuperviseError::RegistrationStillActive { .. }
15572        ));
15573        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15574        assert!(child.is_none());
15575        registry.deregister_connection(connection).unwrap();
15576        let err = set_child_enabled(
15577            &spec,
15578            &runtime,
15579            &registry,
15580            &supervisor.process_liveness,
15581            &snapshot,
15582            &mut child,
15583            true,
15584            &runtime.failure_context(false),
15585        )
15586        .await
15587        .unwrap_err();
15588        assert!(
15589            matches!(err, SuperviseError::Spawn { .. }),
15590            "second enable must attempt a spawn: {err}"
15591        );
15592        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15593    }
15594
15595    const INCUMBENT: u64 = 1;
15596    const CANDIDATE: u64 = 2;
15597
15598    fn swapped_registry() -> Arc<Registry> {
15599        let registry = Arc::new(Registry::default());
15600        let manifest = ModuleManifest::builder("m", "0.1.0").build();
15601        registry
15602            .register_with_control_ops(
15603                manifest.clone(),
15604                1,
15605                ConnectionId::new(INCUMBENT),
15606                Vec::new(),
15607            )
15608            .unwrap();
15609        registry
15610            .register_candidate_with_control_ops(
15611                manifest,
15612                1,
15613                ConnectionId::new(CANDIDATE),
15614                Vec::new(),
15615            )
15616            .unwrap();
15617        registry
15618    }
15619
15620    /// After a promotion the id's active slot is held by the new process, so an
15621    /// id-keyed wait for the incumbent's release can never succeed; the
15622    /// connection-keyed wait completes as soon as the incumbent deregisters.
15623    #[tokio::test]
15624    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
15625        let registry = swapped_registry();
15626        registry.promote_candidate("m").unwrap().unwrap();
15627
15628        assert!(matches!(
15629            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
15630            Err(SuperviseError::RegistrationStillActive { .. })
15631        ));
15632
15633        // Still held while the incumbent's connection has not deregistered.
15634        assert!(matches!(
15635            wait_for_slot_registration_release(
15636                &registry,
15637                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15638                Duration::from_millis(50),
15639            )
15640            .await,
15641            Err(SuperviseError::RegistrationStillActive { .. })
15642        ));
15643
15644        let releaser = Arc::clone(&registry);
15645        let release = tokio::spawn(async move {
15646            sleep(Duration::from_millis(20)).await;
15647            releaser
15648                .deregister_connection(ConnectionId::new(INCUMBENT))
15649                .unwrap();
15650            notify_registration_release();
15651        });
15652        wait_for_slot_registration_release(
15653            &registry,
15654            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15655            Duration::from_secs(5),
15656        )
15657        .await
15658        .expect("the incumbent's own registration is released");
15659        release.await.unwrap();
15660        assert!(registry.get_module("m").unwrap().is_some());
15661    }
15662
15663    /// The candidate slot is waited on separately from the active slot: the
15664    /// incumbent's registration neither holds up nor stands in for it.
15665    #[tokio::test]
15666    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
15667        let registry = swapped_registry();
15668        assert!(matches!(
15669            wait_for_slot_registration_release(
15670                &registry,
15671                RegistrationSlot::Candidate("m"),
15672                Duration::from_millis(50),
15673            )
15674            .await,
15675            Err(SuperviseError::RegistrationStillActive { .. })
15676        ));
15677        registry
15678            .deregister_connection(ConnectionId::new(CANDIDATE))
15679            .unwrap();
15680        wait_for_slot_registration_release(
15681            &registry,
15682            RegistrationSlot::Candidate("m"),
15683            Duration::from_millis(50),
15684        )
15685        .await
15686        .expect("a candidate slot with no candidate is released");
15687        assert!(registry
15688            .registration(RegistrationSlot::Active("m"))
15689            .unwrap()
15690            .is_some());
15691    }
15692}
15693
15694fn classify_exit(status: &ExitStatus) -> ExitReport {
15695    ExitReport {
15696        kind: if status.success() {
15697            ExitKind::Clean
15698        } else {
15699            ExitKind::Crash
15700        },
15701        code: status.code(),
15702        signal: exit_signal(status),
15703        at_ms: unix_ms_now(),
15704    }
15705}
15706
15707/// The terminal record for a module whose `wait()` call itself errored (e.g. the
15708/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
15709/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
15710/// disposition still must be `Failed` so the terminal ring is not silently missing
15711/// an entry, matching what `fail_snapshot` records for this same arm.
15712fn wait_error_exit_report() -> ExitReport {
15713    ExitReport {
15714        kind: ExitKind::Crash,
15715        code: None,
15716        signal: None,
15717        at_ms: unix_ms_now(),
15718    }
15719}
15720
15721#[cfg(unix)]
15722fn exit_signal(status: &ExitStatus) -> Option<i32> {
15723    use std::os::unix::process::ExitStatusExt;
15724
15725    status.signal()
15726}
15727
15728#[cfg(not(unix))]
15729fn exit_signal(_status: &ExitStatus) -> Option<i32> {
15730    None
15731}
15732
15733/// Give an operator-touched module its full crash budget back.
15734///
15735/// Named for the counter it used to zero; it now empties the in-window ring,
15736/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
15737/// ledger of what happened survives every operator action.
15738fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
15739    update_snapshot(snapshot, Some(module_id), |state| {
15740        state.clear_crash_restarts();
15741    })
15742}
15743
15744fn set_running(
15745    snapshot: &SharedSnapshot,
15746    child: &SupervisedChild,
15747    module_id: &str,
15748    spawn_events: &SpawnEventFeed,
15749) -> Result<(), SuperviseError> {
15750    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15751        module_id: Some(module_id.to_string()),
15752    })?;
15753    state.spawn_generation = spawn_events.emit_spawned(
15754        module_id,
15755        child
15756            .registration_spawn
15757            .as_ref()
15758            .map(|spawn| spawn.authority.spawn_number()),
15759        child.pid,
15760        child.spawned_at_ms,
15761    );
15762    if std::mem::take(&mut state.coalesced_restart_pending) {
15763        let generation = state.spawn_generation;
15764        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
15765    }
15766    state.drain_disposition_detail = None;
15767    state.spawn_failure = None;
15768    state.admission_refused = None;
15769    state.config_hold_parked = false;
15770    state.config_hold_daemon_initiated = false;
15771    state.config_hold_approval = None;
15772    state.legacy_registration_spawn =
15773        child.registration_spawn.is_some() && child.admission_base.is_none();
15774    state.registration_declaration = child.registration_declaration.clone();
15775    state.stop_reason = None;
15776    state.config_exit_failed_approval = None;
15777    state.waiting_for_previous_pid = None;
15778    state.waiting_for_previous_detail = None;
15779    // Every caller of this is a plain spawn, which always uses the primary key;
15780    // a promoted swap candidate sets the flag itself after this returns.
15781    state.in_alternate_slot = false;
15782    state.configuration_updated_since_spawn = false;
15783    state.spawned_protocol = Some(child.protocol);
15784    state.state = ModuleState::Running;
15785    state.approved_start_failure = None;
15786    state.enabled = true;
15787    state.process_alive = true;
15788    state.pid = child.id();
15789    state.subtree_target = child.subtree_target.clone();
15790    // Probe samples belong to one process. Keep module-level escalation and
15791    // late-answer history, but never charge a new child for its predecessor's misses.
15792    state.health.status = SupervisorHealthStatus::Unknown;
15793    state.last_probe_evidence = None;
15794    state.health.consecutive_failures = 0;
15795    state.health.detail = None;
15796    state.health.metrics = None;
15797    state.health.last_probe_ms = None;
15798    state.failed_retry.healthy_since = None;
15799    state.failed_retry.failed_at = None;
15800    state.failed_retry.running_since = Some(child.spawned_at);
15801    #[cfg(windows)]
15802    {
15803        state.pipe_consumed = child
15804            .nonce_delivery
15805            .as_ref()
15806            .map(|delivery| delivery.consumed());
15807    }
15808    #[cfg(target_os = "macos")]
15809    {
15810        state.report_ready = Some(Arc::clone(&child.report_ready));
15811    }
15812    state.spawned_at_ms = Some(child.spawned_at_ms);
15813    state.exec_outcome = Some(Arc::clone(&child.exec.0));
15814    state.spawned_from = Some(child.spawned_from.clone());
15815    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
15816    state.spawned_file_identity = child.spawned_file_identity.clone();
15817    state.process_start_time = child.process_start_time;
15818    Ok(())
15819}
15820
15821fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
15822    state.legacy_registration_spawn = false;
15823    state.registration_declaration = None;
15824    state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
15825    state.failed_retry.healthy_since = None;
15826    state.failed_retry.running_since = None;
15827    state.waiting_for_previous_pid = None;
15828    state.waiting_for_previous_detail = None;
15829    state.process_alive = false;
15830    state.spawned_protocol = None;
15831    state.pid = None;
15832    #[cfg(windows)]
15833    {
15834        state.pipe_consumed = None;
15835    }
15836    #[cfg(target_os = "macos")]
15837    {
15838        state.report_ready = None;
15839    }
15840    state.spawned_at_ms = None;
15841    state.exec_outcome = None;
15842    state.spawned_from = None;
15843    state.launch_spec_digest = None;
15844    state.spawned_file_identity = None;
15845    state.process_start_time = None;
15846    state.deliberate_severance = None;
15847}
15848
15849#[cfg(test)]
15850fn record_deliberate_severance(
15851    snapshot: &SharedSnapshot,
15852    identity: ProcessIdentity,
15853) -> Result<(), SuperviseError> {
15854    update_snapshot(snapshot, None, |state| {
15855        state.deliberate_severance = Some(identity);
15856    })
15857}
15858
15859fn apply_deliberate_severance_marker(
15860    snapshot: &SharedSnapshot,
15861    exited_identity: Option<ProcessIdentity>,
15862    mut exit_report: ExitReport,
15863) -> ExitReport {
15864    let marker = lock_snapshot(snapshot)
15865        .ok()
15866        .and_then(|mut state| state.deliberate_severance.take());
15867    if marker.is_some() && marker == exited_identity {
15868        exit_report.kind = ExitKind::DeliberateSeverance;
15869    }
15870    exit_report
15871}
15872
15873fn classify_reaped_child_exit(
15874    snapshot: &SharedSnapshot,
15875    child: &SupervisedChild,
15876    status: &ExitStatus,
15877) -> ExitReport {
15878    let _ = update_snapshot(snapshot, None, |state| {
15879        state.reaped_pid = Some(child.pid);
15880        state.spawn_failure = child.spawn_failure.clone();
15881    });
15882    if matches!(
15883        child.spawn_failure.as_deref(),
15884        Some("registration_timeout" | "registration_channel_closed")
15885    ) {
15886        registration_failure_exit_report(classify_exit(status))
15887    } else {
15888        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
15889    }
15890}
15891
15892fn fail_snapshot(
15893    snapshot: &SharedSnapshot,
15894    module_id: Option<&str>,
15895    last_exit: Option<ExitReport>,
15896    failure: &FailureContext,
15897) {
15898    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
15899        state.state = ModuleState::Failed;
15900        state.config_hold_parked = false;
15901        state.config_hold_daemon_initiated = false;
15902        state.config_hold_approval = None;
15903        clear_current_process_facts(state);
15904        if failure.may_retry() && state.enabled {
15905            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
15906        } else if failure.shutdown.is_set() {
15907            state.failed_retry.cancel();
15908        }
15909        if let Some(last_exit) = last_exit {
15910            state.last_exit = Some(last_exit);
15911        }
15912    }) {
15913        error!(error = %err, "failed to mark supervisor state failed");
15914    }
15915}
15916
15917fn update_snapshot(
15918    snapshot: &SharedSnapshot,
15919    module_id: Option<&str>,
15920    update: impl FnOnce(&mut SupervisorSnapshot),
15921) -> Result<(), SuperviseError> {
15922    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15923        module_id: module_id.map(ToOwned::to_owned),
15924    })?;
15925    update(&mut state);
15926    Ok(())
15927}
15928
15929const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
15930
15931fn lock_snapshot_for_control<'a>(
15932    snapshot: &'a SharedSnapshot,
15933    module_id: &str,
15934    caller: &'static str,
15935) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
15936    let started_at = Instant::now();
15937    let guard = lock_snapshot(snapshot)?;
15938    let waited = started_at.elapsed();
15939    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
15940        warn!(
15941            module_id = %module_id,
15942            waited_ms = waited.as_millis() as u64,
15943            caller = %caller,
15944            "slow snapshot lock"
15945        );
15946    }
15947    Ok(guard)
15948}
15949
15950fn lock_snapshot(
15951    snapshot: &SharedSnapshot,
15952) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
15953    snapshot
15954        .lock()
15955        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
15956}
15957
15958#[cfg(test)]
15959mod terminal_history_tests {
15960    include!("supervise_route_open_recovery_tests.rs");
15961    use std::{
15962        path::PathBuf,
15963        sync::Arc,
15964        time::{Duration, Instant},
15965    };
15966
15967    use tokio::{sync::mpsc, time::sleep};
15968
15969    use super::{
15970        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
15971        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
15972        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
15973        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
15974        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
15975        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
15976        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
15977        SupervisorHealthStatus, SupervisorSnapshot,
15978    };
15979    // The supervisor's clock, distinct from the `std::time::Instant` these tests
15980    // use for their own wall-clock deadlines: crash-restart instants must be on
15981    // the same clock the production code stamps them with, which is tokio's (and
15982    // is what `start_paused` tests can move).
15983    use super::Instant as ClockInstant;
15984    use crate::{
15985        registry::Registry,
15986        terminal_ring::{TerminalRing, TerminalRingConfig},
15987    };
15988    use std::sync::Mutex;
15989    use subc_control::TerminalDisposition;
15990
15991    /// See the twin in `control.rs` for why this derives the path from
15992    /// `current_exe()` and why the existence check is here: `--lib` alone does
15993    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
15994    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
15995    pub(super) fn fake_aft_stub_path() -> PathBuf {
15996        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
15997        assert!(
15998            path.exists(),
15999            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16000             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16001            path.display()
16002        );
16003        path
16004    }
16005
16006    #[test]
16007    fn reserved_never_spawned_refuses_every_hello() {
16008        // The canary hole: a reserved id whose module has never spawned had NO
16009        // gate entry and admitted anyone -- the reservation protected the nonce
16010        // holder, not the NAME. Now the entry is present with no legitimate
16011        // holder and refuses all comers.
16012        let supervisor = SupervisorHandle::default();
16013        supervisor.apply_identity_configuration(&ModuleSpec {
16014            module_id: "never-spawned".to_string(),
16015            program: PathBuf::from("/usr/bin/false"),
16016            args: Vec::new(),
16017            env: Vec::new(),
16018            reserved: true,
16019            reserved_prefixes: Vec::new(),
16020            protocol: ModuleProtocol::Subc,
16021            overlap: Default::default(),
16022            carry_containment: false,
16023        });
16024        assert!(
16025            supervisor
16026                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
16027                .is_err(),
16028            "forged nonce must refuse on a reserved never-spawned id"
16029        );
16030        assert!(
16031            supervisor
16032                .reserved_hello_verification("never-spawned", None)
16033                .is_err(),
16034            "absent nonce must refuse on a reserved never-spawned id"
16035        );
16036        // And a real spawn nonce minted later admits exactly that nonce.
16037        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
16038        supervisor.apply_identity_configuration(&ModuleSpec {
16039            module_id: "never-spawned".to_string(),
16040            program: PathBuf::from("/usr/bin/false"),
16041            args: Vec::new(),
16042            env: Vec::new(),
16043            reserved: true,
16044            reserved_prefixes: Vec::new(),
16045            protocol: ModuleProtocol::Subc,
16046            overlap: Default::default(),
16047            carry_containment: false,
16048        });
16049        assert!(supervisor
16050            .reserved_hello_verification("never-spawned", Some("minted"))
16051            .is_ok());
16052        assert!(supervisor
16053            .reserved_hello_verification("never-spawned", Some("forged"))
16054            .is_err());
16055    }
16056
16057    /// Put `count` crash restarts on a snapshot's ring as if they had all just
16058    /// happened, which is what "spent budget" looks like to every reader.
16059    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
16060        let now = ClockInstant::now();
16061        for _ in 0..count {
16062            state.crash_restarts.push_back(now);
16063        }
16064    }
16065
16066    /// Age the oldest recorded restart out of `window`, standing in for the hours
16067    /// that would otherwise have to pass. Injecting the instant is the point: a
16068    /// test that slept a real window would take ten minutes and still prove less.
16069    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
16070        let aged = state
16071            .crash_restarts
16072            .front()
16073            .expect("a crash restart must be recorded before it can be aged")
16074            .checked_sub(window + Duration::from_secs(1))
16075            .expect("the test clock is far enough from its origin to age an instant");
16076        state.crash_restarts[0] = aged;
16077    }
16078
16079    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
16080        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
16081        seed_crash_restarts(&mut state, count);
16082        state
16083    }
16084
16085    #[test]
16086    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
16087        let policy = RestartPolicy::new(3, Duration::ZERO);
16088        let now = ClockInstant::now();
16089        assert!(daemon_will_restart(
16090            &mut snapshot_with_restarts(true, 2),
16091            &policy,
16092            now
16093        ));
16094        assert!(!daemon_will_restart(
16095            &mut snapshot_with_restarts(true, 3),
16096            &policy,
16097            now
16098        ));
16099        assert!(!daemon_will_restart(
16100            &mut snapshot_with_restarts(false, 0),
16101            &policy,
16102            now
16103        ));
16104    }
16105
16106    #[test]
16107    fn crash_restart_backoff_escalates_with_in_window_count() {
16108        let policy = RestartPolicy::new(4, Duration::from_millis(100))
16109            .with_max_backoff(Duration::from_secs(30));
16110        let now = ClockInstant::now();
16111        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16112        let schedules = (0..4)
16113            .map(|_| {
16114                state
16115                    .next_crash_restart(&policy, now)
16116                    .expect("the test policy allows four crash restarts")
16117            })
16118            .collect::<Vec<_>>();
16119
16120        assert_eq!(
16121            schedules
16122                .iter()
16123                .map(|schedule| schedule.restart_in_window)
16124                .collect::<Vec<_>>(),
16125            vec![0, 1, 2, 3]
16126        );
16127        assert_eq!(
16128            schedules
16129                .iter()
16130                .map(|schedule| schedule.delay)
16131                .collect::<Vec<_>>(),
16132            vec![
16133                Duration::from_millis(100),
16134                Duration::from_secs(1),
16135                Duration::from_secs(10),
16136                Duration::from_secs(30),
16137            ]
16138        );
16139    }
16140
16141    #[test]
16142    fn crash_restart_backoff_resets_after_ring_clear() {
16143        let policy = RestartPolicy::new(3, Duration::from_millis(100));
16144        let now = ClockInstant::now();
16145        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16146        assert_eq!(
16147            state.next_crash_restart(&policy, now).unwrap().delay,
16148            Duration::from_millis(100)
16149        );
16150        assert_eq!(
16151            state.next_crash_restart(&policy, now).unwrap().delay,
16152            Duration::from_secs(1)
16153        );
16154
16155        state.clear_crash_restarts();
16156        let schedule = state
16157            .next_crash_restart(&policy, now)
16158            .expect("a cleared ring must allow another restart");
16159        assert_eq!(schedule.restart_in_window, 0);
16160        assert_eq!(schedule.delay, Duration::from_millis(100));
16161    }
16162
16163    #[test]
16164    fn crash_restart_backoff_ignores_aged_restarts() {
16165        let policy = RestartPolicy::new(3, Duration::from_millis(100));
16166        let now = ClockInstant::now();
16167        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16168        state
16169            .next_crash_restart(&policy, now)
16170            .expect("the first restart is allowed");
16171        state
16172            .next_crash_restart(&policy, now)
16173            .expect("the second restart is allowed");
16174        state.crash_restarts[0] = now
16175            .checked_sub(policy.window + Duration::from_secs(1))
16176            .expect("the fake clock can age a restart past the window");
16177
16178        let schedule = state
16179            .next_crash_restart(&policy, now)
16180            .expect("an aged restart must release its slot");
16181        assert_eq!(schedule.restart_in_window, 1);
16182        assert_eq!(schedule.delay, Duration::from_secs(1));
16183        assert_eq!(state.crash_restarts.len(), 2);
16184    }
16185
16186    /// The budget is a rate: the same three spent restarts refuse a respawn
16187    /// while they are recent and allow one once they have aged past the window.
16188    /// Nothing about the module changed in between, which is the whole point.
16189    #[test]
16190    fn a_budget_spent_before_the_window_no_longer_refuses() {
16191        let policy = RestartPolicy::new(3, Duration::ZERO);
16192        let mut state = snapshot_with_restarts(true, 3);
16193        let now = ClockInstant::now();
16194        assert!(!daemon_will_restart(&mut state, &policy, now));
16195
16196        assert!(daemon_will_restart(
16197            &mut state,
16198            &policy,
16199            now + policy.window + Duration::from_secs(1)
16200        ));
16201        assert!(
16202            state.crash_restarts.is_empty(),
16203            "reading the budget must drop the instants that left the window"
16204        );
16205    }
16206
16207    fn module_with_recovery_snapshot(
16208        state: ModuleState,
16209        enabled: bool,
16210        restart_count: u32,
16211    ) -> SupervisedModule {
16212        let registry = Arc::new(Registry::default());
16213        let supervisor =
16214            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
16215        let runtime = supervisor.runtime_config();
16216        let spec = ModuleSpec {
16217            module_id: "recovery-snapshot".to_string(),
16218            program: fake_aft_stub_path(),
16219            args: Vec::new(),
16220            env: Vec::new(),
16221            reserved: false,
16222            reserved_prefixes: Vec::new(),
16223            protocol: ModuleProtocol::Subc,
16224            overlap: Default::default(),
16225            carry_containment: false,
16226        };
16227        let mut snapshot = SupervisorSnapshot::new(state, enabled);
16228        seed_crash_restarts(&mut snapshot, restart_count);
16229        // These tests read synthetic snapshots. A real child and monitor would
16230        // race those reads by replacing the requested state during startup.
16231        let (commands, _rx) = mpsc::channel(4);
16232        let configured_enabled = snapshot.enabled;
16233        SupervisedModule {
16234            inner: Arc::new(SupervisedModuleInner {
16235                module_id: spec.module_id.clone(),
16236                registry,
16237                snapshot: Arc::new(Mutex::new(snapshot)),
16238                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
16239                    spec,
16240                    enabled: configured_enabled,
16241                    health: runtime.health,
16242                    register_timeout: runtime.register_timeout,
16243                    restart_policy: runtime.restart_policy,
16244                })),
16245                stderr_ring: runtime.stderr_ring,
16246                terminal_ring: runtime.terminal_ring,
16247                commands,
16248                monitor: Mutex::new(None),
16249                effective_drain_timeout: runtime.effective_drain_timeout,
16250                provenance_probe: supervisor.provenance_probe.clone(),
16251                retained: runtime.child_roster.retained_view(),
16252            }),
16253        }
16254    }
16255
16256    #[cfg(target_os = "linux")]
16257    #[tokio::test]
16258    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
16259        let supervisor =
16260            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16261                .with_cgroup_placement(None);
16262        let result = supervisor.spawn(ModuleSpec {
16263            module_id: "no-cgroup-placement".to_string(),
16264            program: fake_aft_stub_path(),
16265            args: Vec::new(),
16266            env: Vec::new(),
16267            reserved: false,
16268            reserved_prefixes: Vec::new(),
16269            protocol: ModuleProtocol::Subc,
16270            overlap: Default::default(),
16271            carry_containment: false,
16272        });
16273
16274        assert!(
16275            result.is_ok(),
16276            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
16277        );
16278    }
16279
16280    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16281    async fn undecided_snapshot_uses_shared_restart_predicate() {
16282        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
16283            .will_recover_after_connection_loss()
16284            .unwrap());
16285        assert!(
16286            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
16287                .will_recover_after_connection_loss()
16288                .unwrap()
16289        );
16290    }
16291
16292    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16293    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
16294        assert!(
16295            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
16296                .will_recover_after_connection_loss()
16297                .unwrap()
16298        );
16299    }
16300
16301    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16302    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
16303        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
16304            .will_recover_after_connection_loss()
16305            .unwrap());
16306        assert!(
16307            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
16308                .will_recover_after_connection_loss()
16309                .unwrap()
16310        );
16311    }
16312
16313    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16314    async fn warming_snapshot_is_limited_to_startup_phases() {
16315        for state in [
16316            ModuleState::Starting,
16317            ModuleState::Running,
16318            ModuleState::Restarting,
16319        ] {
16320            assert!(
16321                module_with_recovery_snapshot(state, true, 0)
16322                    .is_warming()
16323                    .unwrap(),
16324                "{state:?} should be warming"
16325            );
16326        }
16327        for state in [
16328            ModuleState::Unresponsive,
16329            ModuleState::Draining,
16330            ModuleState::Stopped,
16331            ModuleState::Failed,
16332            ModuleState::Disabled,
16333        ] {
16334            assert!(
16335                !module_with_recovery_snapshot(state, true, 0)
16336                    .is_warming()
16337                    .unwrap(),
16338                "{state:?} should not be warming"
16339            );
16340        }
16341    }
16342
16343    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16344    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
16345        let registry = Arc::new(Registry::default());
16346        let supervisor =
16347            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
16348        let module = supervisor
16349            .spawn(ModuleSpec {
16350                module_id: "terminal-history".to_string(),
16351                program: fake_aft_stub_path(),
16352                args: Vec::new(),
16353                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16354                reserved: false,
16355                reserved_prefixes: Vec::new(),
16356                protocol: ModuleProtocol::Subc,
16357                overlap: Default::default(),
16358                carry_containment: false,
16359            })
16360            .unwrap();
16361
16362        let deadline = Instant::now() + Duration::from_secs(5);
16363        loop {
16364            let history = module.terminal_history();
16365            if history.entries.len() == 2 {
16366                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
16367                assert_eq!(history.dropped, 0);
16368                assert_eq!(
16369                    history
16370                        .entries
16371                        .iter()
16372                        .map(|entry| entry.exit_code)
16373                        .collect::<Vec<_>>(),
16374                    vec![Some(23), Some(23)]
16375                );
16376                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
16377                return;
16378            }
16379            assert!(
16380                Instant::now() < deadline,
16381                "module did not retain two terminal exits: {history:?}"
16382            );
16383            sleep(Duration::from_millis(10)).await;
16384        }
16385    }
16386
16387    /// A disable issued while a crash respawn is still backing off must preempt
16388    /// that respawn: the operator's stop wins, the disable must not queue behind
16389    /// the backoff, and the module must never come back up afterwards.
16390    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16391    async fn disable_during_crash_backoff_cancels_pending_respawn() {
16392        let backoff = Duration::from_secs(2);
16393        let supervisor = Supervisor::new_for_test(
16394            Arc::new(Registry::default()),
16395            RestartPolicy::new(10, backoff),
16396        );
16397        let module = supervisor
16398            .spawn(ModuleSpec {
16399                module_id: "disable-during-backoff".to_string(),
16400                program: fake_aft_stub_path(),
16401                args: Vec::new(),
16402                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16403                reserved: false,
16404                reserved_prefixes: Vec::new(),
16405                protocol: ModuleProtocol::Subc,
16406                overlap: Default::default(),
16407                carry_containment: false,
16408            })
16409            .unwrap();
16410
16411        // Wait for the first crash to put the module into its backoff window.
16412        let deadline = Instant::now() + Duration::from_secs(5);
16413        loop {
16414            if module.status().unwrap().state == ModuleState::Restarting {
16415                break;
16416            }
16417            assert!(
16418                Instant::now() < deadline,
16419                "module never entered the crash backoff"
16420            );
16421            sleep(Duration::from_millis(10)).await;
16422        }
16423
16424        let started = Instant::now();
16425        module.set_enabled(false).await.unwrap();
16426        let waited = started.elapsed();
16427
16428        assert!(
16429            waited < backoff / 2,
16430            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
16431        );
16432        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
16433
16434        // Outlast the backoff: the respawn it was counting down to must never run.
16435        sleep(backoff + Duration::from_millis(500)).await;
16436        let status = module.status().unwrap();
16437        assert_eq!(status.state, ModuleState::Disabled);
16438        assert_eq!(
16439            status.spawn_generation, 1,
16440            "module respawned after the operator disabled it"
16441        );
16442    }
16443
16444    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
16445    /// the shape of nats-server, the program this rule exists for.
16446    #[cfg(unix)]
16447    fn protocol_none_sigterm_exits_clean_spec(
16448        module_id: &str,
16449        dir: &std::path::Path,
16450    ) -> (ModuleSpec, PathBuf, PathBuf) {
16451        let ready = dir.join("ready");
16452        let marker = dir.join("sigterm");
16453        let spec = ModuleSpec {
16454            module_id: module_id.to_string(),
16455            program: fake_aft_stub_path(),
16456            args: Vec::new(),
16457            env: vec![
16458                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16459                (
16460                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
16461                    marker.display().to_string(),
16462                ),
16463                (
16464                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
16465                    ready.display().to_string(),
16466                ),
16467            ],
16468            reserved: false,
16469            reserved_prefixes: Vec::new(),
16470            protocol: ModuleProtocol::None,
16471            overlap: Default::default(),
16472            carry_containment: false,
16473        };
16474        (spec, ready, marker)
16475    }
16476
16477    /// Wait for a file the child writes, so a signal is never sent before the
16478    /// child's SIGTERM handler is installed (the default disposition would
16479    /// kill it by signal and the exit would not be clean).
16480    #[cfg(unix)]
16481    async fn wait_for_file(path: &std::path::Path) {
16482        let deadline = Instant::now() + Duration::from_secs(10);
16483        while !path.exists() {
16484            assert!(
16485                Instant::now() < deadline,
16486                "{} never appeared",
16487                path.display()
16488            );
16489            sleep(Duration::from_millis(10)).await;
16490        }
16491    }
16492
16493    /// A protocol-none module that exits 0 because something OUTSIDE the
16494    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
16495    /// the crash-path disposition rather than `stopped`.
16496    #[cfg(unix)]
16497    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16498    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
16499        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
16500        let (spec, ready, marker) =
16501            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
16502        let supervisor = Supervisor::new_for_test(
16503            Arc::new(Registry::default()),
16504            RestartPolicy::new(3, Duration::ZERO),
16505        );
16506        let module = supervisor.spawn(spec).unwrap();
16507        wait_for_file(&ready).await;
16508        // The ready file proves the child installed its SIGTERM handler, not
16509        // that the supervisor has processed the privacy trampoline's exec
16510        // acknowledgement. On macOS status withholds the pid until then.
16511        let deadline = Instant::now() + Duration::from_secs(10);
16512        let first_pid = loop {
16513            let status = module.status().unwrap();
16514            if status.state == ModuleState::Running {
16515                if let Some(pid) = status.pid {
16516                    break pid;
16517                }
16518            }
16519            assert!(
16520                Instant::now() < deadline,
16521                "a running module must report its pid after exec confirmation: {status:?}"
16522            );
16523            sleep(Duration::from_millis(10)).await;
16524        };
16525
16526        rustix::process::kill_process(
16527            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
16528            rustix::process::Signal::TERM,
16529        )
16530        .unwrap();
16531
16532        let deadline = Instant::now() + Duration::from_secs(10);
16533        let respawned = loop {
16534            let status = module.status().unwrap();
16535            if status.state == ModuleState::Running
16536                && status.pid.is_some_and(|pid| pid != first_pid)
16537            {
16538                break status;
16539            }
16540            assert!(
16541                Instant::now() < deadline,
16542                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
16543            );
16544            sleep(Duration::from_millis(10)).await;
16545        };
16546        assert_eq!(respawned.spawn_generation, 2);
16547        assert!(
16548            marker.exists(),
16549            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
16550        );
16551
16552        let history = module.terminal_history();
16553        assert_eq!(history.entries.len(), 1, "{history:?}");
16554        let entry = &history.entries[0];
16555        assert_eq!(entry.exit_code, Some(0));
16556        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
16557        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
16558
16559        module.stop().await.unwrap();
16560    }
16561
16562    /// Repeated unrequested clean exits of a protocol-none module spend the
16563    /// restart budget exactly as crashes do, and the module ends `failed` with
16564    /// the budget named.
16565    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16566    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
16567        let supervisor = Supervisor::new_for_test(
16568            Arc::new(Registry::default()),
16569            RestartPolicy::new(1, Duration::ZERO),
16570        );
16571        let module = supervisor
16572            .spawn(ModuleSpec {
16573                module_id: "none-clean-exit-budget".to_string(),
16574                program: fake_aft_stub_path(),
16575                args: Vec::new(),
16576                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
16577                reserved: false,
16578                reserved_prefixes: Vec::new(),
16579                protocol: ModuleProtocol::None,
16580                overlap: Default::default(),
16581                carry_containment: false,
16582            })
16583            .unwrap();
16584
16585        // Failed follows the terminal write; this deadline only bounds a hang,
16586        // not an assumed duration for the two launches or their exit recording.
16587        let deadline = Instant::now() + Duration::from_secs(10);
16588        loop {
16589            let status = module.status().unwrap();
16590            if status.state == ModuleState::Failed {
16591                break;
16592            }
16593            assert!(
16594                Instant::now() < deadline,
16595                "module never exhausted its budget: {status:?} {:?}",
16596                module.terminal_history()
16597            );
16598            sleep(Duration::from_millis(10)).await;
16599        }
16600        let history = module.terminal_history();
16601        assert_eq!(
16602            history
16603                .entries
16604                .iter()
16605                .map(|entry| (entry.exit_code, entry.disposition.clone()))
16606                .collect::<Vec<_>>(),
16607            vec![
16608                (Some(0), TerminalDisposition::Restarting),
16609                (Some(0), TerminalDisposition::Failed),
16610            ]
16611        );
16612        let detail = history.entries[1]
16613            .disposition_detail
16614            .as_deref()
16615            .expect("a budget failure names the budget");
16616        assert!(detail.contains("max_restarts=1"), "{detail}");
16617        assert_eq!(module.status().unwrap().spawn_generation, 2);
16618    }
16619
16620    #[test]
16621    fn restart_budget_failure_is_published_after_its_terminal_record() {
16622        use std::io::{Read, Write};
16623        let supervisor = Supervisor::new_for_test(
16624            Arc::new(Registry::default()),
16625            RestartPolicy::new(0, Duration::ZERO),
16626        );
16627        let runtime = supervisor.runtime_config();
16628        let spec = ModuleSpec {
16629            protocol: ModuleProtocol::None,
16630            ..windowed_crash_spec("budget-publication-order")
16631        };
16632        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16633            ModuleState::Running,
16634            true,
16635        )));
16636        lock_snapshot(&snapshot).unwrap().process_alive = true;
16637        let reaped_snapshot = snapshot.clone();
16638        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16639        let history = runtime.terminal_ring.clone();
16640        let reap = std::thread::spawn(move || {
16641            tokio::runtime::Builder::new_current_thread()
16642                .enable_all()
16643                .build()
16644                .unwrap()
16645                .block_on(on_child_exit(
16646                    &spec,
16647                    runtime.restart_policy,
16648                    &supervisor.registry,
16649                    &reaped_snapshot,
16650                    &runtime.terminal_ring,
16651                    &runtime.spawn_events,
16652                    &runtime.child_roster,
16653                    ExitReport {
16654                        kind: ExitKind::Clean,
16655                        code: Some(0),
16656                        signal: None,
16657                        at_ms: 1,
16658                    },
16659                    true,
16660                ))
16661        });
16662        let mut reached = [0];
16663        barrier.read_exact(&mut reached).unwrap();
16664        assert_eq!(&reached, b"R");
16665        let (before_state, before_alive) = {
16666            let state = lock_snapshot(&snapshot).unwrap();
16667            (state.state, state.process_alive)
16668        };
16669        let before_history = history.lock().unwrap().snapshot();
16670        barrier.write_all(b"X").unwrap();
16671        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
16672        assert!(before_history.entries.is_empty());
16673        assert_ne!(
16674            before_state,
16675            ModuleState::Failed,
16676            "Failed was visible before its terminal record could be written"
16677        );
16678        assert!(before_alive);
16679        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16680        let history = history.lock().unwrap().snapshot();
16681        assert_eq!(history.entries.len(), 1);
16682        assert_eq!(history.entries[0].exit_code, Some(0));
16683        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16684    }
16685
16686    #[test]
16687    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
16688        use std::io::{Read, Write};
16689
16690        let supervisor = Supervisor::new_for_test(
16691            Arc::new(Registry::default()),
16692            RestartPolicy::new(3, Duration::ZERO),
16693        );
16694        let mut runtime = supervisor.runtime_config();
16695        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
16696        let snapshot = module.inner.snapshot.clone();
16697        lock_snapshot(&snapshot).unwrap().process_alive = true;
16698        runtime.terminal_ring = module.inner.terminal_ring.clone();
16699        let spec = windowed_crash_spec(module.module_id());
16700        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16701        let reaped_snapshot = snapshot.clone();
16702        let reap = std::thread::spawn(move || {
16703            tokio::runtime::Builder::new_current_thread()
16704                .enable_all()
16705                .build()
16706                .unwrap()
16707                .block_on(on_child_exit(
16708                    &spec,
16709                    runtime.restart_policy,
16710                    &supervisor.registry,
16711                    &reaped_snapshot,
16712                    &runtime.terminal_ring,
16713                    &runtime.spawn_events,
16714                    &runtime.child_roster,
16715                    ExitReport {
16716                        kind: ExitKind::Crash,
16717                        code: Some(2),
16718                        signal: None,
16719                        at_ms: 1,
16720                    },
16721                    true,
16722                ))
16723        });
16724
16725        // Hold the crash handler between its budget decision and publication.
16726        // Connection cleanup must still see the recovery that this crash chose,
16727        // even when that recovery consumes the last available restart slot.
16728        let mut reached = [0];
16729        barrier.read_exact(&mut reached).unwrap();
16730        assert_eq!(&reached, b"R");
16731        let before_status = module.status().unwrap();
16732        let before_history = module.terminal_history();
16733        let before_recovers = module.will_recover_after_connection_loss().unwrap();
16734        barrier.write_all(b"X").unwrap();
16735        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
16736
16737        assert_eq!(before_status.state, ModuleState::Running);
16738        assert!(before_status.process_alive);
16739        assert_eq!(before_status.restart_count, 3);
16740        assert!(before_history.entries.is_empty());
16741        assert!(
16742            before_recovers,
16743            "connection cleanup would report a terminal close for a scheduled crash restart"
16744        );
16745        let restarting = module.status().unwrap();
16746        assert_eq!(restarting.state, ModuleState::Restarting);
16747        assert_eq!(restarting.restart_count, 3);
16748        assert!(module.will_recover_after_connection_loss().unwrap());
16749        let history = module.terminal_history();
16750        assert_eq!(history.entries.len(), 1);
16751        assert_eq!(
16752            history.entries[0].disposition,
16753            TerminalDisposition::Restarting
16754        );
16755
16756        // Once the replacement is running, its next crash must be judged
16757        // against the exhausted budget rather than the previous recovery.
16758        update_snapshot(&snapshot, Some(module.module_id()), |state| {
16759            state.state = ModuleState::Running;
16760            state.respawn_pending = false;
16761        })
16762        .unwrap();
16763        assert!(!module.will_recover_after_connection_loss().unwrap());
16764        update_snapshot(&snapshot, Some(module.module_id()), |state| {
16765            state.enabled = false;
16766            state.respawn_pending = true;
16767        })
16768        .unwrap();
16769        assert!(!module.will_recover_after_connection_loss().unwrap());
16770    }
16771
16772    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
16773        use std::net::{TcpListener, TcpStream};
16774        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
16775        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
16776        let (barrier, _) = listener.accept().unwrap();
16777        for stream in [&writer, &barrier] {
16778            stream
16779                .set_read_timeout(Some(Duration::from_secs(10)))
16780                .unwrap();
16781            stream
16782                .set_write_timeout(Some(Duration::from_secs(10)))
16783                .unwrap();
16784        }
16785        *events.1.lock().unwrap() = Some(writer);
16786        barrier
16787    }
16788
16789    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
16790        use std::io::{Read, Write};
16791        let barrier = events.1.lock().unwrap().take();
16792        if let Some(mut barrier) = barrier {
16793            barrier.write_all(b"R").unwrap();
16794            let mut release = [0];
16795            barrier.read_exact(&mut release).unwrap();
16796            assert_eq!(&release, b"X");
16797        }
16798    }
16799
16800    #[test]
16801    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
16802        use std::io::{Read, Write};
16803
16804        let supervisor = Supervisor::new_for_test(
16805            Arc::new(Registry::default()),
16806            RestartPolicy::new(0, Duration::ZERO),
16807        );
16808        let mut runtime = supervisor.runtime_config();
16809        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
16810        let snapshot = module.inner.snapshot.clone();
16811        runtime.terminal_ring = module.inner.terminal_ring.clone();
16812        let spec = ModuleSpec {
16813            module_id: module.module_id().into(),
16814            program: fake_aft_stub_path(),
16815            protocol: ModuleProtocol::None,
16816            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16817            ..windowed_crash_spec(module.module_id())
16818        };
16819        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16820        let reap = std::thread::spawn(move || {
16821            tokio::runtime::Builder::new_current_thread()
16822                .enable_all()
16823                .build()
16824                .unwrap()
16825                .block_on(async {
16826                    let mut child =
16827                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16828                    health_restart_child(
16829                        &spec,
16830                        &runtime,
16831                        &supervisor.registry,
16832                        &supervisor.process_liveness,
16833                        &snapshot,
16834                        &mut child,
16835                        SupervisorHealthStatus::Failing,
16836                        Some("health probe did not answer"),
16837                        1,
16838                    )
16839                    .await
16840                    .unwrap();
16841                    assert!(child.is_none());
16842                });
16843        });
16844        // The loopback barrier acknowledges entry into terminal recording by
16845        // health_restart_child's drain, then waits for the test's release byte.
16846        // Status and history are observed while that writer is held, not after
16847        // an elapsed-time guess about when the health action has completed.
16848        let mut reached = [0];
16849        barrier.read_exact(&mut reached).unwrap();
16850        assert_eq!(&reached, b"R");
16851        let before_status = module.status().unwrap();
16852        let before_history = module.terminal_history();
16853        barrier.write_all(b"X").unwrap();
16854        reap.join().unwrap();
16855        assert!(before_history.entries.is_empty());
16856        assert_ne!(
16857            before_status.state,
16858            ModuleState::Failed,
16859            "Failed was visible before its terminal record could be written"
16860        );
16861        assert!(
16862            before_status.process_alive,
16863            "process death was visible before its terminal record could be written"
16864        );
16865        let failed = module.status().unwrap();
16866        assert_eq!(failed.state, ModuleState::Failed);
16867        assert!(!failed.process_alive);
16868        let history = module.terminal_history();
16869        assert_eq!(history.entries.len(), 1);
16870        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16871        assert_eq!(
16872            history.entries[0].disposition_detail.as_deref(),
16873            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
16874        );
16875    }
16876
16877    #[test]
16878    fn observed_exits_publish_process_death_after_terminal_history() {
16879        use std::io::{Read, Write};
16880        for (kind, enabled, shutdown, wait_error, expected_state) in [
16881            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
16882            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
16883            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
16884            (
16885                ExitKind::DeliberateSeverance,
16886                true,
16887                false,
16888                false,
16889                ModuleState::Restarting,
16890            ),
16891            #[cfg(unix)]
16892            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
16893            (ExitKind::Crash, true, false, true, ModuleState::Failed),
16894        ] {
16895            let supervisor = Supervisor::new_for_test(
16896                Arc::new(Registry::default()),
16897                RestartPolicy::new(1, Duration::ZERO),
16898            );
16899            let runtime = supervisor.runtime_config();
16900            #[cfg(unix)]
16901            if shutdown {
16902                runtime.child_roster.close();
16903            }
16904            #[cfg(not(unix))]
16905            let _ = shutdown;
16906            let spec = windowed_crash_spec("exit-publication-order");
16907            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16908                ModuleState::Running,
16909                enabled,
16910            )));
16911            lock_snapshot(&snapshot).unwrap().process_alive = true;
16912            let reaped_snapshot = snapshot.clone();
16913            let history = runtime.terminal_ring.clone();
16914            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16915            let reap = std::thread::spawn(move || {
16916                if wait_error {
16917                    super::fail_after_wait_error(
16918                        &spec.module_id,
16919                        &reaped_snapshot,
16920                        &runtime.terminal_ring,
16921                        &runtime.spawn_events,
16922                        "wait failed".into(),
16923                        &runtime.failure_context(true),
16924                    );
16925                } else {
16926                    tokio::runtime::Builder::new_current_thread()
16927                        .enable_all()
16928                        .build()
16929                        .unwrap()
16930                        .block_on(on_child_exit(
16931                            &spec,
16932                            runtime.restart_policy,
16933                            &supervisor.registry,
16934                            &reaped_snapshot,
16935                            &runtime.terminal_ring,
16936                            &runtime.spawn_events,
16937                            &runtime.child_roster,
16938                            ExitReport {
16939                                kind,
16940                                code: Some(0),
16941                                signal: None,
16942                                at_ms: 1,
16943                            },
16944                            true,
16945                        ));
16946                }
16947            });
16948            let mut reached = [0];
16949            barrier.read_exact(&mut reached).unwrap();
16950            assert_eq!(&reached, b"R");
16951            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
16952            let before_history = history.lock().unwrap().snapshot();
16953            barrier.write_all(b"X").unwrap();
16954            reap.join().unwrap();
16955            assert!(
16956                before_alive,
16957                "{kind:?}: death was published before recording"
16958            );
16959            assert!(before_history.entries.is_empty());
16960            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
16961            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
16962            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
16963        }
16964    }
16965
16966    #[tokio::test]
16967    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
16968        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
16969        let path = dir.join("terminals.jsonl");
16970        std::fs::create_dir(&path).unwrap();
16971        let supervisor = Supervisor::new_for_test(
16972            Arc::new(Registry::default()),
16973            RestartPolicy::new(0, Duration::ZERO),
16974        )
16975        .with_terminal_journal(path, "health-budget-journal-failure".into());
16976        let runtime = supervisor.runtime_config();
16977        let spec = ModuleSpec {
16978            program: fake_aft_stub_path(),
16979            protocol: ModuleProtocol::None,
16980            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16981            ..windowed_crash_spec("health-budget-journal-failure")
16982        };
16983        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16984        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16985        health_restart_child(
16986            &spec,
16987            &runtime,
16988            &supervisor.registry,
16989            &supervisor.process_liveness,
16990            &snapshot,
16991            &mut child,
16992            SupervisorHealthStatus::Failing,
16993            None,
16994            1,
16995        )
16996        .await
16997        .unwrap();
16998        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16999        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17000        let history = runtime
17001            .terminal_ring
17002            .lock()
17003            .unwrap()
17004            .durable_history(&spec.module_id);
17005        assert!(history.journal_write_failures > 0);
17006        assert_eq!(history.entries.len(), 1);
17007        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17008    }
17009
17010    #[test]
17011    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
17012        let supervisor = Supervisor::new_for_test(
17013            Arc::new(Registry::default()),
17014            RestartPolicy::new(0, Duration::ZERO),
17015        );
17016        let runtime = supervisor.runtime_config();
17017        let spec = ModuleSpec {
17018            program: fake_aft_stub_path(),
17019            protocol: ModuleProtocol::None,
17020            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17021            ..windowed_crash_spec("health-budget-recording-panic")
17022        };
17023        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17024        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17025            tokio::runtime::Builder::new_current_thread()
17026                .enable_all()
17027                .build()
17028                .unwrap()
17029                .block_on(async {
17030                    let mut child =
17031                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17032                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17033                    health_restart_child(
17034                        &spec,
17035                        &runtime,
17036                        &supervisor.registry,
17037                        &supervisor.process_liveness,
17038                        &snapshot,
17039                        &mut child,
17040                        SupervisorHealthStatus::Failing,
17041                        None,
17042                        1,
17043                    )
17044                    .await
17045                    .unwrap();
17046                });
17047        }));
17048        let panic = result.expect_err("recording must still unwind");
17049        assert_eq!(
17050            panic.downcast_ref::<String>().map(String::as_str),
17051            Some("spawn event sequence exhausted")
17052        );
17053        let state = lock_snapshot(&snapshot).unwrap();
17054        assert_eq!(state.state, ModuleState::Failed);
17055        assert!(!state.process_alive);
17056        assert!(state.last_exit.is_some());
17057    }
17058
17059    #[tokio::test]
17060    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
17061        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
17062        let path = dir.join("terminals.jsonl");
17063        std::fs::create_dir(&path).unwrap();
17064        let supervisor = Supervisor::new_for_test(
17065            Arc::new(Registry::default()),
17066            RestartPolicy::new(0, Duration::ZERO),
17067        )
17068        .with_terminal_journal(path, "budget-journal-failure".into());
17069        let runtime = supervisor.runtime_config();
17070        let spec = windowed_crash_spec("budget-journal-failure");
17071        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17072        assert!(matches!(
17073            on_child_exit(
17074                &spec,
17075                runtime.restart_policy,
17076                &supervisor.registry,
17077                &snapshot,
17078                &runtime.terminal_ring,
17079                &runtime.spawn_events,
17080                &runtime.child_roster,
17081                crash_exit_report(1),
17082                true,
17083            )
17084            .await,
17085            NextAction::Stop { .. }
17086        ));
17087        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17088        let history = runtime
17089            .terminal_ring
17090            .lock()
17091            .unwrap()
17092            .durable_history(&spec.module_id);
17093        assert!(history.journal_write_failures > 0);
17094        assert_eq!(history.entries.len(), 1);
17095        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17096    }
17097
17098    #[test]
17099    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
17100        let supervisor = Supervisor::new_for_test(
17101            Arc::new(Registry::default()),
17102            RestartPolicy::new(0, Duration::ZERO),
17103        );
17104        let runtime = supervisor.runtime_config();
17105        let spec = windowed_crash_spec("budget-recording-panic");
17106        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17107        runtime
17108            .spawn_events
17109            .emit_spawned(&spec.module_id, None, 1, 1);
17110        // Exhausting the event sequence makes emit_exited panic before the
17111        // terminal write, exercising publication on the recording unwind.
17112        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17113        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17114            tokio::runtime::Builder::new_current_thread()
17115                .enable_all()
17116                .build()
17117                .unwrap()
17118                .block_on(on_child_exit(
17119                    &spec,
17120                    runtime.restart_policy,
17121                    &supervisor.registry,
17122                    &snapshot,
17123                    &runtime.terminal_ring,
17124                    &runtime.spawn_events,
17125                    &runtime.child_roster,
17126                    crash_exit_report(1),
17127                    true,
17128                ))
17129        }));
17130        let panic = result.err().expect("recording must still unwind");
17131        assert_eq!(
17132            panic.downcast_ref::<String>().map(String::as_str),
17133            Some("spawn event sequence exhausted")
17134        );
17135        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17136    }
17137
17138    /// A stop the supervisor itself requests still stops a protocol-none
17139    /// module, even though the child answers the SIGTERM with exit 0.
17140    #[cfg(unix)]
17141    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17142    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
17143        for disable in [false, true] {
17144            let label = if disable {
17145                "none-requested-disable"
17146            } else {
17147                "none-requested-stop"
17148            };
17149            let dir = cortexkit_test_support::ScratchDir::new(label);
17150            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
17151            let supervisor = Supervisor::new_for_test(
17152                Arc::new(Registry::default()),
17153                RestartPolicy::new(3, Duration::ZERO),
17154            );
17155            let module = supervisor.spawn(spec).unwrap();
17156            wait_for_file(&ready).await;
17157
17158            if disable {
17159                module.set_enabled(false).await.unwrap();
17160            } else {
17161                module.stop().await.unwrap();
17162            }
17163            assert!(
17164                marker.exists(),
17165                "{label}: the child must have left through its SIGTERM handler with exit 0"
17166            );
17167
17168            // Long enough for a zero-backoff respawn to have happened if the
17169            // exit had been treated as a crash.
17170            sleep(Duration::from_millis(500)).await;
17171            let status = module.status().unwrap();
17172            let expected = if disable {
17173                ModuleState::Disabled
17174            } else {
17175                ModuleState::Stopped
17176            };
17177            assert_eq!(status.state, expected, "{label}");
17178            assert_eq!(
17179                status.spawn_generation, 1,
17180                "{label}: respawned after a requested stop"
17181            );
17182            let history = module.terminal_history();
17183            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
17184            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
17185            assert_ne!(
17186                history.entries[0].disposition,
17187                TerminalDisposition::Restarting,
17188                "{label}"
17189            );
17190        }
17191    }
17192
17193    /// A subc-wire module that exits 0 on its own is still a stop: the
17194    /// protocol-none rule must not reach it.
17195    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17196    async fn subc_wire_clean_exit_is_still_a_stop() {
17197        let supervisor = Supervisor::new_for_test(
17198            Arc::new(Registry::default()),
17199            RestartPolicy::new(3, Duration::ZERO),
17200        );
17201        let module = supervisor
17202            .spawn(ModuleSpec {
17203                module_id: "wire-clean-exit".to_string(),
17204                program: fake_aft_stub_path(),
17205                args: Vec::new(),
17206                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
17207                reserved: false,
17208                reserved_prefixes: Vec::new(),
17209                protocol: ModuleProtocol::Subc,
17210                overlap: Default::default(),
17211                carry_containment: false,
17212            })
17213            .unwrap();
17214
17215        let deadline = Instant::now() + Duration::from_secs(10);
17216        while module.terminal_history().entries.is_empty() {
17217            assert!(Instant::now() < deadline, "module never exited");
17218            sleep(Duration::from_millis(10)).await;
17219        }
17220        // Long enough for a zero-backoff respawn to have happened.
17221        sleep(Duration::from_millis(500)).await;
17222        let status = module.status().unwrap();
17223        assert_eq!(status.state, ModuleState::Stopped);
17224        assert_eq!(status.spawn_generation, 1);
17225        let history = module.terminal_history();
17226        assert_eq!(history.entries.len(), 1, "{history:?}");
17227        assert_eq!(history.entries[0].exit_code, Some(0));
17228        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
17229    }
17230
17231    #[cfg(unix)]
17232    #[tokio::test]
17233    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
17234        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
17235        let record = dir.join("live-children.json");
17236        let supervisor = Supervisor::new_for_test(
17237            Arc::new(Registry::default()),
17238            RestartPolicy::new(0, Duration::ZERO),
17239        );
17240        let mut runtime = supervisor.runtime_config();
17241        runtime.child_roster.record_to(record.clone());
17242        let gate = Arc::new(super::ReloadExitRecordGate::default());
17243        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
17244        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17245        let spec = ModuleSpec {
17246            module_id: "reload-exit-roster".into(),
17247            program: fake_aft_stub_path(),
17248            args: Vec::new(),
17249            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
17250            reserved: false,
17251            reserved_prefixes: Vec::new(),
17252            protocol: ModuleProtocol::Subc,
17253            overlap: Default::default(),
17254            carry_containment: false,
17255        };
17256        let mut child = None;
17257        let reload = super::finish_reload_child(
17258            &spec,
17259            &runtime,
17260            &supervisor.registry,
17261            &supervisor.process_liveness,
17262            &snapshot,
17263            &mut child,
17264        );
17265        tokio::pin!(reload);
17266        tokio::select! {
17267            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
17268            _ = gate.reached.notified() => {}
17269        }
17270        assert!(runtime
17271            .terminal_ring
17272            .lock()
17273            .unwrap()
17274            .snapshot()
17275            .entries
17276            .is_empty());
17277        assert_eq!(
17278            crate::live_children::read_record(&record).unwrap().len(),
17279            1,
17280            "shutdown must still wait for the reaped child until its terminal record exists"
17281        );
17282        runtime.child_roster.close();
17283        gate.resume.notify_one();
17284        assert!(reload.await.is_err());
17285        assert!(crate::live_children::read_record(&record)
17286            .unwrap()
17287            .is_empty());
17288        let history = runtime.terminal_ring.lock().unwrap().snapshot();
17289        assert_eq!(history.entries.len(), 1);
17290        assert_eq!(
17291            history.entries[0].disposition,
17292            TerminalDisposition::DaemonShutdown
17293        );
17294    }
17295
17296    /// Each restart-producing arm has its own state transition. Keeping their
17297    /// lifetime count assertions adjacent prevents a later new arm from silently
17298    /// spending budget without recording the historical restart.
17299    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17300    async fn every_restart_increment_path_advances_lifetime_count() {
17301        let supervisor = Supervisor::new_for_test(
17302            Arc::new(Registry::default()),
17303            RestartPolicy::new(1, Duration::ZERO),
17304        );
17305        let runtime = supervisor.runtime_config();
17306        let spec = ModuleSpec {
17307            module_id: "lifetime-increment-path".to_string(),
17308            program: PathBuf::from("/unused/lifetime-increment-path"),
17309            args: Vec::new(),
17310            env: Vec::new(),
17311            reserved: false,
17312            reserved_prefixes: Vec::new(),
17313            protocol: ModuleProtocol::Subc,
17314            overlap: Default::default(),
17315            carry_containment: false,
17316        };
17317
17318        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17319        assert!(matches!(
17320            on_child_exit(
17321                &spec,
17322                runtime.restart_policy,
17323                &supervisor.registry,
17324                &crash_snapshot,
17325                &runtime.terminal_ring,
17326                &runtime.spawn_events,
17327                &runtime.child_roster,
17328                ExitReport {
17329                    kind: ExitKind::Crash,
17330                    code: Some(1),
17331                    signal: None,
17332                    at_ms: 1,
17333                },
17334                true,
17335            )
17336            .await,
17337            NextAction::Restart { schedule: _ }
17338        ));
17339        let (crash_restarts, crash_lifetime) = {
17340            let state = lock_snapshot(&crash_snapshot).unwrap();
17341            (state.crash_restarts.len(), state.lifetime_restarts)
17342        };
17343        assert_eq!(crash_restarts, 1);
17344        assert_eq!(crash_lifetime, 1);
17345
17346        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17347        let mut health_child = None;
17348        assert!(matches!(
17349            health_restart_child(
17350                &spec,
17351                &runtime,
17352                &supervisor.registry,
17353                &supervisor.process_liveness,
17354                &health_snapshot,
17355                &mut health_child,
17356                SupervisorHealthStatus::Failing,
17357                None,
17358                2,
17359            )
17360            .await,
17361            Ok(())
17362        ));
17363        assert!(health_child.is_none());
17364        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
17365        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
17366        let (health_restarts, health_lifetime) = {
17367            let state = lock_snapshot(&health_snapshot).unwrap();
17368            (state.crash_restarts.len(), state.lifetime_restarts)
17369        };
17370        assert_eq!(health_restarts, 1);
17371        assert_eq!(health_lifetime, 1);
17372
17373        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17374        let mut reload_child = None;
17375        assert!(matches!(
17376            handle_reload_spawn_failure(
17377                &spec,
17378                &runtime,
17379                &supervisor.process_liveness,
17380                &reload_snapshot,
17381                &mut reload_child,
17382                "forced reload spawn failure".to_string(),
17383            )
17384            .await,
17385            Err(SuperviseError::ReloadFailed { .. })
17386        ));
17387        let (reload_restarts, reload_lifetime) = {
17388            let state = lock_snapshot(&reload_snapshot).unwrap();
17389            (state.crash_restarts.len(), state.lifetime_restarts)
17390        };
17391        assert_eq!(reload_restarts, 1);
17392        assert_eq!(reload_lifetime, 1);
17393    }
17394
17395    #[tokio::test]
17396    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
17397        let supervisor = Supervisor::new_for_test(
17398            Arc::new(Registry::default()),
17399            RestartPolicy::new(3, Duration::ZERO),
17400        );
17401        let runtime = supervisor.runtime_config();
17402        let spec = ModuleSpec {
17403            module_id: "deliberately-severed".to_string(),
17404            program: PathBuf::from("/unused/deliberately-severed"),
17405            args: Vec::new(),
17406            env: Vec::new(),
17407            reserved: false,
17408            reserved_prefixes: Vec::new(),
17409            protocol: ModuleProtocol::Subc,
17410            overlap: Default::default(),
17411            carry_containment: false,
17412        };
17413        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17414        let process = ProcessIdentity {
17415            pid: 41,
17416            start_time: 101,
17417        };
17418        record_deliberate_severance(&snapshot, process).unwrap();
17419        let exit_report = apply_deliberate_severance_marker(
17420            &snapshot,
17421            Some(process),
17422            ExitReport {
17423                kind: ExitKind::Crash,
17424                code: Some(1),
17425                signal: None,
17426                at_ms: 1,
17427            },
17428        );
17429        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
17430
17431        assert!(matches!(
17432            on_child_exit(
17433                &spec,
17434                runtime.restart_policy,
17435                &supervisor.registry,
17436                &snapshot,
17437                &runtime.terminal_ring,
17438                &runtime.spawn_events,
17439                &runtime.child_roster,
17440                exit_report,
17441                true,
17442            )
17443            .await,
17444            NextAction::Restart { schedule: _ }
17445        ));
17446        let state = lock_snapshot(&snapshot).unwrap();
17447        assert_eq!(state.lifetime_restarts, 1);
17448        assert_eq!(state.crash_restarts.len(), 0);
17449    }
17450
17451    #[tokio::test]
17452    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
17453        let supervisor = Supervisor::new_for_test(
17454            Arc::new(Registry::default()),
17455            RestartPolicy::new(3, Duration::ZERO),
17456        );
17457        let runtime = supervisor.runtime_config();
17458        let spec = ModuleSpec {
17459            module_id: "genuine-crash".to_string(),
17460            program: PathBuf::from("/unused/genuine-crash"),
17461            args: Vec::new(),
17462            env: Vec::new(),
17463            reserved: false,
17464            reserved_prefixes: Vec::new(),
17465            protocol: ModuleProtocol::Subc,
17466            overlap: Default::default(),
17467            carry_containment: false,
17468        };
17469        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17470
17471        assert!(matches!(
17472            on_child_exit(
17473                &spec,
17474                runtime.restart_policy,
17475                &supervisor.registry,
17476                &snapshot,
17477                &runtime.terminal_ring,
17478                &runtime.spawn_events,
17479                &runtime.child_roster,
17480                ExitReport {
17481                    kind: ExitKind::Crash,
17482                    code: Some(1),
17483                    signal: None,
17484                    at_ms: 1,
17485                },
17486                true,
17487            )
17488            .await,
17489            NextAction::Restart { schedule: _ }
17490        ));
17491        let state = lock_snapshot(&snapshot).unwrap();
17492        assert_eq!(state.lifetime_restarts, 1);
17493        assert_eq!(state.crash_restarts.len(), 1);
17494    }
17495
17496    fn crash_exit_report(at_ms: u64) -> ExitReport {
17497        ExitReport {
17498            kind: ExitKind::Crash,
17499            code: Some(1),
17500            signal: None,
17501            at_ms,
17502        }
17503    }
17504
17505    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
17506        ModuleSpec {
17507            module_id: module_id.to_string(),
17508            program: PathBuf::from("/unused").join(module_id),
17509            args: Vec::new(),
17510            env: Vec::new(),
17511            reserved: false,
17512            reserved_prefixes: Vec::new(),
17513            protocol: ModuleProtocol::Subc,
17514            overlap: Default::default(),
17515            carry_containment: false,
17516        }
17517    }
17518
17519    /// A real crash loop still stops. Three crashes with nothing aging out spend
17520    /// a budget of two and the third respawn is refused, and both surfaces an
17521    /// operator has -- the log line and the retained terminal record -- name the
17522    /// window rather than only the cap, because `max_restarts=2` alone is what
17523    /// this budget used to mean.
17524    #[tokio::test]
17525    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
17526        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
17527        let supervisor = Supervisor::new_for_test(
17528            Arc::new(Registry::default()),
17529            RestartPolicy::new(2, Duration::ZERO),
17530        );
17531        let runtime = supervisor.runtime_config();
17532        let spec = windowed_crash_spec("crash-loop-in-window");
17533        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17534
17535        for attempt in 1..=2 {
17536            assert!(
17537                matches!(
17538                    on_child_exit(
17539                        &spec,
17540                        runtime.restart_policy,
17541                        &supervisor.registry,
17542                        &snapshot,
17543                        &runtime.terminal_ring,
17544                        &runtime.spawn_events,
17545                        &runtime.child_roster,
17546                        crash_exit_report(attempt),
17547                        true,
17548                    )
17549                    .await,
17550                    NextAction::Restart { schedule: _ }
17551                ),
17552                "crash {attempt} is inside the budget and must respawn"
17553            );
17554        }
17555
17556        assert!(matches!(
17557            on_child_exit(
17558                &spec,
17559                runtime.restart_policy,
17560                &supervisor.registry,
17561                &snapshot,
17562                &runtime.terminal_ring,
17563                &runtime.spawn_events,
17564                &runtime.child_roster,
17565                crash_exit_report(3),
17566                true,
17567            )
17568            .await,
17569            NextAction::Stop { .. }
17570        ));
17571
17572        {
17573            let state = lock_snapshot(&snapshot).unwrap();
17574            assert_eq!(state.state, ModuleState::Failed);
17575            assert_eq!(state.crash_restarts.len(), 2);
17576            assert_eq!(state.lifetime_restarts, 2);
17577        }
17578
17579        let history = runtime
17580            .terminal_ring
17581            .lock()
17582            .expect("terminal ring is not poisoned")
17583            .snapshot();
17584        let last = history
17585            .entries
17586            .last()
17587            .expect("the refused crash is retained");
17588        assert_eq!(last.disposition, TerminalDisposition::Failed);
17589        assert_eq!(
17590            last.disposition_detail.as_deref(),
17591            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
17592        );
17593
17594        let captured = crate::router::test_log::captured_logs(&logs);
17595        assert!(
17596            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
17597            "the stop must be logged with its window: {captured}"
17598        );
17599    }
17600
17601    /// The rate, stated as a test: three crashes where the first has aged past
17602    /// the window are two crashes as far as the budget is concerned, so the
17603    /// third respawn is allowed and the ring holds only the two recent ones.
17604    ///
17605    /// This is the case a lifetime counter got wrong -- and the case the daemon
17606    /// now hits routinely, since a module exits non-zero every time its
17607    /// connection to the daemon drops.
17608    #[tokio::test]
17609    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
17610        let supervisor = Supervisor::new_for_test(
17611            Arc::new(Registry::default()),
17612            RestartPolicy::new(2, Duration::ZERO),
17613        );
17614        let runtime = supervisor.runtime_config();
17615        let spec = windowed_crash_spec("crash-across-windows");
17616        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17617
17618        for attempt in 1..=2 {
17619            assert!(matches!(
17620                on_child_exit(
17621                    &spec,
17622                    runtime.restart_policy,
17623                    &supervisor.registry,
17624                    &snapshot,
17625                    &runtime.terminal_ring,
17626                    &runtime.spawn_events,
17627                    &runtime.child_roster,
17628                    crash_exit_report(attempt),
17629                    true,
17630                )
17631                .await,
17632                NextAction::Restart { schedule: _ }
17633            ));
17634        }
17635
17636        // The oldest crash moves out of the window; nothing else about the
17637        // module changes.
17638        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17639            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
17640        })
17641        .unwrap();
17642
17643        assert!(
17644            matches!(
17645                on_child_exit(
17646                    &spec,
17647                    runtime.restart_policy,
17648                    &supervisor.registry,
17649                    &snapshot,
17650                    &runtime.terminal_ring,
17651                    &runtime.spawn_events,
17652                    &runtime.child_roster,
17653                    crash_exit_report(3),
17654                    true,
17655                )
17656                .await,
17657                NextAction::Restart { schedule: _ }
17658            ),
17659            "a crash older than the window must not hold a budget slot"
17660        );
17661
17662        let state = lock_snapshot(&snapshot).unwrap();
17663        assert_eq!(state.state, ModuleState::Restarting);
17664        assert_eq!(
17665            state.crash_restarts.len(),
17666            2,
17667            "the aged instant is dropped and the new one takes its place"
17668        );
17669        assert_eq!(
17670            state.lifetime_restarts, 3,
17671            "the ledger counts every restart, including the ones the window forgot"
17672        );
17673    }
17674
17675    /// An operator restart hands the budget back whole, and the ledger keeps
17676    /// counting. Those are different questions -- "how close is this module to
17677    /// being stopped" and "how many times has it been replaced" -- and the
17678    /// operator action answers only the first.
17679    #[tokio::test]
17680    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
17681        let supervisor = Supervisor::new_for_test(
17682            Arc::new(Registry::default()),
17683            RestartPolicy::new(2, Duration::ZERO),
17684        );
17685        let runtime = supervisor.runtime_config();
17686        let spec = windowed_crash_spec("operator-cleared-budget");
17687        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17688
17689        for attempt in 1..=2 {
17690            assert!(matches!(
17691                on_child_exit(
17692                    &spec,
17693                    runtime.restart_policy,
17694                    &supervisor.registry,
17695                    &snapshot,
17696                    &runtime.terminal_ring,
17697                    &runtime.spawn_events,
17698                    &runtime.child_roster,
17699                    crash_exit_report(attempt),
17700                    true,
17701                )
17702                .await,
17703                NextAction::Restart { schedule: _ }
17704            ));
17705        }
17706
17707        reset_restart_count(&snapshot, &spec.module_id).unwrap();
17708        {
17709            let state = lock_snapshot(&snapshot).unwrap();
17710            assert!(
17711                state.crash_restarts.is_empty(),
17712                "an operator restart returns the full budget"
17713            );
17714            assert_eq!(
17715                state.lifetime_restarts, 2,
17716                "clearing the budget must not unmake the crashes"
17717            );
17718        }
17719
17720        assert!(
17721            matches!(
17722                on_child_exit(
17723                    &spec,
17724                    runtime.restart_policy,
17725                    &supervisor.registry,
17726                    &snapshot,
17727                    &runtime.terminal_ring,
17728                    &runtime.spawn_events,
17729                    &runtime.child_roster,
17730                    crash_exit_report(3),
17731                    true,
17732                )
17733                .await,
17734                NextAction::Restart { schedule: _ }
17735            ),
17736            "the cleared budget must be spendable again"
17737        );
17738        let state = lock_snapshot(&snapshot).unwrap();
17739        assert_eq!(state.crash_restarts.len(), 1);
17740        assert_eq!(state.lifetime_restarts, 3);
17741    }
17742
17743    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17744    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
17745        let severed = ProcessIdentity {
17746            pid: 41,
17747            start_time: 101,
17748        };
17749        let successor = ProcessIdentity {
17750            pid: 41,
17751            start_time: 202,
17752        };
17753        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
17754        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
17755            state.pid = Some(successor.pid);
17756            state.process_start_time = Some(successor.start_time);
17757        })
17758        .unwrap();
17759        assert!(!module.record_deliberate_severance(severed).unwrap());
17760
17761        let exit_report = apply_deliberate_severance_marker(
17762            &module.inner.snapshot,
17763            Some(successor),
17764            ExitReport {
17765                kind: ExitKind::Crash,
17766                code: Some(1),
17767                signal: None,
17768                at_ms: 1,
17769            },
17770        );
17771
17772        assert_eq!(exit_report.kind, ExitKind::Crash);
17773    }
17774
17775    #[tokio::test]
17776    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
17777        let registry = Registry::default();
17778        let supervisor = Supervisor::new_for_test(
17779            Arc::new(Registry::default()),
17780            RestartPolicy::new(3, Duration::ZERO),
17781        );
17782        let runtime = supervisor.runtime_config();
17783        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17784        let spec = ModuleSpec {
17785            module_id: "drain-deliberate-severance".to_string(),
17786            program: fake_aft_stub_path(),
17787            args: Vec::new(),
17788            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17789            reserved: false,
17790            reserved_prefixes: Vec::new(),
17791            protocol: ModuleProtocol::Subc,
17792            overlap: Default::default(),
17793            carry_containment: false,
17794        };
17795        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17796        let process = ProcessIdentity {
17797            pid: 41,
17798            start_time: 101,
17799        };
17800        child.process_identity = Some(process);
17801        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17802            state.pid = Some(process.pid);
17803            state.process_start_time = Some(process.start_time);
17804        })
17805        .unwrap();
17806        record_deliberate_severance(&snapshot, process).unwrap();
17807
17808        drain_child_to_state(
17809            &spec.module_id,
17810            spec.protocol,
17811            // The child exits on its own; no signal may change the exit this
17812            // test classifies.
17813            StopNotice::SentOverConnection,
17814            &registry,
17815            None,
17816            &snapshot,
17817            &runtime.terminal_ring,
17818            &runtime.spawn_events,
17819            child,
17820            Duration::from_secs(1),
17821            ModuleState::Stopped,
17822            Some(false),
17823            &super::FailureContext::operator(),
17824        )
17825        .await
17826        .unwrap();
17827
17828        let state = lock_snapshot(&snapshot).unwrap();
17829        assert_eq!(
17830            state.last_exit.as_ref().map(|exit| exit.kind),
17831            Some(ExitKind::DeliberateSeverance)
17832        );
17833        assert_eq!(state.lifetime_restarts, 1);
17834        assert_eq!(state.crash_restarts.len(), 0);
17835        drop(state);
17836        let history = runtime.terminal_ring.lock().unwrap().snapshot();
17837        assert_eq!(
17838            history.entries[0].exit_kind,
17839            subc_control::TerminalExitKind::DeliberateSeverance
17840        );
17841    }
17842
17843    #[tokio::test]
17844    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
17845        let registry = Registry::default();
17846        let supervisor = Supervisor::new_for_test(
17847            Arc::new(Registry::default()),
17848            RestartPolicy::new(3, Duration::ZERO),
17849        );
17850        let runtime = supervisor.runtime_config();
17851        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17852        let spec = ModuleSpec {
17853            module_id: "ordinary-drain".to_string(),
17854            program: fake_aft_stub_path(),
17855            args: Vec::new(),
17856            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17857            reserved: false,
17858            reserved_prefixes: Vec::new(),
17859            protocol: ModuleProtocol::Subc,
17860            overlap: Default::default(),
17861            carry_containment: false,
17862        };
17863        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17864
17865        drain_child_to_state(
17866            &spec.module_id,
17867            spec.protocol,
17868            // The child exits on its own; no signal may change the exit this
17869            // test classifies.
17870            StopNotice::SentOverConnection,
17871            &registry,
17872            None,
17873            &snapshot,
17874            &runtime.terminal_ring,
17875            &runtime.spawn_events,
17876            child,
17877            Duration::from_secs(1),
17878            ModuleState::Stopped,
17879            Some(false),
17880            &super::FailureContext::operator(),
17881        )
17882        .await
17883        .unwrap();
17884
17885        let state = lock_snapshot(&snapshot).unwrap();
17886        assert_eq!(
17887            state.last_exit.as_ref().map(|exit| exit.kind),
17888            Some(ExitKind::Crash)
17889        );
17890        assert_eq!(state.lifetime_restarts, 0);
17891        assert_eq!(state.crash_restarts.len(), 0);
17892    }
17893
17894    #[test]
17895    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
17896        // The server's generic fatal-routing branch only knows that the
17897        // connection failed; it does not know that the daemon deliberately
17898        // initiated a process-killing severance. Keep this seam explicit so a
17899        // future connection error path cannot silently reintroduce the stale
17900        // exemption that mislabels a later genuine crash.
17901        assert!(!include_str!("server.rs")
17902            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
17903    }
17904
17905    /// The `route.closed` `drained` value must be the quiescence wait's own
17906    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
17907    /// measurement at all and `false` is the one honest constant. This is the exact
17908    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
17909    /// on every return path, including the one that used to return early via `?`
17910    /// with `route.closing` already sent and no `route.closed` ever following.
17911    #[test]
17912    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
17913        assert!(drained_after_quiescence_wait(&Ok(true)));
17914        assert!(!drained_after_quiescence_wait(&Ok(false)));
17915        assert!(!drained_after_quiescence_wait(&Err(
17916            SuperviseError::StatePoisoned { module_id: None }
17917        )));
17918    }
17919
17920    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
17921    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
17922    /// already reaped out-of-band) still leaves a terminal record rather than none
17923    /// at all. Triggering the real `wait()` I/O error from an integration test would
17924    /// need a genuine already-reaped-child race, which is OS-specific and not
17925    /// something this suite attempts elsewhere; this test instead verifies the
17926    /// record produced for that arm end-to-end through the real `TerminalRing`, and
17927    /// the call site itself is verified by inspection to sit in that exact arm.
17928    #[test]
17929    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
17930        let ring = Arc::new(Mutex::new(TerminalRing::new(
17931            TerminalRingConfig::default(),
17932            0,
17933        )));
17934        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
17935
17936        let snapshot = ring.lock().unwrap().snapshot();
17937        assert_eq!(snapshot.entries.len(), 1);
17938        let entry = &snapshot.entries[0];
17939        assert_eq!(entry.exit_code, None);
17940        assert_eq!(entry.exit_signal, None);
17941        assert_eq!(entry.disposition, TerminalDisposition::Failed);
17942    }
17943
17944    /// A launch is numbered before it starts. One that never runs keeps its
17945    /// number, and the next launch of the module is numbered after it.
17946    #[test]
17947    fn spawn_generation_of_a_failed_launch_is_never_reused() {
17948        let feed = super::SpawnEventFeed::default();
17949        feed.configure_incarnation("launch-numbering".to_string());
17950        let (incarnation, failed) = feed.allocate_generation("relaunched");
17951        assert_eq!(incarnation, "launch-numbering");
17952        let (_, next) = feed.allocate_generation("relaunched");
17953        assert!(next > failed);
17954        assert_eq!(feed.emit_spawned("relaunched", Some(next), 7, 1), next);
17955        // A process with no pre-allocated number is numbered after both.
17956        assert!(feed.emit_spawned("relaunched", None, 8, 2) > next);
17957        assert_eq!(
17958            feed.allocate_generation("other").1,
17959            1,
17960            "numbering is per module"
17961        );
17962    }
17963
17964    #[test]
17965    fn wait_error_exit_path_preserves_spawn_event_density() {
17966        let feed = super::SpawnEventFeed::default();
17967        feed.configure_incarnation("wait-error-density".to_string());
17968        feed.emit_spawned("wait-error", None, 41, 1);
17969        let ring = Arc::new(Mutex::new(TerminalRing::new(
17970            TerminalRingConfig::default(),
17971            0,
17972        )));
17973
17974        record_wait_error_terminal("wait-error", &ring, &feed);
17975        feed.emit_spawned("after-wait-error", None, 42, 2);
17976
17977        let state = feed.0.lock().unwrap();
17978        let sequences = state
17979            .events
17980            .iter()
17981            .map(|event| event.cursor.seq)
17982            .collect::<Vec<_>>();
17983        assert_eq!(sequences, vec![1, 2, 3]);
17984        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
17985        assert_eq!(state.events[1].exit_code, None);
17986        assert_eq!(state.events[1].exit_signal, None);
17987    }
17988
17989    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
17990    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
17991    /// not a clean exit it never actually observed.
17992    #[test]
17993    fn wait_error_exit_report_is_classified_as_a_crash() {
17994        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
17995    }
17996}
17997
17998#[cfg(all(test, unix))]
17999mod restart_ordering_tests {
18000    use super::*;
18001    use std::io::Write;
18002    use tokio::io::AsyncReadExt;
18003
18004    #[test]
18005    #[ignore = "child process fixture"]
18006    fn serving_child() {
18007        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
18008            return;
18009        };
18010        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
18011        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
18012        for stream in listener.incoming() {
18013            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
18014        }
18015    }
18016
18017    async fn restart_before_publication(requests: usize) {
18018        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
18019        let address_file = home.path().join("address");
18020        let spec = ModuleSpec {
18021            module_id: "restart-serving-child".into(),
18022            program: std::env::current_exe().unwrap(),
18023            args: vec![
18024                "--exact".into(),
18025                "supervise::restart_ordering_tests::serving_child".into(),
18026                "--ignored".into(),
18027                "--nocapture".into(),
18028            ],
18029            env: vec![(
18030                "RESTART_TEST_ADDRESS_FILE".into(),
18031                address_file.display().to_string(),
18032            )],
18033            reserved: false,
18034            reserved_prefixes: vec![],
18035            protocol: ModuleProtocol::None,
18036            overlap: Default::default(),
18037            carry_containment: false,
18038        };
18039        let mut supervisor =
18040            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18041        let (spawned_tx, spawned_rx) = oneshot::channel();
18042        let spawned_tx = Mutex::new(Some(spawned_tx));
18043        let (release_tx, release_rx) = std::sync::mpsc::channel();
18044        let release_rx = Mutex::new(release_rx);
18045        // Hold publication, not the child: it must answer a real request before
18046        // the restart enters the queue, even though status still has generation 0.
18047        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
18048            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
18049            release_rx
18050                .lock()
18051                .unwrap()
18052                .recv_timeout(Duration::from_secs(20))
18053                .unwrap();
18054        })));
18055        let registered = supervisor
18056            .register_configured_with_health(
18057                spec.clone(),
18058                true,
18059                HealthConfig::default(),
18060                None,
18061                RestartPolicy::new(3, Duration::ZERO),
18062                DEFAULT_REGISTER_TIMEOUT,
18063            )
18064            .unwrap();
18065        let module = registered.module.clone();
18066        let startup = tokio::task::spawn_blocking(move || {
18067            supervisor.start_registered_configured(registered, None, true)
18068        });
18069        spawned_rx.await.unwrap();
18070        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18071            let address = loop {
18072                if let Ok(address) = std::fs::read_to_string(&address_file) {
18073                    if !address.is_empty() {
18074                        break address;
18075                    }
18076                }
18077                tokio::time::sleep(Duration::from_millis(1)).await;
18078            };
18079            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
18080            let mut reply = String::new();
18081            stream.read_to_string(&mut reply).await.unwrap();
18082            reply.trim().parse::<u32>().unwrap()
18083        })
18084        .await
18085        .expect("child must serve before generation publication");
18086        assert_eq!(
18087            lock_snapshot(&module.inner.snapshot)
18088                .unwrap()
18089                .spawn_generation,
18090            0
18091        );
18092        let mut replies = Vec::new();
18093        for _ in 0..requests {
18094            let mut restart = Box::pin(module.restart(Some(0)));
18095            // Poll the public API through queue admission, leaving only its ACK
18096            // pending. The monitor cannot consume the command until released.
18097            std::future::poll_fn(|cx| {
18098                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18099                std::task::Poll::Ready(())
18100            })
18101            .await;
18102            replies.push(restart);
18103        }
18104        release_tx.send(()).unwrap();
18105        startup.await.unwrap().unwrap();
18106        for reply in replies {
18107            reply.await.unwrap();
18108        }
18109        // Restart schedules its replacement after completing the handler. Wait
18110        // for that separate actor turn before placing the command barrier.
18111        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
18112            loop {
18113                if lock_snapshot(&module.inner.snapshot)
18114                    .unwrap()
18115                    .spawn_generation
18116                    > 1
18117                {
18118                    break;
18119                }
18120                tokio::time::sleep(Duration::from_millis(1)).await;
18121            }
18122        })
18123        .await;
18124        // Commands are handled serially. The update-spec reply proves every
18125        // restart handler has finished before we inspect the snapshot; restart's
18126        // own ACK only acknowledges initiation, not replacement completion.
18127        module.update_spec_for_test(spec).await.unwrap();
18128        let (generation, pid) = {
18129            let state = lock_snapshot(&module.inner.snapshot).unwrap();
18130            (state.spawn_generation, state.pid)
18131        };
18132        module.stop().await.unwrap();
18133        replacement.expect("restart must publish a replacement for the already-serving child");
18134        assert_ne!(
18135            pid,
18136            Some(old_pid),
18137            "a serving child predating the restart must be replaced"
18138        );
18139        assert_eq!(generation, 2, "queued restarts must share one replacement");
18140    }
18141
18142    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18143    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
18144        restart_before_publication(1).await;
18145    }
18146
18147    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18148    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
18149        restart_before_publication(2).await;
18150    }
18151
18152    #[tokio::test]
18153    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
18154        let supervisor =
18155            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18156        let registered = supervisor
18157            .register_configured_with_health(
18158                ModuleSpec {
18159                    module_id: "queued-before-replacement".into(),
18160                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
18161                    // macOS a module whose running image is not the configured
18162                    // program is refused once its exec is confirmed (see
18163                    // `record_write_stall_tests::sleeping_spec`).
18164                    program: "/bin/sleep".into(),
18165                    args: vec!["600".into()],
18166                    env: vec![],
18167                    reserved: false,
18168                    reserved_prefixes: vec![],
18169                    protocol: ModuleProtocol::None,
18170                    overlap: Default::default(),
18171                    carry_containment: false,
18172                },
18173                true,
18174                HealthConfig::default(),
18175                None,
18176                RestartPolicy::default(),
18177                DEFAULT_REGISTER_TIMEOUT,
18178            )
18179            .unwrap();
18180        let RegisteredConfiguredModule {
18181            mut spec,
18182            mut runtime,
18183            module,
18184            mut commands,
18185        } = registered;
18186        let snapshot = &module.inner.snapshot;
18187        let mut restart = Box::pin(module.restart(Some(0)));
18188        std::future::poll_fn(|cx| {
18189            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18190            std::task::Poll::Ready(())
18191        })
18192        .await;
18193        // Drive the actor explicitly so a replacement is definitely spawned
18194        // after receipt, but before this queued command is dequeued.
18195        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
18196        let replacement_pid = child.as_ref().unwrap().id();
18197        let command = commands.recv().await.unwrap();
18198        assert!(
18199            handle_supervisor_command(
18200                command,
18201                &mut spec,
18202                &mut runtime,
18203                &supervisor.registry,
18204                &supervisor.process_liveness,
18205                snapshot,
18206                &mut child,
18207                &mut commands,
18208                &mut VecDeque::new(),
18209            )
18210            .await
18211        );
18212        restart.await.unwrap();
18213        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
18214        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
18215        if let Some(mut child) = child {
18216            child.start_kill().unwrap();
18217            child.wait().await.unwrap();
18218        }
18219        assert_eq!(
18220            kept_pid, replacement_pid,
18221            "a newer replacement must survive"
18222        );
18223        assert!(
18224            !pending,
18225            "a satisfied restart must not schedule another spawn"
18226        );
18227    }
18228}
18229
18230#[cfg(all(test, unix))]
18231mod health_event_tests {
18232    use super::*;
18233    use cortexkit_test_support::ScratchDir;
18234    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
18235
18236    struct Actor {
18237        module: SupervisedModule,
18238        registry: Arc<Registry>,
18239        forwarding: Arc<ForwardingTable>,
18240        spec: ModuleSpec,
18241        health: HealthConfig,
18242        rx: mpsc::Receiver<crate::router::OutboundFrame>,
18243        roster: ChildRoster,
18244        _home: ScratchDir,
18245    }
18246
18247    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
18248        // OS exec and socket readiness are real I/O. Stay runnable while waiting
18249        // for an observed condition, rather than letting paused time auto-advance
18250        // deadlines or assuming a fixed number of yields finishes the work.
18251        let deadline = std::time::Instant::now() + Duration::from_secs(10);
18252        let tick = Instant::now();
18253        loop {
18254            if let Some(value) = observe() {
18255                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
18256                return value;
18257            }
18258            assert!(
18259                std::time::Instant::now() < deadline,
18260                "timed out waiting for {reason}"
18261            );
18262            tokio::task::yield_now().await;
18263        }
18264    }
18265
18266    impl Actor {
18267        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
18268            Self::start_with_policy(
18269                protocol,
18270                cadence,
18271                DEFAULT_REGISTER_TIMEOUT,
18272                RestartPolicy::new(3, Duration::ZERO),
18273            )
18274            .await
18275        }
18276
18277        async fn start_with_policy(
18278            protocol: ModuleProtocol,
18279            cadence: Duration,
18280            register_timeout: Duration,
18281            policy: RestartPolicy,
18282        ) -> Self {
18283            let home = ScratchDir::new("health-events");
18284            let registry = Arc::new(Registry::default());
18285            let forwarding = Arc::new(ForwardingTable::default());
18286            let health = HealthConfig {
18287                cadence,
18288                deadline: Duration::from_secs(3600),
18289                ..HealthConfig::default()
18290            };
18291            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
18292                .with_forwarding(forwarding.clone())
18293                .with_health_config(health.clone());
18294            let spec = ModuleSpec {
18295                module_id: "event-health-child".into(),
18296                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
18297                // the privacy trampoline refuses a module whose running image
18298                // is not the configured program, and `start` below waits for
18299                // the process to be confirmed (`reported_pid`). The shell form
18300                // passed only while the image check ran before the shell had
18301                // replaced itself with `sleep` (see
18302                // `record_write_stall_tests::sleeping_spec`).
18303                program: PathBuf::from("/bin/sleep"),
18304                // The process stays alive but has no real bus peer. Registrations
18305                // below exercise the same registry writes as accepted HELLOs.
18306                args: vec!["600".into()],
18307                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18308                    .into_iter()
18309                    .map(|key| (key.into(), home.path().display().to_string()))
18310                    .collect(),
18311                reserved: false,
18312                reserved_prefixes: vec![],
18313                protocol,
18314                overlap: Default::default(),
18315                carry_containment: false,
18316            };
18317            let module = supervisor
18318                .supervise_configured_requested(
18319                    spec.clone(),
18320                    true,
18321                    health.clone(),
18322                    Some(0),
18323                    policy,
18324                    register_timeout,
18325                    None,
18326                )
18327                .unwrap();
18328            let (_, rx) = mpsc::channel(8);
18329            let actor = Self {
18330                module,
18331                registry,
18332                forwarding,
18333                spec,
18334                health,
18335                rx,
18336                roster: supervisor.child_roster.clone(),
18337                _home: home,
18338            };
18339            actor
18340                .wait_for_select(|checkpoint| checkpoint.generation > 0)
18341                .await;
18342            actor
18343        }
18344
18345        async fn expire_registration_wait(&self, remaining: Duration) {
18346            tokio::time::advance(remaining).await;
18347            wait_for("registration timeout TERM grace", || {
18348                lock_snapshot(&self.module.inner.snapshot)
18349                    .unwrap()
18350                    .actor_select
18351                    .is_none()
18352                    .then_some(())
18353            })
18354            .await;
18355            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
18356        }
18357
18358        fn turns(&self) -> u64 {
18359            lock_snapshot(&self.module.inner.snapshot)
18360                .unwrap()
18361                .actor_turns
18362        }
18363
18364        async fn wait_for_select(
18365            &self,
18366            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
18367        ) -> ActorSelectCheckpoint {
18368            wait_for("actor select checkpoint", || {
18369                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
18370                state.actor_select.filter(|checkpoint| {
18371                    checkpoint.generation == state.spawn_generation
18372                        && state.state == ModuleState::Running
18373                        && state.process_alive
18374                        && state.reported_pid().is_some()
18375                        && expected(checkpoint)
18376                })
18377            })
18378            .await
18379        }
18380
18381        fn hello(&mut self, connection: u64, health: bool) {
18382            let manifest =
18383                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
18384                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
18385                    .build();
18386            let connection = ConnectionId::new(connection);
18387            let (tx, rx) = mpsc::channel(8);
18388            self.rx = rx;
18389            self.forwarding
18390                .register_module_connection(
18391                    connection,
18392                    self.spec.module_id.clone(),
18393                    subc_protocol::PROTOCOL_VERSION,
18394                    Concurrency::ModuleManaged,
18395                    FrameSink::new(tx),
18396                )
18397                .unwrap();
18398            self.registry
18399                .register_with_control_ops(
18400                    manifest,
18401                    subc_protocol::PROTOCOL_VERSION,
18402                    connection,
18403                    if health {
18404                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
18405                    } else {
18406                        vec![]
18407                    },
18408                )
18409                .unwrap();
18410        }
18411
18412        async fn probe(&mut self) -> crate::router::OutboundFrame {
18413            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
18414                Ok(frame) => Some(frame),
18415                Err(mpsc::error::TryRecvError::Empty) => None,
18416                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
18417            })
18418            .await;
18419            let body: Value = serde_json::from_slice(&frame.body).unwrap();
18420            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
18421            frame
18422        }
18423
18424        fn no_probe(&mut self) {
18425            assert!(matches!(
18426                self.rx.try_recv(),
18427                Err(mpsc::error::TryRecvError::Empty)
18428            ));
18429        }
18430
18431        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
18432            self.forwarding
18433                .complete_module_control_rpc(
18434                    ConnectionId::new(connection),
18435                    frame.header.corr,
18436                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
18437                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
18438                        status: HealthStatus::Ok,
18439                        detail: None,
18440                        metrics: None,
18441                    }),
18442                )
18443                .unwrap();
18444        }
18445    }
18446
18447    #[tokio::test(start_paused = true)]
18448    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
18449        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18450        let initial = actor.module.status().unwrap();
18451        tokio::time::advance(Duration::from_secs(119)).await;
18452        tokio::task::yield_now().await;
18453        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
18454        assert!(actor.module.terminal_history().entries.is_empty());
18455        actor.expire_registration_wait(Duration::from_secs(1)).await;
18456        actor
18457            .wait_for_select(|at| at.generation > initial.spawn_generation)
18458            .await;
18459        let status = actor.module.status().unwrap();
18460        assert_ne!(status.pid, initial.pid);
18461        assert!(!cortexkit_test_support::process_alive(
18462            initial.pid.unwrap() as i32
18463        ));
18464        assert_eq!(status.restart_count, 1);
18465        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
18466        let entries = actor.module.terminal_history().entries;
18467        assert_eq!(entries.len(), 1);
18468        assert_eq!(
18469            entries[0].disposition_detail.as_deref(),
18470            Some("registration_timeout")
18471        );
18472        actor.module.stop().await.unwrap();
18473    }
18474
18475    #[tokio::test(start_paused = true)]
18476    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
18477        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18478        let pid = actor.module.status().unwrap().pid;
18479        tokio::time::advance(Duration::from_secs(119)).await;
18480        actor.hello(1, false);
18481        let registered = actor
18482            .wait_for_select(|at| at.registered_connection.is_some())
18483            .await;
18484        assert_eq!(
18485            registered.wake_after, None,
18486            "HELLO must cancel the deadline"
18487        );
18488        tokio::time::advance(Duration::from_secs(300)).await;
18489        tokio::task::yield_now().await;
18490        assert_eq!(actor.module.status().unwrap().pid, pid);
18491        assert!(actor.module.terminal_history().entries.is_empty());
18492        // A disconnect after the first HELLO must not re-arm a startup timer.
18493        actor
18494            .registry
18495            .deregister_connection(ConnectionId::new(1))
18496            .unwrap();
18497        actor
18498            .wait_for_select(|at| at.registered_connection.is_none())
18499            .await;
18500        tokio::time::advance(Duration::from_secs(300)).await;
18501        tokio::task::yield_now().await;
18502        assert_eq!(actor.module.status().unwrap().pid, pid);
18503        assert!(actor.module.terminal_history().entries.is_empty());
18504        actor.module.stop().await.unwrap();
18505
18506        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18507        let pid = transient.module.status().unwrap().pid;
18508        tokio::time::advance(Duration::from_secs(119)).await;
18509        let turns = transient.turns();
18510        // Both writes happen before the supervisor can refresh the registry.
18511        // An accepted HELLO must remain evidence after its connection is gone.
18512        transient.hello(1, false);
18513        transient
18514            .registry
18515            .deregister_connection(ConnectionId::new(1))
18516            .unwrap();
18517        let parked = transient.wait_for_select(|at| at.turn > turns).await;
18518        assert_eq!(
18519            parked.wake_after, None,
18520            "transient HELLO must cancel the deadline"
18521        );
18522        tokio::time::advance(Duration::from_secs(300)).await;
18523        tokio::task::yield_now().await;
18524        assert_eq!(transient.module.status().unwrap().pid, pid);
18525        assert!(transient.module.terminal_history().entries.is_empty());
18526        transient.module.stop().await.unwrap();
18527    }
18528
18529    #[tokio::test(start_paused = true)]
18530    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
18531        let actor = Actor::start_with_policy(
18532            ModuleProtocol::Subc,
18533            Duration::from_secs(30),
18534            Duration::from_secs(10),
18535            RestartPolicy::new(1, Duration::ZERO),
18536        )
18537        .await;
18538        let generation = actor.module.status().unwrap().spawn_generation;
18539        actor
18540            .expire_registration_wait(Duration::from_secs(10))
18541            .await;
18542        actor.wait_for_select(|at| at.generation > generation).await;
18543        actor
18544            .expire_registration_wait(Duration::from_secs(10))
18545            .await;
18546        let retry = wait_for("failed retry schedule", || {
18547            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
18548            (state.state == ModuleState::Failed)
18549                .then_some(state.failed_retry.pending)
18550                .flatten()
18551        })
18552        .await;
18553        assert_eq!(retry.delay, Duration::from_secs(600));
18554        assert_eq!(retry.attempt, 1);
18555        let entries = actor.module.terminal_history().entries;
18556        assert_eq!(entries.len(), 2);
18557        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
18558        assert!(entries[1]
18559            .disposition_detail
18560            .as_deref()
18561            .unwrap()
18562            .contains("registration_timeout"));
18563        tokio::time::advance(Duration::from_secs(599)).await;
18564        tokio::task::yield_now().await;
18565        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
18566        tokio::time::advance(Duration::from_secs(1)).await;
18567        actor
18568            .wait_for_select(|at| at.generation > generation + 1)
18569            .await;
18570        // The slow retry is another ordinary spawn, with its own HELLO budget.
18571        actor
18572            .expire_registration_wait(Duration::from_secs(10))
18573            .await;
18574        wait_for("slow retry registration timeout", || {
18575            (actor.module.terminal_history().entries.len() == 3).then_some(())
18576        })
18577        .await;
18578        actor.module.stop().await.unwrap();
18579    }
18580
18581    #[tokio::test(start_paused = true)]
18582    async fn registration_timeout_honours_ten_second_override() {
18583        let actor = Actor::start_with_policy(
18584            ModuleProtocol::Subc,
18585            Duration::from_secs(30),
18586            Duration::from_secs(10),
18587            RestartPolicy::new(3, Duration::ZERO),
18588        )
18589        .await;
18590        let generation = actor.module.status().unwrap().spawn_generation;
18591        tokio::time::advance(Duration::from_secs(9)).await;
18592        tokio::task::yield_now().await;
18593        assert!(actor.module.terminal_history().entries.is_empty());
18594        actor.expire_registration_wait(Duration::from_secs(1)).await;
18595        actor.wait_for_select(|at| at.generation > generation).await;
18596        assert_eq!(actor.module.terminal_history().entries.len(), 1);
18597        actor.module.stop().await.unwrap();
18598    }
18599
18600    #[tokio::test]
18601    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
18602        use std::os::unix::process::{CommandExt, ExitStatusExt};
18603
18604        struct Helper(std::process::Child);
18605        impl Drop for Helper {
18606            fn drop(&mut self) {
18607                // This handle still owns the helper until wait returns. Never
18608                // clean up its old process group after the leader is reaped.
18609                if matches!(self.0.try_wait(), Ok(None)) {
18610                    let _ = self.0.kill();
18611                    let _ = self.0.wait();
18612                }
18613            }
18614        }
18615
18616        let home = ScratchDir::new("registration-timeout-helper");
18617        let supervisor = Supervisor::new_for_test(
18618            Arc::new(Registry::default()),
18619            RestartPolicy::new(0, Duration::ZERO),
18620        );
18621        let spec = ModuleSpec {
18622            module_id: "registration-timeout-group".into(),
18623            program: "/bin/sleep".into(),
18624            args: vec!["120".into()],
18625            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18626                .into_iter()
18627                .map(|key| (key.into(), home.path().display().to_string()))
18628                .collect(),
18629            reserved: false,
18630            reserved_prefixes: vec![],
18631            protocol: ModuleProtocol::Subc,
18632            overlap: Default::default(),
18633            carry_containment: false,
18634        };
18635        let module = supervisor
18636            .supervise_configured_requested(
18637                spec,
18638                true,
18639                HealthConfig::default(),
18640                None,
18641                RestartPolicy::new(0, Duration::ZERO),
18642                Duration::from_secs(10),
18643                None,
18644            )
18645            .unwrap();
18646        let leader = timeout(Duration::from_secs(5), async {
18647            loop {
18648                if let Some(pid) = module.status().unwrap().pid {
18649                    break pid;
18650                }
18651                sleep(Duration::from_millis(5)).await;
18652            }
18653        })
18654        .await
18655        .expect("supervised group leader must be reported before the deadline");
18656        assert!(leader > 1);
18657        let ready = home.join("helper.ready");
18658        // Own the helper separately so even a broken group kill can be cleaned
18659        // up by its own unreaped child handle, without signalling a stale pgid.
18660        let mut helper = Helper(
18661            std::process::Command::new("/bin/sh")
18662                .args([
18663                    "-c",
18664                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
18665                    "helper",
18666                ])
18667                .arg(&ready)
18668                .process_group(i32::try_from(leader).unwrap())
18669                .stdout(std::process::Stdio::null())
18670                .stderr(std::process::Stdio::null())
18671                .spawn()
18672                .unwrap(),
18673        );
18674        timeout(Duration::from_secs(5), async {
18675            while !ready.exists() {
18676                assert!(helper.0.try_wait().unwrap().is_none());
18677                sleep(Duration::from_millis(5)).await;
18678            }
18679        })
18680        .await
18681        .expect("helper must install TERM ignore before the registration deadline");
18682        let helper_pid =
18683            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
18684        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
18685        sleep(Duration::from_millis(20)).await;
18686        assert!(
18687            helper.0.try_wait().unwrap().is_none(),
18688            "helper must ignore TERM"
18689        );
18690
18691        timeout(Duration::from_secs(20), async {
18692            while module.state().unwrap() != ModuleState::Failed {
18693                sleep(Duration::from_millis(5)).await;
18694            }
18695        })
18696        .await
18697        .expect("registration timeout must stop and reap the group leader");
18698        let helper_exit = timeout(Duration::from_secs(3), async {
18699            loop {
18700                if let Some(status) = helper.0.try_wait().unwrap() {
18701                    break status;
18702                }
18703                sleep(Duration::from_millis(5)).await;
18704            }
18705        })
18706        .await
18707        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
18708        assert_eq!(helper_exit.signal(), Some(9));
18709        assert!(!cortexkit_test_support::process_alive(
18710            i32::try_from(leader).unwrap()
18711        ));
18712        assert_eq!(
18713            module.status().unwrap().last_exit.unwrap().kind,
18714            ExitKind::Crash
18715        );
18716        assert!(module.terminal_history().entries[0]
18717            .disposition_detail
18718            .as_deref()
18719            .unwrap()
18720            .contains("registration_timeout"));
18721        module.stop().await.unwrap();
18722    }
18723
18724    #[tokio::test(start_paused = true)]
18725    async fn registration_timeout_does_not_count_an_earlier_exit() {
18726        let actor = Actor::start_with_policy(
18727            ModuleProtocol::Subc,
18728            Duration::from_secs(30),
18729            DEFAULT_REGISTER_TIMEOUT,
18730            RestartPolicy::new(0, Duration::ZERO),
18731        )
18732        .await;
18733        let pid = actor.module.status().unwrap().pid.unwrap();
18734        rustix::process::kill_process(
18735            rustix::process::Pid::from_raw(pid as i32).unwrap(),
18736            rustix::process::Signal::TERM,
18737        )
18738        .unwrap();
18739        wait_for("early exit", || {
18740            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
18741        })
18742        .await;
18743        tokio::time::advance(Duration::from_secs(121)).await;
18744        tokio::task::yield_now().await;
18745        let entries = actor.module.terminal_history().entries;
18746        assert_eq!(entries.len(), 1);
18747        assert!(!entries[0]
18748            .disposition_detail
18749            .as_deref()
18750            .unwrap()
18751            .contains("registration_timeout"));
18752        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
18753        actor.module.stop().await.unwrap();
18754    }
18755
18756    #[tokio::test(start_paused = true)]
18757    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
18758        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18759        let pid = actor.module.status().unwrap().pid;
18760        tokio::time::advance(Duration::from_secs(119)).await;
18761        actor.roster.close();
18762        let turns = actor.turns();
18763        tokio::time::advance(Duration::from_secs(300)).await;
18764        let parked = actor.wait_for_select(|at| at.turn > turns).await;
18765        assert_eq!(parked.wake_after, None);
18766        assert_eq!(actor.module.status().unwrap().pid, pid);
18767        assert!(actor.module.terminal_history().entries.is_empty());
18768        actor.module.stop().await.unwrap();
18769        let status = actor.module.status().unwrap();
18770        assert_eq!(status.restart_count, 0);
18771        assert_eq!(
18772            actor.module.terminal_history().entries[0].disposition,
18773            TerminalDisposition::DaemonShutdown
18774        );
18775    }
18776
18777    #[tokio::test(start_paused = true)]
18778    async fn no_health_children_park_for_sixty_seconds() {
18779        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18780        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18781        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18782        unadvertised.hello(1, false);
18783        // Include any one-off registration wake in the budget; receiving a
18784        // no-health HELLO must not turn a parked child into a polling child.
18785        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
18786        // a single 60 s jump would only observe one expired timer.
18787        for _ in 0..6000 {
18788            tokio::time::advance(Duration::from_millis(10)).await;
18789            tokio::task::yield_now().await;
18790        }
18791        for actor in [&none, &unregistered, &unadvertised] {
18792            assert!(
18793                actor.turns() <= 5,
18794                "no-health actor woke {} times",
18795                actor.turns()
18796            );
18797            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
18798        }
18799    }
18800
18801    #[tokio::test(start_paused = true)]
18802    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
18803        // With a zero cadence the first probe is due at once, so this measures
18804        // only how fast the registration notification starts health probing,
18805        // not the normal 30-33 s wait before a first probe (checked below).
18806        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
18807        let tick = Instant::now();
18808        actor.hello(1, true);
18809        actor.probe().await;
18810        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
18811    }
18812
18813    #[tokio::test(start_paused = true)]
18814    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
18815        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18816        let hello_tick = Instant::now();
18817        actor.hello(1, true);
18818        let armed = actor
18819            .wait_for_select(|checkpoint| {
18820                checkpoint.registered_connection == Some(ConnectionId::new(1))
18821                    && checkpoint.next_probe_at.is_some()
18822            })
18823            .await;
18824        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18825        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
18826        let first_deadline = hello_tick + delay;
18827        assert_eq!(armed.next_probe_at, Some(first_deadline));
18828        tokio::time::advance(Duration::from_secs(29)).await;
18829        let turns = actor.turns();
18830        actor
18831            .registry
18832            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
18833            .unwrap();
18834        let updated = actor
18835            .wait_for_select(|checkpoint| checkpoint.turn > turns)
18836            .await;
18837        assert_eq!(
18838            updated.next_probe_at,
18839            Some(first_deadline),
18840            "catalog update must not reset cadence"
18841        );
18842        actor.no_probe();
18843        tokio::time::advance(first_deadline - Instant::now()).await;
18844        let frame = actor.probe().await;
18845        actor.answer(1, frame);
18846        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
18847        let next_deadline = Instant::now() + next;
18848        let rearmed = actor
18849            .wait_for_select(|checkpoint| {
18850                checkpoint
18851                    .next_probe_at
18852                    .is_some_and(|deadline| deadline > first_deadline)
18853            })
18854            .await;
18855        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
18856        tokio::time::advance(next - Duration::from_nanos(1)).await;
18857        actor.no_probe();
18858        tokio::time::advance(Duration::from_nanos(1)).await;
18859        actor.probe().await;
18860    }
18861
18862    #[tokio::test(start_paused = true)]
18863    async fn disconnect_disarms_and_reconnect_rearms_health() {
18864        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18865        actor.hello(1, true);
18866        actor
18867            .wait_for_select(|checkpoint| {
18868                checkpoint.registered_connection == Some(ConnectionId::new(1))
18869                    && checkpoint.next_probe_at.is_some()
18870            })
18871            .await;
18872        let turns = actor.turns();
18873        let tick = Instant::now();
18874        actor
18875            .registry
18876            .deregister_connection(ConnectionId::new(1))
18877            .unwrap();
18878        let disarmed = actor
18879            .wait_for_select(|checkpoint| {
18880                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
18881            })
18882            .await;
18883        assert_eq!(disarmed.next_probe_at, None);
18884        assert_eq!(disarmed.wake_after, None);
18885        assert_eq!(
18886            Instant::now(),
18887            tick,
18888            "disconnect must disarm in the notification tick"
18889        );
18890        tokio::time::advance(Duration::from_secs(60)).await;
18891        actor.no_probe();
18892        let reconnect_tick = Instant::now();
18893        actor.hello(2, true);
18894        let rearmed = actor
18895            .wait_for_select(|checkpoint| {
18896                checkpoint.registered_connection == Some(ConnectionId::new(2))
18897                    && checkpoint.next_probe_at.is_some()
18898            })
18899            .await;
18900        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18901        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
18902        tokio::time::advance(delay).await;
18903        actor.probe().await;
18904    }
18905
18906    #[tokio::test(start_paused = true)]
18907    async fn reregistration_without_health_disarms_the_old_schedule() {
18908        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18909        actor.hello(1, true);
18910        actor
18911            .wait_for_select(|checkpoint| {
18912                checkpoint.registered_connection == Some(ConnectionId::new(1))
18913                    && checkpoint.next_probe_at.is_some()
18914            })
18915            .await;
18916        actor
18917            .registry
18918            .deregister_connection(ConnectionId::new(1))
18919            .unwrap();
18920        actor.hello(2, false);
18921        let disarmed = actor
18922            .wait_for_select(|checkpoint| {
18923                checkpoint.registered_connection == Some(ConnectionId::new(2))
18924            })
18925            .await;
18926        assert_eq!(disarmed.next_probe_at, None);
18927        assert_eq!(disarmed.wake_after, None);
18928        tokio::time::advance(Duration::from_secs(60)).await;
18929        actor.no_probe();
18930    }
18931
18932    #[tokio::test(start_paused = true)]
18933    async fn restart_waits_for_its_own_hello_before_rearming_health() {
18934        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18935        actor.hello(1, true);
18936        let armed = actor
18937            .wait_for_select(|checkpoint| {
18938                checkpoint.registered_connection == Some(ConnectionId::new(1))
18939                    && checkpoint.next_probe_at.is_some()
18940            })
18941            .await;
18942        // Simulate the old process's connection closing before the restart
18943        // tears the process down. The replacement process has started but has
18944        // not sent its HELLO, so it is not registered.
18945        actor
18946            .registry
18947            .deregister_connection(ConnectionId::new(1))
18948            .unwrap();
18949        actor.module.restart(Some(0)).await.unwrap();
18950        let parked = actor
18951            .wait_for_select(|checkpoint| {
18952                checkpoint.generation > armed.generation
18953                    && checkpoint.registered_connection.is_none()
18954                    && checkpoint.next_probe_at.is_none()
18955            })
18956            .await;
18957        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
18958        actor.rx.close();
18959        // Drain the stop notice the restart queued for the old process's connection,
18960        // so it isn't mistaken for traffic to the replacement.
18961        while actor.rx.try_recv().is_ok() {}
18962        let turns = actor.turns();
18963        tokio::time::advance(Duration::from_secs(60)).await;
18964        let after = actor.turns();
18965        eprintln!(
18966            "restart park turns: before={turns} after={after} delta={}",
18967            after - turns
18968        );
18969        assert_eq!(after, turns, "replacement must park before HELLO");
18970        let hello_tick = Instant::now();
18971        actor.hello(2, true);
18972        let rearmed = actor
18973            .wait_for_select(|checkpoint| {
18974                checkpoint.registered_connection == Some(ConnectionId::new(2))
18975                    && checkpoint.next_probe_at.is_some()
18976            })
18977            .await;
18978        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18979        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
18980        tokio::time::advance(delay).await;
18981        actor.probe().await;
18982    }
18983
18984    #[tokio::test(start_paused = true)]
18985    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
18986        use tokio::{
18987            io::{AsyncReadExt, AsyncWriteExt},
18988            net::TcpListener,
18989        };
18990        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18991        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
18992        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
18993        let (started_tx, mut started_rx) = mpsc::channel(8);
18994        let peer = tokio::spawn(async move {
18995            loop {
18996                let (mut socket, _) = listener.accept().await.unwrap();
18997                let mut request = [0; 1024];
18998                assert!(socket.read(&mut request).await.unwrap() > 0);
18999                started_tx.send(Instant::now()).await.unwrap();
19000                socket
19001                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19002                    .await
19003                    .unwrap();
19004            }
19005        });
19006        actor
19007            .module
19008            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19009            .await
19010            .unwrap();
19011        let first_tick = Instant::now();
19012        let first = actor
19013            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
19014            .await;
19015        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
19016        assert!(started_rx.try_recv().is_err());
19017
19018        // Restart just before the old deadline: reusing it would give the new
19019        // process only one second to start, rather than a full cadence.
19020        tokio::time::advance(
19021            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
19022        )
19023        .await;
19024        actor.module.restart(Some(0)).await.unwrap();
19025        let spawn_tick = Instant::now();
19026        let replacement = actor
19027            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
19028            .await;
19029        let deadline = replacement.next_probe_at.unwrap();
19030        assert!(
19031            deadline >= spawn_tick + actor.health.cadence,
19032            "a replacement must not inherit its predecessor's probe deadline"
19033        );
19034        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
19035        assert!(
19036            started_rx.try_recv().is_err(),
19037            "no HTTP probe during startup grace"
19038        );
19039        tokio::time::advance(deadline - Instant::now()).await;
19040        let landed = wait_for("first replacement HTTP probe", || {
19041            started_rx.try_recv().ok()
19042        })
19043        .await;
19044        assert!(landed >= spawn_tick + actor.health.cadence);
19045        wait_for("healthy replacement", || {
19046            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19047                .then_some(())
19048        })
19049        .await;
19050        peer.abort();
19051    }
19052
19053    #[tokio::test(start_paused = true)]
19054    async fn rescan_adds_and_removes_http_health_without_polling() {
19055        use tokio::{
19056            io::{AsyncReadExt, AsyncWriteExt},
19057            net::TcpListener,
19058        };
19059        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19060        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19061        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
19062        // One millisecond separates successive probes without a polling delay.
19063        actor.health.cadence = Duration::from_millis(1);
19064        let (started_tx, mut started_rx) = mpsc::channel(8);
19065        let peer = tokio::spawn(async move {
19066            loop {
19067                let (mut socket, _) = listener.accept().await.unwrap();
19068                let mut request = [0; 1024];
19069                assert!(socket.read(&mut request).await.unwrap() > 0);
19070                started_tx.send(()).await.unwrap();
19071                socket
19072                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19073                    .await
19074                    .unwrap();
19075            }
19076        });
19077        let tick = Instant::now();
19078        let turns = actor.turns();
19079        actor
19080            .module
19081            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19082            .await
19083            .unwrap();
19084        let armed = actor
19085            .wait_for_select(|checkpoint| {
19086                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
19087            })
19088            .await;
19089        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
19090        assert_eq!(Instant::now(), tick);
19091        tokio::time::advance(Duration::from_millis(1)).await;
19092        let mut started = false;
19093        wait_for("successful HTTP probe", || {
19094            started |= started_rx.try_recv().is_ok();
19095            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19096                .then_some(())
19097        })
19098        .await;
19099        assert!(started, "added HTTP check must start at its first deadline");
19100        assert_eq!(
19101            actor.module.status().unwrap().health.status,
19102            SupervisorHealthStatus::Ok
19103        );
19104        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
19105        actor.health.http = None;
19106        let turns = actor.turns();
19107        actor
19108            .module
19109            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19110            .await
19111            .unwrap();
19112        let parked = actor
19113            .wait_for_select(|checkpoint| {
19114                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
19115            })
19116            .await;
19117        assert_eq!(parked.wake_after, None);
19118        let turns = actor.turns();
19119        tokio::time::advance(Duration::from_secs(60)).await;
19120        assert_eq!(
19121            actor.turns(),
19122            turns,
19123            "removed HTTP check must park the actor"
19124        );
19125        assert!(
19126            started_rx.try_recv().is_err(),
19127            "removed HTTP check must stay disarmed"
19128        );
19129        assert_eq!(
19130            actor.module.status().unwrap().health,
19131            ModuleHealthStatus::default()
19132        );
19133        peer.abort();
19134    }
19135}
19136
19137#[cfg(test)]
19138mod drain_busy_reprobe_tests {
19139    //! The drain's busy-gauge wait, driven against an in-process module
19140    //! connection whose health answers the test scripts. Time is paused, so
19141    //! every elapsed duration below is exact virtual time.
19142    use super::*;
19143    use std::collections::BTreeMap;
19144    use std::sync::Mutex as StdMutex;
19145    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
19146
19147    const MODULE_ID: &str = "drain-reprobe-module";
19148    const CONNECTION: u64 = 7;
19149    const GAUGE: &str = "runs_in_flight";
19150    /// The production default for both the health cadence and the drain
19151    /// ceiling, which is the pairing that used to cost a full ceiling.
19152    const CADENCE: Duration = Duration::from_secs(30);
19153    const CEILING: Duration = Duration::from_secs(30);
19154
19155    /// How the scripted module answers each `health.check`.
19156    #[derive(Clone, Copy)]
19157    enum Answer {
19158        /// Report the gauge busy until this instant, quiet from then on, after
19159        /// taking `latency` to answer.
19160        BusyUntil { until: Instant, latency: Duration },
19161        /// Answer at once, with metrics that leave the declared gauge out.
19162        OmitsGauge,
19163        /// Never answer.
19164        Silent,
19165    }
19166
19167    struct Drain {
19168        forwarding: Arc<ForwardingTable>,
19169        runtime: SupervisorRuntimeConfig,
19170        endpoint: crate::ModuleEndpointId,
19171        /// When each probe reached the module.
19172        probes: Arc<StdMutex<Vec<Instant>>>,
19173        responder: tokio::task::JoinHandle<()>,
19174    }
19175
19176    impl Drain {
19177        fn start(answer: Answer) -> Self {
19178            let registry = Arc::new(Registry::default());
19179            let forwarding = Arc::new(ForwardingTable::default());
19180            let health = HealthConfig {
19181                cadence: CADENCE,
19182                ..HealthConfig::default()
19183            };
19184            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
19185                .with_forwarding(forwarding.clone())
19186                .with_health_config(health)
19187                .runtime_config();
19188            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
19189            let endpoint = forwarding
19190                .register_module_connection(
19191                    ConnectionId::new(CONNECTION),
19192                    MODULE_ID.to_string(),
19193                    subc_protocol::PROTOCOL_VERSION,
19194                    Concurrency::ModuleManaged,
19195                    FrameSink::new(tx),
19196                )
19197                .unwrap();
19198            let probes = Arc::new(StdMutex::new(Vec::new()));
19199            let responder = tokio::spawn({
19200                let forwarding = forwarding.clone();
19201                let probes = probes.clone();
19202                async move {
19203                    while let Some(frame) = rx.recv().await {
19204                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
19205                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
19206                        probes.lock().unwrap().push(Instant::now());
19207                        let metrics = match answer {
19208                            Answer::BusyUntil { until, latency } => {
19209                                sleep(latency).await;
19210                                let busy = u64::from(Instant::now() < until);
19211                                serde_json::json!({ GAUGE: busy })
19212                            }
19213                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
19214                            Answer::Silent => continue,
19215                        };
19216                        // A probe abandoned at its deadline has no RPC left to
19217                        // complete, so a late answer may be refused.
19218                        let _ = forwarding.complete_module_control_rpc(
19219                            ConnectionId::new(CONNECTION),
19220                            frame.header.corr,
19221                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
19222                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19223                                status: HealthStatus::Ok,
19224                                detail: None,
19225                                metrics: Some(metrics),
19226                            }),
19227                        );
19228                    }
19229                }
19230            });
19231            Self {
19232                forwarding,
19233                runtime,
19234                endpoint,
19235                probes,
19236                responder,
19237            }
19238        }
19239
19240        /// Run one by-id drain wait with `ceiling`, returning its verdict and
19241        /// how long it took.
19242        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
19243            let (drained, elapsed, _) = self.run_traced(ceiling).await;
19244            (drained, elapsed)
19245        }
19246
19247        /// [`Self::run`], also returning what the drain's probes read.
19248        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
19249            let started = Instant::now();
19250            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
19251            let drained = wait_for_forwarding_quiescence(
19252                &self.forwarding,
19253                MODULE_ID,
19254                &self.runtime,
19255                self.endpoint,
19256                started + ceiling,
19257                &[GAUGE.to_string()],
19258                DrainScope::Active,
19259                &mut trace,
19260            )
19261            .await
19262            .unwrap();
19263            (drained, started.elapsed(), trace)
19264        }
19265
19266        fn probes(&self) -> Vec<Instant> {
19267            self.probes.lock().unwrap().clone()
19268        }
19269    }
19270
19271    impl Drop for Drain {
19272        fn drop(&mut self) {
19273            self.responder.abort();
19274        }
19275    }
19276
19277    #[tokio::test(start_paused = true)]
19278    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
19279        let drain = Drain::start(Answer::BusyUntil {
19280            until: Instant::now() + Duration::from_secs(1),
19281            latency: Duration::ZERO,
19282        });
19283        let (drained, elapsed) = drain.run(CEILING).await;
19284        assert!(
19285            drained,
19286            "gauges that read zero during the drain must let it finish drained, \
19287             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
19288        );
19289        assert!(
19290            elapsed < Duration::from_secs(3),
19291            "drain took {elapsed:?} after the gauges went quiet at 1s"
19292        );
19293        let probes = drain.probes();
19294        assert!(
19295            probes.len() >= 2,
19296            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
19297        );
19298    }
19299
19300    /// The same shape in real time on a multi-threaded runtime: the first
19301    /// probe is answered busy at once, within a millisecond of the drain
19302    /// starting, and later probes read idle. The drain must keep probing
19303    /// after that first instant answer and end soon after the gauge falls.
19304    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19305    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
19306        let drain = Drain::start(Answer::BusyUntil {
19307            until: Instant::now() + Duration::from_millis(600),
19308            latency: Duration::ZERO,
19309        });
19310        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
19311        assert!(drained, "the gauge fell at 600ms: {trace}");
19312        assert!(
19313            elapsed < Duration::from_secs(3),
19314            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
19315        );
19316        let probes = drain.probes();
19317        assert!(
19318            probes.len() >= 2 && trace.probes as usize == probes.len(),
19319            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
19320        );
19321    }
19322
19323    #[tokio::test(start_paused = true)]
19324    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
19325        let started = Instant::now();
19326        let drain = Drain::start(Answer::BusyUntil {
19327            until: started + Duration::from_secs(3600),
19328            latency: Duration::ZERO,
19329        });
19330        let (drained, elapsed) = drain.run(CEILING).await;
19331        assert!(!drained, "a module busy throughout must not drain");
19332        assert!(
19333            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
19334            "a busy module is held until the ceiling and no longer: {elapsed:?}"
19335        );
19336        // Re-probing is paced by the interval, not a busy loop: at most one
19337        // probe per interval across the ceiling, plus the first at time zero.
19338        let probes = drain.probes();
19339        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
19340        assert!(
19341            probes.len() as u128 <= max_probes,
19342            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
19343            probes.len()
19344        );
19345        for pair in probes.windows(2) {
19346            assert!(
19347                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
19348                "consecutive probes {:?} apart",
19349                pair[1] - pair[0]
19350            );
19351        }
19352    }
19353
19354    #[tokio::test(start_paused = true)]
19355    async fn a_slow_probe_is_never_overlapped_by_the_next() {
19356        // Each answer takes longer than the re-probe interval; the next probe
19357        // must wait for the previous answer and then the interval.
19358        let latency = Duration::from_secs(1);
19359        let drain = Drain::start(Answer::BusyUntil {
19360            until: Instant::now() + Duration::from_secs(3600),
19361            latency,
19362        });
19363        let (drained, _) = drain.run(Duration::from_secs(5)).await;
19364        assert!(!drained);
19365        let probes = drain.probes();
19366        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
19367        for pair in probes.windows(2) {
19368            assert!(
19369                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
19370                "a probe started {:?} after the previous one, before it could finish",
19371                pair[1] - pair[0]
19372            );
19373        }
19374    }
19375
19376    #[tokio::test(start_paused = true)]
19377    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
19378        let drain = Drain::start(Answer::Silent);
19379        let ceiling = Duration::from_secs(2);
19380        let (drained, elapsed) = drain.run(ceiling).await;
19381        assert!(!drained, "an unanswered probe counts as busy");
19382        assert!(
19383            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
19384            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
19385        );
19386        assert_eq!(
19387            drain.probes().len(),
19388            1,
19389            "the unanswered probe holds the only slot until the ceiling"
19390        );
19391    }
19392
19393    /// Every event logged while it is installed, as `message` plus fields.
19394    #[derive(Clone, Default)]
19395    struct LogCapture {
19396        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
19397    }
19398
19399    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
19400        fn on_event(
19401            &self,
19402            event: &tracing::Event<'_>,
19403            _context: tracing_subscriber::layer::Context<'_, S>,
19404        ) {
19405            struct Fields(BTreeMap<String, String>);
19406            impl tracing::field::Visit for Fields {
19407                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
19408                    self.0
19409                        .insert(field.name().to_string(), format!("{value:?}"));
19410                }
19411            }
19412            let mut fields = Fields(BTreeMap::new());
19413            event.record(&mut fields);
19414            self.events.lock().unwrap().push(fields.0);
19415        }
19416    }
19417
19418    impl LogCapture {
19419        fn install(&self) -> tracing::subscriber::DefaultGuard {
19420            use tracing_subscriber::layer::SubscriberExt;
19421            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
19422        }
19423
19424        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
19425            self.events
19426                .lock()
19427                .unwrap()
19428                .iter()
19429                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
19430                .cloned()
19431                .collect()
19432        }
19433    }
19434
19435    /// A module that declares a busy gauge but leaves it out of its health
19436    /// answers holds a drain to its ceiling. The timeout line must say so by
19437    /// name, and the drain must warn about the missing gauge once, not on
19438    /// every probe.
19439    #[tokio::test(start_paused = true)]
19440    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
19441        let capture = LogCapture::default();
19442        let _guard = capture.install();
19443        let drain = Drain::start(Answer::OmitsGauge);
19444        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
19445        assert!(!drained, "a gauge never reported cannot read quiet");
19446        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
19447
19448        let missing = capture.with_message("lacks a declared busy gauge");
19449        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
19450        assert_eq!(
19451            missing[0].get("missing_gauges").map(String::as_str),
19452            Some(GAUGE)
19453        );
19454
19455        warn_drain_timed_out(
19456            MODULE_ID,
19457            Duration::from_secs(2),
19458            RouteCloseReason::Restart,
19459            &Default::default(),
19460            &trace,
19461        );
19462        let timeouts = capture.with_message("route drain timed out");
19463        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
19464        let gauges = &timeouts[0]["busy_gauges"];
19465        assert!(
19466            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
19467            "the timeout line must name the gauge and why it read busy: {gauges}"
19468        );
19469        assert!(
19470            gauges.ends_with("; no value ever read"),
19471            "no value was ever read for the gauge: {gauges}"
19472        );
19473        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
19474    }
19475
19476    /// The trace names a busy value and when it was read. For a probe that
19477    /// never got an answer, it names the probe's error kind and says that no
19478    /// value was ever read.
19479    #[tokio::test(start_paused = true)]
19480    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
19481        let busy = Drain::start(Answer::BusyUntil {
19482            until: Instant::now() + Duration::from_secs(3600),
19483            latency: Duration::ZERO,
19484        });
19485        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
19486        let rendered = trace.to_string();
19487        assert!(
19488            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
19489                && rendered.contains("; last value non_zero(1) @"),
19490            "{rendered}"
19491        );
19492
19493        let silent = Drain::start(Answer::Silent);
19494        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
19495        // The probe is capped at the 1 s ceiling, and its reading is stamped
19496        // when it gave up.
19497        assert_eq!(
19498            trace.to_string(),
19499            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
19500        );
19501        assert_eq!(trace.probes, 1);
19502    }
19503
19504    /// A gauge value that is not a count reads busy and is shown as sent.
19505    #[test]
19506    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
19507        let gauges = [GAUGE.to_string()];
19508        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
19509        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
19510        assert_eq!(
19511            busy_gauge_observation(&verdicts),
19512            BusyGaugeObservation::Busy
19513        );
19514        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
19515        assert_eq!(
19516            busy_gauge_observation(&zero),
19517            BusyGaugeObservation::Quiescent
19518        );
19519        let none = gauge_verdicts(None, &gauges);
19520        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
19521    }
19522}
19523
19524#[cfg(test)]
19525mod health_evidence_tests {
19526    use super::{HealthProbeError, HealthProbeEvidence};
19527    use std::collections::HashSet;
19528
19529    /// The evidential asymmetry, asserted rather than described.
19530    ///
19531    /// Exactly ONE observation is proof a module cannot serve, and the one that
19532    /// fires under CPU starvation is not it. Before the split, all fifteen
19533    /// construction sites collapsed into a single String, so a timeout carried the
19534    /// same weight as a dead lane -- which is how a healthy module was restarted
19535    /// three times in one day.
19536    #[test]
19537    fn only_a_dead_lane_is_proof_of_death() {
19538        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
19539        // Three non-proof classes, each for a different reason: silence is
19540        // consistent with health, a bad answer proves the module ALIVE, and a
19541        // daemon-side fault never reached the module at all.
19542        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
19543        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
19544        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
19545    }
19546
19547    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
19548    ///
19549    /// A shared label renders two different observations identically in the line an
19550    /// operator reads after an unexplained restart -- the exact confusion this
19551    /// change removes.
19552    #[test]
19553    fn every_evidence_class_has_a_distinct_label() {
19554        let labels = [
19555            HealthProbeError::lane_dead("").label(),
19556            HealthProbeError::no_answer("").label(),
19557            HealthProbeError::bad_answer("").label(),
19558            HealthProbeError::misconfigured("").label(),
19559        ];
19560        let unique: HashSet<_> = labels.iter().collect();
19561        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
19562    }
19563
19564    /// The class is additional information, not a replacement.
19565    ///
19566    /// An operator needs both "this was silence" and the specific text saying how
19567    /// long we waited; a classification that swallowed the message would trade one
19568    /// missing distinction for another.
19569    #[test]
19570    fn classification_preserves_the_original_message() {
19571        let err = HealthProbeError::no_answer("module did not answer within 5s");
19572        assert_eq!(err.to_string(), "module did not answer within 5s");
19573        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19574    }
19575}
19576
19577#[cfg(test)]
19578mod health_tombstone_tests {
19579    use std::{path::PathBuf, sync::Arc, time::Duration};
19580
19581    use subc_protocol::{
19582        manifest::Concurrency,
19583        session::{HealthStatus, ModuleControlResponse},
19584    };
19585    use tokio::sync::mpsc;
19586
19587    use super::{
19588        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
19589        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
19590    };
19591    use crate::{
19592        control::ControlHandler,
19593        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
19594        registry::{ConnectionId, Registry},
19595        router::FrameSink,
19596    };
19597
19598    struct ProbeHarness {
19599        spec: ModuleSpec,
19600        runtime: SupervisorRuntimeConfig,
19601        forwarding: Arc<ForwardingTable>,
19602        module_connection: ConnectionId,
19603        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19604        handler: ControlHandler,
19605        module: super::SupervisedModule,
19606    }
19607
19608    fn probe_harness() -> ProbeHarness {
19609        let registry = Arc::new(Registry::default());
19610        let forwarding = Arc::new(ForwardingTable::default());
19611        let supervisor_handle = super::SupervisorHandle::new();
19612        let health = HealthConfig {
19613            http: None,
19614            cadence: Duration::from_secs(30),
19615            deadline: Duration::from_secs(5),
19616            failure_threshold: 3,
19617            on_degraded: HealthAction::Report,
19618            on_failing: HealthAction::Report,
19619            critical: false,
19620        };
19621        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
19622            .with_forwarding(Arc::clone(&forwarding))
19623            .with_handle(supervisor_handle.clone())
19624            .with_health_config(health);
19625        let spec = ModuleSpec {
19626            module_id: "late-health-module".to_string(),
19627            program: PathBuf::from("disabled-module"),
19628            args: Vec::new(),
19629            env: Vec::new(),
19630            reserved: false,
19631            reserved_prefixes: Vec::new(),
19632            protocol: ModuleProtocol::Subc,
19633            overlap: Default::default(),
19634            carry_containment: false,
19635        };
19636        let module = supervisor
19637            .supervise_configured(spec.clone(), false)
19638            .unwrap();
19639        let runtime = supervisor.runtime_config();
19640        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
19641            .with_supervisor(supervisor_handle);
19642        let module_connection = ConnectionId::new(700);
19643        let (module_tx, module_rx) = mpsc::channel(8);
19644        forwarding
19645            .register_module_connection(
19646                module_connection,
19647                spec.module_id.clone(),
19648                subc_protocol::PROTOCOL_VERSION,
19649                Concurrency::ModuleManaged,
19650                FrameSink::new(module_tx),
19651            )
19652            .unwrap();
19653
19654        ProbeHarness {
19655            spec,
19656            runtime,
19657            forwarding,
19658            module_connection,
19659            module_rx,
19660            handler,
19661            module,
19662        }
19663    }
19664
19665    async fn finish_after(
19666        harness: &mut ProbeHarness,
19667        stall: Duration,
19668    ) -> ModuleControlRpcCompletion {
19669        assert!(stall > harness.runtime.health.deadline);
19670        let deadline = harness.runtime.health.deadline;
19671        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19672        let answer = async {
19673            let frame = harness.module_rx.recv().await.expect("health.check frame");
19674            tokio::time::advance(deadline).await;
19675            tokio::task::yield_now().await;
19676            tokio::time::advance(stall - deadline).await;
19677            harness
19678                .forwarding
19679                .complete_module_control_rpc(
19680                    harness.module_connection,
19681                    frame.header.corr,
19682                    Some("health.check"),
19683                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19684                        status: HealthStatus::Ok,
19685                        detail: None,
19686                        metrics: None,
19687                    }),
19688                )
19689                .unwrap()
19690        };
19691        let (probe_result, completion) = tokio::join!(probe, answer);
19692        let err = probe_result.expect_err("probe must miss its deadline");
19693        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19694        completion
19695    }
19696
19697    async fn time_out_without_answer(harness: &mut ProbeHarness) {
19698        let deadline = harness.runtime.health.deadline;
19699        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19700        let exhaust_deadline = async {
19701            let _frame = harness.module_rx.recv().await.expect("health.check frame");
19702            tokio::time::advance(deadline).await;
19703            tokio::task::yield_now().await;
19704        };
19705        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
19706        let err = probe_result.expect_err("probe must miss its deadline");
19707        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19708    }
19709
19710    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
19711        let registry = Arc::clone(&harness.module.inner.registry);
19712        let snapshot = Arc::clone(&harness.module.inner.snapshot);
19713        let process_liveness = super::SupervisorProcessLiveness::default();
19714        let mut child = None;
19715        let cycle = super::run_health_probe_cycle(
19716            &harness.spec,
19717            &harness.runtime,
19718            &registry,
19719            &process_liveness,
19720            &snapshot,
19721            &mut child,
19722        );
19723        let peer = async {
19724            let frame = harness.module_rx.recv().await.expect("health.check frame");
19725            if answer {
19726                harness
19727                    .forwarding
19728                    .complete_module_control_rpc(
19729                        harness.module_connection,
19730                        frame.header.corr,
19731                        Some("health.check"),
19732                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19733                            status: HealthStatus::Ok,
19734                            detail: None,
19735                            metrics: Some(serde_json::json!({"ready": true})),
19736                        }),
19737                    )
19738                    .unwrap();
19739            } else {
19740                tokio::time::advance(harness.runtime.health.deadline).await;
19741                tokio::task::yield_now().await;
19742            }
19743        };
19744        tokio::join!(cycle, peer);
19745    }
19746
19747    #[tokio::test(start_paused = true)]
19748    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
19749        let mut harness = probe_harness();
19750        // Drive the probe cycle directly with an in-memory wire peer. Stop the
19751        // disabled module's monitor so only this test owns lifecycle transitions;
19752        // no OS process is launched, and a restart is observed at scheduling.
19753        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
19754        monitor.abort();
19755        let _ = monitor.await;
19756        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
19757            state.enabled = true;
19758            state.state = super::ModuleState::Running;
19759            state.process_alive = true;
19760        })
19761        .unwrap();
19762
19763        run_probe_cycle(&mut harness, true).await;
19764        assert_eq!(
19765            harness.module.status().unwrap().health.status,
19766            super::SupervisorHealthStatus::Ok
19767        );
19768
19769        for failures in 1..harness.runtime.health.failure_threshold {
19770            run_probe_cycle(&mut harness, false).await;
19771            let status = harness.module.status().unwrap();
19772            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19773            assert_eq!(status.health.consecutive_failures, failures);
19774            assert!(status.health.last_probe_ms.is_some());
19775            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
19776            assert!(status.health.metrics.is_none());
19777            assert_eq!(status.state, super::ModuleState::Running);
19778            assert!(status.process_alive);
19779            assert_eq!(status.restart_count, 0);
19780            assert_eq!(status.lifetime_restarts, 0);
19781            assert!(status.health.last_action.is_none());
19782            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19783        }
19784
19785        run_probe_cycle(&mut harness, true).await;
19786        let recovered = harness.module.status().unwrap();
19787        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
19788        assert_eq!(recovered.health.consecutive_failures, 0);
19789        assert!(recovered.health.detail.is_none());
19790        assert_eq!(
19791            recovered.health.metrics,
19792            Some(serde_json::json!({"ready": true}))
19793        );
19794        assert_eq!(recovered.lifetime_restarts, 0);
19795
19796        for failures in 1..=harness.runtime.health.failure_threshold {
19797            run_probe_cycle(&mut harness, false).await;
19798            let status = harness.module.status().unwrap();
19799            assert_eq!(status.health.consecutive_failures, failures);
19800            if failures < harness.runtime.health.failure_threshold {
19801                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19802                assert_eq!(status.state, super::ModuleState::Running);
19803                assert_eq!(status.lifetime_restarts, 0);
19804                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19805            } else {
19806                assert_eq!(
19807                    status.health.status,
19808                    super::SupervisorHealthStatus::Unresponsive
19809                );
19810                assert_eq!(status.state, super::ModuleState::Restarting);
19811                assert_eq!(status.restart_count, 1);
19812                assert_eq!(status.lifetime_restarts, 1);
19813                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
19814                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
19815            }
19816        }
19817    }
19818
19819    #[tokio::test(start_paused = true)]
19820    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
19821        let mut harness = probe_harness();
19822
19823        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
19824        let first_latency = match &first {
19825            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19826            other => panic!("late answer was not retained: {other:?}"),
19827        };
19828        assert!(harness.handler.observe_module_control_completion(first));
19829
19830        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
19831        let second_latency = match &second {
19832            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19833            other => panic!("late answer was not retained: {other:?}"),
19834        };
19835        assert!(harness.handler.observe_module_control_completion(second));
19836
19837        assert_eq!(first_latency, Duration::from_secs(8));
19838        assert_eq!(
19839            second_latency - first_latency,
19840            Duration::from_secs(3),
19841            "latency must grow linearly with the additional stall"
19842        );
19843        let health = harness.module.status().unwrap().health;
19844        assert_eq!(health.late_answer_count, 2);
19845        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
19846    }
19847
19848    /// A module that answers every probe late must never march to the kill
19849    /// threshold: the late answer proves it is alive, so it must clear the miss
19850    /// streak the timeout recorded. Without the reset, a CPU-starved module
19851    /// that serves every probe seconds past the deadline accumulates
19852    /// `consecutive_failures` to the threshold and is killed — the exact
19853    /// sequence from the 2026-08-14 aft disable, where the daemon logged
19854    /// "proves the module is alive" five times while counting five misses.
19855    #[tokio::test(start_paused = true)]
19856    async fn late_answer_clears_the_consecutive_failure_streak() {
19857        let mut harness = probe_harness();
19858
19859        // Timeout recorded first: the probe path saw no answer in time.
19860        time_out_without_answer(&mut harness).await;
19861        harness
19862            .module
19863            .record_health_probe_failure_for_test("[no-answer] test miss")
19864            .unwrap();
19865        assert_eq!(
19866            harness.module.status().unwrap().health.consecutive_failures,
19867            1,
19868            "precondition: the miss must be on the streak before the late answer"
19869        );
19870
19871        // The stalled reply then lands: proof of life.
19872        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
19873        assert!(matches!(
19874            late,
19875            ModuleControlRpcCompletion::LateHealthAnswer { .. }
19876        ));
19877        assert!(harness.handler.observe_module_control_completion(late));
19878
19879        let health = harness.module.status().unwrap().health;
19880        assert_eq!(
19881            health.consecutive_failures, 0,
19882            "a late answer is an answer: the streak must reset"
19883        );
19884        assert_eq!(health.late_answer_count, 1);
19885    }
19886
19887    #[tokio::test(start_paused = true)]
19888    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
19889        let mut harness = probe_harness();
19890
19891        for _ in 0..20 {
19892            time_out_without_answer(&mut harness).await;
19893            assert_eq!(
19894                harness.forwarding.health_probe_tombstone_count().unwrap(),
19895                1
19896            );
19897        }
19898    }
19899}
19900
19901#[cfg(test)]
19902mod child_env_tests {
19903    #[cfg(windows)]
19904    use super::{
19905        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
19906    };
19907    #[cfg(windows)]
19908    use std::sync::{Arc, OnceLock};
19909    #[cfg(windows)]
19910    #[test]
19911    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
19912        let spec = spec(Vec::new());
19913        let mut command = Command::new("unused");
19914        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
19915            .unwrap()
19916            .unwrap();
19917        let name = command
19918            .as_std()
19919            .get_envs()
19920            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
19921            .and_then(|(_, value)| value)
19922            .unwrap();
19923        assert_eq!(name, OsStr::new(handoff.name()));
19924        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
19925        assert!(command.as_std().get_envs().any(|(key, value)| key
19926            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
19927            && value == Some(OsStr::new("env"))));
19928        assert!(command
19929            .as_std()
19930            .get_envs()
19931            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
19932    }
19933
19934    #[cfg(windows)]
19935    #[test]
19936    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
19937        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
19938        snapshot.process_alive = true;
19939        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
19940        assert_eq!(
19941            observed_launch_nonce_source(&snapshot).as_deref(),
19942            Some("env")
19943        );
19944        let consumed = Arc::new(OnceLock::new());
19945        snapshot.pipe_consumed = Some(consumed.clone());
19946        assert_eq!(
19947            observed_launch_nonce_source(&snapshot).as_deref(),
19948            Some("env")
19949        );
19950        consumed.set(()).unwrap();
19951        assert_eq!(
19952            observed_launch_nonce_source(&snapshot).as_deref(),
19953            Some("pipe")
19954        );
19955        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
19956        assert_eq!(
19957            observed_launch_nonce_source(&snapshot).as_deref(),
19958            Some("env")
19959        );
19960        clear_current_process_facts(&mut snapshot);
19961        assert_eq!(observed_launch_nonce_source(&snapshot), None);
19962    }
19963    use super::{
19964        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
19965        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
19966        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
19967    };
19968    use std::{ffi::OsStr, path::PathBuf};
19969    use tokio::process::Command;
19970
19971    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
19972        ModuleSpec {
19973            module_id: "env-plan".to_string(),
19974            program: PathBuf::from("/nonexistent"),
19975            args: Vec::new(),
19976            env,
19977            reserved: false,
19978            reserved_prefixes: Vec::new(),
19979            protocol: ModuleProtocol::Subc,
19980            overlap: Default::default(),
19981            carry_containment: false,
19982        }
19983    }
19984
19985    #[test]
19986    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
19987        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
19988        let digest = LaunchSpecDigest::from_spec(&base);
19989        let mut reordered = base.clone();
19990        reordered.env.reverse();
19991        reordered.module_id = "different-id".into();
19992        reordered.reserved = true;
19993        reordered.overlap = Default::default();
19994        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
19995        let mut delimiter_collision = base.clone();
19996        delimiter_collision.env[0] = ("A=B".into(), "C".into());
19997        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
19998        let mut changed = base.clone();
19999        changed.protocol = ModuleProtocol::None;
20000        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20001        changed = base.clone();
20002        changed.args.push("--flag".into());
20003        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20004        changed = base.clone();
20005        changed.program = PathBuf::from("/other");
20006        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20007        changed = base.clone();
20008        changed.env[1].1 = "other".into();
20009        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20010        assert!(!format!("{digest:?}").contains("B=C"));
20011        assert!(!format!("{digest:?}").contains("[u8"));
20012    }
20013
20014    #[cfg(unix)]
20015    #[test]
20016    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
20017        use std::os::unix::ffi::OsStringExt;
20018        let mut left = spec(Vec::new());
20019        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
20020        let mut right = left.clone();
20021        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
20022        assert_eq!(
20023            left.program.to_string_lossy(),
20024            right.program.to_string_lossy()
20025        );
20026        assert_ne!(
20027            LaunchSpecDigest::from_spec(&left),
20028            LaunchSpecDigest::from_spec(&right)
20029        );
20030    }
20031
20032    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
20033    /// one still gets its own.
20034    ///
20035    /// This is the narrow goal `env_clear()` was reached for, and the reason the
20036    /// fix is `env_remove` rather than deleting the line: an operator's ambient
20037    /// filter silently becoming an unconfigured module's log level is a real
20038    /// defect, just a much smaller one than clearing the environment.
20039    ///
20040    /// Asserted on the command plan rather than a spawned child because proving
20041    /// the ABSENCE of an inherited variable needs the parent's environment
20042    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
20043    /// removal as `(key, None)`, which is exactly the distinction wanted: not
20044    /// "absent because nobody set it" but "explicitly unset for the child".
20045    #[test]
20046    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
20047        let mut command = Command::new("/nonexistent");
20048        apply_child_env(&mut command, &spec(Vec::new()));
20049        let removed = command
20050            .as_std()
20051            .get_envs()
20052            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
20053        assert!(
20054            removed,
20055            "ambient CK_LOG must be explicitly removed for an unconfigured module"
20056        );
20057
20058        let mut configured = Command::new("/nonexistent");
20059        apply_child_env(
20060            &mut configured,
20061            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
20062        );
20063        let effective = configured
20064            .as_std()
20065            .get_envs()
20066            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
20067            .last()
20068            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20069        assert_eq!(
20070            effective,
20071            Some(Some("debug".to_string())),
20072            "a module's configured CK_LOG must survive the ambient removal"
20073        );
20074    }
20075
20076    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
20077    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
20078    /// the same reason as the CK_LOG test above.
20079    ///
20080    /// The argument is the load-bearing half: a stock binary exits on an
20081    /// unknown flag before it listens, so with `--subc` appended the mode
20082    /// could not supervise the one process it exists for. Found by the first
20083    /// conformance run (nats-server: `flag provided but not defined: -subc`).
20084    #[test]
20085    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
20086        let connection_file = std::path::Path::new("/run/subc-connection.json");
20087        let handle = SupervisorHandle::new();
20088
20089        let mut none_spec = spec(Vec::new());
20090        none_spec.protocol = ModuleProtocol::None;
20091        let mut none = Command::new("/nonexistent");
20092        let none_handoff =
20093            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
20094                .expect("protocol-none spawn args apply");
20095        assert!(
20096            none_handoff.is_none(),
20097            "protocol:none spawn must not receive a nonce descriptor"
20098        );
20099        #[cfg(windows)]
20100        assert!(!none.as_std().get_envs().any(|(key, value)| key
20101            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20102            && value.is_some()));
20103        assert!(
20104            !none.as_std().get_envs().any(|(key, value)| key
20105                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
20106                && value.is_some()),
20107            "protocol:none spawn must not name a nonce descriptor"
20108        );
20109        let none_args: Vec<String> = none
20110            .as_std()
20111            .get_args()
20112            .map(|a| a.to_string_lossy().into_owned())
20113            .collect();
20114        assert!(
20115            !none_args.iter().any(|a| a == SUBC_ARG),
20116            "protocol:none argv must not carry --subc; got {none_args:?}"
20117        );
20118        let none_has_nonce = none
20119            .as_std()
20120            .get_envs()
20121            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
20122        assert!(
20123            !none_has_nonce,
20124            "protocol:none spawn must not receive a launch nonce"
20125        );
20126        let none_has_module_id = none
20127            .as_std()
20128            .get_envs()
20129            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
20130        assert!(
20131            none_has_module_id,
20132            "SUBC_MODULE_ID is inert and stays on every path"
20133        );
20134        assert!(
20135            handle.spawn_nonce(&none_spec.module_id).is_none(),
20136            "no nonce record for a process that will never present one"
20137        );
20138
20139        // Control: the subc-wire path is unchanged by the branch above.
20140        let wire_spec = spec(Vec::new());
20141        let mut wire = Command::new("/nonexistent");
20142        let wire_handoff =
20143            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
20144                .expect("subc-wire spawn args apply");
20145        let wire_fd_env = wire
20146            .as_std()
20147            .get_envs()
20148            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
20149            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20150        #[cfg(unix)]
20151        assert_eq!(
20152            wire_fd_env,
20153            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
20154            "a subc-wire spawn names the pipe it will receive at descriptor 3"
20155        );
20156        #[cfg(windows)]
20157        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
20158        #[cfg(not(any(unix, windows)))]
20159        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
20160        let wire_args: Vec<String> = wire
20161            .as_std()
20162            .get_args()
20163            .map(|a| a.to_string_lossy().into_owned())
20164            .collect();
20165        assert_eq!(
20166            wire_args,
20167            vec![
20168                SUBC_ARG.to_string(),
20169                connection_file.to_string_lossy().into_owned()
20170            ],
20171            "a subc-wire spawn still carries --subc <path>"
20172        );
20173        assert_eq!(
20174            wire.as_std()
20175                .get_envs()
20176                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
20177            !cfg!(unix),
20178            "only Windows supplies the environment nonce"
20179        );
20180        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
20181    }
20182
20183    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
20184    /// spec tries to set it; only a swap candidate carries it.
20185    ///
20186    /// "Set it only on candidates" is not enough, because spawn applies the
20187    /// spec's env verbatim and the daemon's own environment is inherited: either
20188    /// could hand a plain restart the swap role, and a module reading it would
20189    /// warm on its long swap budget while callers wait. Asserted as an explicit
20190    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
20191    /// test above gives.
20192    #[test]
20193    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
20194        let role = |command: &Command| {
20195            command
20196                .as_std()
20197                .get_envs()
20198                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
20199                .last()
20200                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
20201        };
20202        let forged = spec(vec![(
20203            SUBC_SPAWN_ROLE_ENV.to_string(),
20204            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
20205        )]);
20206
20207        let mut plain = Command::new("/nonexistent");
20208        apply_child_env(&mut plain, &forged);
20209        apply_spawn_role(&mut plain, SpawnRole::Plain);
20210        assert_eq!(
20211            role(&plain),
20212            Some(None),
20213            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
20214        );
20215
20216        let mut candidate = Command::new("/nonexistent");
20217        apply_child_env(&mut candidate, &spec(Vec::new()));
20218        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
20219        assert_eq!(
20220            role(&candidate),
20221            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
20222        );
20223    }
20224
20225    /// Daemon-private capture retention keys never reach the child.
20226    ///
20227    /// cortexkit-log exposes retention as a Rust struct with no environment
20228    /// names, so these entries are supervisor metadata. Passing them through
20229    /// would invent a public child-process contract by accident.
20230    #[test]
20231    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
20232        let mut command = Command::new("/nonexistent");
20233        apply_child_env(
20234            &mut command,
20235            &spec(vec![
20236                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
20237                ("KEPT".to_string(), "yes".to_string()),
20238            ]),
20239        );
20240        let keys: Vec<String> = command
20241            .as_std()
20242            .get_envs()
20243            .filter(|(_, value)| value.is_some())
20244            .map(|(key, _)| key.to_string_lossy().into_owned())
20245            .collect();
20246        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
20247        assert!(
20248            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
20249            "daemon-private capture key leaked to the child: {keys:?}"
20250        );
20251    }
20252}
20253
20254#[cfg(test)]
20255mod jitter_tests {
20256    use super::jittered_health_delay;
20257    use std::{collections::HashSet, time::Duration};
20258
20259    /// Module ids drawn from a real fleet, so the dispersal claim is about names
20260    /// that actually occur rather than invented ones.
20261    ///
20262    /// This is a SAMPLE, not a registry: the property under test is that distinct
20263    /// ids disperse, which holds for any set of distinct strings. Several entries
20264    /// are already historical (modules get renamed), and that costs nothing here --
20265    /// but it means a reader must not mistake this for the live module set, and a
20266    /// rename sweep will match it without there being anything to change.
20267    const FLEET: [&str; 14] = [
20268        "aft",
20269        "alfonso-core",
20270        "magic-context",
20271        "broca",
20272        "thalamus",
20273        "quota",
20274        "engram",
20275        "plexus",
20276        "cerebellum",
20277        "astrocyte",
20278        "synapse",
20279        "subc-mcp",
20280        "cortexkit-credentials",
20281        "subc-federation",
20282    ];
20283
20284    /// Probes must not converge after a fleet-wide restart.
20285    ///
20286    /// This is the property the jitter exists for: every module reconnects at
20287    /// once, and without dispersal all fourteen would then probe on the same
20288    /// tick forever. Nothing failed visibly when this went untested -- a
20289    /// convergent fleet still probes correctly, just in a burst, so the symptom
20290    /// is a periodic load spike that looks like whatever else is running.
20291    #[test]
20292    fn probe_delays_disperse_across_the_fleet() {
20293        let cadence = Duration::from_secs(30);
20294        let delays: HashSet<Duration> = FLEET
20295            .iter()
20296            .map(|id| jittered_health_delay(id, 0, cadence))
20297            .collect();
20298        assert_eq!(
20299            delays.len(),
20300            FLEET.len(),
20301            "every supervised module must land on its own probe offset"
20302        );
20303    }
20304
20305    /// The offset may only ever DELAY a probe, never bring it forward.
20306    ///
20307    /// A delay below the cadence would probe a module more often than
20308    /// configured, which is the opposite of what an operator asked for and
20309    /// would tighten the failure budget without anyone changing it.
20310    #[test]
20311    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
20312        let cadence = Duration::from_secs(30);
20313        let span = cadence / 10;
20314        for id in FLEET {
20315            for probe_index in 0..8 {
20316                let delay = jittered_health_delay(id, probe_index, cadence);
20317                assert!(
20318                    delay >= cadence,
20319                    "{id}#{probe_index}: jitter must not shorten the cadence"
20320                );
20321                assert!(
20322                    delay < cadence + span,
20323                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
20324                );
20325            }
20326        }
20327    }
20328
20329    /// A module keeps its offset across daemon restarts.
20330    ///
20331    /// The delay is derived rather than randomised precisely so a restart does
20332    /// not re-roll every module into a fresh chance of collision. A random
20333    /// source would satisfy the dispersal test above and quietly lose this.
20334    #[test]
20335    fn a_module_offset_is_stable_across_restarts() {
20336        let cadence = Duration::from_secs(30);
20337        for id in FLEET {
20338            assert_eq!(
20339                jittered_health_delay(id, 0, cadence),
20340                jittered_health_delay(id, 0, cadence),
20341                "{id}: the same module and probe index must produce the same offset"
20342            );
20343        }
20344    }
20345
20346    /// A zero cadence disables probing rather than producing a busy loop.
20347    #[test]
20348    fn zero_cadence_yields_zero_delay() {
20349        assert_eq!(
20350            jittered_health_delay("aft", 0, Duration::ZERO),
20351            Duration::ZERO
20352        );
20353    }
20354}
20355
20356#[cfg(all(test, target_os = "linux"))]
20357mod cgroup_placement_tests {
20358    use super::{
20359        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
20360        SuperviseError, SupervisedChild,
20361    };
20362    use crate::stderr_tail::{StderrRing, StderrTailConfig};
20363    use cortexkit_test_support::ScratchDir;
20364    use std::{
20365        fs, io,
20366        path::{Path, PathBuf},
20367        sync::{Arc, Mutex},
20368    };
20369    use tokio::process::Command;
20370
20371    #[tokio::test]
20372    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
20373        use super::*;
20374        let dir = ScratchDir::new("unique-spawn-cgroups");
20375        let root = PathBuf::from(format!(
20376            "/sys/fs/cgroup/subc-unique-test-{}-{}",
20377            std::process::id(),
20378            unix_ms_now()
20379        ));
20380        if let Err(error) = fs::create_dir(&root) {
20381            assert!(
20382                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
20383                "required cgroup test cannot execute: {error}"
20384            );
20385            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
20386            return;
20387        }
20388        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
20389        let group_count = || {
20390            fs::read_dir(root.join("subc-modules"))
20391                .unwrap()
20392                .map(|entry| entry.unwrap().file_type().unwrap())
20393                .filter(|kind| kind.is_dir())
20394                .count()
20395        };
20396        let supervisor =
20397            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
20398                .with_cgroup_placement(Some(placement.clone()));
20399        let runtime = supervisor.runtime_config();
20400        let mut spec = ModuleSpec {
20401            module_id: "unique-spawn".into(),
20402            program: PathBuf::from("/bin/sleep"),
20403            args: vec!["60".into()],
20404            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
20405                .into_iter()
20406                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
20407                .collect(),
20408            reserved: false,
20409            reserved_prefixes: vec![],
20410            protocol: ModuleProtocol::None,
20411            overlap: Default::default(),
20412            carry_containment: false,
20413        };
20414        let spawn = |spec: &ModuleSpec| {
20415            spawn_child(
20416                spec,
20417                None,
20418                None,
20419                &runtime.stderr_ring,
20420                None,
20421                &runtime.child_roster,
20422                runtime.spawn_delivery(),
20423                Some(&placement),
20424                runtime.register_timeout,
20425            )
20426            .unwrap()
20427        };
20428        let mut live = spawn(&spec);
20429        for _ in 0..3 {
20430            // A new process can enter the old slot while retirement is pending.
20431            let next = spawn(&spec);
20432            assert_ne!(live.module_id, next.module_id);
20433            live.start_kill().unwrap();
20434            live.wait().await.unwrap();
20435            live = next;
20436            assert!(
20437                live.child.try_wait().unwrap().is_none(),
20438                "retiring the old slot must not kill the replacement"
20439            );
20440            assert_eq!(
20441                group_count(),
20442                1,
20443                "only the live spawn's cgroup should remain"
20444            );
20445        }
20446        supervisor.begin_daemon_shutdown();
20447        let reap = tokio::spawn(async move {
20448            live.wait().await.unwrap();
20449        });
20450        supervisor
20451            .end_children_for_daemon_shutdown(false, std::future::pending())
20452            .await;
20453        reap.await.unwrap();
20454        assert_eq!(group_count(), 0);
20455        // A normal exit uses the same tree-cleanup path as a killed spawn.
20456        spec.program = PathBuf::from("/bin/true");
20457        spec.args.clear();
20458        let fresh_roster = ChildRoster::default();
20459        let mut short = spawn_child(
20460            &spec,
20461            None,
20462            None,
20463            &runtime.stderr_ring,
20464            None,
20465            &fresh_roster,
20466            None,
20467            Some(&placement),
20468            runtime.register_timeout,
20469        )
20470        .unwrap();
20471        short.wait().await.unwrap();
20472        assert_eq!(group_count(), 0);
20473        spec.module_id = "_".repeat(255);
20474        let mut long_id = spawn_child(
20475            &spec,
20476            None,
20477            None,
20478            &runtime.stderr_ring,
20479            None,
20480            &fresh_roster,
20481            None,
20482            Some(&placement),
20483            runtime.register_timeout,
20484        )
20485        .unwrap();
20486        long_id.wait().await.unwrap();
20487        assert_eq!(
20488            group_count(),
20489            0,
20490            "valid long module IDs must not exceed cgroup NAME_MAX"
20491        );
20492        fs::remove_dir(root.join("subc-modules")).unwrap();
20493        fs::remove_dir(root).unwrap();
20494        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
20495    }
20496
20497    #[test]
20498    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
20499        let path = Path::new("/definitely-missing-subc-cgroup");
20500        let mut command = Command::new("true");
20501        let error = apply_cgroup_placement(
20502            &mut command,
20503            &ModuleSpec {
20504                module_id: "broken-cgroup".to_string(),
20505                program: PathBuf::from("true"),
20506                args: Vec::new(),
20507                env: Vec::new(),
20508                reserved: false,
20509                reserved_prefixes: Vec::new(),
20510                protocol: ModuleProtocol::Subc,
20511                overlap: Default::default(),
20512                carry_containment: false,
20513            },
20514            path,
20515        )
20516        .expect_err("a parent cgroup open failure must reject the supervised spawn");
20517        let reason = error.to_string();
20518
20519        assert!(
20520            matches!(error, SuperviseError::Cgroup { .. }),
20521            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
20522        );
20523        assert!(
20524            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
20525            "parent cgroup open failure must name cgroup.procs: {reason}"
20526        );
20527    }
20528
20529    #[tokio::test]
20530    async fn reaping_a_child_removes_its_empty_module_cgroup() {
20531        let root = ScratchDir::new("supervisor-reap-cgroup");
20532        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20533        let placement = subc_cgroup::prepare_at(&root)
20534            .expect("prepare scratch cgroup root")
20535            .expect("scratch root has a cgroup.procs marker");
20536        let module_id = "reaped-module";
20537        let module = placement
20538            .module_path(module_id)
20539            .expect("create scratch module cgroup");
20540        let child = Command::new("true")
20541            .env("XDG_DATA_HOME", root.path())
20542            .env("XDG_RUNTIME_DIR", root.path())
20543            .env("XDG_CONFIG_HOME", root.path())
20544            .spawn()
20545            .expect("spawn short-lived child");
20546        let pid = child.id().expect("spawned child has pid");
20547        let mut child = SupervisedChild {
20548            registration_channel: None,
20549            registration_spawn: None,
20550            live_program_binding: None,
20551            registered_config_connection: None,
20552            admission_base: None,
20553            registration_declaration: None,
20554            _program_binding: None,
20555            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
20556            child: crate::supervise::containment::ProcessChild::plain(child, pid),
20557            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
20558            protocol: ModuleProtocol::Subc,
20559            module_id: module_id.to_string(),
20560            cgroup_placement: Some(placement),
20561            stdout_pump: None,
20562            stderr_pump: None,
20563            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
20564            spawned_at: tokio::time::Instant::now(),
20565            config_delivery: None,
20566            registration_deadline: None,
20567            registration_watch: None,
20568            spawned_at_ms: 0,
20569            spawned_from: PathBuf::from("true"),
20570            launch_spec_digest: LaunchSpecDigest([0; 32]),
20571            spawned_file_identity: None,
20572            process_start_time: None,
20573            process_identity: None,
20574            pid,
20575            roster_guard: None,
20576            #[cfg(target_os = "macos")]
20577            privacy_exec: None,
20578            spawn_failure: None,
20579            admission_kill_sent: false,
20580        };
20581
20582        child.wait().await.expect("reap short-lived child");
20583
20584        assert!(
20585            !module.exists(),
20586            "waiting for a non-carried child must remove its empty cgroup before returning"
20587        );
20588        child.finish_exit(false).await.unwrap();
20589    }
20590
20591    #[test]
20592    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
20593        let root = ScratchDir::new("supervisor-non-empty-cgroup");
20594        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20595        let placement = subc_cgroup::prepare_at(&root)
20596            .expect("prepare scratch cgroup root")
20597            .expect("scratch root has a cgroup.procs marker");
20598        let module = placement
20599            .module_path("surviving-module")
20600            .expect("create scratch module cgroup");
20601        fs::write(module.join("surviving-process"), b"still present")
20602            .expect("make scratch cgroup non-empty");
20603        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
20604
20605        remove_module_cgroup(&placement, "surviving-module");
20606
20607        let logs = crate::router::test_log::captured_logs(&logs);
20608        assert!(
20609            module.exists(),
20610            "failed removal must leave the cgroup intact"
20611        );
20612        assert!(
20613            logs.contains("could not remove module cgroup after process exit; continuing teardown")
20614                && logs.contains("surviving-module"),
20615            "best-effort removal must report the failure without returning it: {logs}"
20616        );
20617    }
20618
20619    #[test]
20620    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
20621        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
20622        let reason = SuperviseError::Spawn {
20623            program: PathBuf::from("/bin/true"),
20624            source: io::Error::from_raw_os_error(13),
20625            cgroup_path: Some(cgroup_path.clone()),
20626        }
20627        .to_string();
20628
20629        assert!(
20630            reason.contains(&cgroup_path.display().to_string()),
20631            "a pre_exec spawn failure must name the cgroup path: {reason}"
20632        );
20633    }
20634}
20635
20636#[cfg(test)]
20637mod spawn_subscriber_lag_tests {
20638    use super::*;
20639
20640    /// A subscriber whose connection stops draining is dropped once its frame
20641    /// channel fills. The client must learn that from a terminal Error frame
20642    /// after the frames already queued for it, not from a stream that simply
20643    /// goes quiet.
20644    #[tokio::test]
20645    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
20646        let feed = SpawnEventFeed::default();
20647        feed.configure_incarnation("lag-incarnation".to_string());
20648        // A one-slot connection queue that nobody reads until the emits are
20649        // done: the forwarder parks on it and the subscriber channel fills.
20650        let (tx, mut rx) = mpsc::channel(1);
20651        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
20652            .expect("subscribe");
20653        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
20654        for index in 0..emitted {
20655            feed.emit_spawned(&format!("lag-module-{index}"), None, 1000, 0);
20656            // Let the forwarder take what it can so the fill point is the
20657            // subscriber channel, not a scheduling accident.
20658            tokio::task::yield_now().await;
20659        }
20660        assert_eq!(
20661            feed.subscriber_count(),
20662            0,
20663            "the lagged subscriber must be removed"
20664        );
20665
20666        let mut data = Vec::new();
20667        let mut last = None;
20668        loop {
20669            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
20670                .await
20671                .expect("the forwarder must finish once the subscriber is dropped");
20672            let Some(outbound) = next else { break };
20673            let frame = outbound.frame;
20674            if frame.header.ty == FrameType::StreamData {
20675                assert!(last.is_none(), "no data may follow the terminal frame");
20676                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
20677                data.push(event.cursor.seq);
20678            } else {
20679                assert!(last.is_none(), "exactly one terminal frame");
20680                last = Some(frame);
20681            }
20682        }
20683        assert!(!data.is_empty(), "queued frames drain before the terminal");
20684        for pair in data.windows(2) {
20685            assert_eq!(
20686                pair[1],
20687                pair[0] + 1,
20688                "queued frames arrive dense and in order"
20689            );
20690        }
20691        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
20692        assert_eq!(terminal.header.ty, FrameType::Error);
20693        assert_eq!(terminal.header.corr, 7);
20694        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
20695        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
20696        let detail = body.detail.expect("lagged error carries detail");
20697        assert_eq!(
20698            detail["first_undelivered_cursor"]["seq"],
20699            data.last().unwrap() + 1,
20700            "the named cursor is the first event the subscriber did not receive"
20701        );
20702        assert_eq!(
20703            detail["first_undelivered_cursor"]["daemon_incarnation"],
20704            "lag-incarnation"
20705        );
20706    }
20707}
20708
20709#[cfg(test)]
20710mod terminal_history_read_concurrency_tests {
20711    use super::*;
20712    use crate::terminal_journal::read_pause;
20713    use cortexkit_test_support::ScratchDir;
20714    use std::sync::mpsc as std_mpsc;
20715
20716    fn journaled_ring(
20717        journal: &Arc<crate::terminal_journal::TerminalJournal>,
20718    ) -> Arc<Mutex<TerminalRing>> {
20719        Arc::new(Mutex::new(
20720            TerminalRing::new(TerminalRingConfig::default(), 1)
20721                .with_journal(Some(Arc::clone(journal))),
20722        ))
20723    }
20724
20725    fn crash(at_ms: u64) -> ExitReport {
20726        ExitReport {
20727            kind: ExitKind::Crash,
20728            code: Some(1),
20729            signal: None,
20730            at_ms,
20731        }
20732    }
20733
20734    /// Record an exit on another thread and report whether it finished within
20735    /// `bound`. The recorder thread is left running if it did not.
20736    fn record_within(
20737        module_id: &'static str,
20738        ring: &Arc<Mutex<TerminalRing>>,
20739        at_ms: u64,
20740        bound: Duration,
20741    ) -> bool {
20742        let ring = Arc::clone(ring);
20743        let (done, done_rx) = std_mpsc::channel();
20744        std::thread::spawn(move || {
20745            record_terminal(
20746                module_id,
20747                &ring,
20748                &SpawnEventFeed::default(),
20749                &crash(at_ms),
20750                TerminalDisposition::Restarting,
20751            );
20752            let _ = done.send(());
20753        });
20754        done_rx.recv_timeout(bound).is_ok()
20755    }
20756
20757    /// A history read in progress must not hold the journal writer (which every
20758    /// module's exit recording needs) or the module's own ring. Exits recorded
20759    /// while the read is paused complete promptly; the paused read answers as of
20760    /// the moment it started, and the next read has each exit exactly once.
20761    #[test]
20762    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
20763        let dir = ScratchDir::new("terminal-history-concurrent-read");
20764        let path = dir.join("terminals.jsonl");
20765        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
20766            path.clone(),
20767            "daemon".into(),
20768        ));
20769        let reader_ring = journaled_ring(&journal);
20770        let other_ring = journaled_ring(&journal);
20771        assert!(record_within(
20772            "reader-module",
20773            &reader_ring,
20774            10,
20775            Duration::from_secs(5)
20776        ));
20777
20778        let (started, release) = read_pause::install(&path);
20779        let reading = {
20780            let ring = Arc::clone(&reader_ring);
20781            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
20782        };
20783        started
20784            .recv_timeout(Duration::from_secs(5))
20785            .expect("the history read reached its pause");
20786
20787        let bound = Duration::from_secs(1);
20788        assert!(
20789            record_within("other-module", &other_ring, 20, bound),
20790            "another module's exit waited on a history read (journal writer held)"
20791        );
20792        assert!(
20793            record_within("reader-module", &reader_ring, 30, bound),
20794            "the read module's own exit waited on its history read (ring held)"
20795        );
20796
20797        drop(release);
20798        let paused = reading.join().unwrap();
20799        assert_eq!(
20800            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20801            vec![10],
20802            "an exit recorded after the read began lands in neither half of it"
20803        );
20804        assert_eq!(paused.journal_skipped_lines, 0);
20805        assert_eq!(paused.journal_read_errors, 0);
20806
20807        let after = durable_terminal_history_of(&reader_ring, "reader-module");
20808        assert_eq!(
20809            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20810            vec![10, 30],
20811            "the next read merges ring and journal with no duplicate"
20812        );
20813        assert_eq!(after.journal_skipped_lines, 0);
20814    }
20815}
20816
20817/// What a restart does with the exited process's output readers. These drive
20818/// the same settle functions the supervisor calls, with a reader the test
20819/// holds, so a reader that has not been scheduled by the bound is a controlled
20820/// input rather than something only a loaded machine produces.
20821#[cfg(test)]
20822mod stderr_settle_tests {
20823    use std::{
20824        future::Future,
20825        io,
20826        pin::Pin,
20827        sync::{Arc, Mutex},
20828        task::{Context, Poll},
20829        time::Duration,
20830    };
20831
20832    use tokio::{
20833        io::{AsyncRead, ReadBuf},
20834        sync::oneshot,
20835        time::Instant,
20836    };
20837
20838    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
20839    use crate::stderr_tail::{
20840        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
20841        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
20842    };
20843
20844    const BOUND: Duration = Duration::from_millis(250);
20845
20846    /// Yields `before`, then stays pending until the gate is released, then
20847    /// yields `after` and reaches EOF. The bytes after the gate were written
20848    /// by a process that has already exited; only the reader is behind.
20849    struct HeldReader {
20850        before: Option<Vec<u8>>,
20851        gate: Option<oneshot::Receiver<()>>,
20852        after: io::Cursor<Vec<u8>>,
20853    }
20854
20855    impl AsyncRead for HeldReader {
20856        fn poll_read(
20857            mut self: Pin<&mut Self>,
20858            cx: &mut Context<'_>,
20859            buf: &mut ReadBuf<'_>,
20860        ) -> Poll<io::Result<()>> {
20861            if let Some(bytes) = self.before.take() {
20862                buf.put_slice(&bytes);
20863                return Poll::Ready(Ok(()));
20864            }
20865            if let Some(gate) = self.gate.as_mut() {
20866                match Pin::new(gate).poll(cx) {
20867                    Poll::Pending => return Poll::Pending,
20868                    Poll::Ready(_) => self.gate = None,
20869                }
20870            }
20871            Pin::new(&mut self.after).poll_read(cx, buf)
20872        }
20873    }
20874
20875    struct DiscardSink;
20876
20877    impl OutputSink for DiscardSink {
20878        fn write_line(&mut self, _line: &[u8]) {}
20879    }
20880
20881    fn line(text: &str) -> TailEntry {
20882        TailEntry::Line {
20883            text: text.to_string(),
20884            truncated: false,
20885            at_ms: None,
20886        }
20887    }
20888
20889    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
20890        ring.lock().unwrap()
20891    }
20892
20893    /// Start a reader for a new process generation that delivers `before`
20894    /// immediately and `after` only once the returned sender fires (or is
20895    /// dropped).
20896    fn held_pump(
20897        ring: &Arc<Mutex<StderrRing>>,
20898        before: &str,
20899        after: &str,
20900    ) -> (StderrPump, oneshot::Sender<()>) {
20901        let generation = lock(ring).begin_process();
20902        let (release, gate) = oneshot::channel();
20903        let reader = HeldReader {
20904            before: Some(before.as_bytes().to_vec()),
20905            gate: Some(gate),
20906            after: io::Cursor::new(after.as_bytes().to_vec()),
20907        };
20908        let task = tokio::spawn(pump_stderr_to(
20909            reader,
20910            Arc::clone(ring),
20911            generation,
20912            DiscardSink,
20913        ));
20914        (StderrPump { task, generation }, release)
20915    }
20916
20917    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
20918        for _ in 0..1000 {
20919            if done(&lock(ring)) {
20920                return;
20921            }
20922            tokio::time::sleep(Duration::from_millis(1)).await;
20923        }
20924        panic!(
20925            "ring never reached the expected state: {:?}",
20926            lock(ring).snapshot(None, None)
20927        );
20928    }
20929
20930    #[tokio::test(start_paused = true)]
20931    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
20932        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
20933        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
20934
20935        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
20936        let before_release = lock(&ring).snapshot(None, None);
20937        assert!(
20938            matches!(before_release.capture, CaptureState::Incomplete { .. }),
20939            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
20940        );
20941
20942        // The restart: the next process starts and writes before the old
20943        // reader catches up.
20944        let next = lock(&ring).begin_process();
20945        lock(&ring).push_line_from(next, "next process booting");
20946        release.send(()).unwrap();
20947        wait_until(&ring, |ring| {
20948            ring.snapshot(None, None).capture == CaptureState::Captured
20949        })
20950        .await;
20951
20952        assert_eq!(
20953            untimed(lock(&ring).snapshot(None, None).entries),
20954            vec![
20955                line("booting"),
20956                line("config error: missing storage"),
20957                TailEntry::ProcessStart,
20958                line("next process booting"),
20959            ],
20960            "the crash's last line must survive a slow reader and stay in the crashed process's section"
20961        );
20962    }
20963
20964    #[tokio::test(start_paused = true)]
20965    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
20966    ) {
20967        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
20968        // `_held` is never fired: a descendant keeps the pipe open for the
20969        // whole test.
20970        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
20971
20972        let started = Instant::now();
20973        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
20974        assert_eq!(
20975            started.elapsed(),
20976            BOUND,
20977            "the restart must wait exactly the bound for a pipe that stays open, no longer"
20978        );
20979
20980        let next = lock(&ring).begin_process();
20981        lock(&ring).push_line_from(next, "next process booting");
20982        tokio::time::sleep(Duration::from_secs(60)).await;
20983
20984        let snapshot = lock(&ring).snapshot(None, None);
20985        match &snapshot.capture {
20986            CaptureState::Incomplete { reason } => assert!(
20987                reason.contains("had not reached EOF") && reason.contains("250ms"),
20988                "the reason must say what is missing and after how long: {reason}"
20989            ),
20990            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
20991        }
20992        assert_eq!(
20993            untimed(snapshot.entries),
20994            vec![
20995                line("parent exiting"),
20996                TailEntry::ProcessStart,
20997                line("next process booting"),
20998            ]
20999        );
21000    }
21001
21002    #[tokio::test(start_paused = true)]
21003    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
21004        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21005        let (pump, release) = held_pump(&ring, "one\n", "two\n");
21006        release.send(()).unwrap();
21007
21008        settle_stderr_pump("clean", &ring, pump, BOUND).await;
21009
21010        let snapshot = lock(&ring).snapshot(None, None);
21011        assert_eq!(snapshot.capture, CaptureState::Captured);
21012        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
21013    }
21014
21015    #[tokio::test(start_paused = true)]
21016    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
21017        const BURST: usize = 150;
21018        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
21019        let path = capture.join("two-pipe.stderr.log");
21020        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
21021        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21022        let generation = lock(&ring).begin_process();
21023        let stdout = (0..BURST)
21024            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
21025            .collect::<String>();
21026        let stderr = (0..BURST)
21027            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
21028            .collect::<String>();
21029        // Model a child that has written both bursts and exited. Hold the stdout
21030        // reader after one line and a partial prefix, as if pipe readiness or
21031        // capture-file writes had delayed it. The final line also needs EOF to
21032        // flush: neither complete nor partial buffered output may be discarded.
21033        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
21034        let (release, gate) = oneshot::channel();
21035        let reader = HeldReader {
21036            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
21037            gate: Some(gate),
21038            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
21039        };
21040        let (finished, completion) = oneshot::channel();
21041        let stdout_sink = sink.clone();
21042        let pump = tokio::spawn(async move {
21043            pump_stdout_to(reader, stdout_sink).await;
21044            finished.send(()).unwrap();
21045        });
21046        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
21047
21048        let started = Instant::now();
21049        settle_stdout_pump("two-pipe", pump, BOUND).await;
21050        assert_eq!(
21051            started.elapsed(),
21052            BOUND,
21053            "a late reader must not hold up restart"
21054        );
21055        assert_eq!(
21056            std::fs::read_to_string(&path).unwrap().lines().count(),
21057            BURST + 1
21058        );
21059
21060        release
21061            .send(())
21062            .expect("stdout reader must survive the exit drain bound");
21063        tokio::time::timeout(Duration::from_secs(1), completion)
21064            .await
21065            .expect("released stdout reader must reach EOF")
21066            .expect("stdout capture must finish at EOF, not be cancelled");
21067        let contents = std::fs::read_to_string(&path).unwrap();
21068        let mut lines = contents
21069            .lines()
21070            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
21071            .collect::<Vec<_>>();
21072        let mut expected = (0..BURST)
21073            .flat_map(|i| {
21074                [
21075                    format!("out-{i:04}-{}", "o".repeat(64)),
21076                    format!("err-{i:04}-{}", "e".repeat(64)),
21077                ]
21078            })
21079            .collect::<Vec<_>>();
21080        lines.sort_unstable();
21081        expected.sort_unstable();
21082        assert_eq!(
21083            lines, expected,
21084            "both pipes must retain every complete line exactly once"
21085        );
21086    }
21087}
21088
21089/// Containment of a module's process tree (issue #109).
21090///
21091/// The behaviour these defend against is a module helper surviving its module:
21092/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
21093/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
21094/// compounds it.
21095///
21096/// They run against the SUPERVISOR rather than the job-object crate because the
21097/// claim is about teardown: a crate-level test proves a job can reap a tree, not
21098/// that the daemon's drain path reaches it.
21099///
21100/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
21101/// lane there is a separate containment path with its own tests.
21102#[cfg(all(test, windows))]
21103mod job_containment_tests {
21104    use super::*;
21105    use cortexkit_test_support::ScratchDir;
21106    use std::{
21107        path::{Path, PathBuf},
21108        sync::{Arc, Mutex},
21109        time::{Duration, Instant},
21110    };
21111
21112    /// The stub, expected beside this test executable.
21113    ///
21114    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
21115    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
21116    /// failure then reads as a broken test rather than an unbuilt dependency.
21117    fn stub_path() -> PathBuf {
21118        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
21119        assert!(
21120            path.exists(),
21121            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
21122             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
21123            path.display()
21124        );
21125        path
21126    }
21127
21128    /// Poll for the grandchild pid the stub records, and parse it.
21129    fn read_grandchild_pid(path: &Path) -> u32 {
21130        let deadline = Instant::now() + Duration::from_secs(10);
21131        loop {
21132            if let Ok(contents) = std::fs::read_to_string(path) {
21133                if let Ok(pid) = contents.trim().parse() {
21134                    return pid;
21135                }
21136            }
21137            assert!(
21138                Instant::now() < deadline,
21139                "the stub never recorded a grandchild pid at {}",
21140                path.display()
21141            );
21142            std::thread::sleep(Duration::from_millis(10));
21143        }
21144    }
21145
21146    /// Everything one fixture run needs, so the two tests below differ in exactly
21147    /// one place: whether the child is contained.
21148    struct Fixture {
21149        _dir: ScratchDir,
21150        _roster: ChildRoster,
21151        module_id: String,
21152        grandchild: u32,
21153        child: Option<SupervisedChild>,
21154        registry: Arc<Registry>,
21155        snapshot: Arc<Mutex<SupervisorSnapshot>>,
21156        terminal_ring: Arc<Mutex<TerminalRing>>,
21157        spawn_events: SpawnEventFeed,
21158    }
21159
21160    fn fixture(label: &str, module_id: &str) -> Fixture {
21161        fixture_with_carry(label, module_id, false)
21162    }
21163
21164    fn fixture_with_carry(label: &str, module_id: &str, carry_containment: bool) -> Fixture {
21165        let dir = ScratchDir::new(label);
21166        let pid_file = dir.join("grandchild.pid");
21167        let supervisor = Supervisor::new_for_test(
21168            Arc::new(Registry::default()),
21169            RestartPolicy::new(3, Duration::ZERO),
21170        );
21171        let runtime = supervisor.runtime_config();
21172        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21173        let spec = ModuleSpec {
21174            module_id: module_id.to_string(),
21175            program: stub_path(),
21176            // Zero args deliberately: a `--subc` argument would make the stub dial
21177            // a daemon that is not there, and the failure would land in the same
21178            // stderr ring this fixture exists to keep quiet.
21179            args: Vec::new(),
21180            env: vec![
21181                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
21182                (
21183                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
21184                    pid_file.display().to_string(),
21185                ),
21186            ],
21187            reserved: false,
21188            reserved_prefixes: Vec::new(),
21189            protocol: ModuleProtocol::Subc,
21190            overlap: Default::default(),
21191            carry_containment,
21192        };
21193        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
21194            .expect("spawn the supervised fixture");
21195        let grandchild = read_grandchild_pid(&pid_file);
21196        Fixture {
21197            _dir: dir,
21198            _roster: runtime.child_roster.clone(),
21199            module_id: module_id.to_string(),
21200            grandchild,
21201            child: Some(child),
21202            registry: Arc::new(Registry::default()),
21203            snapshot,
21204            terminal_ring: Arc::clone(&runtime.terminal_ring),
21205            spawn_events: SpawnEventFeed::default(),
21206        }
21207    }
21208
21209    async fn assert_admission_kill_reaps_grandchild(carry: bool) {
21210        let label = if carry {
21211            "admission-kill-carried-tree"
21212        } else {
21213            "admission-kill-plain-tree"
21214        };
21215        let mut fixture = fixture_with_carry(label, label, carry);
21216        let descendant = subc_os::Process::open(fixture.grandchild)
21217            .unwrap()
21218            .expect("the stub descendant must be alive before admission kill");
21219        assert!(descendant.observe().is_some());
21220        let child = fixture.child.as_mut().unwrap();
21221        assert_eq!(child.child.container().is_some(), carry);
21222        assert_eq!(child.job.is_some(), !carry);
21223
21224        child.start_admission_kill().unwrap();
21225        assert!(child.admission_kill_sent);
21226        let status = timeout(Duration::from_secs(10), child.wait())
21227            .await
21228            .unwrap()
21229            .unwrap();
21230        assert_eq!(
21231            status.code(),
21232            Some(admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE),
21233            "containment teardown must preserve the leader's distinctive status"
21234        );
21235        // Keep the child and its job/container alive: later exit finalization or
21236        // handle drop must not be what makes the descendant-death claim pass.
21237        assert!(
21238            descendant.wait_for_exit(Duration::from_secs(10)).unwrap(),
21239            "admission kill left a descendant running before exit finalization"
21240        );
21241        assert_eq!(child.child.container().is_some(), carry);
21242        assert_eq!(child.job.is_some(), !carry);
21243        child.finish_exit(false).await.unwrap();
21244    }
21245
21246    #[tokio::test]
21247    async fn admission_kill_reaps_grandchild_with_plain_job() {
21248        assert_admission_kill_reaps_grandchild(false).await;
21249    }
21250
21251    #[tokio::test]
21252    async fn admission_kill_reaps_grandchild_with_carried_job() {
21253        assert_admission_kill_reaps_grandchild(true).await;
21254    }
21255
21256    #[tokio::test]
21257    async fn admission_kill_retires_registration_before_wait() {
21258        let mut fixture = fixture("admission-registration", "admission-registration");
21259        let handle = SupervisorHandle::new();
21260        let spawn =
21261            RegistrationSpawn::new(&handle, "admission-registration", SpawnRole::Plain, None);
21262        spawn.ready();
21263        let child = fixture.child.as_mut().unwrap();
21264        child.registration_spawn = Some(spawn.clone());
21265        child.start_admission_kill().unwrap();
21266        assert!(
21267            !spawn.launch_ready(),
21268            "admission kill must retire before its caller can await the child"
21269        );
21270        timeout(Duration::from_secs(10), child.wait())
21271            .await
21272            .unwrap()
21273            .unwrap();
21274        child.finish_exit(false).await.unwrap();
21275    }
21276
21277    impl Fixture {
21278        /// Drain through the supervisor's own teardown path.
21279        async fn drain(&mut self) {
21280            let child = self
21281                .child
21282                .take()
21283                .expect("the fixture child is still present");
21284            drain_child_to_state(
21285                &self.module_id,
21286                ModuleProtocol::Subc,
21287                // No forwarding table in this fixture, so nothing reaches the
21288                // child over a connection.
21289                StopNotice::NotSent,
21290                &self.registry,
21291                None,
21292                &self.snapshot,
21293                &self.terminal_ring,
21294                &self.spawn_events,
21295                child,
21296                Duration::from_millis(500),
21297                ModuleState::Stopped,
21298                Some(false),
21299                &FailureContext::operator(),
21300            )
21301            .await
21302            .expect("drain the supervised fixture");
21303        }
21304    }
21305
21306    /// Teardown reaps the grandchild, not merely the direct child.
21307    ///
21308    /// This is the assertion the change exists for. Before containment the
21309    /// grandchild survived: it is a separate process, and `start_kill` is
21310    /// `TerminateProcess` scoped to one pid.
21311    #[tokio::test]
21312    async fn teardown_reaps_the_grandchild() {
21313        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
21314        let grandchild = fixture.grandchild;
21315
21316        assert!(
21317            subc_jobobject::process_exists(grandchild),
21318            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
21319        );
21320
21321        fixture.drain().await;
21322
21323        assert!(
21324            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21325            "grandchild {grandchild} outlived module teardown: the tree was not contained"
21326        );
21327    }
21328
21329    /// The mutation control: with containment withheld, the grandchild survives
21330    /// the same kill.
21331    ///
21332    /// This is the defect reproduction from #109 — a direct-child kill reaches
21333    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
21334    /// supervisor because `spawn_and_mark_running` now always contains on
21335    /// Windows, which is the point: there is no longer a path that spawns
21336    /// uncontained, so the control has to construct one.
21337    ///
21338    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
21339    /// grandchild ever dies here, that test is passing for a reason unrelated to
21340    /// the job object and the containment claim is unproven.
21341    #[test]
21342    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
21343        let dir = ScratchDir::new("teardown-uncontained");
21344        let pid_file = dir.join("grandchild.pid");
21345        let mut child = std::process::Command::new(stub_path())
21346            .env("FAKE_AFT_NEVER_CONNECT", "1")
21347            .env(
21348                "FAKE_AFT_GRANDCHILD_PID_FILE",
21349                pid_file.display().to_string(),
21350            )
21351            .stdin(std::process::Stdio::null())
21352            .stdout(std::process::Stdio::null())
21353            .stderr(std::process::Stdio::null())
21354            .spawn()
21355            .expect("spawn the uncontained fixture");
21356        let grandchild = read_grandchild_pid(&pid_file);
21357
21358        // Exactly what the pre-fix teardown did: kill the direct child.
21359        child.kill().expect("kill the direct child");
21360        let _ = child.wait();
21361
21362        assert!(
21363            subc_jobobject::process_exists(grandchild),
21364            "grandchild {grandchild} died with the direct child, so this control no longer \
21365             distinguishes contained from uncontained teardown and the regression test is \
21366             passing vacuously"
21367        );
21368
21369        // The orphan this control demonstrates is the leak the fix prevents, so
21370        // the control must not leave one behind.
21371        kill_tree(grandchild);
21372    }
21373
21374    /// Crash durability: closing the containment handle reaps the tree with no
21375    /// teardown code running at all.
21376    ///
21377    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
21378    /// call anything — and it is why containment is a kernel property of the
21379    /// handle rather than a step in the drain. Discovered by getting the
21380    /// mutation control wrong: clearing `job` to "disable" containment instead
21381    /// killed the tree, which is the guarantee, not a mistake.
21382    #[tokio::test]
21383    async fn dropping_containment_reaps_the_grandchild() {
21384        let mut fixture = fixture("drop-containment", "tree-drop");
21385        let grandchild = fixture.grandchild;
21386
21387        assert!(subc_jobobject::process_exists(grandchild));
21388
21389        // No `drain` call, no kill: dropping the handle is the entire mechanism.
21390        fixture.child.as_mut().expect("child present").job = None;
21391
21392        assert!(
21393            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21394            "grandchild {grandchild} survived the containment handle closing, so a daemon \
21395             crash would leave the tree behind"
21396        );
21397    }
21398
21399    /// Kill a pid and its tree, then confirm it is gone.
21400    fn kill_tree(pid: u32) {
21401        let _ = std::process::Command::new("taskkill.exe")
21402            .args(["/PID", &pid.to_string(), "/T", "/F"])
21403            .stdin(std::process::Stdio::null())
21404            .stdout(std::process::Stdio::null())
21405            .stderr(std::process::Stdio::null())
21406            .status();
21407        assert!(
21408            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
21409            "could not clean up grandchild {pid}"
21410        );
21411    }
21412}
21413
21414#[cfg(test)]
21415mod privacy_trampoline_configuration_tests {
21416    #[tokio::test]
21417    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21418    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
21419        #[cfg(target_os = "macos")]
21420        {
21421            let supervisor = super::Supervisor::new(
21422                std::sync::Arc::new(crate::Registry::default()),
21423                super::RestartPolicy::default(),
21424            );
21425            let error = supervisor.spawn(spec()).unwrap_err();
21426            assert!(
21427                error
21428                    .to_string()
21429                    .contains("no privacy trampoline configured"),
21430                "{error}"
21431            );
21432        }
21433    }
21434
21435    #[tokio::test]
21436    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21437    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
21438        #[cfg(target_os = "macos")]
21439        {
21440            let supervisor = super::Supervisor::new(
21441                std::sync::Arc::new(crate::Registry::default()),
21442                super::RestartPolicy::default(),
21443            )
21444            .with_privacy_trampoline(std::env::current_exe().unwrap());
21445            let error = supervisor.spawn(spec()).unwrap_err();
21446            assert!(
21447                error
21448                    .to_string()
21449                    .contains("binary does not implement the privacy trampoline protocol"),
21450                "{error}"
21451            );
21452        }
21453    }
21454
21455    #[cfg(target_os = "macos")]
21456    fn spec() -> super::ModuleSpec {
21457        super::ModuleSpec {
21458            module_id: "privacy-configuration".into(),
21459            program: "/bin/sleep".into(),
21460            args: vec!["30".into()],
21461            env: vec![],
21462            reserved: false,
21463            reserved_prefixes: vec![],
21464            protocol: subc_control::ModuleProtocol::None,
21465            overlap: super::ModuleOverlap::Exclusive,
21466            carry_containment: false,
21467        }
21468    }
21469}
21470
21471#[cfg(test)]
21472mod privacy_exec_boundary_tests {
21473    #[cfg(target_os = "macos")]
21474    use super::*;
21475    #[cfg(target_os = "macos")]
21476    use std::{
21477        io::{Read, Write},
21478        net::{TcpListener, TcpStream},
21479    };
21480
21481    /// Unit-test-only pause at the actual early image read, not at a later
21482    /// status read. Production supervisors never inspect this environment key.
21483    #[cfg(target_os = "macos")]
21484    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
21485        if let Some((_, path)) = spec
21486            .env
21487            .iter()
21488            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
21489        {
21490            let mut barrier = TcpStream::connect(path).unwrap();
21491            barrier
21492                .set_read_timeout(Some(Duration::from_secs(30)))
21493                .unwrap();
21494            barrier.write_all(&pid.to_ne_bytes()).unwrap();
21495            let mut release = [0];
21496            barrier.read_exact(&mut release).unwrap();
21497            assert_eq!(&release, b"X");
21498        }
21499    }
21500
21501    #[cfg(target_os = "macos")]
21502    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
21503        ModuleSpec {
21504            module_id: "privacy-boundary".into(),
21505            program: program.into(),
21506            args: args.iter().map(|arg| (*arg).into()).collect(),
21507            env: vec![],
21508            reserved: false,
21509            reserved_prefixes: vec![],
21510            protocol: ModuleProtocol::None,
21511            overlap: ModuleOverlap::Exclusive,
21512            carry_containment: false,
21513        }
21514    }
21515
21516    #[cfg(target_os = "macos")]
21517    async fn accept(listener: TcpListener) -> TcpStream {
21518        // Socket readiness, not elapsed time, establishes both pause points.
21519        let listener = tokio::net::TcpListener::from_std({
21520            listener.set_nonblocking(true).unwrap();
21521            listener
21522        })
21523        .unwrap();
21524        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
21525            .await
21526            .unwrap()
21527            .unwrap();
21528        let stream = stream.into_std().unwrap();
21529        stream.set_nonblocking(false).unwrap();
21530        stream
21531            .set_read_timeout(Some(Duration::from_secs(30)))
21532            .unwrap();
21533        stream
21534    }
21535
21536    #[tokio::test]
21537    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21538    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
21539        #[cfg(target_os = "macos")]
21540        {
21541            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
21542            // Loopback sockets also work when the replay adapter's TMPDIR is
21543            // longer than Darwin's Unix-domain socket path limit.
21544            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21545            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21546            let record = root.join("live-children.json");
21547            let supervisor =
21548                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21549                    .with_live_children_record(&record);
21550            let runtime = supervisor.runtime_config();
21551            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21552            let mut spec = spec("/bin/sleep", &["30"]);
21553            spec.env = vec![
21554                (
21555                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21556                    exec_listener.local_addr().unwrap().to_string(),
21557                ),
21558                (
21559                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
21560                    sample_listener.local_addr().unwrap().to_string(),
21561                ),
21562            ];
21563            let spawn = tokio::task::spawn_blocking(move || {
21564                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
21565            });
21566            let mut sample = accept(sample_listener).await;
21567            let mut pid = [0; 4];
21568            sample.read_exact(&mut pid).unwrap();
21569            let pid = u32::from_ne_bytes(pid);
21570            let mut exec = accept(exec_listener).await;
21571            let mut ready = [0];
21572            exec.read_exact(&mut ready).unwrap();
21573            assert_eq!(&ready, b"R");
21574            // The early read is guaranteed to see a real, non-null trampoline
21575            // image: the fixture has reached its barrier and cannot exec yet.
21576            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
21577            assert_eq!(
21578                observe_spawned_image(pid).unwrap().executable,
21579                Some(trampoline)
21580            );
21581            sample.write_all(b"X").unwrap();
21582            let mut child = spawn.await.unwrap();
21583            let early = crate::live_children::read_record(&record).unwrap();
21584            assert_eq!(early.len(), 1);
21585            assert_eq!(early[0].pid, pid);
21586            assert_eq!(
21587                early[0].executable, None,
21588                "unconfirmed trampoline image entered the roster"
21589            );
21590            assert!(child.report_ready.get().is_none());
21591            // The barrier's duration is unrelated to the production five-second
21592            // exec budget. Start the test's confirmation budget upon release.
21593            child.privacy_exec.as_mut().unwrap().deadline =
21594                tokio::time::Instant::now() + Duration::from_secs(30);
21595            exec.write_all(b"X").unwrap();
21596            child.confirm_privacy_exec().await;
21597            assert_eq!(child.spawn_failure, None);
21598            assert!(child.report_ready.get().is_some());
21599            let confirmed = crate::live_children::read_record(&record).unwrap();
21600            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
21601            assert_ne!(module, trampoline);
21602            assert_eq!(confirmed[0].executable, Some(module.into()));
21603            child.start_kill().unwrap();
21604            child.wait().await.unwrap();
21605            child.release_roster();
21606        }
21607    }
21608
21609    #[tokio::test]
21610    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21611    async fn macos_unstamped_reexec_records_the_observed_native_image() {
21612        #[cfg(target_os = "macos")]
21613        {
21614            let root = cortexkit_test_support::ScratchDir::new("privacy-reexec");
21615            let record = root.join("live-children.json");
21616            let supervisor =
21617                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21618                    .with_live_children_record(&record);
21619            let runtime = supervisor.runtime_config();
21620            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21621            let fixture = crate::config_gate::program::native_fixture_for_test();
21622            let mut spec = spec(
21623                "/bin/sh",
21624                &["-c", "exec \"$1\" 600", "reexec", fixture.to_str().unwrap()],
21625            );
21626            spec.env
21627                .push(("SUBC_TEST_PRIVACY_IMAGE_DELAY_MS".into(), "5".into()));
21628            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21629            let native_image = subc_os::file_identity(&fixture).unwrap();
21630            // Native image installation, not shell startup speed, establishes
21631            // that this launch has crossed the re-exec boundary before sampling.
21632            let reexec = tokio::time::timeout(Duration::from_secs(5), async {
21633                while observe_spawned_image(child.pid).and_then(|image| image.executable)
21634                    != Some(native_image)
21635                {
21636                    tokio::time::sleep(Duration::from_millis(1)).await;
21637                }
21638            })
21639            .await;
21640            child.confirm_privacy_exec().await;
21641            let failure = child.spawn_failure.clone();
21642            let ready = child.report_ready.get().is_some();
21643            let confirmed = crate::live_children::read_record(&record).unwrap();
21644            child.start_kill().unwrap();
21645            tokio::time::timeout(Duration::from_secs(5), child.wait())
21646                .await
21647                .unwrap()
21648                .unwrap();
21649            child.release_roster();
21650
21651            reexec.expect("the shell must re-exec the native fixture");
21652            assert_eq!(
21653                failure, None,
21654                "an unstamped program may re-exec before observation"
21655            );
21656            assert!(ready);
21657            assert_eq!(confirmed.len(), 1);
21658            assert_eq!(confirmed[0].executable, Some(native_image.into()));
21659        }
21660    }
21661
21662    #[tokio::test]
21663    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21664    async fn macos_unstamped_ack_without_exec_still_refuses_the_trampoline() {
21665        #[cfg(target_os = "macos")]
21666        {
21667            let listener = TcpListener::bind("127.0.0.1:0").unwrap();
21668            let supervisor =
21669                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
21670            let runtime = supervisor.runtime_config();
21671            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21672            let fixture = crate::config_gate::program::native_fixture_for_test();
21673            let mut spec = spec(fixture.to_str().unwrap(), &["600"]);
21674            spec.env.push((
21675                "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21676                listener.local_addr().unwrap().to_string(),
21677            ));
21678            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21679            let mut exec = accept(listener).await;
21680            let mut ready = [0];
21681            exec.read_exact(&mut ready).unwrap();
21682            assert_eq!(&ready, b"R");
21683            assert_eq!(
21684                observe_spawned_image(child.pid).unwrap().executable,
21685                subc_os::file_identity(&test_privacy_trampoline())
21686            );
21687            // Simulate a broken trampoline that acknowledges exec but keeps
21688            // running its own image, held at the fixture's pre-exec barrier.
21689            let (reader, writer) = std::io::pipe().unwrap();
21690            drop(writer);
21691            child.privacy_exec.as_mut().unwrap().reader =
21692                tokio::io::unix::AsyncFd::new(reader).unwrap();
21693            child.confirm_privacy_exec().await;
21694            let failure = child.spawn_failure.clone();
21695            let ready = child.report_ready.get().is_some();
21696            child.start_kill().unwrap();
21697            tokio::time::timeout(Duration::from_secs(5), child.wait())
21698                .await
21699                .unwrap()
21700                .unwrap();
21701            child.release_roster();
21702            assert_eq!(
21703                failure.as_deref(),
21704                Some("privacy identity trampoline did not exec the module")
21705            );
21706            assert!(!ready);
21707        }
21708    }
21709
21710    #[tokio::test]
21711    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21712    async fn macos_resumed_121_is_a_module_exit_not_a_trampoline_refusal() {
21713        #[cfg(target_os = "macos")]
21714        {
21715            let registry = Arc::new(Registry::default());
21716            let policy = RestartPolicy::new(0, Duration::ZERO);
21717            let fixture = crate::config_gate::program::native_fixture_for_test();
21718            let spec = spec(fixture.to_str().unwrap(), &["exit121"]);
21719            let scratch = cortexkit_test_support::ScratchDir::new("resumed-exit");
21720            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy)
21721                .with_config_runtime(suspended_spawn_tests::gated_owner(&scratch, &spec, true));
21722            let runtime = supervisor.runtime_config();
21723            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21724            // A module cannot exit before suspended admission. After resume,
21725            // retain the real exit status even when it resembles a refusal.
21726            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21727            child.confirm_privacy_exec().await;
21728            assert_eq!(child.spawn_failure, None);
21729            assert!(child.report_ready.get().is_some());
21730            let pid = child.pid;
21731            tokio::task::spawn_blocking(move || {
21732                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
21733            })
21734            .await
21735            .unwrap()
21736            .unwrap();
21737            let status = child.wait().await.unwrap();
21738            assert_eq!(status.code(), Some(121));
21739            assert!(child.privacy_exec.is_none());
21740            let report = classify_reaped_child_exit(&snapshot, &child, &status);
21741            on_child_exit(
21742                &spec,
21743                policy,
21744                &registry,
21745                &snapshot,
21746                &runtime.terminal_ring,
21747                &runtime.spawn_events,
21748                &runtime.child_roster,
21749                report,
21750                true,
21751            )
21752            .await;
21753            let state = lock_snapshot(&snapshot).unwrap();
21754            assert_eq!(state.state, ModuleState::Failed);
21755            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
21756            assert_eq!(state.reported_pid(), None);
21757            drop(state);
21758            let history = runtime.terminal_ring.lock().unwrap().snapshot();
21759            assert_eq!(history.entries.len(), 1);
21760            let terminal = &history.entries[0];
21761            assert_eq!(terminal.exit_code, Some(121));
21762            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
21763            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
21764            assert_eq!(
21765                terminal.disposition_detail.as_deref(),
21766                Some(policy.budget_exhausted_detail().as_str()),
21767                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
21768            );
21769            assert_eq!(child.spawn_failure, None);
21770            child.release_roster();
21771        }
21772    }
21773}
21774
21775#[cfg(all(test, target_os = "macos"))]
21776#[path = "supervise/suspended_spawn_tests.rs"]
21777mod suspended_spawn_tests;
21778
21779/// The daemon's real spawn path hands a subc-wire child its launch nonce on
21780/// descriptor 3, without an environment copy. The shell records the nonce
21781/// and its environment after exec so these tests observe the real handover.
21782#[cfg(all(test, unix))]
21783mod launch_nonce_descriptor_tests {
21784    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
21785    use crate::stderr_tail::{StderrRing, StderrTailConfig};
21786    use cortexkit_test_support::ScratchDir;
21787    use std::{
21788        path::PathBuf,
21789        sync::{Arc, Mutex},
21790        time::{Duration, Instant},
21791    };
21792
21793    async fn probe(role: super::SpawnRole) {
21794        let scratch = ScratchDir::new("launch-nonce-descriptor");
21795        let fd_copy = scratch.join("from-descriptor");
21796        let env_copy = scratch.join("environment");
21797        let script = format!(
21798            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
21799            fd = fd_copy.display(), env = env_copy.display(),
21800        );
21801        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
21802        let spec = ModuleSpec {
21803            module_id: "nonce-descriptor-probe".to_string(),
21804            program: PathBuf::from("/bin/sh"),
21805            args: vec!["-c".to_string(), script],
21806            env: vec![
21807                xdg("XDG_DATA_HOME"),
21808                xdg("XDG_RUNTIME_DIR"),
21809                xdg("XDG_CONFIG_HOME"),
21810                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
21811            ],
21812            reserved: true,
21813            reserved_prefixes: Vec::new(),
21814            protocol: ModuleProtocol::Subc,
21815            overlap: Default::default(),
21816            carry_containment: false,
21817        };
21818        let handle = SupervisorHandle::new();
21819        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21820        let roster = ChildRoster::default();
21821        #[cfg(target_os = "macos")]
21822        {
21823            let path = super::test_privacy_trampoline();
21824            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
21825        }
21826        let child = super::spawn_child_in_slot(
21827            &spec,
21828            None,
21829            Some(&handle),
21830            &ring,
21831            None,
21832            &roster,
21833            None,
21834            #[cfg(target_os = "linux")]
21835            None,
21836            role,
21837            matches!(role, super::SpawnRole::SwapCandidate),
21838            super::DEFAULT_REGISTER_TIMEOUT,
21839        )
21840        .expect("spawn probe");
21841        let deadline = Instant::now() + Duration::from_secs(10);
21842        while !(fd_copy.exists() && env_copy.exists()) {
21843            assert!(Instant::now() < deadline, "probe never wrote its copies");
21844            tokio::time::sleep(Duration::from_millis(20)).await;
21845        }
21846        let nonce = std::fs::read_to_string(fd_copy).unwrap();
21847        assert!(!nonce.is_empty());
21848        let environment = std::fs::read_to_string(env_copy).unwrap();
21849        assert!(environment
21850            .lines()
21851            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
21852        let copy = environment
21853            .lines()
21854            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
21855        assert_eq!(
21856            copy, None,
21857            "Unix children must never receive the environment nonce"
21858        );
21859        if matches!(role, super::SpawnRole::Plain) {
21860            assert_eq!(
21861                handle.spawn_nonce(&spec.module_id).as_deref(),
21862                Some(nonce.as_str())
21863            );
21864        }
21865        drop(child);
21866    }
21867
21868    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
21869    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
21870        probe(super::SpawnRole::Plain).await;
21871    }
21872
21873    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
21874    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
21875        probe(super::SpawnRole::SwapCandidate).await;
21876    }
21877}
21878
21879#[cfg(all(test, target_os = "linux"))]
21880#[path = "supervise/cgroup_reap_tests.rs"]
21881mod cgroup_reap_tests;
21882
21883#[cfg(all(test, target_os = "linux"))]
21884mod cgroup_containment_tests {
21885    use super::*;
21886    use cortexkit_test_support::ScratchDir;
21887
21888    #[tokio::test]
21889    async fn linux_subtree_memory_covers_module_children() {
21890        let dir = ScratchDir::new("subtree-memory");
21891        let root = PathBuf::from(format!(
21892            "/sys/fs/cgroup/subc-memory-test-{}-{}",
21893            std::process::id(),
21894            unix_ms_now()
21895        ));
21896        if let Err(error) = std::fs::create_dir(&root) {
21897            assert!(
21898                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
21899                "required cgroup test cannot execute: {error}"
21900            );
21901            eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
21902            return;
21903        }
21904        let placement = subc_cgroup::prepare_at(&root)
21905            .unwrap()
21906            .expect("delegated cgroup");
21907        let supervisor = Supervisor::new_for_test(
21908            Arc::new(Registry::default()),
21909            RestartPolicy::new(3, Duration::ZERO),
21910        )
21911        .with_cgroup_placement(Some(placement));
21912        let ready = dir.join("ready");
21913        let module = supervisor
21914            .spawn(ModuleSpec {
21915                module_id: "memory-tree".into(),
21916                program: PathBuf::from("/bin/sh"),
21917                args: vec![
21918                    "-c".into(),
21919                    "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
21920                    "fixture".into(),
21921                    ready.display().to_string(),
21922                ],
21923                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
21924                    .into_iter()
21925                    .map(|key| (key.into(), dir.display().to_string()))
21926                    .collect(),
21927                reserved: false,
21928                reserved_prefixes: Vec::new(),
21929                protocol: ModuleProtocol::None,
21930                overlap: Default::default(),
21931                carry_containment: false,
21932            })
21933            .unwrap();
21934        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
21935        while !ready.exists() {
21936            assert!(
21937                tokio::time::Instant::now() < deadline,
21938                "children never started"
21939            );
21940            tokio::time::sleep(Duration::from_millis(10)).await;
21941        }
21942        let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
21943        let reading = &readings[0];
21944        assert!(
21945            reading.process_count.is_some_and(|count| count >= 4),
21946            "{reading:?}"
21947        );
21948        assert!(
21949            reading.memory_bytes.is_some_and(|bytes| bytes > 0),
21950            "{reading:?}"
21951        );
21952        module.stop().await.unwrap();
21953        drop(module);
21954        std::fs::remove_dir(root.join("subc-modules")).unwrap();
21955        std::fs::remove_dir(&root).unwrap();
21956        eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
21957    }
21958
21959    fn running(pid: u32) -> bool {
21960        // An orphan can remain a zombie until the container init reaps it.
21961        std::fs::read_to_string(format!("/proc/{pid}/stat"))
21962            .ok()
21963            .and_then(|stat| {
21964                stat.rsplit_once(") ")
21965                    .map(|(_, rest)| rest.starts_with('Z'))
21966            })
21967            .is_some_and(|zombie| !zombie)
21968    }
21969
21970    #[tokio::test]
21971    async fn linux_teardown_reaps_the_grandchild() {
21972        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
21973    }
21974
21975    #[tokio::test]
21976    async fn linux_shutdown_straggler_reaps_the_grandchild() {
21977        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
21978    }
21979
21980    async fn teardown_tree(test_name: &str, shutdown: bool) {
21981        let dir = ScratchDir::new(test_name);
21982        let root = PathBuf::from(format!(
21983            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
21984            std::process::id(),
21985            unix_ms_now()
21986        ));
21987        if let Err(error) = std::fs::create_dir(&root) {
21988            assert!(
21989                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
21990                "required cgroup test cannot execute: {error}"
21991            );
21992            eprintln!(
21993                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
21994                root.display()
21995            );
21996            return;
21997        }
21998        let placement = subc_cgroup::prepare_at(&root)
21999            .expect("prepare isolated kernel cgroup")
22000            .expect("isolated cgroup is delegated");
22001        let module_id = "tree-teardown";
22002        let module = placement
22003            .module_path(module_id)
22004            .expect("create isolated module cgroup");
22005        if !module.join("cgroup.kill").exists() {
22006            std::fs::remove_dir(&module).unwrap();
22007            std::fs::remove_dir(root.join("subc-modules")).unwrap();
22008            std::fs::remove_dir(&root).unwrap();
22009            assert!(
22010                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22011                "required cgroup.kill interface unavailable"
22012            );
22013            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
22014            return;
22015        }
22016        let supervisor = Supervisor::new_for_test(
22017            Arc::new(Registry::default()),
22018            RestartPolicy::new(3, Duration::ZERO),
22019        )
22020        .with_cgroup_placement(Some(placement));
22021        let mut runtime = supervisor.runtime_config();
22022        runtime.child_roster = runtime
22023            .child_roster
22024            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
22025        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
22026        let pid_file = dir.join("grandchild.pid");
22027        let spec = ModuleSpec {
22028            module_id: module_id.to_string(),
22029            program: PathBuf::from("/bin/sh"),
22030            args: vec![
22031                "-c".into(),
22032                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
22033                "fixture".into(),
22034                pid_file.display().to_string(),
22035            ],
22036            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22037                .into_iter()
22038                .map(|key| (key.to_string(), dir.display().to_string()))
22039                .collect(),
22040            reserved: false,
22041            reserved_prefixes: Vec::new(),
22042            protocol: ModuleProtocol::None,
22043            overlap: Default::default(),
22044            carry_containment: false,
22045        };
22046        let child =
22047            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
22048        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22049        let grandchild: u32 = loop {
22050            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
22051                if let Ok(pid) = contents.trim().parse() {
22052                    break pid;
22053                }
22054            }
22055            assert!(
22056                tokio::time::Instant::now() < deadline,
22057                "grandchild pid was not recorded"
22058            );
22059            tokio::time::sleep(Duration::from_millis(10)).await;
22060        };
22061        assert!(
22062            running(grandchild),
22063            "grandchild must be alive before teardown"
22064        );
22065        if shutdown {
22066            let mut child = child;
22067            crate::child_roster::end_children_for_daemon_shutdown(
22068                &runtime.child_roster,
22069                false,
22070                std::future::pending(),
22071            )
22072            .await;
22073            child.wait().await.expect("reap shutdown straggler");
22074        } else {
22075            drain_child_to_state(
22076                module_id,
22077                ModuleProtocol::None,
22078                StopNotice::NotSent,
22079                &Registry::default(),
22080                None,
22081                &snapshot,
22082                &runtime.terminal_ring,
22083                &SpawnEventFeed::default(),
22084                child,
22085                Duration::from_millis(100),
22086                ModuleState::Stopped,
22087                Some(false),
22088                &FailureContext::operator(),
22089            )
22090            .await
22091            .expect("real supervisor teardown");
22092        }
22093        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
22094        while running(grandchild) && tokio::time::Instant::now() < deadline {
22095            tokio::time::sleep(Duration::from_millis(10)).await;
22096        }
22097        let survived = running(grandchild);
22098        // Kill a surviving grandchild so a failed test does not leave it behind.
22099        if survived {
22100            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
22101            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
22102            tokio::time::sleep(Duration::from_millis(100)).await;
22103        }
22104        if module.exists() {
22105            std::fs::remove_dir(&module).expect("remove empty module cgroup");
22106        }
22107        std::fs::remove_dir(root.join("subc-modules")).unwrap();
22108        std::fs::remove_dir(&root).unwrap();
22109        assert!(
22110            !survived,
22111            "grandchild {grandchild} outlived module teardown"
22112        );
22113        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
22114    }
22115}
22116
22117/// A slow live-children record write on the restart path must not stop the
22118/// daemon from answering the replacement module's authentication handshake.
22119#[cfg(all(test, unix))]
22120mod record_write_stall_tests {
22121    use super::*;
22122    use std::sync::atomic::{AtomicBool, Ordering};
22123    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
22124
22125    /// `subc-client-rs` gives the whole handshake this long before the module
22126    /// exits with `Timeout { stage: ServerProof }`.
22127    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
22128    /// How long the replacement's admission write is held: longer than the
22129    /// module's deadline, standing in for the delay a busy disk can add
22130    /// across the two syncs in that write.
22131    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
22132
22133    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
22134    /// On macOS every module is launched through the privacy trampoline, and
22135    /// a module that is not a `#!` script is admitted only if its running
22136    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
22137    /// the shell soon replaces itself with `/bin/sleep`, which does not match
22138    /// the configured `/bin/sh`. The image is read only after the spawn's
22139    /// live-children record write, and here that write lasted long enough for
22140    /// the shell to have exec'd every time, so each process was refused and
22141    /// killed right after spawn, and the supervisor crash-looped it with
22142    /// backoff until its restart budget ran out. The restart this test
22143    /// measures would then never produce a module at all.
22144    fn sleeping_spec() -> ModuleSpec {
22145        ModuleSpec {
22146            module_id: "record-write-stall".into(),
22147            program: "/bin/sleep".into(),
22148            args: vec!["600".into()],
22149            env: vec![],
22150            reserved: false,
22151            reserved_prefixes: vec![],
22152            protocol: ModuleProtocol::None,
22153            overlap: Default::default(),
22154            carry_containment: false,
22155        }
22156    }
22157
22158    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22159    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
22160        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
22161        let supervisor =
22162            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
22163                .with_live_children_record(home.join("live-children.json"));
22164        let module = supervisor.spawn(sleeping_spec()).unwrap();
22165        // `reported_pid` names a process only once it is confirmed as the
22166        // module (on macOS, once the trampoline's image check has passed), so
22167        // a process the daemon is about to refuse and kill never counts as
22168        // the running module, here or in the final wait below.
22169        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
22170            loop {
22171                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
22172                    .unwrap()
22173                    .reported_pid()
22174                {
22175                    return pid;
22176                }
22177                tokio::time::sleep(Duration::from_millis(10)).await;
22178            }
22179        })
22180        .await
22181        .expect("the first process was admitted as the module");
22182
22183        // The daemon's listener, served by this runtime's workers like the
22184        // supervisor's own tasks.
22185        let key = vec![0x42; 32];
22186        let daemon_id = [0x24; 16];
22187        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22188        let port = listener.local_addr().unwrap().port();
22189        let server = tokio::spawn(crate::server::serve_listener(
22190            listener,
22191            Arc::new(crate::router::Router::with_default_self_handler()),
22192            crate::server::ServerAuth::with_limits(
22193                key.clone(),
22194                daemon_id,
22195                "test",
22196                crate::server::DEFAULT_AUTH_DEADLINE,
22197                4,
22198            ),
22199        ));
22200
22201        // Hold the write that admits the replacement, once, and say when it
22202        // has started.
22203        let (held_tx, held_rx) = std::sync::mpsc::channel();
22204        let armed = AtomicBool::new(true);
22205        supervisor
22206            .child_roster
22207            .set_record_write_hook_for_test(move |live| {
22208                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
22209                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
22210                    let _ = held_tx.send(());
22211                    std::thread::sleep(HELD_RECORD_WRITE);
22212                }
22213            });
22214
22215        // The replacement module, as a separate process would be: its own
22216        // thread and runtime, so only the daemon's runtime can be stalled.
22217        let module_process = std::thread::spawn(move || {
22218            held_rx
22219                .recv_timeout(Duration::from_secs(30))
22220                .expect("the restart reached the replacement's record write");
22221            let conn = ConnectionInfo {
22222                schema: SCHEMA_VERSION,
22223                wire_version: None,
22224                endpoints: vec![Endpoint {
22225                    host: "127.0.0.1".to_owned(),
22226                    port,
22227                }],
22228                key,
22229                daemon_id,
22230                pid: std::process::id(),
22231                daemon_ver: "test".to_owned(),
22232            };
22233            tokio::runtime::Builder::new_current_thread()
22234                .enable_all()
22235                .build()
22236                .unwrap()
22237                .block_on(async {
22238                    let started = Instant::now();
22239                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
22240                        .await
22241                        .unwrap();
22242                    let result =
22243                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
22244                    (result, started.elapsed())
22245                })
22246        });
22247
22248        module.restart(Some(0)).await.unwrap();
22249        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
22250            .await
22251            .unwrap();
22252        assert!(
22253            auth.is_ok(),
22254            "the daemon must answer a module's handshake while a restart waits on the \
22255             live-children record write; got {auth:?} after {elapsed:?}"
22256        );
22257
22258        tokio::time::timeout(Duration::from_secs(20), async {
22259            while lock_snapshot(&module.inner.snapshot)
22260                .unwrap()
22261                .reported_pid()
22262                .is_none_or(|pid| pid == old_pid)
22263            {
22264                tokio::time::sleep(Duration::from_millis(10)).await;
22265            }
22266        })
22267        .await
22268        .expect("the restart published its replacement");
22269        module.stop().await.unwrap();
22270        server.abort();
22271    }
22272}
22273
22274#[cfg(all(test, unix))]
22275mod retained_status_tests {
22276    use super::*;
22277
22278    #[tokio::test]
22279    async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
22280        use crate::live_children::{ContainerIdentity, RetainedReason};
22281        for (reason, carried, expected) in [
22282            (
22283                RetainedReason::ContainerSurvivors(vec![4242]),
22284                true,
22285                Some("waiting: container survivors (pids [4242])".to_owned()),
22286            ),
22287            (
22288                RetainedReason::ContainerUnverifiable("member inspection denied".into()),
22289                true,
22290                Some(
22291                    "waiting: container survivors could not be verified: member inspection denied"
22292                        .to_owned(),
22293                ),
22294            ),
22295            (RetainedReason::Survived, false, None),
22296        ] {
22297            let supervisor =
22298                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
22299            let spec = ModuleSpec {
22300                module_id: "retained-status-fixture".into(),
22301                program: PathBuf::from("/not-spawned"),
22302                args: Vec::new(),
22303                env: Vec::new(),
22304                reserved: false,
22305                reserved_prefixes: Vec::new(),
22306                protocol: ModuleProtocol::None,
22307                overlap: ModuleOverlap::Exclusive,
22308                carry_containment: carried,
22309            };
22310            let module = supervisor
22311                .supervise_configured(spec.clone(), false)
22312                .unwrap();
22313            // No process is launched. Invalid group zero is deliberate: if a
22314            // status read wrongly rechecks, verification changes the reason
22315            // and fails the assertions without signalling any real process.
22316            let guard = supervisor
22317                .child_roster
22318                .admit(
22319                    spec.module_id,
22320                    0,
22321                    ModuleProtocol::None,
22322                    None,
22323                    crate::child_roster::RecordedIdentity {
22324                        container: carried.then_some(ContainerIdentity {
22325                            pgid: Some(0),
22326                            created_start_time: Some(0),
22327                            cgroup_path: None,
22328                            ..ContainerIdentity::test_metadata()
22329                        }),
22330                        ..crate::child_roster::RecordedIdentity::default()
22331                    },
22332                )
22333                .unwrap();
22334            guard.retain(reason);
22335            let status = module.status().unwrap();
22336            assert_eq!(status.waiting_for_previous_detail, expected.clone());
22337            assert_eq!(status.health.detail, expected);
22338            assert!(!status.process_alive);
22339        }
22340    }
22341}
22342
22343fn registration_child_is_running(child: &mut SupervisedChild) -> bool {
22344    child.try_wait().is_ok_and(|status| status.is_none())
22345}