1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use subc_control::{
14 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
15 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
16};
17use subc_protocol::{
18 manifest::{SelfSignalKind, SignalAnchor},
19 session::{
20 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
21 MODULE_CONTROL_OP_HEALTH_CHECK,
22 },
23 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
24};
25use tokio::{
26 process::{Child, Command},
27 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
28 task::JoinHandle,
29 time::{sleep, sleep_until, timeout, timeout_at, Instant},
30};
31use tracing::{debug, error, info, warn};
32
33use crate::{
34 child_roster::ChildRoster,
35 daemon_config::{
36 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
37 },
38 forwarding::{
39 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
40 ModuleDrainTarget, PendingModuleControlRpc,
41 },
42 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
43 registry::{ConnectionId, RegistrationEndReason, RegistryError},
44 stderr_tail::{
45 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
46 StderrTailSnapshot,
47 },
48 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
49 Frame, FrameSink, Registry,
50};
51
52#[cfg(not(windows))]
53use crate::provenance::spawned_file_identity;
54
55#[path = "supervise_swap.rs"]
56mod swap;
57
58#[path = "supervise_activity.rs"]
59mod activity;
60
61pub const SUBC_ARG: &str = "--subc";
67
68const DEFAULT_MAX_RESTARTS: u32 = 3;
69const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
70const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
71const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
75pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
86const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
87const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
88const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
111pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
113const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
114pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
119
120#[cfg(target_os = "macos")]
121struct PrivacyExec {
122 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
123 deadline: tokio::time::Instant,
124 expected: Option<subc_os::FileIdentity>,
125 trampoline: Option<subc_os::FileIdentity>,
126 script: bool,
127 module_id: String,
128}
129
130#[cfg(all(test, target_os = "macos"))]
131pub(crate) fn test_privacy_trampoline() -> PathBuf {
132 let path = std::env::current_exe()
135 .unwrap()
136 .parent()
137 .unwrap()
138 .parent()
139 .unwrap()
140 .join("privacy-trampoline-fixture");
141 assert!(
145 path.exists(),
146 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
147 --bins --features test-support` or `cargo test -p subc-daemon` first",
148 path.display()
149 );
150 path
151}
152
153#[cfg(target_os = "macos")]
154fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
155 use std::io::Read;
156 let mut probe = std::process::Command::new(path)
157 .args(["__disclaim-exec", "--probe"])
158 .stdin(Stdio::null())
159 .stdout(Stdio::piped())
160 .stderr(Stdio::piped())
161 .spawn()
162 .map_err(|error| {
163 format!(
164 "privacy trampoline probe failed for {}: {error}",
165 path.display()
166 )
167 })?;
168 let deadline = std::time::Instant::now() + Duration::from_secs(5);
169 let status = loop {
170 match probe.try_wait() {
171 Ok(Some(status)) => break status,
172 Ok(None) if std::time::Instant::now() < deadline => {
173 std::thread::sleep(Duration::from_millis(5))
174 }
175 result => {
176 let _ = probe.kill();
177 let _ = probe.wait();
178 return Err(format!(
179 "privacy trampoline probe failed or timed out for {}: {result:?}",
180 path.display()
181 ));
182 }
183 }
184 };
185 let mut answer = String::new();
186 if let Some(stdout) = probe.stdout.take() {
187 let _ = stdout.take(256).read_to_string(&mut answer);
188 }
189 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
190 return Ok(());
191 }
192 let mut diagnostic = String::new();
193 if let Some(stderr) = probe.stderr.take() {
194 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
195 }
196 let cause = diagnostic
197 .trim()
198 .strip_prefix("ck-subc: own privacy identity refused: ")
199 .unwrap_or("binary does not implement the privacy trampoline protocol");
200 Err(format!(
201 "{cause}: probe of {} exited {status}",
202 path.display()
203 ))
204}
205
206#[cfg(target_os = "macos")]
207fn privacy_command(
208 spec: &ModuleSpec,
209 roster: &ChildRoster,
210) -> Result<
211 (
212 Command,
213 Option<PrivacyExec>,
214 subc_os::privacy_identity::ExecAcknowledgement,
215 ),
216 SuperviseError,
217> {
218 let failure = |cause: String| {
219 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
220 SuperviseError::Spawn {
221 program: spec.program.clone(),
222 source: io::Error::other(cause),
223 cgroup_path: None,
224 }
225 };
226 let trampoline = roster.privacy_trampoline().map_err(failure)?;
227 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
232 let path = spec
233 .env
234 .iter()
235 .find(|(key, _)| key == "PATH")
236 .map(|(_, value)| std::ffi::OsString::from(value))
237 .or_else(|| std::env::var_os("PATH"))
238 .unwrap_or_else(|| "/usr/bin:/bin".into());
239 std::env::split_paths(&path)
240 .map(|dir| dir.join(&spec.program))
241 .find(|path| path.is_file())
242 .unwrap_or_else(|| spec.program.clone())
243 } else {
244 spec.program.clone()
245 };
246 let expected = subc_os::file_identity(&program);
247 let trampoline_image = subc_os::file_identity(&trampoline);
248 let script = {
249 use std::io::Read;
250 let mut prefix = [0u8; 2];
251 std::fs::File::open(&program)
252 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
253 };
254 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
255 return Err(failure(
256 "privacy identity module executable is missing or is the trampoline itself".to_string(),
257 ));
258 }
259 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
260 .map_err(|error| failure(error.to_string()))?;
261 let reader =
262 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
263 let mut command = Command::new(&trampoline);
264 command
265 .arg("__disclaim-exec")
266 .arg(ack.fd().to_string())
267 .arg(&program);
268 ack.install(command.as_std_mut());
269 Ok((
270 command,
271 Some(PrivacyExec {
272 reader,
273 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
274 expected,
275 trampoline: trampoline_image,
276 script,
277 module_id: spec.module_id.clone(),
278 }),
279 ack,
280 ))
281}
282
283struct SupervisedChild {
284 child: Child,
285 #[cfg(windows)]
286 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
287 #[cfg(target_os = "macos")]
288 privacy_exec: Option<PrivacyExec>,
289 #[cfg(target_os = "macos")]
296 report_ready: Arc<OnceLock<()>>,
297 spawn_failure: Option<String>,
299 protocol: ModuleProtocol,
304 #[cfg(target_os = "linux")]
310 module_id: String,
311 #[cfg(target_os = "linux")]
312 cgroup_placement: Option<subc_cgroup::Placement>,
313 #[cfg(windows)]
335 job: Option<subc_jobobject::JobObject>,
336 stdout_pump: Option<JoinHandle<()>>,
337 stderr_pump: Option<StderrPump>,
338 stderr_ring: Arc<Mutex<StderrRing>>,
339 spawned_at_ms: u64,
340 spawned_from: PathBuf,
341 spawned_file_identity: Option<SpawnedFileIdentity>,
342 process_start_time: Option<u64>,
343 process_identity: Option<ProcessIdentity>,
344 pid: u32,
345 roster_guard: Option<crate::child_roster::RosterGuard>,
348}
349
350impl SupervisedChild {
351 fn id(&self) -> Option<u32> {
352 Some(self.pid)
353 }
354
355 fn process_identity(&self) -> Option<ProcessIdentity> {
356 self.process_identity
357 }
358
359 async fn wait(&mut self) -> io::Result<ExitStatus> {
360 #[cfg(target_os = "macos")]
361 self.confirm_privacy_exec().await;
362 let result = self.child.wait().await;
370 #[cfg(target_os = "linux")]
371 if result.is_ok() {
372 if let Some(placement) = self.cgroup_placement.as_ref() {
373 cleanup_reaped_cgroup(placement, &self.module_id).await;
374 self.cgroup_placement = None;
377 }
378 }
379 result
380 }
381
382 #[cfg(target_os = "macos")]
383 async fn confirm_privacy_exec(&mut self) {
384 let Some(pending) = &mut self.privacy_exec else {
385 return;
386 };
387 let result = tokio::time::timeout_at(pending.deadline, async {
388 let mut record = Vec::new();
389 loop {
390 let mut ready = pending.reader.readable().await?;
391 let read = ready.try_io(|reader| {
392 use std::io::Read;
393 let mut reader = reader.get_ref();
394 let mut buffer = [0u8; 256];
395 reader.read(&mut buffer).map(|count| (count, buffer))
396 });
397 match read {
398 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
399 Ok(Ok((count, buffer))) => {
400 if record.len() + count > 1024 {
401 return Err(io::Error::other(
402 "privacy exec refusal record is too long",
403 ));
404 }
405 record.extend_from_slice(&buffer[..count]);
406 }
407 Ok(Err(error)) => return Err(error),
408 Err(_) => continue,
409 }
410 }
411 })
412 .await;
413 let pending = self.privacy_exec.as_ref().expect("pending exec");
416 let cause = match result {
417 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
418 Ok(Err(error)) => Some(format!(
419 "privacy identity exec acknowledgement failed: {error}"
420 )),
421 Ok(Ok(record)) if !record.is_empty() => Some(
422 std::str::from_utf8(&record)
423 .ok()
424 .and_then(|record| {
425 record
426 .trim()
427 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
428 })
429 .filter(|cause| !cause.is_empty())
430 .unwrap_or("invalid privacy exec refusal record")
431 .to_string(),
432 ),
433 Ok(Ok(_)) => match self.child.try_wait() {
434 Ok(Some(_status)) => None,
438 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
439 Ok(None) => {
440 let image = observe_spawned_image(self.pid);
441 if let Some(image) = image.filter(|image| {
442 image.executable.is_some()
443 && image.executable != pending.trampoline
444 && (image.executable == pending.expected || pending.script)
445 }) {
446 if let Some(guard) = &self.roster_guard {
447 guard.confirm_executable(image);
448 }
449 let _ = self.report_ready.set(());
450 info!(module_id = %pending.module_id, pid = self.pid,
451 "module spawned with own privacy identity (responsibility disclaimed)");
452 None
453 } else if image.is_none()
454 || image.is_some_and(|image| image.executable.is_none())
455 {
456 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
463 Ok(Ok(_status)) => None,
464 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
465 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
466 }
467 } else {
468 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
469 }
470 }
471 },
472 };
473 let pending = self.privacy_exec.take().expect("pending exec");
474 if let Some(cause) = cause {
475 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
476 self.spawn_failure = Some(cause);
477 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
480 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
481 }
482 let _ = self.child.start_kill();
483 }
484 }
485
486 fn release_roster(&mut self) {
490 self.roster_guard = None;
491 }
492
493 fn start_kill(&mut self) -> io::Result<()> {
506 #[cfg(windows)]
507 if let Some(job) = &self.job {
508 if let Err(error) = job.terminate() {
509 debug!(
510 error = %error,
511 "job termination failed; the direct-child kill still owns the outcome"
512 );
513 }
514 }
515 #[cfg(target_os = "linux")]
516 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
517 self.child.start_kill()
518 }
519
520 async fn drain_stderr(&mut self, module_id: &str) {
521 if let Some(pump) = self.stdout_pump.take() {
522 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
523 }
524
525 let Some(pump) = self.stderr_pump.take() else {
526 return;
527 };
528 settle_stderr_pump(
529 module_id,
530 &self.stderr_ring,
531 pump,
532 STDERR_PUMP_DRAIN_TIMEOUT,
533 )
534 .await;
535 }
536}
537
538async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
542 match timeout(bound, &mut pump).await {
543 Ok(Ok(())) => {}
544 Ok(Err(error)) => {
545 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
546 }
547 Err(_) => {
548 drop(pump);
549 warn!(
550 module_id,
551 waited = ?bound,
552 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
553 );
554 }
555 }
556}
557
558struct StderrPump {
561 task: JoinHandle<()>,
562 generation: u64,
563}
564
565async fn settle_stderr_pump(
571 module_id: &str,
572 ring: &Arc<Mutex<StderrRing>>,
573 pump: StderrPump,
574 bound: Duration,
575) {
576 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
577 let StderrPump {
578 mut task,
579 generation,
580 } = pump;
581 lock().retire_pump(generation);
582 match timeout(bound, &mut task).await {
583 Ok(Ok(())) => {}
584 Ok(Err(err)) => {
585 let mut ring = lock();
586 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
587 ring.finish_pump(generation);
588 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
589 }
590 Err(_) => {
591 drop(task);
593 lock().mark_pump_late(
594 generation,
595 format!(
596 "stderr of the exited process had not reached EOF {bound:?} after it was \
597 retired (a descendant may still hold the pipe open); lines it still \
598 writes are kept in that process's section"
599 ),
600 );
601 warn!(
602 module_id,
603 waited = ?bound,
604 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
605 );
606 }
607 }
608}
609
610fn registration_release_events() -> &'static watch::Sender<u64> {
611 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
612 EVENTS.get_or_init(|| {
613 let (sender, _receiver) = watch::channel(0);
614 sender
615 })
616}
617
618pub(crate) fn notify_registration_release() {
619 let events = registration_release_events();
620 let next_generation = (*events.borrow()).wrapping_add(1);
621 events.send_replace(next_generation);
622}
623
624#[derive(Debug, Clone, PartialEq, Eq)]
626pub struct ModuleSpec {
627 pub module_id: String,
628 pub program: PathBuf,
629 pub args: Vec<String>,
630 pub env: Vec<(String, String)>,
631 pub reserved: bool,
636 pub reserved_prefixes: Vec<String>,
641 pub protocol: ModuleProtocol,
660 pub overlap: ModuleOverlap,
665}
666
667#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
674pub enum ModuleOverlap {
675 #[default]
677 Exclusive,
678 Safe,
690}
691
692impl ModuleOverlap {
693 pub fn as_str(self) -> &'static str {
694 match self {
695 Self::Exclusive => "exclusive",
696 Self::Safe => "safe",
697 }
698 }
699}
700
701pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
711pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
713pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
718
719#[derive(Debug, Clone, Copy, PartialEq, Eq)]
737pub struct RestartPolicy {
738 pub max_restarts: u32,
739 pub backoff: Duration,
742 pub max_backoff: Duration,
744 pub window: Duration,
748}
749
750impl RestartPolicy {
751 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
755 Self {
756 max_restarts,
757 backoff,
758 max_backoff: DEFAULT_MAX_BACKOFF,
759 window: DEFAULT_RESTART_WINDOW,
760 }
761 }
762
763 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
764 self.max_backoff = max_backoff;
765 self
766 }
767
768 pub fn with_window(mut self, window: Duration) -> Self {
769 self.window = window;
770 self
771 }
772
773 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
778 if self.backoff.is_zero() || self.max_backoff.is_zero() {
779 return Duration::ZERO;
780 }
781
782 let mut delay = self.backoff;
783 for _ in 0..restart_in_window {
784 if delay >= self.max_backoff {
785 return self.max_backoff;
786 }
787 delay = delay
788 .checked_mul(10)
789 .unwrap_or(self.max_backoff)
790 .min(self.max_backoff);
791 }
792 delay.min(self.max_backoff)
793 }
794
795 fn budget_exhausted_detail(&self) -> String {
800 format!(
801 "crash budget exhausted: max_restarts={} within window_secs={}",
802 self.max_restarts,
803 self.window.as_secs()
804 )
805 }
806}
807
808impl Default for RestartPolicy {
809 fn default() -> Self {
810 Self {
811 max_restarts: DEFAULT_MAX_RESTARTS,
812 backoff: DEFAULT_BACKOFF,
813 max_backoff: DEFAULT_MAX_BACKOFF,
814 window: DEFAULT_RESTART_WINDOW,
815 }
816 }
817}
818
819#[derive(Debug, Clone, Copy, PartialEq, Eq)]
820struct CrashRestartSchedule {
821 restart_in_window: u32,
822 delay: Duration,
823}
824
825fn daemon_will_restart(
832 state: &mut SupervisorSnapshot,
833 policy: &RestartPolicy,
834 now: Instant,
835) -> bool {
836 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
837}
838
839const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
840const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
841const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
842const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
843
844#[derive(Debug, Clone, Copy, PartialEq, Eq)]
845pub enum HealthAction {
846 Report,
847 Restart,
848 Alert,
849}
850
851impl fmt::Display for HealthAction {
852 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
853 f.write_str(match self {
854 Self::Report => "report",
855 Self::Restart => "restart",
856 Self::Alert => "alert",
857 })
858 }
859}
860
861#[derive(Debug, Clone, PartialEq, Eq)]
862pub struct HealthConfig {
863 pub http: Option<String>,
867 pub cadence: Duration,
868 pub deadline: Duration,
869 pub failure_threshold: u32,
870 pub on_degraded: HealthAction,
871 pub on_failing: HealthAction,
872 pub critical: bool,
873}
874
875impl Default for HealthConfig {
876 fn default() -> Self {
877 Self {
878 http: None,
879 cadence: DEFAULT_HEALTH_CADENCE,
880 deadline: DEFAULT_HEALTH_DEADLINE,
881 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
882 on_degraded: HealthAction::Report,
883 on_failing: HealthAction::Report,
884 critical: false,
885 }
886 }
887}
888
889#[derive(Debug, Clone, PartialEq)]
907pub struct ModuleHealthStatus {
908 pub status: SupervisorHealthStatus,
909 pub last_probe_ms: Option<u64>,
910 pub detail: Option<String>,
911 pub metrics: Option<Value>,
912 pub consecutive_failures: u32,
913 pub late_answer_count: u64,
916 pub last_late_answer_latency_ms: Option<u64>,
918 pub last_action: Option<String>,
919 pub last_action_ms: Option<u64>,
923}
924
925impl Default for ModuleHealthStatus {
926 fn default() -> Self {
927 Self {
928 status: SupervisorHealthStatus::Unknown,
929 last_probe_ms: None,
930 detail: None,
931 metrics: None,
932 consecutive_failures: 0,
933 late_answer_count: 0,
934 last_late_answer_latency_ms: None,
935 last_action: None,
936 last_action_ms: None,
937 }
938 }
939}
940
941#[derive(Debug, Clone, Copy, PartialEq, Eq)]
943pub enum ModuleState {
944 Starting,
945 Running,
946 Unresponsive,
947 Restarting,
948 Draining,
949 Stopped,
950 Failed,
951 Disabled,
952}
953
954impl fmt::Display for ModuleState {
955 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
956 f.write_str(match self {
957 Self::Starting => "starting",
958 Self::Running => "running",
959 Self::Unresponsive => "unresponsive",
960 Self::Restarting => "restarting",
961 Self::Draining => "draining",
962 Self::Stopped => "stopped",
963 Self::Failed => "failed",
964 Self::Disabled => "disabled",
965 })
966 }
967}
968
969#[derive(Debug, Clone, Copy, PartialEq, Eq)]
971pub enum ExitKind {
972 Clean,
973 Crash,
974 DeliberateSeverance,
975}
976
977impl From<ExitKind> for TerminalExitKind {
978 fn from(kind: ExitKind) -> Self {
979 match kind {
980 ExitKind::Clean => Self::Clean,
981 ExitKind::Crash => Self::Crash,
982 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
983 }
984 }
985}
986
987#[derive(Debug, Clone, Copy, PartialEq, Eq)]
990pub(crate) struct ProcessIdentity {
991 pub(crate) pid: u32,
992 pub(crate) start_time: u64,
993}
994
995#[derive(Debug, Clone, PartialEq, Eq)]
997pub struct ExitReport {
998 pub kind: ExitKind,
999 pub code: Option<i32>,
1000 pub signal: Option<i32>,
1001 pub at_ms: u64,
1002}
1003
1004#[derive(Debug, Clone, PartialEq)]
1007pub struct ModuleStatus {
1008 pub module_id: String,
1009 pub waiting_for_previous_pid: Option<u32>,
1011 pub launch_nonce_source: Option<String>,
1016 pub state: ModuleState,
1017 pub enabled: bool,
1018 pub process_alive: bool,
1019 pub registration_active: bool,
1020 pub protocol: ModuleProtocol,
1025 pub live: bool,
1036 pub restart_count: u32,
1040 pub lifetime_restarts: u32,
1044 pub spawn_generation: u64,
1045 pub max_restarts: u32,
1050 pub restart_window: Duration,
1054 pub drain_timeout: Duration,
1058 pub restart_backoff: Duration,
1059 pub restart_max_backoff: Duration,
1060 pub pid: Option<u32>,
1065 pub spawned_at_ms: Option<u64>,
1066 pub spawned_from: Option<PathBuf>,
1067 pub process_start_time: Option<u64>,
1068 pub last_exit: Option<ExitReport>,
1069 pub health: ModuleHealthStatus,
1070}
1071
1072#[cfg(test)]
1073#[derive(Debug, Clone, Copy, PartialEq)]
1074struct ActorSelectCheckpoint {
1075 generation: u64,
1076 turn: u64,
1077 registered_connection: Option<ConnectionId>,
1078 next_probe_at: Option<Instant>,
1079 wake_after: Option<Duration>,
1080}
1081
1082#[derive(Debug, Clone, PartialEq)]
1083struct SupervisorSnapshot {
1084 #[cfg(windows)]
1085 pipe_consumed: Option<Arc<OnceLock<()>>>,
1086 #[cfg(test)]
1088 actor_turns: u64,
1089 #[cfg(test)]
1092 actor_select: Option<ActorSelectCheckpoint>,
1093 state: ModuleState,
1094 enabled: bool,
1095 process_alive: bool,
1096 spawned_protocol: Option<ModuleProtocol>,
1097 spawn_failure: Option<String>,
1098 waiting_for_previous_pid: Option<u32>,
1099 crash_restarts: VecDeque<Instant>,
1105 lifetime_restarts: u32,
1106 spawn_generation: u64,
1115 pid: Option<u32>,
1116 #[cfg(target_os = "macos")]
1117 report_ready: Option<Arc<OnceLock<()>>>,
1118 reaped_pid: Option<u32>,
1120 respawn_pending: bool,
1122 coalesced_restart_pending: bool,
1124 spawned_at_ms: Option<u64>,
1125 spawned_from: Option<PathBuf>,
1126 spawned_file_identity: Option<SpawnedFileIdentity>,
1127 process_start_time: Option<u64>,
1128 deliberate_severance: Option<ProcessIdentity>,
1129 last_exit: Option<ExitReport>,
1130 drain_disposition_detail: Option<String>,
1132 health: ModuleHealthStatus,
1133 in_alternate_slot: bool,
1138 draining_to_replace: bool,
1145 configuration_updated_since_spawn: bool,
1151}
1152
1153impl SupervisorSnapshot {
1154 fn reported_pid(&self) -> Option<u32> {
1159 #[cfg(target_os = "macos")]
1160 if self
1161 .report_ready
1162 .as_ref()
1163 .is_some_and(|ready| ready.get().is_none())
1164 {
1165 return None;
1166 }
1167 self.pid
1168 }
1169
1170 fn starting() -> Self {
1171 Self::new(ModuleState::Starting, true)
1172 }
1173
1174 fn disabled() -> Self {
1175 Self::new(ModuleState::Disabled, false)
1176 }
1177
1178 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1182 while let Some(oldest) = self.crash_restarts.front() {
1183 if now.duration_since(*oldest) > window {
1184 self.crash_restarts.pop_front();
1185 } else {
1186 break;
1187 }
1188 }
1189 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1190 }
1191
1192 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1198 self.crash_restarts.push_back(now);
1199 while self.crash_restarts.len() > policy.max_restarts as usize {
1200 self.crash_restarts.pop_front();
1201 }
1202 self.lifetime_restarts += 1;
1203 }
1204
1205 fn next_crash_restart(
1209 &mut self,
1210 policy: &RestartPolicy,
1211 now: Instant,
1212 ) -> Option<CrashRestartSchedule> {
1213 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1214 if restart_in_window >= policy.max_restarts {
1215 return None;
1216 }
1217 self.record_crash_restart(policy, now);
1218 Some(CrashRestartSchedule {
1219 restart_in_window,
1220 delay: policy.delay_for_restart(restart_in_window),
1221 })
1222 }
1223
1224 fn clear_crash_restarts(&mut self) {
1229 self.crash_restarts.clear();
1230 }
1231
1232 fn new(state: ModuleState, enabled: bool) -> Self {
1233 Self {
1234 #[cfg(test)]
1235 actor_turns: 0,
1236 #[cfg(test)]
1237 actor_select: None,
1238 state,
1239 enabled,
1240 process_alive: false,
1241 #[cfg(windows)]
1242 pipe_consumed: None,
1243 spawned_protocol: None,
1244 spawn_failure: None,
1245 waiting_for_previous_pid: None,
1246 crash_restarts: VecDeque::new(),
1247 lifetime_restarts: 0,
1248 spawn_generation: 0,
1249 pid: None,
1250 #[cfg(target_os = "macos")]
1251 report_ready: None,
1252 reaped_pid: None,
1253 respawn_pending: false,
1254 coalesced_restart_pending: false,
1255 spawned_at_ms: None,
1256 spawned_from: None,
1257 spawned_file_identity: None,
1258 process_start_time: None,
1259 deliberate_severance: None,
1260 last_exit: None,
1261 drain_disposition_detail: None,
1262 health: ModuleHealthStatus::default(),
1263 in_alternate_slot: false,
1264 draining_to_replace: false,
1265 configuration_updated_since_spawn: false,
1266 }
1267 }
1268}
1269
1270type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1271
1272type SpawnSubscriberKey = (ConnectionId, u64);
1273
1274#[derive(Debug)]
1275struct SpawnSubscriber {
1276 version: u8,
1277 frames: mpsc::Sender<Frame>,
1278 lagged: Option<oneshot::Sender<SpawnCursor>>,
1282}
1283
1284#[derive(Debug)]
1285struct SpawnEventState {
1286 daemon_incarnation: String,
1287 seq: u64,
1288 capacity: usize,
1289 live: HashMap<String, LiveSpawn>,
1290 generations: HashMap<String, u64>,
1291 events: VecDeque<SpawnEvent>,
1292 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1293}
1294
1295impl Default for SpawnEventState {
1296 fn default() -> Self {
1297 Self {
1298 daemon_incarnation: "unconfigured".to_string(),
1299 seq: 0,
1300 capacity: SPAWN_EVENT_RING_CAPACITY,
1301 live: HashMap::new(),
1302 generations: HashMap::new(),
1303 events: VecDeque::new(),
1304 subscribers: HashMap::new(),
1305 }
1306 }
1307}
1308
1309#[derive(Debug, Clone, Default)]
1310struct SpawnEventFeed(
1311 Arc<Mutex<SpawnEventState>>,
1312 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1313);
1314
1315#[derive(Debug, Clone, PartialEq, Eq)]
1316pub(crate) enum SpawnSubscribeRefusal {
1317 ForeignIncarnation { current: String },
1318 TooOld { oldest: SpawnCursor },
1319 Frame(String),
1320}
1321
1322impl SpawnEventFeed {
1323 fn configure_incarnation(&self, daemon_incarnation: String) {
1324 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1325 state.daemon_incarnation = daemon_incarnation;
1326 state.seq = 0;
1327 state.live.clear();
1328 state.generations.clear();
1329 state.events.clear();
1330 state.subscribers.clear();
1331 }
1332
1333 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1334 SpawnCursor {
1335 daemon_incarnation: state.daemon_incarnation.clone(),
1336 seq: state.seq,
1337 }
1338 }
1339
1340 fn snapshot(&self) -> SpawnSnapshot {
1341 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1342 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1343 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1344 SpawnSnapshot {
1345 cursor: Self::cursor(&state),
1346 ring_bound: state.capacity as u64,
1347 live,
1348 }
1349 }
1350
1351 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1352 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1353 let generation = state
1354 .generations
1355 .get(module_id)
1356 .copied()
1357 .unwrap_or(0)
1358 .checked_add(1)
1359 .expect("spawn generation exhausted");
1360 state.generations.insert(module_id.to_string(), generation);
1361 let live = LiveSpawn {
1362 module_id: module_id.to_string(),
1363 spawn_generation: generation,
1364 pid,
1365 spawned_at_ms,
1366 };
1367 state.live.insert(module_id.to_string(), live);
1368 Self::emit_locked(
1369 &mut state,
1370 SpawnEventKind::Spawned,
1371 module_id.to_string(),
1372 generation,
1373 pid,
1374 None,
1375 None,
1376 );
1377 generation
1378 }
1379
1380 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1381 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1382 let Some(live) = state.live.remove(module_id) else {
1383 warn!(
1384 module_id,
1385 "terminal record had no live spawn event identity"
1386 );
1387 return;
1388 };
1389 Self::emit_locked(
1390 &mut state,
1391 SpawnEventKind::Exited,
1392 module_id.to_string(),
1393 live.spawn_generation,
1394 live.pid,
1395 exit_code,
1396 exit_signal,
1397 );
1398 }
1399
1400 fn emit_superseded_exited(
1407 &self,
1408 module_id: &str,
1409 spawn_generation: u64,
1410 pid: u32,
1411 exit_code: Option<i32>,
1412 exit_signal: Option<i32>,
1413 ) {
1414 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1415 if state
1416 .live
1417 .get(module_id)
1418 .is_some_and(|live| live.spawn_generation == spawn_generation)
1419 {
1420 state.live.remove(module_id);
1421 }
1422 Self::emit_locked(
1423 &mut state,
1424 SpawnEventKind::Exited,
1425 module_id.to_string(),
1426 spawn_generation,
1427 pid,
1428 exit_code,
1429 exit_signal,
1430 );
1431 }
1432
1433 #[allow(clippy::too_many_arguments)]
1434 fn emit_locked(
1435 state: &mut SpawnEventState,
1436 kind: SpawnEventKind,
1437 module_id: String,
1438 spawn_generation: u64,
1439 pid: u32,
1440 exit_code: Option<i32>,
1441 exit_signal: Option<i32>,
1442 ) {
1443 state.seq = state
1444 .seq
1445 .checked_add(1)
1446 .expect("spawn event sequence exhausted");
1447 let event = SpawnEvent {
1448 cursor: Self::cursor(state),
1449 kind,
1450 module_id,
1451 spawn_generation,
1452 pid,
1453 exit_code,
1454 exit_signal,
1455 };
1456 state.events.push_back(event.clone());
1457 while state.events.len() > state.capacity {
1458 state.events.pop_front();
1459 }
1460 let body = match serde_json::to_vec(&event) {
1461 Ok(body) => body,
1462 Err(error) => {
1463 error!(%error, "failed to serialize supervisor spawn event");
1464 return;
1465 }
1466 };
1467 state.subscribers.retain(|(connection_id, corr), subscriber| {
1468 let frame = Frame::build_with_version(
1469 subscriber.version,
1470 FrameType::StreamData,
1471 control_flags(),
1472 0,
1473 0,
1474 *corr,
1475 body.clone(),
1476 );
1477 match frame {
1478 Ok(frame) => {
1479 if subscriber.frames.try_send(frame).is_ok() {
1480 true
1481 } else {
1482 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1483 if let Some(lagged) = subscriber.lagged.take() {
1484 let _ = lagged.send(event.cursor.clone());
1485 }
1486 false
1487 }
1488 }
1489 Err(error) => {
1490 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1491 false
1492 }
1493 }
1494 });
1495 }
1496
1497 fn subscribe(
1498 &self,
1499 connection_id: ConnectionId,
1500 corr: u64,
1501 version: u8,
1502 since: Option<SpawnCursor>,
1503 sink: FrameSink,
1504 ) -> Result<(), SpawnSubscribeRefusal> {
1505 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1506 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1507 {
1508 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1509 let replay = if let Some(since) = since {
1510 if since.daemon_incarnation != state.daemon_incarnation {
1511 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1512 current: state.daemon_incarnation.clone(),
1513 });
1514 }
1515 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1516 if since.seq < oldest.seq.saturating_sub(1) {
1517 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1518 }
1519 }
1520 state
1521 .events
1522 .iter()
1523 .filter(|event| event.cursor.seq > since.seq)
1524 .cloned()
1525 .collect::<Vec<_>>()
1526 } else {
1527 Vec::new()
1528 };
1529 for event in replay {
1530 let body = serde_json::to_vec(&event)
1531 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1532 let frame = Frame::build_with_version(
1533 version,
1534 FrameType::StreamData,
1535 control_flags(),
1536 0,
1537 0,
1538 corr,
1539 body,
1540 )
1541 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1542 frames
1543 .try_send(frame)
1544 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1545 }
1546 state.subscribers.insert(
1547 (connection_id, corr),
1548 SpawnSubscriber {
1549 version,
1550 frames,
1551 lagged: Some(lagged),
1552 },
1553 );
1554 }
1555 tokio::spawn(async move {
1566 while let Some(frame) = receiver.recv().await {
1567 if sink.send(frame).await.is_err() {
1568 return;
1569 }
1570 }
1571 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1572 return;
1573 };
1574 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1575 Ok(frame) => {
1576 let _ = sink.send(frame).await;
1577 }
1578 Err(error) => {
1579 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1580 }
1581 }
1582 });
1583 Ok(())
1584 }
1585
1586 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1587 let Some(subscriber) = self
1588 .0
1589 .lock()
1590 .unwrap_or_else(|p| p.into_inner())
1591 .subscribers
1592 .remove(&(connection_id, corr))
1593 else {
1594 return false;
1595 };
1596 if let Ok(frame) = Frame::build_with_version(
1597 subscriber.version,
1598 FrameType::StreamEnd,
1599 control_flags(),
1600 0,
1601 0,
1602 corr,
1603 Vec::new(),
1604 ) {
1605 tokio::spawn(async move {
1606 let _ = subscriber.frames.send(frame).await;
1607 });
1608 }
1609 true
1610 }
1611
1612 fn remove_connection(&self, connection_id: ConnectionId) {
1613 self.0
1614 .lock()
1615 .unwrap_or_else(|p| p.into_inner())
1616 .subscribers
1617 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1618 }
1619
1620 #[cfg(any(test, feature = "test-support"))]
1621 fn set_capacity(&self, capacity: usize) {
1622 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1623 }
1624
1625 #[cfg(any(test, feature = "test-support"))]
1626 fn subscriber_count(&self) -> usize {
1627 self.0
1628 .lock()
1629 .unwrap_or_else(|p| p.into_inner())
1630 .subscribers
1631 .len()
1632 }
1633}
1634
1635fn spawn_subscriber_lagged_frame(
1638 version: u8,
1639 corr: u64,
1640 first_undelivered: SpawnCursor,
1641) -> Result<Frame, String> {
1642 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1643 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1644 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1645 .to_string(),
1646 detail: Some(serde_json::json!({
1647 "first_undelivered_cursor": first_undelivered
1648 })),
1649 })
1650 .map_err(|error| error.to_string())?;
1651 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1652 .map_err(|error| error.to_string())
1653}
1654
1655pub trait ModuleProcessLiveness: Send + Sync {
1656 fn process_live(&self, module_id: &str) -> Option<bool>;
1657
1658 fn process_replacing(&self, _module_id: &str) -> bool {
1664 false
1665 }
1666}
1667
1668#[derive(Debug, Clone, Default)]
1670pub struct SupervisorProcessLiveness {
1671 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1672}
1673
1674impl SupervisorProcessLiveness {
1675 pub fn new() -> Self {
1676 Self::default()
1677 }
1678
1679 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1680 let mut snapshots = self
1681 .snapshots
1682 .lock()
1683 .unwrap_or_else(|poisoned| poisoned.into_inner());
1684 snapshots.insert(module_id, snapshot);
1685 }
1686
1687 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1688 let mut snapshots = self
1689 .snapshots
1690 .lock()
1691 .unwrap_or_else(|poisoned| poisoned.into_inner());
1692 let is_current = snapshots
1693 .get(module_id)
1694 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1695 .unwrap_or(false);
1696 if is_current {
1697 snapshots.remove(module_id);
1698 }
1699 }
1700}
1701
1702impl ModuleProcessLiveness for SupervisorProcessLiveness {
1703 fn process_live(&self, module_id: &str) -> Option<bool> {
1704 let snapshot = {
1705 let snapshots = self
1706 .snapshots
1707 .lock()
1708 .unwrap_or_else(|poisoned| poisoned.into_inner());
1709 snapshots.get(module_id).cloned()
1710 }?;
1711 let snapshot = snapshot
1712 .lock()
1713 .unwrap_or_else(|poisoned| poisoned.into_inner());
1714 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1715 }
1716
1717 fn process_replacing(&self, module_id: &str) -> bool {
1718 let Some(snapshot) = self
1719 .snapshots
1720 .lock()
1721 .unwrap_or_else(|poisoned| poisoned.into_inner())
1722 .get(module_id)
1723 .cloned()
1724 else {
1725 return false;
1726 };
1727 let snapshot = snapshot
1728 .lock()
1729 .unwrap_or_else(|poisoned| poisoned.into_inner());
1730 snapshot.enabled
1731 && match snapshot.state {
1732 ModuleState::Restarting => true,
1733 ModuleState::Draining => snapshot.draining_to_replace,
1734 ModuleState::Starting
1735 | ModuleState::Running
1736 | ModuleState::Unresponsive
1737 | ModuleState::Stopped
1738 | ModuleState::Failed
1739 | ModuleState::Disabled => false,
1740 }
1741 }
1742}
1743
1744#[cfg(test)]
1745#[derive(Debug, Default)]
1746struct ReloadExitRecordGate {
1747 reached: tokio::sync::Notify,
1748 resume: tokio::sync::Notify,
1749}
1750
1751#[derive(Debug, Clone, Copy)]
1752enum RespawnKind {
1753 Spawn,
1754 Reload,
1755}
1756
1757#[derive(Debug, Clone, Copy)]
1758struct PendingRespawn {
1759 deadline: Instant,
1760 kind: RespawnKind,
1761}
1762
1763type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1764
1765#[derive(Debug, Clone)]
1766struct SupervisorRuntimeConfig {
1767 start_requester: Option<Principal>,
1768 pending_activity_restart: bool,
1769 provenance_probe: ExecutableIdentityProbe,
1770 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1772 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1774 restart_policy: RestartPolicy,
1775 drain_timeout: Duration,
1778 effective_drain_timeout: Arc<Mutex<Duration>>,
1781 default_drain_timeout: Duration,
1784 health: HealthConfig,
1785 connection_file_path: Option<PathBuf>,
1786 capture_logs_dir: Option<PathBuf>,
1787 forwarding: Option<Arc<ForwardingTable>>,
1788 supervisor_handle: Option<SupervisorHandle>,
1791 stderr_ring: Arc<Mutex<StderrRing>>,
1798 terminal_ring: Arc<Mutex<TerminalRing>>,
1799 spawn_events: SpawnEventFeed,
1800 child_roster: ChildRoster,
1801 #[cfg(target_os = "linux")]
1802 cgroup_placement: Option<subc_cgroup::Placement>,
1803 #[cfg(test)]
1804 test_seed_stale_facts_before_enable_spawn: bool,
1805 #[cfg(test)]
1806 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1807}
1808
1809#[derive(Debug, Clone, PartialEq, Eq)]
1810struct SupervisedConfiguration {
1811 spec: ModuleSpec,
1812 health: HealthConfig,
1813}
1814
1815#[derive(Debug, Clone, Default)]
1821pub struct SupervisorHandle {
1822 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1823 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1824 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1825 configured_ids: Arc<Mutex<HashSet<String>>>,
1837 spawn_events: SpawnEventFeed,
1838 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
1849 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
1855 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
1859 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
1867 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
1873 promotion_observer: PromotionObserverSlot,
1875 operation_lock: Arc<AsyncMutex<()>>,
1879}
1880
1881pub(crate) trait SwapPromotionObserver: Send + Sync {
1890 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
1891}
1892
1893#[derive(Clone, Default)]
1897struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
1898
1899impl fmt::Debug for PromotionObserverSlot {
1900 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1901 f.write_str("PromotionObserverSlot")
1902 }
1903}
1904
1905#[derive(Debug, Clone)]
1907struct OpenSwap {
1908 candidate_nonce: String,
1911 incumbent_nonce: Option<String>,
1916 candidate_admitted: bool,
1920}
1921
1922#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1925pub(crate) enum SwapHelloAdmission {
1926 NotSwapping,
1929 Candidate,
1931 Refused,
1934}
1935
1936#[derive(Debug, Clone, PartialEq, Eq)]
1937pub(crate) enum ReservedHelloRejection {
1938 Exact {
1939 module_id: String,
1940 },
1941 Prefix {
1942 prefix: String,
1943 owner_module_id: String,
1944 },
1945}
1946
1947impl SupervisorHandle {
1948 pub fn new() -> Self {
1949 Self::default()
1950 }
1951
1952 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
1953 self.spawn_events.snapshot()
1954 }
1955
1956 pub(crate) fn subscribe_spawns(
1957 &self,
1958 connection_id: ConnectionId,
1959 corr: u64,
1960 version: u8,
1961 since: Option<SpawnCursor>,
1962 sink: FrameSink,
1963 ) -> Result<(), SpawnSubscribeRefusal> {
1964 self.spawn_events
1965 .subscribe(connection_id, corr, version, since, sink)
1966 }
1967
1968 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
1969 self.spawn_events.cancel(connection_id, corr)
1970 }
1971
1972 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
1973 self.spawn_events.remove_connection(connection_id);
1974 }
1975
1976 #[cfg(any(test, feature = "test-support"))]
1977 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
1978 assert!(capacity > 0, "spawn event capacity must be non-zero");
1979 self.spawn_events.set_capacity(capacity);
1980 }
1981
1982 #[cfg(any(test, feature = "test-support"))]
1983 pub fn spawn_subscriber_count_for_test(&self) -> usize {
1984 self.spawn_events.subscriber_count()
1985 }
1986
1987 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
1990 self.spawn_nonces
1991 .lock()
1992 .unwrap_or_else(|poisoned| poisoned.into_inner())
1993 .insert(module_id.to_string(), nonce);
1994 }
1995
1996 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
1999 self.reserved_nonces
2000 .lock()
2001 .unwrap_or_else(|poisoned| poisoned.into_inner())
2002 .insert(module_id.to_string(), Some(nonce));
2003 }
2004
2005 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2007 let mut owners = self
2008 .reserved_prefix_owners
2009 .lock()
2010 .unwrap_or_else(|poisoned| poisoned.into_inner());
2011 owners.retain(|_, owner| owner != owner_module_id);
2012 for prefix in prefixes {
2013 owners.insert(prefix.clone(), owner_module_id.to_string());
2014 }
2015 }
2016
2017 #[cfg(test)]
2019 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2020 self.spawn_nonces
2021 .lock()
2022 .unwrap_or_else(|poisoned| poisoned.into_inner())
2023 .get(module_id)
2024 .cloned()
2025 }
2026
2027 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2028 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2029 let spawn_nonce = self
2030 .spawn_nonces
2031 .lock()
2032 .unwrap_or_else(|poisoned| poisoned.into_inner())
2033 .get(&spec.module_id)
2034 .cloned();
2035 let mut reserved_nonces = self
2036 .reserved_nonces
2037 .lock()
2038 .unwrap_or_else(|poisoned| poisoned.into_inner());
2039 if spec.reserved {
2040 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2045 }
2046 drop(reserved_nonces);
2047 self.removal_tombstones
2051 .lock()
2052 .unwrap_or_else(|poisoned| poisoned.into_inner())
2053 .remove(&spec.module_id);
2054 }
2055
2056 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2061 self.reserved_hello_rejection(module_id, presented)
2062 .is_none()
2063 }
2064
2065 pub(crate) fn reserved_hello_rejection(
2066 &self,
2067 module_id: &str,
2068 presented: Option<&str>,
2069 ) -> Option<ReservedHelloRejection> {
2070 let nonces = self
2071 .reserved_nonces
2072 .lock()
2073 .unwrap_or_else(|poisoned| poisoned.into_inner());
2074 if let Some(expected) = nonces.get(module_id) {
2075 let authorized = match expected {
2079 Some(expected) => {
2080 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2081 }
2082 None => false,
2083 };
2084 if authorized {
2085 return None;
2086 }
2087 return Some(ReservedHelloRejection::Exact {
2088 module_id: module_id.to_string(),
2089 });
2090 }
2091 drop(nonces);
2092
2093 let matched_prefix = self
2094 .reserved_prefix_owners
2095 .lock()
2096 .unwrap_or_else(|poisoned| poisoned.into_inner())
2097 .iter()
2098 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2099 .max_by_key(|(prefix, _)| prefix.len())
2100 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2101 let (prefix, owner_module_id) = matched_prefix?;
2102
2103 let authorized = presented.is_some_and(|presented| {
2104 self.spawn_nonces
2105 .lock()
2106 .unwrap_or_else(|poisoned| poisoned.into_inner())
2107 .get(&owner_module_id)
2108 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2109 || self.swap_nonce_matches(&owner_module_id, presented)
2112 });
2113 if authorized {
2114 None
2115 } else {
2116 Some(ReservedHelloRejection::Prefix {
2117 prefix,
2118 owner_module_id,
2119 })
2120 }
2121 }
2122
2123 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2128 if presented.is_empty() {
2129 return false;
2130 }
2131 let nonces = self
2132 .spawn_nonces
2133 .lock()
2134 .unwrap_or_else(|poisoned| poisoned.into_inner());
2135 let current = nonces
2136 .get(module_id)
2137 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2138 drop(nonces);
2139 current || self.swap_nonce_matches(module_id, presented)
2144 }
2145
2146 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2148 let swaps = self
2149 .swaps
2150 .lock()
2151 .unwrap_or_else(|poisoned| poisoned.into_inner());
2152 swaps.get(module_id).is_some_and(|swap| {
2153 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2154 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2155 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2156 })
2157 })
2158 }
2159
2160 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2163 let incumbent_nonce = self
2164 .spawn_nonces
2165 .lock()
2166 .unwrap_or_else(|poisoned| poisoned.into_inner())
2167 .get(module_id)
2168 .cloned();
2169 self.swaps
2170 .lock()
2171 .unwrap_or_else(|poisoned| poisoned.into_inner())
2172 .insert(
2173 module_id.to_string(),
2174 OpenSwap {
2175 candidate_nonce,
2176 incumbent_nonce,
2177 candidate_admitted: false,
2178 },
2179 );
2180 }
2181
2182 pub(crate) fn close_swap(&self, module_id: &str) {
2185 self.swaps
2186 .lock()
2187 .unwrap_or_else(|poisoned| poisoned.into_inner())
2188 .remove(module_id);
2189 }
2190
2191 pub(crate) fn set_swap_promotion_observer(
2194 &self,
2195 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2196 ) {
2197 *self
2198 .promotion_observer
2199 .0
2200 .lock()
2201 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2202 }
2203
2204 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2207 let observer = self
2208 .promotion_observer
2209 .0
2210 .lock()
2211 .unwrap_or_else(|poisoned| poisoned.into_inner())
2212 .as_ref()
2213 .and_then(std::sync::Weak::upgrade);
2214 if let Some(observer) = observer {
2215 observer.swap_promoted(registration);
2216 }
2217 }
2218
2219 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2221 self.swaps
2222 .lock()
2223 .unwrap_or_else(|poisoned| poisoned.into_inner())
2224 .contains_key(module_id)
2225 }
2226
2227 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2232 let candidate_nonce = self
2233 .swaps
2234 .lock()
2235 .unwrap_or_else(|poisoned| poisoned.into_inner())
2236 .get(module_id)
2237 .map(|swap| swap.candidate_nonce.clone());
2238 let Some(nonce) = candidate_nonce else {
2239 return;
2240 };
2241 self.set_spawn_nonce(module_id, nonce.clone());
2242 if reserved {
2243 self.set_reserved_nonce(module_id, nonce);
2244 }
2245 }
2246
2247 pub(crate) fn swap_hello_admission(
2262 &self,
2263 module_id: &str,
2264 presented: Option<&str>,
2265 ) -> SwapHelloAdmission {
2266 let swaps = self
2267 .swaps
2268 .lock()
2269 .unwrap_or_else(|poisoned| poisoned.into_inner());
2270 let Some(swap) = swaps.get(module_id) else {
2271 return SwapHelloAdmission::NotSwapping;
2272 };
2273 let Some(presented) = presented else {
2274 return SwapHelloAdmission::Refused;
2275 };
2276 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2277 return if swap.candidate_admitted {
2278 SwapHelloAdmission::Refused
2279 } else {
2280 SwapHelloAdmission::Candidate
2281 };
2282 }
2283 if swap
2284 .incumbent_nonce
2285 .as_deref()
2286 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2287 {
2288 return SwapHelloAdmission::NotSwapping;
2289 }
2290 SwapHelloAdmission::Refused
2291 }
2292
2293 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2296 if let Some(swap) = self
2297 .swaps
2298 .lock()
2299 .unwrap_or_else(|poisoned| poisoned.into_inner())
2300 .get_mut(module_id)
2301 {
2302 swap.candidate_admitted = true;
2303 }
2304 }
2305
2306 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2308 self.spawn_nonces
2309 .lock()
2310 .unwrap_or_else(|poisoned| poisoned.into_inner())
2311 .get(module_id)
2312 .cloned()
2313 }
2314
2315 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2317 self.reserved_nonces
2318 .lock()
2319 .unwrap_or_else(|poisoned| poisoned.into_inner())
2320 .get(module_id)
2321 .cloned()
2322 .flatten()
2323 }
2324
2325 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2326 self.mark_configured(module.module_id());
2330 let mut modules = self
2331 .modules
2332 .lock()
2333 .unwrap_or_else(|poisoned| poisoned.into_inner());
2334 modules.insert(module.module_id().to_string(), module)
2335 }
2336
2337 fn mark_configured(&self, module_id: &str) {
2341 self.configured_ids
2342 .lock()
2343 .unwrap_or_else(|poisoned| poisoned.into_inner())
2344 .insert(module_id.to_string());
2345 }
2346
2347 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2351 let modules = self
2352 .modules
2353 .lock()
2354 .unwrap_or_else(|poisoned| poisoned.into_inner());
2355 if !modules.contains_key(module_id) {
2356 self.configured_ids
2357 .lock()
2358 .unwrap_or_else(|poisoned| poisoned.into_inner())
2359 .remove(module_id);
2360 }
2361 }
2362
2363 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2371 self.configured_ids
2372 .lock()
2373 .unwrap_or_else(|poisoned| poisoned.into_inner())
2374 .contains(module_id)
2375 }
2376
2377 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2378 let modules = self
2379 .modules
2380 .lock()
2381 .unwrap_or_else(|poisoned| poisoned.into_inner());
2382 modules.get(module_id).cloned()
2383 }
2384
2385 pub(crate) fn record_late_health_answer(
2386 &self,
2387 module_id: &str,
2388 latency_ms: u64,
2389 ) -> Result<bool, SuperviseError> {
2390 let Some(module) = self.get(module_id) else {
2391 return Ok(false);
2392 };
2393 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2394 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2395 state.health.last_late_answer_latency_ms = Some(latency_ms);
2396 state.health.consecutive_failures = 0;
2404 })?;
2405 Ok(true)
2406 }
2407
2408 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2414 let Some(module) = self.get(module_id) else {
2415 return Ok(false);
2416 };
2417 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2418 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2419 return Ok(false);
2420 };
2421 drop(snapshot);
2422 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2423 }
2424
2425 pub fn list(&self) -> Vec<SupervisedModule> {
2426 let modules = self
2427 .modules
2428 .lock()
2429 .unwrap_or_else(|poisoned| poisoned.into_inner());
2430 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2431 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2432 modules
2433 }
2434
2435 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2436 self.activity_images
2437 .lock()
2438 .unwrap_or_else(|p| p.into_inner())
2439 .remove(module_id);
2440 self.spawn_nonces
2441 .lock()
2442 .unwrap_or_else(|poisoned| poisoned.into_inner())
2443 .remove(module_id);
2444 self.close_swap(module_id);
2445 let mut reserved_nonces = self
2446 .reserved_nonces
2447 .lock()
2448 .unwrap_or_else(|poisoned| poisoned.into_inner());
2449 if reserved_nonces.contains_key(module_id) {
2450 reserved_nonces.insert(module_id.to_string(), None);
2453 }
2454 drop(reserved_nonces);
2455 self.reserved_prefix_owners
2456 .lock()
2457 .unwrap_or_else(|poisoned| poisoned.into_inner())
2458 .retain(|_, owner| owner != module_id);
2459 let removed = self
2460 .modules
2461 .lock()
2462 .unwrap_or_else(|poisoned| poisoned.into_inner())
2463 .remove(module_id);
2464 self.configured_ids
2465 .lock()
2466 .unwrap_or_else(|poisoned| poisoned.into_inner())
2467 .remove(module_id);
2468 removed
2469 }
2470
2471 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2474 self.removal_tombstones
2475 .lock()
2476 .unwrap_or_else(|poisoned| poisoned.into_inner())
2477 .insert(module_id.to_string(), unix_ms_now());
2478 }
2479
2480 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2482 self.removal_tombstones
2483 .lock()
2484 .unwrap_or_else(|poisoned| poisoned.into_inner())
2485 .get(module_id)
2486 .copied()
2487 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2488 }
2489
2490 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2495 if self.get(module_id).is_some() {
2496 return false;
2497 }
2498 let mut reserved_nonces = self
2499 .reserved_nonces
2500 .lock()
2501 .unwrap_or_else(|poisoned| poisoned.into_inner());
2502 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2503 return false;
2504 }
2505 reserved_nonces.remove(module_id);
2506 true
2507 }
2508
2509 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2510 Arc::clone(&self.operation_lock)
2511 }
2512}
2513
2514#[derive(Debug, Clone)]
2516pub struct Supervisor {
2517 registry: Arc<Registry>,
2518 restart_policy: RestartPolicy,
2519 drain_timeout: Duration,
2520 connection_file_path: Option<PathBuf>,
2521 capture_logs_dir: Option<PathBuf>,
2522 forwarding: Option<Arc<ForwardingTable>>,
2523 process_liveness: Arc<SupervisorProcessLiveness>,
2524 supervisor_handle: Option<SupervisorHandle>,
2525 health: HealthConfig,
2526 daemon_start_clock: crate::clock::StartClock,
2527 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2528 spawn_events: SpawnEventFeed,
2529 provenance_probe: ExecutableIdentityProbe,
2530 child_roster: ChildRoster,
2533 #[cfg(target_os = "linux")]
2534 cgroup_placement: Option<subc_cgroup::Placement>,
2535 #[cfg(test)]
2536 test_after_first_spawn: AfterFirstSpawnHook,
2537}
2538
2539pub(crate) struct RegisteredConfiguredModule {
2543 spec: ModuleSpec,
2544 runtime: SupervisorRuntimeConfig,
2545 module: SupervisedModule,
2546 commands: mpsc::Receiver<SupervisorCommand>,
2547}
2548
2549#[cfg(test)]
2555#[derive(Clone, Default)]
2556struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2557
2558#[cfg(test)]
2559type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2560
2561#[cfg(test)]
2562impl fmt::Debug for AfterFirstSpawnHook {
2563 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2564 f.write_str("AfterFirstSpawnHook")
2565 }
2566}
2567
2568#[cfg(test)]
2569impl AfterFirstSpawnHook {
2570 fn run(&self, module_id: &str) {
2571 if let Some(hook) = &self.0 {
2572 hook(module_id);
2573 }
2574 }
2575}
2576
2577impl Supervisor {
2578 #[cfg(test)]
2579 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2580 let supervisor = Self::new(registry, policy);
2581 #[cfg(target_os = "macos")]
2582 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2583 supervisor
2584 }
2585 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2590 let path = path.into();
2591 #[cfg(target_os = "macos")]
2592 {
2593 let result = probe_privacy_trampoline(&path).map(|()| path);
2594 if let Err(cause) = &result {
2595 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2596 }
2597 self.child_roster.set_privacy_trampoline(result);
2598 }
2599 #[cfg(not(target_os = "macos"))]
2600 let _ = path;
2601 self
2602 }
2603 #[cfg(unix)]
2614 pub(crate) fn begin_daemon_shutdown(&self) {
2615 self.child_roster.close();
2616 if let Some(journal) = &self.terminal_journal {
2617 journal.stamp_shutdown();
2618 }
2619 }
2620
2621 #[cfg(unix)]
2625 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2626 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2627 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2628 let Some(forwarding) = &self.forwarding else {
2629 return Ok(());
2630 };
2631 let modules = forwarding
2632 .begin_daemon_drain()
2633 .map_err(SuperviseError::Forwarding)?;
2634 let deadline_ms =
2635 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2636 let mut notices = tokio::task::JoinSet::new();
2637 let mut drains = Vec::new();
2638 for (module_id, endpoint) in modules {
2639 let Some(target) = forwarding
2640 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2641 .map_err(SuperviseError::Forwarding)?
2642 else {
2643 continue;
2644 };
2645 let routes = forwarding
2646 .endpoint_routes(target.endpoint)
2647 .map_err(SuperviseError::Forwarding)?;
2648 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2654 reason: RouteCloseReason::Restart,
2655 deadline_ms,
2656 })
2657 .expect("module draining serializes");
2658 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2659 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2660 for route in routes {
2661 let client = route.goodbye_target;
2662 if let Some((_, channels)) = clients
2663 .iter_mut()
2664 .find(|(existing, _)| existing.connection_id == client.connection_id)
2665 {
2666 channels.push(client.channel);
2667 } else {
2668 let channel = client.channel;
2669 clients.push((client, vec![channel]));
2670 }
2671 }
2672 for (client, mut channels) in clients {
2673 channels.sort_unstable();
2674 channels.dedup();
2675 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2676 module_id: module_id.clone(),
2677 channels,
2678 reason: RouteCloseReason::Restart,
2679 })
2680 .expect("route closing serializes");
2681 recipients.push((client.sink, client.negotiated_ver, closing));
2682 }
2683 for (sink, version, body) in recipients {
2684 notices.spawn(async move {
2685 let frame = Frame::build_with_version(
2686 version,
2687 FrameType::Push,
2688 control_flags(),
2689 0,
2690 0,
2691 0,
2692 body,
2693 )
2694 .expect("bounded lifecycle notice frame builds");
2695 sink.send_flushed(frame).await
2696 });
2697 }
2698 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2699 drains.push((module_id, target.endpoint, gauges));
2700 }
2701 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2704 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2705 if !matches!(result, Ok(Ok(()))) {
2706 warn!(?result, "daemon shutdown notice delivery failed");
2707 }
2708 }
2709 notices.abort_all();
2710 let deadline = Instant::now() + DRAIN_BUDGET;
2711 let mut waits = tokio::task::JoinSet::new();
2712 for (module_id, endpoint, gauges) in drains {
2713 let forwarding = Arc::clone(forwarding);
2714 let mut runtime = self.runtime_config();
2715 runtime.health.cadence = Duration::from_millis(100);
2716 waits.spawn(async move {
2717 let wait_result = wait_for_forwarding_quiescence(
2718 &forwarding,
2719 &module_id,
2720 &runtime,
2721 endpoint,
2722 deadline,
2723 &gauges,
2724 DrainScope::Endpoint(endpoint),
2725 )
2726 .await;
2727 forwarding
2731 .record_daemon_drain_result(
2732 endpoint,
2733 drained_after_quiescence_wait(&wait_result),
2734 )
2735 .map_err(SuperviseError::Forwarding)?;
2736 wait_result
2737 });
2738 }
2739 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2740 if !matches!(result, Ok(Ok(true))) {
2741 warn!(?result, "daemon shutdown drain did not reach quiescence");
2742 }
2743 }
2744 Ok(())
2745 }
2746
2747 #[cfg(unix)]
2759 pub(crate) async fn end_children_for_daemon_shutdown(
2760 &self,
2761 already_escalated: bool,
2762 escalate: impl std::future::Future<Output = ()>,
2763 ) {
2764 tokio::pin!(escalate);
2765 let mut escalated = already_escalated;
2766 if let Some(forwarding) = &self.forwarding {
2767 let reason = CloseReason::new(
2768 "daemon_shutdown",
2769 "the daemon is exiting after its shutdown notice and drain",
2770 );
2771 if escalated {
2772 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2775 } else {
2776 tokio::select! {
2777 biased;
2778 _ = escalate.as_mut() => {
2779 info!("second SIGTERM: abandoning module GOODBYE delivery");
2780 escalated = true;
2781 }
2782 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2783 }
2784 }
2785 let closed = forwarding.close_all_connections(&reason);
2786 debug!(closed, "closed established connections for daemon shutdown");
2787 }
2788 let escalated_here = escalated && !already_escalated;
2792 let remaining_escalate = async move {
2793 if escalated_here {
2794 std::future::pending::<()>().await;
2795 } else {
2796 escalate.await;
2797 }
2798 };
2799 crate::child_roster::end_children_for_daemon_shutdown(
2800 &self.child_roster,
2801 escalated,
2802 remaining_escalate,
2803 )
2804 .await;
2805 }
2806
2807 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2808 Self {
2809 registry,
2810 restart_policy,
2811 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2812 connection_file_path: None,
2813 capture_logs_dir: None,
2814 forwarding: None,
2815 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2816 supervisor_handle: None,
2817 health: HealthConfig::default(),
2818 daemon_start_clock: crate::clock::StartClock::capture(),
2819 terminal_journal: None,
2820 spawn_events: SpawnEventFeed::default(),
2821 provenance_probe: ExecutableIdentityProbe::default(),
2822 child_roster: ChildRoster::default(),
2823 #[cfg(target_os = "linux")]
2824 cgroup_placement: None,
2825 #[cfg(test)]
2826 test_after_first_spawn: AfterFirstSpawnHook::default(),
2827 }
2828 }
2829
2830 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2831 self.drain_timeout = drain_timeout;
2832 self
2833 }
2834
2835 pub fn with_process_liveness(
2836 mut self,
2837 process_liveness: Arc<SupervisorProcessLiveness>,
2838 ) -> Self {
2839 self.process_liveness = process_liveness;
2840 self
2841 }
2842
2843 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2844 self.connection_file_path = Some(connection_file_path.into());
2845 self
2846 }
2847
2848 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
2850 self.capture_logs_dir = Some(logs_dir.into());
2851 self
2852 }
2853
2854 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
2857 self.spawn_events.configure_incarnation(daemon_incarnation);
2861 self
2862 }
2863
2864 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
2867 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
2868 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
2869 path,
2870 daemon_incarnation,
2871 )));
2872 this
2873 }
2874
2875 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
2876 self.forwarding = Some(forwarding);
2877 self
2878 }
2879
2880 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
2881 self.spawn_events = supervisor_handle.spawn_events.clone();
2882 self.supervisor_handle = Some(supervisor_handle);
2883 self
2884 }
2885
2886 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
2887 self.health = health;
2888 self
2889 }
2890
2891 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
2895 self.child_roster.record_to(path.into());
2896 self
2897 }
2898
2899 #[cfg(target_os = "linux")]
2900 pub fn with_cgroup_placement(
2901 mut self,
2902 cgroup_placement: Option<subc_cgroup::Placement>,
2903 ) -> Self {
2904 self.cgroup_placement = cgroup_placement;
2905 self
2906 }
2907
2908 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
2914 validate_spec(&spec)?;
2915 self.establish_identity(&spec);
2916
2917 let runtime = self.runtime_config();
2918 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
2919 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
2920 let spawned = spawn_child(
2921 &spec,
2922 runtime.connection_file_path.as_deref(),
2923 self.supervisor_handle.as_ref(),
2924 &runtime.stderr_ring,
2925 runtime.capture_logs_dir.as_deref(),
2926 &runtime.child_roster,
2927 #[cfg(target_os = "linux")]
2928 runtime.cgroup_placement.as_ref(),
2929 );
2930 #[cfg(test)]
2931 self.test_after_first_spawn.run(&spec.module_id);
2932 let child = match spawned {
2933 Ok(child) => child,
2934 Err(err) => {
2935 self.abandon_unrostered(&spec.module_id);
2938 return Err(err);
2939 }
2940 };
2941 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
2942 if let Some(spawn) = activity_spawn {
2943 spawn.complete(&snapshot);
2944 }
2945
2946 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
2947 }
2948
2949 fn establish_identity(&self, spec: &ModuleSpec) {
2962 if let Some(supervisor_handle) = &self.supervisor_handle {
2963 supervisor_handle.apply_identity_configuration(spec);
2964 supervisor_handle.mark_configured(&spec.module_id);
2965 }
2966 }
2967
2968 fn abandon_unrostered(&self, module_id: &str) {
2971 if let Some(supervisor_handle) = &self.supervisor_handle {
2972 supervisor_handle.unmark_configured_unless_rostered(module_id);
2973 }
2974 }
2975
2976 fn mark_first_process_running(
2979 &self,
2980 spec: &ModuleSpec,
2981 runtime: &SupervisorRuntimeConfig,
2982 snapshot: &SharedSnapshot,
2983 child: &SupervisedChild,
2984 ) -> Result<(), SuperviseError> {
2985 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
2986 self.abandon_unrostered(&spec.module_id);
2987 return Err(err);
2988 }
2989 self.process_liveness
2990 .track(spec.module_id.clone(), Arc::clone(snapshot));
2991 Ok(())
2992 }
2993
2994 pub fn supervise_configured(
3000 &self,
3001 spec: ModuleSpec,
3002 enabled: bool,
3003 ) -> Result<SupervisedModule, SuperviseError> {
3004 self.supervise_configured_requested(
3005 spec,
3006 enabled,
3007 self.health.clone(),
3008 None,
3009 self.restart_policy,
3010 None,
3011 )
3012 }
3013
3014 pub fn supervise_configured_with_health(
3020 &self,
3021 spec: ModuleSpec,
3022 enabled: bool,
3023 health: HealthConfig,
3024 drain_timeout_ms: Option<u64>,
3025 restart_policy: RestartPolicy,
3026 ) -> Result<SupervisedModule, SuperviseError> {
3027 self.supervise_configured_requested(
3028 spec,
3029 enabled,
3030 health,
3031 drain_timeout_ms,
3032 restart_policy,
3033 None,
3034 )
3035 }
3036
3037 pub(crate) fn supervise_configured_requested(
3038 &self,
3039 spec: ModuleSpec,
3040 enabled: bool,
3041 health: HealthConfig,
3042 drain_timeout_ms: Option<u64>,
3043 restart_policy: RestartPolicy,
3044 requester: Option<Principal>,
3045 ) -> Result<SupervisedModule, SuperviseError> {
3046 let registered = self.register_configured_with_health(
3047 spec,
3048 enabled,
3049 health,
3050 drain_timeout_ms,
3051 restart_policy,
3052 )?;
3053 self.start_registered_configured(registered, requester)
3054 }
3055
3056 pub(crate) fn register_configured_with_health(
3061 &self,
3062 spec: ModuleSpec,
3063 enabled: bool,
3064 health: HealthConfig,
3065 drain_timeout_ms: Option<u64>,
3066 restart_policy: RestartPolicy,
3067 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3068 validate_spec(&spec)?;
3069 self.establish_identity(&spec);
3070
3071 let mut runtime = self.runtime_config();
3072 runtime.health = health;
3073 runtime.restart_policy = restart_policy;
3074 if let Some(ms) = drain_timeout_ms {
3075 runtime.drain_timeout = Duration::from_millis(ms);
3076 *runtime
3077 .effective_drain_timeout
3078 .lock()
3079 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3080 }
3081 let snapshot = if enabled {
3082 SupervisorSnapshot::starting()
3083 } else {
3084 SupervisorSnapshot::disabled()
3085 };
3086 #[cfg(target_os = "linux")]
3087 let snapshot = {
3088 let mut snapshot = snapshot;
3089 if enabled {
3090 snapshot.waiting_for_previous_pid =
3091 runtime.child_roster.previous_instance_pid(&spec.module_id);
3092 }
3093 snapshot
3094 };
3095 let (module, commands) =
3096 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3097 Ok(RegisteredConfiguredModule {
3098 spec,
3099 runtime,
3100 module,
3101 commands,
3102 })
3103 }
3104
3105 pub(crate) fn start_registered_configured(
3108 &self,
3109 registered: RegisteredConfiguredModule,
3110 requester: Option<Principal>,
3111 ) -> Result<SupervisedModule, SuperviseError> {
3112 let RegisteredConfiguredModule {
3113 spec,
3114 mut runtime,
3115 module,
3116 commands,
3117 } = registered;
3118 runtime.start_requester = requester;
3119 let snapshot = &module.inner.snapshot;
3120 let should_spawn = {
3121 let state = lock_snapshot(snapshot)?;
3122 state.enabled && state.waiting_for_previous_pid.is_none()
3123 };
3124 let child = if should_spawn {
3125 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3126 let spawned = spawn_child(
3127 &spec,
3128 runtime.connection_file_path.as_deref(),
3129 self.supervisor_handle.as_ref(),
3130 &runtime.stderr_ring,
3131 runtime.capture_logs_dir.as_deref(),
3132 &runtime.child_roster,
3133 #[cfg(target_os = "linux")]
3134 runtime.cgroup_placement.as_ref(),
3135 );
3136 #[cfg(test)]
3137 self.test_after_first_spawn.run(&spec.module_id);
3138 match spawned {
3139 Ok(child) => {
3140 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3141 if let Some(spawn) = activity_spawn {
3142 spawn.complete(snapshot);
3143 }
3144 Some(child)
3145 }
3146 Err(err) => {
3147 if runtime.health.critical {
3148 error!(
3149 module_id = %spec.module_id,
3150 program = %spec.program.display(),
3151 error = %err,
3152 "critical configured module failed to spawn; marking failed and alerting"
3153 );
3154 } else {
3155 error!(
3156 module_id = %spec.module_id,
3157 program = %spec.program.display(),
3158 error = %err,
3159 "configured module failed to spawn; marking failed and continuing"
3160 );
3161 }
3162 update_snapshot(snapshot, Some(&spec.module_id), |state| {
3163 state.state = ModuleState::Failed;
3164 clear_current_process_facts(state);
3165 })?;
3166 None
3167 }
3168 }
3169 } else {
3170 None
3171 };
3172 self.start_monitor(spec, runtime, &module, child, commands);
3173 Ok(module)
3174 }
3175
3176 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3177 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3178 SupervisorRuntimeConfig {
3179 start_requester: None,
3180 pending_activity_restart: false,
3181 provenance_probe: self.provenance_probe.clone(),
3182 scheduled_respawn: Arc::default(),
3183 deferred_reload_reply: Arc::default(),
3184 restart_policy: self.restart_policy,
3185 drain_timeout: self.drain_timeout,
3186 child_roster: self
3189 .child_roster
3190 .for_module(Arc::clone(&effective_drain_timeout)),
3191 effective_drain_timeout,
3192 default_drain_timeout: self.drain_timeout,
3193 health: self.health.clone(),
3194 connection_file_path: self.connection_file_path.clone(),
3195 capture_logs_dir: self.capture_logs_dir.clone(),
3196 forwarding: self.forwarding.clone(),
3197 supervisor_handle: self.supervisor_handle.clone(),
3198 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3199 terminal_ring: Arc::new(Mutex::new(
3200 TerminalRing::new(
3201 TerminalRingConfig::default(),
3202 self.daemon_start_clock.started_at_ms(),
3203 )
3204 .with_start_clock(self.daemon_start_clock)
3205 .with_journal(self.terminal_journal.clone())
3206 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3207 )),
3208 spawn_events: self.spawn_events.clone(),
3209 #[cfg(target_os = "linux")]
3210 cgroup_placement: self.cgroup_placement.clone(),
3211 #[cfg(test)]
3212 test_seed_stale_facts_before_enable_spawn: false,
3213 #[cfg(test)]
3214 test_reload_exit_record_gate: None,
3215 }
3216 }
3217
3218 fn supervised_module(
3219 &self,
3220 spec: ModuleSpec,
3221 runtime: SupervisorRuntimeConfig,
3222 snapshot: SharedSnapshot,
3223 child: Option<SupervisedChild>,
3224 ) -> SupervisedModule {
3225 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3226 self.start_monitor(spec, runtime, &module, child, commands);
3227 module
3228 }
3229
3230 fn register_module(
3231 &self,
3232 spec: &ModuleSpec,
3233 runtime: &SupervisorRuntimeConfig,
3234 snapshot: SharedSnapshot,
3235 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3236 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3237 spec: spec.clone(),
3238 health: runtime.health.clone(),
3239 }));
3240 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3241 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3242 let restart_policy = runtime.restart_policy;
3246 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3247 let (tx, rx) = mpsc::channel(4);
3248 let module_id = spec.module_id.clone();
3249 let module = SupervisedModule {
3250 inner: Arc::new(SupervisedModuleInner {
3251 module_id: module_id.clone(),
3252 registry: Arc::clone(&self.registry),
3253 snapshot,
3254 configuration,
3255 stderr_ring,
3256 terminal_ring,
3257 commands: tx,
3258 monitor: Mutex::new(None),
3259 restart_policy,
3260 effective_drain_timeout,
3261 provenance_probe: self.provenance_probe.clone(),
3262 }),
3263 };
3264 if let Some(supervisor_handle) = &self.supervisor_handle {
3267 supervisor_handle.insert(module.clone());
3268 }
3269 (module, rx)
3270 }
3271
3272 fn start_monitor(
3273 &self,
3274 spec: ModuleSpec,
3275 runtime: SupervisorRuntimeConfig,
3276 module: &SupervisedModule,
3277 child: Option<SupervisedChild>,
3278 commands: mpsc::Receiver<SupervisorCommand>,
3279 ) {
3280 let monitor = tokio::spawn(supervise_loop(
3281 spec,
3282 runtime,
3283 Arc::clone(&self.registry),
3284 Arc::clone(&self.process_liveness),
3285 Arc::clone(&module.inner.snapshot),
3286 child,
3287 commands,
3288 ));
3289 *module
3290 .inner
3291 .monitor
3292 .lock()
3293 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3294 }
3295}
3296
3297impl Default for Supervisor {
3298 fn default() -> Self {
3299 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3300 }
3301}
3302
3303#[derive(Clone)]
3305pub struct SupervisedModule {
3306 inner: Arc<SupervisedModuleInner>,
3307}
3308
3309struct SupervisedModuleInner {
3310 module_id: String,
3311 registry: Arc<Registry>,
3312 snapshot: SharedSnapshot,
3313 configuration: Arc<Mutex<SupervisedConfiguration>>,
3314 stderr_ring: Arc<Mutex<StderrRing>>,
3315 terminal_ring: Arc<Mutex<TerminalRing>>,
3316 commands: mpsc::Sender<SupervisorCommand>,
3317 monitor: Mutex<Option<JoinHandle<()>>>,
3318 restart_policy: RestartPolicy,
3322 effective_drain_timeout: Arc<Mutex<Duration>>,
3323 provenance_probe: ExecutableIdentityProbe,
3324}
3325
3326impl fmt::Debug for SupervisedModule {
3327 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3328 f.debug_struct("SupervisedModule")
3329 .field("module_id", &self.inner.module_id)
3330 .field("status", &self.status())
3331 .finish_non_exhaustive()
3332 }
3333}
3334
3335impl SupervisedModule {
3336 pub fn module_id(&self) -> &str {
3337 &self.inner.module_id
3338 }
3339
3340 #[cfg(test)]
3344 pub(crate) fn record_health_probe_failure_for_test(
3345 &self,
3346 detail: &str,
3347 ) -> Result<(), SuperviseError> {
3348 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3349 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3350 state.health.detail = Some(detail.to_string());
3351 })
3352 }
3353
3354 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3355 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3356 }
3357
3358 pub(crate) fn registration_end_reason(
3359 &self,
3360 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3361 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3362 Ok(match snapshot.state {
3363 ModuleState::Draining if snapshot.draining_to_replace => {
3364 Some(RegistrationEndReason::SupervisorRestart)
3365 }
3366 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3367 ModuleState::Restarting if snapshot.draining_to_replace => {
3368 Some(RegistrationEndReason::SupervisorRestart)
3369 }
3370 _ => None,
3371 })
3372 }
3373
3374 pub fn stderr_tail(
3381 &self,
3382 max_lines: Option<usize>,
3383 max_bytes: Option<usize>,
3384 ) -> StderrTailSnapshot {
3385 self.inner
3386 .stderr_ring
3387 .lock()
3388 .unwrap_or_else(|poisoned| poisoned.into_inner())
3389 .snapshot(max_lines, max_bytes)
3390 }
3391
3392 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3397 self.inner
3398 .terminal_ring
3399 .lock()
3400 .unwrap_or_else(|poisoned| poisoned.into_inner())
3401 .snapshot()
3402 }
3403
3404 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3409 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3410 }
3411
3412 pub(crate) async fn read_durable_terminal_history(
3417 &self,
3418 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3419 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3420 let module_id = self.inner.module_id.clone();
3421 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3422 .await
3423 }
3424
3425 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3426 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3427 .map(|(status, _)| status)
3428 }
3429
3430 pub(crate) fn record_deliberate_severance(
3431 &self,
3432 identity: ProcessIdentity,
3433 ) -> Result<bool, SuperviseError> {
3434 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3435 if snapshot.pid != Some(identity.pid)
3436 || snapshot.process_start_time != Some(identity.start_time)
3437 {
3438 return Ok(false);
3439 }
3440 snapshot.deliberate_severance = Some(identity);
3441 Ok(true)
3442 }
3443
3444 pub(crate) fn status_for_control(
3449 &self,
3450 caller: &'static str,
3451 ) -> Result<ModuleStatus, SuperviseError> {
3452 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3453 .map(|(status, _)| status)
3454 }
3455
3456 fn status_with_snapshot_lock(
3457 &self,
3458 snapshot: &SharedSnapshot,
3459 caller: Option<&'static str>,
3460 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3461 let mut guard = match caller {
3462 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3463 None => lock_snapshot(snapshot)?,
3464 };
3465 let restart_count =
3468 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3469 let snapshot = guard.clone();
3470 drop(guard);
3471 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3472 SuperviseError::StatePoisoned {
3473 module_id: Some(self.inner.module_id.clone()),
3474 }
3475 })?;
3476 let registration_active = self
3477 .inner
3478 .registry
3479 .get_module(&self.inner.module_id)
3480 .map_err(SuperviseError::Registry)?
3481 .is_some();
3482 let protocol = snapshot
3483 .spawned_protocol
3484 .unwrap_or(self.declared_protocol()?);
3485 let running_process =
3486 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3487 let live = match protocol {
3493 ModuleProtocol::Subc => running_process && registration_active,
3494 ModuleProtocol::None => running_process,
3495 };
3496
3497 Ok((
3498 ModuleStatus {
3499 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3500 module_id: self.inner.module_id.clone(),
3501 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3502 state: snapshot.state,
3503 enabled: snapshot.enabled,
3504 process_alive: snapshot.process_alive,
3505 registration_active,
3506 protocol,
3507 live,
3508 restart_count,
3509 lifetime_restarts: snapshot.lifetime_restarts,
3510 spawn_generation: snapshot.spawn_generation,
3511 max_restarts: self.inner.restart_policy.max_restarts,
3512 restart_window: self.inner.restart_policy.window,
3513 drain_timeout,
3514 restart_backoff: self.inner.restart_policy.backoff,
3515 restart_max_backoff: self.inner.restart_policy.max_backoff,
3516 pid: snapshot.reported_pid(),
3517 spawned_at_ms: snapshot.spawned_at_ms,
3518 spawned_from: snapshot.spawned_from,
3519 process_start_time: snapshot.process_start_time,
3520 last_exit: snapshot.last_exit,
3521 health: snapshot.health,
3522 },
3523 snapshot.spawned_file_identity.clone(),
3524 ))
3525 }
3526
3527 #[cfg(test)]
3528 pub(crate) fn hold_snapshot_for_test(
3529 &self,
3530 acquired: std::sync::mpsc::Sender<()>,
3531 hold: Duration,
3532 ) -> std::thread::JoinHandle<()> {
3533 let snapshot = Arc::clone(&self.inner.snapshot);
3534 std::thread::spawn(move || {
3535 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3536 acquired
3537 .send(())
3538 .expect("test receiver waits for snapshot lock");
3539 std::thread::sleep(hold);
3540 })
3541 }
3542
3543 pub(crate) async fn status_and_running_image_agreement(
3549 &self,
3550 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3551 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3552 let image = self
3553 .inner
3554 .provenance_probe
3555 .observe(
3556 status.pid,
3557 status.spawned_from.as_deref(),
3558 identity,
3559 status.process_start_time,
3560 )
3561 .await;
3562 Ok((status, image))
3563 }
3564
3565 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3566 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3567 Ok(snapshot) => snapshot.clone(),
3568 Err(_) => {
3569 return subc_control::RunningImageAgreement::Unavailable {
3570 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3571 };
3572 }
3573 };
3574 self.inner
3575 .provenance_probe
3576 .observe(
3577 snapshot.reported_pid(),
3578 snapshot.spawned_from.as_deref(),
3579 snapshot.spawned_file_identity.clone(),
3580 snapshot.process_start_time,
3581 )
3582 .await
3583 }
3584
3585 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3588 #[cfg(windows)]
3589 {
3590 let image = lock_snapshot(&self.inner.snapshot)
3591 .ok()
3592 .and_then(|snapshot| {
3593 snapshot
3594 .spawned_file_identity
3595 .as_ref()
3596 .and_then(|identity| identity.windows_image.clone())
3597 });
3598 match image {
3599 Some(image) => {
3600 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3601 }
3602 None => subc_control::ChildResourceUsage::Unavailable {
3603 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3604 },
3605 }
3606 }
3607 #[cfg(not(windows))]
3608 {
3609 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3610 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3611 Err(_) => {
3612 return subc_control::ChildResourceUsage::Unavailable {
3613 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3614 }
3615 }
3616 };
3617 crate::child_resources::read(pid, start_time)
3618 }
3619 }
3620
3621 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3622 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3623 Ok(match snapshot.state {
3624 ModuleState::Restarting => true,
3625 ModuleState::Failed | ModuleState::Disabled => false,
3626 _ if snapshot.enabled && snapshot.respawn_pending => true,
3627 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3628 })
3629 }
3630
3631 #[cfg(test)]
3632 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3633 self.is_warming_with_snapshot_lock(None)
3634 }
3635
3636 pub(crate) fn is_warming_for_control(
3637 &self,
3638 caller: &'static str,
3639 ) -> Result<bool, SuperviseError> {
3640 self.is_warming_with_snapshot_lock(Some(caller))
3641 }
3642
3643 fn is_warming_with_snapshot_lock(
3644 &self,
3645 caller: Option<&'static str>,
3646 ) -> Result<bool, SuperviseError> {
3647 let snapshot = match caller {
3648 Some(caller) => {
3649 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3650 }
3651 None => lock_snapshot(&self.inner.snapshot)?,
3652 }
3653 .clone();
3654 Ok(matches!(
3655 snapshot.state,
3656 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3657 ))
3658 }
3659
3660 pub async fn drain(&self) -> Result<(), SuperviseError> {
3662 self.stop().await
3663 }
3664
3665 pub(crate) async fn retire_requested(
3666 &self,
3667 requester: Option<Principal>,
3668 ) -> Result<(), SuperviseError> {
3669 match self.state()? {
3670 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3671 ModuleState::Starting
3672 | ModuleState::Running
3673 | ModuleState::Unresponsive
3674 | ModuleState::Restarting
3675 | ModuleState::Draining
3676 | ModuleState::Disabled => {}
3677 }
3678
3679 let (reply_tx, reply_rx) = oneshot::channel();
3680 self.inner
3681 .commands
3682 .send(SupervisorCommand::Retire {
3683 requester,
3684 reply: reply_tx,
3685 })
3686 .await
3687 .map_err(|_| SuperviseError::CommandClosed {
3688 module_id: self.inner.module_id.clone(),
3689 })?;
3690 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3691 module_id: self.inner.module_id.clone(),
3692 })?
3693 }
3694
3695 pub async fn stop(&self) -> Result<(), SuperviseError> {
3696 self.stop_requested(None).await
3697 }
3698
3699 pub(crate) async fn stop_requested(
3700 &self,
3701 requester: Option<Principal>,
3702 ) -> Result<(), SuperviseError> {
3703 match self.state()? {
3704 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3705 ModuleState::Starting
3706 | ModuleState::Running
3707 | ModuleState::Unresponsive
3708 | ModuleState::Restarting
3709 | ModuleState::Draining
3710 | ModuleState::Disabled => {}
3711 }
3712
3713 let (reply_tx, reply_rx) = oneshot::channel();
3714 self.inner
3715 .commands
3716 .send(SupervisorCommand::Drain {
3717 requester,
3718 reply: reply_tx,
3719 })
3720 .await
3721 .map_err(|_| SuperviseError::CommandClosed {
3722 module_id: self.inner.module_id.clone(),
3723 })?;
3724 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3725 module_id: self.inner.module_id.clone(),
3726 })?
3727 }
3728
3729 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3730 self.restart_requested(drain_timeout_ms, None).await
3731 }
3732
3733 pub(crate) async fn restart_requested(
3734 &self,
3735 drain_timeout_ms: Option<u64>,
3736 requester: Option<Principal>,
3737 ) -> Result<(), SuperviseError> {
3738 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3739 let (reply_tx, reply_rx) = oneshot::channel();
3740 self.inner
3741 .commands
3742 .send(SupervisorCommand::Restart {
3743 requester,
3744 drain_timeout_ms,
3745 received_at_generation,
3746 queued_at: Instant::now(),
3747 reply: reply_tx,
3748 })
3749 .await
3750 .map_err(|_| SuperviseError::CommandClosed {
3751 module_id: self.inner.module_id.clone(),
3752 })?;
3753 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3754 module_id: self.inner.module_id.clone(),
3755 })?
3756 }
3757
3758 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3763 self.swap_requested(ready_timeout, None).await
3764 }
3765
3766 pub(crate) async fn swap_requested(
3767 &self,
3768 ready_timeout: Option<Duration>,
3769 requester: Option<Principal>,
3770 ) -> Result<(), SuperviseError> {
3771 let (reply_tx, reply_rx) = oneshot::channel();
3772 self.inner
3773 .commands
3774 .send(SupervisorCommand::Swap {
3775 requester,
3776 ready_timeout,
3777 reply: reply_tx,
3778 })
3779 .await
3780 .map_err(|_| SuperviseError::CommandClosed {
3781 module_id: self.inner.module_id.clone(),
3782 })?;
3783 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3784 module_id: self.inner.module_id.clone(),
3785 })?
3786 }
3787
3788 pub async fn reload(&self) -> Result<(), SuperviseError> {
3789 self.reload_requested(None).await
3790 }
3791
3792 pub(crate) async fn reload_requested(
3793 &self,
3794 requester: Option<Principal>,
3795 ) -> Result<(), SuperviseError> {
3796 let (reply_tx, reply_rx) = oneshot::channel();
3797 self.inner
3798 .commands
3799 .send(SupervisorCommand::Reload {
3800 requester,
3801 reply: reply_tx,
3802 })
3803 .await
3804 .map_err(|_| SuperviseError::CommandClosed {
3805 module_id: self.inner.module_id.clone(),
3806 })?;
3807 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3808 module_id: self.inner.module_id.clone(),
3809 })?
3810 }
3811
3812 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3813 self.set_enabled_requested(enabled, None).await
3814 }
3815
3816 pub(crate) async fn set_enabled_requested(
3817 &self,
3818 enabled: bool,
3819 requester: Option<Principal>,
3820 ) -> Result<bool, SuperviseError> {
3821 let (reply_tx, reply_rx) = oneshot::channel();
3822 self.inner
3823 .commands
3824 .send(SupervisorCommand::SetEnabled {
3825 requester,
3826 enabled,
3827 reply: reply_tx,
3828 })
3829 .await
3830 .map_err(|_| SuperviseError::CommandClosed {
3831 module_id: self.inner.module_id.clone(),
3832 })?;
3833 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3834 module_id: self.inner.module_id.clone(),
3835 })?
3836 }
3837
3838 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3842 let configured = self
3843 .inner
3844 .configuration
3845 .lock()
3846 .map_err(|_| SuperviseError::StatePoisoned {
3847 module_id: Some(self.inner.module_id.clone()),
3848 })?
3849 .spec
3850 .protocol;
3851 let state = lock_snapshot(&self.inner.snapshot)?;
3852 Ok(state.spawned_protocol.unwrap_or(configured))
3853 }
3854
3855 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
3856 let configuration =
3857 self.inner
3858 .configuration
3859 .lock()
3860 .map_err(|_| SuperviseError::StatePoisoned {
3861 module_id: Some(self.inner.module_id.clone()),
3862 })?;
3863 Ok((configuration.spec.clone(), configuration.health.clone()))
3864 }
3865
3866 #[cfg(any(test, feature = "test-support"))]
3870 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
3871 let (_, health) = self.configuration()?;
3872 let drain_timeout_ms = u64::try_from(
3873 self.inner
3874 .effective_drain_timeout
3875 .lock()
3876 .unwrap_or_else(|poisoned| poisoned.into_inner())
3877 .as_millis(),
3878 )
3879 .ok();
3880 self.update_configuration(spec, health, drain_timeout_ms)
3881 .await
3882 }
3883
3884 pub(crate) async fn update_configuration(
3885 &self,
3886 spec: ModuleSpec,
3887 health: HealthConfig,
3888 drain_timeout_ms: Option<u64>,
3889 ) -> Result<(), SuperviseError> {
3890 if spec.module_id != self.inner.module_id {
3891 return Err(SuperviseError::InvalidSpec {
3892 reason: "a supervised module's module_id cannot be changed".to_string(),
3893 });
3894 }
3895 validate_spec(&spec)?;
3896 let (reply_tx, reply_rx) = oneshot::channel();
3897 self.inner
3898 .commands
3899 .send(SupervisorCommand::UpdateConfiguration {
3900 spec: spec.clone(),
3901 health: health.clone(),
3902 drain_timeout_ms,
3903 reply: reply_tx,
3904 })
3905 .await
3906 .map_err(|_| SuperviseError::CommandClosed {
3907 module_id: self.inner.module_id.clone(),
3908 })?;
3909 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3910 module_id: self.inner.module_id.clone(),
3911 })?;
3912 let mut configuration =
3913 self.inner
3914 .configuration
3915 .lock()
3916 .map_err(|_| SuperviseError::StatePoisoned {
3917 module_id: Some(self.inner.module_id.clone()),
3918 })?;
3919 configuration.spec = spec;
3920 configuration.health = health;
3921 Ok(())
3922 }
3923}
3924
3925impl Drop for SupervisedModuleInner {
3926 fn drop(&mut self) {
3927 let Ok(mut monitor) = self.monitor.lock() else {
3928 return;
3929 };
3930 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
3931 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
3932 state.state = ModuleState::Stopped;
3933 clear_current_process_facts(state);
3934 });
3935 monitor.abort();
3936 }
3937 let _ = monitor.take();
3938 }
3939}
3940
3941#[derive(Debug)]
3942enum SupervisorCommand {
3943 Drain {
3944 requester: Option<Principal>,
3945 reply: oneshot::Sender<Result<(), SuperviseError>>,
3946 },
3947 Retire {
3948 requester: Option<Principal>,
3949 reply: oneshot::Sender<Result<(), SuperviseError>>,
3950 },
3951 Restart {
3952 requester: Option<Principal>,
3953 drain_timeout_ms: Option<u64>,
3958 received_at_generation: u64,
3962 queued_at: Instant,
3965 reply: oneshot::Sender<Result<(), SuperviseError>>,
3966 },
3967 Reload {
3968 requester: Option<Principal>,
3969 reply: oneshot::Sender<Result<(), SuperviseError>>,
3970 },
3971 SetEnabled {
3972 requester: Option<Principal>,
3973 enabled: bool,
3974 reply: oneshot::Sender<Result<bool, SuperviseError>>,
3975 },
3976 UpdateConfiguration {
3977 spec: ModuleSpec,
3978 health: HealthConfig,
3979 drain_timeout_ms: Option<u64>,
3982 reply: oneshot::Sender<()>,
3983 },
3984 Swap {
3985 requester: Option<Principal>,
3986 ready_timeout: Option<Duration>,
3989 reply: oneshot::Sender<Result<(), SuperviseError>>,
3991 },
3992}
3993
3994#[derive(Debug)]
3995pub enum SuperviseError {
3996 InvalidSpec {
3997 reason: String,
3998 },
3999 Spawn {
4000 program: PathBuf,
4001 source: io::Error,
4002 cgroup_path: Option<PathBuf>,
4003 },
4004 Cgroup {
4005 module_id: String,
4006 source: io::Error,
4007 },
4008 LaunchNonce {
4011 reason: String,
4012 },
4013 Wait {
4014 module_id: String,
4015 source: io::Error,
4016 },
4017 Kill {
4018 module_id: String,
4019 source: io::Error,
4020 },
4021 Forwarding(ForwardingError),
4022 Registry(RegistryError),
4023 ReloadUnavailable {
4024 module_id: String,
4025 reason: String,
4026 },
4027 Disabled {
4032 module_id: String,
4033 },
4034 ReloadFailed {
4035 module_id: String,
4036 reason: String,
4037 },
4038 RegistrationStillActive {
4039 module_id: String,
4040 waited: Duration,
4041 },
4042 StatePoisoned {
4043 module_id: Option<String>,
4044 },
4045 CommandClosed {
4046 module_id: String,
4047 },
4048 SwapInProgress {
4052 module_id: String,
4053 },
4054 SwapRefused {
4056 module_id: String,
4057 reason: SwapRefusal,
4058 },
4059 SwapFailed {
4063 module_id: String,
4064 arm: SwapFailureArm,
4065 detail: String,
4066 candidate_exit: Option<ExitReport>,
4069 },
4070}
4071
4072#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4074pub enum SwapRefusal {
4075 OverlapExclusive,
4077 NotRegistered,
4080 ProtocolNone,
4083 NotConfigured,
4086 AlreadySwapping,
4088}
4089
4090impl SwapRefusal {
4091 pub fn as_str(self) -> &'static str {
4092 match self {
4093 Self::OverlapExclusive => "overlap_exclusive",
4094 Self::NotRegistered => "not_registered",
4095 Self::ProtocolNone => "protocol_none",
4096 Self::NotConfigured => "not_configured",
4097 Self::AlreadySwapping => "already_swapping",
4098 }
4099 }
4100}
4101
4102#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4105pub enum SwapFailureArm {
4106 SpawnFailed,
4108 NeverRegistered,
4110 NeverReady,
4112 CandidateExited,
4114 CandidateUnhealthy,
4116 Interrupted,
4120 CutoverLost,
4125}
4126
4127impl SwapFailureArm {
4128 pub fn as_str(self) -> &'static str {
4129 match self {
4130 Self::SpawnFailed => "spawn_failed",
4131 Self::NeverRegistered => "never_registered",
4132 Self::NeverReady => "never_ready",
4133 Self::CandidateExited => "candidate_exited",
4134 Self::CandidateUnhealthy => "candidate_unhealthy",
4135 Self::Interrupted => "interrupted",
4136 Self::CutoverLost => "cutover_lost",
4137 }
4138 }
4139}
4140
4141impl fmt::Display for SuperviseError {
4142 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4143 match self {
4144 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4145 Self::Spawn {
4146 program,
4147 source,
4148 cgroup_path: Some(cgroup_path),
4149 } => write!(
4150 f,
4151 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4152 cgroup_path.display(),
4153 program.display()
4154 ),
4155 Self::Spawn {
4156 program,
4157 source,
4158 cgroup_path: None,
4159 } => write!(
4160 f,
4161 "failed to spawn module '{}': {source}",
4162 program.display()
4163 ),
4164 Self::Cgroup { module_id, source } => {
4165 write!(
4166 f,
4167 "failed to prepare cgroup for module '{module_id}': {source}"
4168 )
4169 }
4170 Self::LaunchNonce { reason } => {
4171 write!(
4172 f,
4173 "failed to generate reserved-module launch nonce: {reason}"
4174 )
4175 }
4176 Self::Wait { module_id, source } => {
4177 write!(f, "failed to wait for module '{module_id}': {source}")
4178 }
4179 Self::Kill { module_id, source } => {
4180 write!(f, "failed to kill module '{module_id}': {source}")
4181 }
4182 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4183 Self::Registry(err) => write!(f, "registry error: {err}"),
4184 Self::ReloadUnavailable { module_id, reason } => {
4185 write!(f, "reload unavailable for module '{module_id}': {reason}")
4186 }
4187 Self::Disabled { module_id } => {
4188 write!(
4189 f,
4190 "module '{module_id}' is disabled; enable it before restart or reload"
4191 )
4192 }
4193 Self::ReloadFailed { module_id, reason } => {
4194 write!(f, "reload failed for module '{module_id}': {reason}")
4195 }
4196 Self::RegistrationStillActive { module_id, waited } => write!(
4197 f,
4198 "module '{module_id}' registration remained active after waiting {waited:?}"
4199 ),
4200 Self::StatePoisoned { module_id } => match module_id {
4201 Some(module_id) => {
4202 write!(f, "supervisor state for module '{module_id}' was poisoned")
4203 }
4204 None => write!(f, "supervisor state was poisoned"),
4205 },
4206 Self::CommandClosed { module_id } => {
4207 write!(
4208 f,
4209 "supervisor command channel for module '{module_id}' is closed"
4210 )
4211 }
4212 Self::SwapInProgress { module_id } => write!(
4213 f,
4214 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4215 ),
4216 Self::SwapRefused { module_id, reason } => match reason {
4217 SwapRefusal::OverlapExclusive => write!(
4218 f,
4219 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4220 ),
4221 SwapRefusal::NotRegistered => write!(
4222 f,
4223 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4224 ),
4225 SwapRefusal::ProtocolNone => write!(
4226 f,
4227 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4228 ),
4229 SwapRefusal::NotConfigured => write!(
4230 f,
4231 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4232 ),
4233 SwapRefusal::AlreadySwapping => {
4234 write!(f, "module '{module_id}' is already being swapped")
4235 }
4236 },
4237 Self::SwapFailed {
4238 module_id,
4239 arm,
4240 detail,
4241 ..
4242 } => write!(
4243 f,
4244 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4245 arm.as_str()
4246 ),
4247 }
4248 }
4249}
4250
4251impl Error for SuperviseError {
4252 fn source(&self) -> Option<&(dyn Error + 'static)> {
4253 match self {
4254 Self::Spawn { source, .. }
4255 | Self::Cgroup { source, .. }
4256 | Self::Wait { source, .. }
4257 | Self::Kill { source, .. } => Some(source),
4258 Self::Forwarding(err) => Some(err),
4259 Self::Registry(err) => Some(err),
4260 Self::LaunchNonce { .. }
4261 | Self::InvalidSpec { .. }
4262 | Self::ReloadUnavailable { .. }
4263 | Self::Disabled { .. }
4264 | Self::ReloadFailed { .. }
4265 | Self::RegistrationStillActive { .. }
4266 | Self::StatePoisoned { .. }
4267 | Self::CommandClosed { .. }
4268 | Self::SwapInProgress { .. }
4269 | Self::SwapRefused { .. }
4270 | Self::SwapFailed { .. } => None,
4271 }
4272 }
4273}
4274
4275pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4276 if spec.module_id.trim().is_empty() {
4277 return Err(SuperviseError::InvalidSpec {
4278 reason: "module_id must not be empty".to_string(),
4279 });
4280 }
4281
4282 Ok(())
4283}
4284
4285#[derive(Debug, Default)]
4286struct HealthProbeRuntime {
4287 configured_health: Option<HealthConfig>,
4288 registered_connection: Option<crate::ConnectionId>,
4289 advertised: bool,
4290 next_probe_at: Option<Instant>,
4291 probe_index: u64,
4292}
4293
4294fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4295 lock_snapshot(snapshot)
4296 .ok()
4297 .and_then(|state| state.spawned_protocol)
4298 .unwrap_or(spec.protocol)
4299}
4300
4301impl HealthProbeRuntime {
4302 fn refresh_registration(
4303 &mut self,
4304 spec: &ModuleSpec,
4305 runtime: &SupervisorRuntimeConfig,
4306 registry: &Registry,
4307 snapshot: &SharedSnapshot,
4308 ) {
4309 if self.configured_health.as_ref() != Some(&runtime.health) {
4310 self.configured_health = Some(runtime.health.clone());
4311 self.next_probe_at = None;
4312 self.registered_connection = None;
4313 self.probe_index = 0;
4314 }
4315 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4319 self.registered_connection = None;
4320 self.advertised = runtime.health.http.is_some();
4321 if !self.advertised {
4322 self.next_probe_at = None;
4323 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4324 let unknown = ModuleHealthStatus::default();
4325 if state.health != unknown {
4326 state.health = unknown;
4327 }
4328 });
4329 } else if self.next_probe_at.is_none() {
4330 self.next_probe_at = Some(
4331 Instant::now()
4332 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4333 );
4334 }
4335 return;
4336 }
4337
4338 let registration = match registry.get_module(&spec.module_id) {
4339 Ok(registration) => registration,
4340 Err(err) => {
4341 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4342 self.advertised = false;
4343 self.next_probe_at = None;
4344 return;
4345 }
4346 };
4347
4348 let Some(registration) = registration else {
4349 self.registered_connection = None;
4350 self.advertised = false;
4351 self.next_probe_at = None;
4352 return;
4353 };
4354
4355 let advertised = registration
4356 .control_ops
4357 .iter()
4358 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4359 if !advertised {
4360 self.registered_connection = Some(registration.connection_id);
4361 self.advertised = false;
4362 self.next_probe_at = None;
4363 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4364 if state.health.status != SupervisorHealthStatus::Unknown
4365 || state.health.consecutive_failures != 0
4366 || state.health.last_probe_ms.is_some()
4367 || state.health.detail.is_some()
4368 || state.health.metrics.is_some()
4369 {
4370 state.health.status = SupervisorHealthStatus::Unknown;
4371 state.health.consecutive_failures = 0;
4372 state.health.last_probe_ms = None;
4373 state.health.detail = None;
4374 state.health.metrics = None;
4375 }
4376 });
4377 return;
4378 }
4379
4380 let reregistered = self.registered_connection != Some(registration.connection_id);
4381 self.registered_connection = Some(registration.connection_id);
4382 self.advertised = true;
4383 if reregistered || self.next_probe_at.is_none() {
4384 self.probe_index = 0;
4385 self.next_probe_at = Some(
4386 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4387 );
4388 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4389 state.health.status = SupervisorHealthStatus::Unknown;
4390 state.health.consecutive_failures = 0;
4391 state.health.detail = None;
4392 state.health.metrics = None;
4393 });
4394 }
4395 }
4396
4397 fn wake_after(&self) -> Option<Duration> {
4398 if !self.advertised {
4399 return None;
4400 }
4401 self.next_probe_at
4402 .map(|next| next.saturating_duration_since(Instant::now()))
4403 }
4404
4405 fn due(&self) -> bool {
4406 self.advertised
4407 && self
4408 .next_probe_at
4409 .is_some_and(|next| Instant::now() >= next)
4410 }
4411
4412 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4413 self.probe_index = self.probe_index.wrapping_add(1);
4414 self.next_probe_at = Some(
4415 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4416 );
4417 }
4418}
4419
4420#[derive(Debug)]
4455enum HealthProbeEvidence {
4456 LaneDead,
4458 NoAnswer,
4460 BadAnswer,
4462 Misconfigured,
4464}
4465
4466#[derive(Debug)]
4467struct HealthProbeError {
4468 evidence: HealthProbeEvidence,
4469 message: String,
4470}
4471
4472impl HealthProbeError {
4473 fn lane_dead(message: impl Into<String>) -> Self {
4474 Self::with(HealthProbeEvidence::LaneDead, message)
4475 }
4476
4477 fn no_answer(message: impl Into<String>) -> Self {
4478 Self::with(HealthProbeEvidence::NoAnswer, message)
4479 }
4480
4481 fn bad_answer(message: impl Into<String>) -> Self {
4482 Self::with(HealthProbeEvidence::BadAnswer, message)
4483 }
4484
4485 fn misconfigured(message: impl Into<String>) -> Self {
4486 Self::with(HealthProbeEvidence::Misconfigured, message)
4487 }
4488
4489 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4490 Self {
4491 evidence,
4492 message: message.into(),
4493 }
4494 }
4495
4496 #[allow(dead_code)]
4510 fn is_proof_of_death(&self) -> bool {
4511 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4512 }
4513
4514 fn label(&self) -> &'static str {
4522 match self.evidence {
4523 HealthProbeEvidence::LaneDead => "lane-dead",
4524 HealthProbeEvidence::NoAnswer => "no-answer",
4525 HealthProbeEvidence::BadAnswer => "bad-answer",
4526 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4527 }
4528 }
4529}
4530
4531impl fmt::Display for HealthProbeError {
4532 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4533 f.write_str(&self.message)
4534 }
4535}
4536
4537async fn run_health_probe_cycle(
4538 spec: &ModuleSpec,
4539 runtime: &SupervisorRuntimeConfig,
4540 registry: &Registry,
4541 process_liveness: &SupervisorProcessLiveness,
4542 snapshot: &SharedSnapshot,
4543 child: &mut Option<SupervisedChild>,
4544) {
4545 let now_ms = unix_ms_now();
4546 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4547 .then_some(runtime.health.http.as_deref())
4548 .flatten();
4549 let result = match http {
4550 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4551 None => probe_module_health(&spec.module_id, runtime, None).await,
4552 };
4553 match result {
4554 Ok(report) => {
4555 handle_health_report(
4556 spec,
4557 runtime,
4558 registry,
4559 process_liveness,
4560 snapshot,
4561 child,
4562 report,
4563 now_ms,
4564 )
4565 .await;
4566 }
4567 Err(err) => {
4568 if http.is_some() {
4569 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4570 state.health.status = SupervisorHealthStatus::Failing;
4571 });
4572 }
4573 handle_health_probe_failure(
4574 spec,
4575 runtime,
4576 registry,
4577 process_liveness,
4578 snapshot,
4579 child,
4580 err,
4581 now_ms,
4582 )
4583 .await;
4584 }
4585 }
4586}
4587
4588pub(crate) struct HttpProbeTarget<'a> {
4589 address: std::net::SocketAddr,
4590 localhost: bool,
4591 authority: &'a str,
4592 path: String,
4593}
4594
4595pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4598 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4599 return Err("must not contain whitespace, controls, or a fragment".into());
4600 }
4601 let rest = url
4602 .strip_prefix("http://")
4603 .ok_or("must use plain http://")?;
4604 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4605 let (authority, suffix) = rest.split_at(split);
4606 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4607 ("::1", rest)
4608 } else {
4609 let split = authority.find(':').unwrap_or(authority.len());
4610 authority.split_at(split)
4611 };
4612 let ip: std::net::IpAddr = match host {
4613 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4614 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4615 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4616 };
4617 let port = if port.is_empty() {
4618 80
4619 } else {
4620 port.strip_prefix(':')
4621 .and_then(|p| p.parse::<u16>().ok())
4622 .filter(|p| *p > 0)
4623 .ok_or("must have a valid nonzero TCP port")?
4624 };
4625 let path = if suffix.is_empty() {
4626 "/".into()
4627 } else if suffix.starts_with('?') {
4628 format!("/{suffix}")
4629 } else {
4630 suffix.into()
4631 };
4632 Ok(HttpProbeTarget {
4633 address: std::net::SocketAddr::new(ip, port),
4634 localhost: host == "localhost",
4635 authority,
4636 path,
4637 })
4638}
4639
4640async fn probe_http_health(
4641 url: &str,
4642 deadline: Duration,
4643) -> Result<HealthReport, HealthProbeError> {
4644 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4645 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4646 let mut response_status = String::new();
4649 let mut body = Vec::new();
4650 let probe = async {
4651 let connection = match tokio::net::TcpStream::connect(target.address).await {
4654 Err(_) if target.localhost => {
4655 tokio::net::TcpStream::connect((
4656 std::net::Ipv6Addr::LOCALHOST,
4657 target.address.port(),
4658 ))
4659 .await
4660 }
4661 result => result,
4662 };
4663 let mut stream = connection.map_err(|error| {
4664 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4665 })?;
4666 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4667 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4668 let mut reader = BufReader::new(stream);
4669 let mut budget = 16 * 1024;
4670 let status = http_line(&mut reader, &mut budget).await?;
4671 let mut words = status.split_ascii_whitespace();
4672 let version = words.next();
4673 let code = words
4674 .next()
4675 .filter(|word| word.len() == 3)
4676 .and_then(|word| word.parse::<u16>().ok());
4677 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4678 || !code.is_some_and(|code| (100..600).contains(&code))
4679 {
4680 return Err(HealthProbeError::bad_answer(format!(
4681 "invalid HTTP status: {status}"
4682 )));
4683 }
4684 let code = code.expect("validated status code");
4685 response_status = status.clone();
4686 let mut length = None;
4687 let mut chunked = false;
4688 loop {
4689 let line = http_line(&mut reader, &mut budget).await?;
4690 if line.is_empty() {
4691 break;
4692 }
4693 if let Some((name, value)) = line.split_once(':') {
4694 if name.eq_ignore_ascii_case("content-length") {
4695 length = Some(value.trim().parse::<u64>().map_err(|_| {
4696 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4697 })?);
4698 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4699 chunked = value.trim().eq_ignore_ascii_case("chunked");
4700 }
4701 }
4702 }
4703 if chunked {
4704 while body.len() < 200 {
4705 let line = http_line(&mut reader, &mut budget).await?;
4706 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4707 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4708 if size == 0 {
4709 break;
4710 }
4711 let count = size.min((200 - body.len()) as u64) as usize;
4712 let start = body.len();
4713 (&mut reader)
4714 .take(count as u64)
4715 .read_to_end(&mut body)
4716 .await
4717 .map_err(|error| {
4718 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4719 })?;
4720 if body.len() - start != count {
4721 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4722 }
4723 if size > count as u64 || body.len() == 200 {
4724 break;
4725 }
4726 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4727 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4728 }
4729 }
4730 } else {
4731 reader
4732 .take(length.unwrap_or(200).min(200))
4733 .read_to_end(&mut body)
4734 .await
4735 .map_err(|error| {
4736 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4737 })?;
4738 }
4739 if (200..300).contains(&code) {
4740 Ok(HealthReport::ok())
4741 } else {
4742 Err(HealthProbeError::bad_answer(
4743 "HTTP health endpoint returned non-2xx",
4744 ))
4745 }
4746 };
4747 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4748 Err(HealthProbeError::no_answer(format!(
4749 "HTTP probe timed out after {deadline:?}"
4750 )))
4751 });
4752 if let Err(error) = &mut result {
4753 if !response_status.is_empty() {
4754 error.message = format!(
4755 "{}; {response_status}: {}",
4756 error.message,
4757 String::from_utf8_lossy(&body)
4758 );
4759 }
4760 }
4761 result
4762}
4763
4764async fn http_line(
4765 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4766 remaining: &mut usize,
4767) -> Result<String, HealthProbeError> {
4768 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4769 let mut line = Vec::new();
4770 (&mut *reader)
4771 .take(*remaining as u64)
4772 .read_until(b'\n', &mut line)
4773 .await
4774 .map_err(|error| {
4775 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4776 })?;
4777 *remaining -= line.len();
4778 if !line.ends_with(b"\r\n") {
4779 return Err(HealthProbeError::bad_answer(
4780 "HTTP headers are incomplete or exceed 16 KiB",
4781 ));
4782 }
4783 line.truncate(line.len() - 2);
4784 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4785}
4786
4787async fn probe_module_health(
4788 module_id: &str,
4789 runtime: &SupervisorRuntimeConfig,
4790 drain_deadline: Option<Instant>,
4791) -> Result<HealthReport, HealthProbeError> {
4792 let Some(forwarding) = runtime.forwarding.as_ref() else {
4793 return Err(HealthProbeError::misconfigured(
4794 "supervisor was not configured with a forwarding table",
4795 ));
4796 };
4797 let probe_started_at = Instant::now();
4798 let mut deadline = probe_started_at + runtime.health.deadline;
4799 if let Some(drain_deadline) = drain_deadline {
4800 deadline = deadline.min(drain_deadline);
4801 }
4802 let pending = if drain_deadline.is_some() {
4803 forwarding.begin_drain_health_probe_rpc_for(
4804 module_id,
4805 MODULE_CONTROL_OP_HEALTH_CHECK,
4806 probe_started_at,
4807 deadline,
4808 )
4809 } else {
4810 forwarding.begin_health_probe_rpc_for(
4811 module_id,
4812 MODULE_CONTROL_OP_HEALTH_CHECK,
4813 probe_started_at,
4814 deadline,
4815 )
4816 }
4817 .map_err(|err| {
4818 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4821 })?;
4822 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4823}
4824
4825async fn probe_endpoint_health(
4832 endpoint: crate::ModuleEndpointId,
4833 runtime: &SupervisorRuntimeConfig,
4834 deadline_cap: Option<Instant>,
4835) -> Result<HealthReport, HealthProbeError> {
4836 let Some(forwarding) = runtime.forwarding.as_ref() else {
4837 return Err(HealthProbeError::misconfigured(
4838 "supervisor was not configured with a forwarding table",
4839 ));
4840 };
4841 let probe_started_at = Instant::now();
4842 let mut deadline = probe_started_at + runtime.health.deadline;
4843 if let Some(cap) = deadline_cap {
4844 deadline = deadline.min(cap);
4845 }
4846 let pending = forwarding
4847 .begin_endpoint_health_probe_rpc_for(
4848 endpoint,
4849 MODULE_CONTROL_OP_HEALTH_CHECK,
4850 probe_started_at,
4851 deadline,
4852 )
4853 .map_err(|err| {
4854 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4855 })?;
4856 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4857}
4858
4859async fn await_health_probe(
4861 forwarding: &ForwardingTable,
4862 pending: PendingModuleControlRpc,
4863 deadline: Instant,
4864 probe_budget: Duration,
4865) -> Result<HealthReport, HealthProbeError> {
4866 let PendingModuleControlRpc {
4867 endpoint,
4868 module_sink,
4869 negotiated_ver,
4870 corr,
4871 receiver,
4872 } = pending;
4873 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4874 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
4875 })?;
4876 let frame = Frame::build_with_version(
4877 negotiated_ver,
4878 FrameType::Request,
4879 control_flags(),
4880 0,
4881 0,
4882 corr,
4883 body,
4884 )
4885 .map_err(|err| {
4886 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
4887 })?;
4888
4889 match timeout_at(deadline, module_sink.send(frame)).await {
4895 Ok(Ok(())) => {}
4896 Ok(Err(err)) => {
4897 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
4898 return Err(HealthProbeError::lane_dead(format!(
4901 "failed to send health.check: {err}"
4902 )));
4903 }
4904 Err(_elapsed) => {
4905 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
4906 return Err(HealthProbeError::no_answer(
4910 "health.check send timed out before enqueue (module egress full)",
4911 ));
4912 }
4913 }
4914
4915 match timeout_at(deadline, receiver).await {
4916 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4920 response.health_report().ok_or_else(|| {
4921 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
4922 })
4923 }
4924 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
4925 format!("health.check rejected: {}", body.message),
4926 )),
4927 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4928 Err(HealthProbeError::lane_dead(message))
4929 }
4930 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4931 Err(HealthProbeError::bad_answer(message))
4932 }
4933 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4934 Err(HealthProbeError::bad_answer(format!(
4935 "expected module-control op '{expected}', got '{actual}'"
4936 )))
4937 }
4938 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
4942 "module answered health.check after its daemon deadline",
4943 )),
4944 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
4945 "health.check waiter was canceled before the module responded",
4946 )),
4947 Err(_) => {
4948 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
4949 Err(HealthProbeError::no_answer(format!(
4950 "module did not answer health.check within {probe_budget:?}"
4951 )))
4952 }
4953 }
4954}
4955
4956#[allow(clippy::too_many_arguments)]
4957async fn handle_health_report(
4958 spec: &ModuleSpec,
4959 runtime: &SupervisorRuntimeConfig,
4960 registry: &Registry,
4961 process_liveness: &SupervisorProcessLiveness,
4962 snapshot: &SharedSnapshot,
4963 child: &mut Option<SupervisedChild>,
4964 report: HealthReport,
4965 now_ms: u64,
4966) {
4967 let status = supervisor_health_status(report.status);
4968 let detail = report.detail.clone();
4969 let metrics = truncate_health_metrics(report.metrics);
4970 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4971 state.health.status = status;
4972 state.health.last_probe_ms = Some(now_ms);
4973 state.health.detail = detail.clone();
4974 state.health.metrics = metrics.clone();
4975 state.health.consecutive_failures = 0;
4976 });
4977
4978 let action = match report.status {
4979 HealthStatus::Ok => return,
4980 HealthStatus::Degraded => runtime.health.on_degraded,
4981 HealthStatus::Failing => runtime.health.on_failing,
4982 };
4983 apply_l3_health_action(
4984 spec,
4985 runtime,
4986 registry,
4987 process_liveness,
4988 snapshot,
4989 child,
4990 status,
4991 detail.as_deref(),
4992 action,
4993 now_ms,
4994 )
4995 .await;
4996}
4997
4998#[allow(clippy::too_many_arguments)]
4999async fn handle_health_probe_failure(
5000 spec: &ModuleSpec,
5001 runtime: &SupervisorRuntimeConfig,
5002 registry: &Registry,
5003 process_liveness: &SupervisorProcessLiveness,
5004 snapshot: &SharedSnapshot,
5005 child: &mut Option<SupervisedChild>,
5006 err: HealthProbeError,
5007 now_ms: u64,
5008) {
5009 let threshold = runtime.health.failure_threshold.max(1);
5010 let mut failures = 0;
5011 let detail = format!("[{}] {err}", err.label());
5016 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5017 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5020 state.health.status = SupervisorHealthStatus::Unknown;
5021 }
5022 state.health.last_probe_ms = Some(now_ms);
5023 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5024 state.health.detail = Some(detail.clone());
5025 state.health.metrics = None;
5026 failures = state.health.consecutive_failures;
5027 });
5028
5029 if failures < threshold {
5030 warn!(
5031 module_id = %spec.module_id,
5032 consecutive_failures = failures,
5033 threshold,
5034 evidence = err.label(),
5035 detail = %detail,
5036 "health.check probe failed"
5037 );
5038 return;
5039 }
5040
5041 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5042 state.state = ModuleState::Unresponsive;
5043 state.health.status = SupervisorHealthStatus::Unresponsive;
5044 });
5045 if runtime.health.critical {
5049 error!(
5050 module_id = %spec.module_id,
5051 status = "unresponsive",
5052 evidence = err.label(),
5053 detail = %detail,
5054 "critical module health alert"
5055 );
5056 } else {
5057 warn!(
5058 module_id = %spec.module_id,
5059 status = "unresponsive",
5060 evidence = err.label(),
5061 detail = %detail,
5062 "module health threshold breached"
5063 );
5064 }
5065 if let Err(err) = health_restart_child(
5066 spec,
5067 runtime,
5068 registry,
5069 process_liveness,
5070 snapshot,
5071 child,
5072 SupervisorHealthStatus::Unresponsive,
5073 Some(&detail),
5074 now_ms,
5075 )
5076 .await
5077 {
5078 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5079 }
5080}
5081
5082#[allow(clippy::too_many_arguments)]
5083async fn apply_l3_health_action(
5084 spec: &ModuleSpec,
5085 runtime: &SupervisorRuntimeConfig,
5086 registry: &Registry,
5087 process_liveness: &SupervisorProcessLiveness,
5088 snapshot: &SharedSnapshot,
5089 child: &mut Option<SupervisedChild>,
5090 status: SupervisorHealthStatus,
5091 detail: Option<&str>,
5092 action: HealthAction,
5093 now_ms: u64,
5094) {
5095 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5096 match action {
5097 HealthAction::Report => {
5098 info!(
5099 module_id = %spec.module_id,
5100 status = ?status,
5101 detail,
5102 "module reported non-ok health"
5103 );
5104 }
5105 HealthAction::Alert => {
5106 error!(
5107 module_id = %spec.module_id,
5108 status = ?status,
5109 detail,
5110 "module health alert"
5111 );
5112 }
5113 HealthAction::Restart => {
5114 if let Err(err) = health_restart_child(
5115 spec,
5116 runtime,
5117 registry,
5118 process_liveness,
5119 snapshot,
5120 child,
5121 status,
5122 detail,
5123 now_ms,
5124 )
5125 .await
5126 {
5127 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5128 }
5129 }
5130 }
5131}
5132
5133#[allow(clippy::too_many_arguments)]
5134async fn health_restart_child(
5135 spec: &ModuleSpec,
5136 runtime: &SupervisorRuntimeConfig,
5137 registry: &Registry,
5138 process_liveness: &SupervisorProcessLiveness,
5139 snapshot: &SharedSnapshot,
5140 child: &mut Option<SupervisedChild>,
5141 status: SupervisorHealthStatus,
5142 detail: Option<&str>,
5143 now_ms: u64,
5144) -> Result<(), SuperviseError> {
5145 let (enabled, schedule) = {
5146 let mut state = lock_snapshot(snapshot)?;
5147 let enabled = state.enabled;
5148 let schedule = if enabled {
5149 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5150 } else {
5151 None
5152 };
5153 (enabled, schedule)
5154 };
5155
5156 if !enabled {
5157 return Err(SuperviseError::Disabled {
5158 module_id: spec.module_id.clone(),
5159 });
5160 }
5161
5162 if schedule.is_none() {
5163 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5164 error!(
5165 module_id = %spec.module_id,
5166 status = ?status,
5167 detail,
5168 max_restarts = runtime.restart_policy.max_restarts,
5169 window_secs = runtime.restart_policy.window.as_secs(),
5170 reason = %runtime.restart_policy.budget_exhausted_detail(),
5171 "health restart budget exhausted; marking module failed"
5172 );
5173 let stop_notice = begin_forwarding_drain_if_configured(
5174 spec,
5175 runtime,
5176 registry,
5177 snapshot,
5178 Some(true),
5179 RouteCloseReason::Disable,
5180 )
5181 .await?;
5182 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5183 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5184 })?;
5185 drain_optional_child(
5186 &spec.module_id,
5187 spec.protocol,
5188 stop_notice,
5189 registry,
5190 runtime.forwarding.as_deref(),
5191 snapshot,
5192 &runtime.terminal_ring,
5193 &runtime.spawn_events,
5194 child,
5195 runtime.drain_timeout,
5196 ModuleState::Failed,
5197 Some(true),
5198 )
5199 .await?;
5200 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5201 return Ok(());
5202 }
5203
5204 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5205 let mut restart_count = 0;
5206 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5207 restart_count = state.crash_restarts.len();
5208 state.state = ModuleState::Unresponsive;
5209 state.health.status = status;
5210 state.health.last_action = Some(HealthAction::Restart.to_string());
5211 state.health.last_action_ms = Some(now_ms);
5212 })?;
5213 warn!(
5214 module_id = %spec.module_id,
5215 status = ?status,
5216 detail,
5217 restart_count,
5218 restart_in_window = schedule.restart_in_window,
5219 delay_ms = schedule.delay.as_millis() as u64,
5220 "health-triggered module restart"
5221 );
5222
5223 let stop_notice = begin_forwarding_drain_if_configured(
5224 spec,
5225 runtime,
5226 registry,
5227 snapshot,
5228 Some(true),
5229 RouteCloseReason::Restart,
5230 )
5231 .await?;
5232 drain_optional_child(
5233 &spec.module_id,
5234 spec.protocol,
5235 stop_notice,
5236 registry,
5237 runtime.forwarding.as_deref(),
5238 snapshot,
5239 &runtime.terminal_ring,
5240 &runtime.spawn_events,
5241 child,
5242 runtime.drain_timeout,
5243 ModuleState::Restarting,
5244 Some(true),
5245 )
5246 .await?;
5247 schedule_respawn(
5248 runtime,
5249 snapshot,
5250 &spec.module_id,
5251 schedule.delay,
5252 RespawnKind::Spawn,
5253 )
5254}
5255
5256fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5257 if let Some(reply) = runtime
5258 .deferred_reload_reply
5259 .lock()
5260 .unwrap_or_else(|p| p.into_inner())
5261 .take()
5262 {
5263 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5264 module_id: module_id.to_string(),
5265 reason: reason.to_string(),
5266 }));
5267 }
5268}
5269
5270fn schedule_respawn(
5271 runtime: &SupervisorRuntimeConfig,
5272 snapshot: &SharedSnapshot,
5273 module_id: &str,
5274 delay: Duration,
5275 kind: RespawnKind,
5276) -> Result<(), SuperviseError> {
5277 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5278 update_snapshot(snapshot, Some(module_id), |state| {
5279 state.respawn_pending = true
5280 })?;
5281 *runtime
5282 .scheduled_respawn
5283 .lock()
5284 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5285 deadline: Instant::now() + delay,
5286 kind,
5287 });
5288 Ok(())
5289}
5290
5291fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5292 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5293 state.health.last_action = Some(action);
5294 state.health.last_action_ms = Some(now_ms);
5295 });
5296}
5297
5298fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5299 match status {
5300 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5301 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5302 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5303 }
5304}
5305
5306fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5318 let metrics = metrics?;
5319 match serde_json::to_vec(&metrics) {
5320 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5321 "truncated": true,
5322 "original_bytes": encoded.len(),
5323 })),
5324 Ok(_) | Err(_) => Some(metrics),
5325 }
5326}
5327
5328fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5334 if cadence.is_zero() {
5335 return Duration::ZERO;
5336 }
5337 let cadence_ms = cadence.as_millis() as u64;
5338 if cadence_ms == 0 {
5354 return cadence;
5355 }
5356 let jitter_span = (cadence_ms / 10).max(1);
5371 let hash = module_id.as_bytes().iter().fold(
5372 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5373 |acc, byte| {
5374 acc.wrapping_mul(1099511628211)
5375 .wrapping_add(u64::from(*byte))
5376 },
5377 );
5378 cadence + Duration::from_millis(hash % jitter_span)
5379}
5380
5381#[cfg(test)]
5382mod tests {
5383 use super::*;
5384
5385 #[test]
5386 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5387 let handle = SupervisorHandle::new();
5388 let module_id = "readded-tombstone";
5389 handle.record_rescan_removal(module_id);
5390 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5391
5392 handle.apply_identity_configuration(&ModuleSpec {
5393 module_id: module_id.to_string(),
5394 program: PathBuf::from("/test/module"),
5395 args: Vec::new(),
5396 env: Vec::new(),
5397 reserved: false,
5398 reserved_prefixes: Vec::new(),
5399 protocol: ModuleProtocol::Subc,
5400 overlap: Default::default(),
5401 });
5402
5403 assert!(
5404 handle.removal_tombstone_age_ms(module_id).is_none(),
5405 "a re-added module must not retain a stale removal tombstone"
5406 );
5407 }
5408
5409 #[derive(Debug, PartialEq, Eq)]
5412 struct OwnerInSpawnWindow {
5413 module_id: String,
5414 configured: bool,
5415 on_roster: bool,
5416 admission_refusal: Option<&'static str>,
5417 }
5418
5419 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5430 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5431 use crate::scopes::ScopeTable;
5432 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5433
5434 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5435 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5436 module_id: module_id.to_string(),
5437 program,
5438 args: Vec::new(),
5439 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5440 .into_iter()
5441 .map(|key| (key.to_string(), dir.path().display().to_string()))
5442 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5443 .collect(),
5444 reserved: false,
5445 reserved_prefixes: Vec::new(),
5446 protocol: ModuleProtocol::Subc,
5447 overlap: Default::default(),
5448 };
5449 let live = super::terminal_history_tests::fake_aft_stub_path();
5450 let missing = dir.path().join("definitely-missing-module");
5451
5452 let handle = SupervisorHandle::new();
5453 let mut supervisor =
5454 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5455 .with_handle(handle.clone());
5456 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5457 let hook_handle = handle.clone();
5458 let hook_observed = Arc::clone(&observed);
5459 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5460 let configured = hook_handle.is_configured(module_id);
5463 let selector = ScopeSelector::new(
5464 Principal::Reserved {
5465 module_id: module_id.to_string(),
5466 },
5467 "s".to_string(),
5468 Some(1),
5469 );
5470 let carrier = Principal::Reserved {
5471 module_id: "carrier".to_string(),
5472 };
5473 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5474 .admit(&carrier, module_id, &selector, configured)
5475 {
5476 Ok(_) => None,
5477 Err(refusal) => Some(refusal.code),
5478 };
5479 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5480 module_id: module_id.to_string(),
5481 configured,
5482 on_roster: hook_handle.get(module_id).is_some(),
5483 admission_refusal,
5484 });
5485 })));
5486
5487 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5488 let configured = supervisor
5489 .supervise_configured(stub("configured", live.clone()), true)
5490 .unwrap();
5491 let with_health = supervisor
5492 .supervise_configured_with_health(
5493 stub("with-health", live.clone()),
5494 true,
5495 HealthConfig::default(),
5496 None,
5497 RestartPolicy::default(),
5498 )
5499 .unwrap();
5500 let failed = supervisor
5503 .supervise_configured_with_health(
5504 stub("failed-spawn", missing.clone()),
5505 true,
5506 HealthConfig::default(),
5507 None,
5508 RestartPolicy::default(),
5509 )
5510 .unwrap();
5511 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5514
5515 let expected = [
5516 "plain",
5517 "configured",
5518 "with-health",
5519 "failed-spawn",
5520 "spawn-error",
5521 ]
5522 .into_iter()
5523 .map(|module_id| OwnerInSpawnWindow {
5524 module_id: module_id.to_string(),
5525 configured: true,
5526 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5527 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5528 })
5529 .collect::<Vec<_>>();
5530 assert_eq!(*observed.lock().unwrap(), expected);
5531
5532 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5533 assert!(
5534 handle.get(module_id).is_some(),
5535 "{module_id} is on the roster"
5536 );
5537 assert!(
5538 handle.is_configured(module_id),
5539 "{module_id} stays configured"
5540 );
5541 }
5542 assert!(handle.get("spawn-error").is_none());
5543 assert!(
5544 !handle.is_configured("spawn-error"),
5545 "a plain spawn that failed must not leave its module marked configured"
5546 );
5547
5548 handle.retire("failed-spawn");
5550 assert!(!handle.is_configured("failed-spawn"));
5551
5552 for module in [plain, configured, with_health] {
5553 module.stop().await.unwrap();
5554 }
5555 drop(failed);
5556 }
5557
5558 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5559 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5560 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5561 snapshot.process_alive = true;
5562 snapshot.pid = Some(41);
5563 snapshot.spawned_at_ms = Some(42);
5564 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5565 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5566 device: 43,
5567 inode: 44,
5568 #[cfg(windows)]
5569 windows_image: None,
5570 });
5571 })
5572 .unwrap();
5573 snapshot
5574 }
5575
5576 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5577 let snapshot = lock_snapshot(snapshot).unwrap();
5578 assert!(!snapshot.process_alive);
5579 assert_eq!(snapshot.pid, None);
5580 assert_eq!(snapshot.spawned_at_ms, None);
5581 assert_eq!(snapshot.spawned_from, None);
5582 assert_eq!(snapshot.spawned_file_identity, None);
5583 }
5584
5585 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5586 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5587 let supervisor =
5588 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5589 let mut runtime = supervisor.runtime_config();
5590 runtime.test_seed_stale_facts_before_enable_spawn = true;
5591 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5592 let mut child = None;
5593 let spec = ModuleSpec {
5594 module_id: "failed-enable-clears-facts".to_string(),
5595 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5596 args: Vec::new(),
5597 env: Vec::new(),
5598 reserved: false,
5599 reserved_prefixes: Vec::new(),
5600 protocol: ModuleProtocol::Subc,
5601 overlap: Default::default(),
5602 };
5603
5604 let result = set_child_enabled(
5605 &spec,
5606 &runtime,
5607 &supervisor.registry,
5608 &supervisor.process_liveness,
5609 &snapshot,
5610 &mut child,
5611 true,
5612 )
5613 .await;
5614
5615 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5616 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5617 assert_snapshot_process_facts_cleared(&snapshot);
5618 }
5619
5620 #[tokio::test]
5621 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5622 let supervisor =
5623 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5624 let runtime = supervisor.runtime_config();
5625 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5626 ModuleState::Restarting,
5627 true,
5628 )));
5629 let spec = ModuleSpec {
5630 module_id: "start-stranded-restarting".to_string(),
5631 program: super::terminal_history_tests::fake_aft_stub_path(),
5632 args: Vec::new(),
5633 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5634 reserved: false,
5635 reserved_prefixes: Vec::new(),
5636 protocol: ModuleProtocol::None,
5637 overlap: Default::default(),
5638 };
5639 let mut child = None;
5640 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5641 assert!(!super::set_child_enabled(
5642 &spec,
5643 &runtime,
5644 &Registry::default(),
5645 &supervisor.process_liveness,
5646 &snapshot,
5647 &mut child,
5648 true
5649 )
5650 .await
5651 .unwrap());
5652 assert!(child.is_none());
5653 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5654 assert!(super::set_child_enabled(
5655 &spec,
5656 &runtime,
5657 &Registry::default(),
5658 &supervisor.process_liveness,
5659 &snapshot,
5660 &mut child,
5661 true
5662 )
5663 .await
5664 .unwrap());
5665 assert_eq!(
5666 lock_snapshot(&snapshot).unwrap().state,
5667 ModuleState::Running
5668 );
5669 let mut child = child.unwrap();
5670 child.start_kill().unwrap();
5671 child.wait().await.unwrap();
5672 }
5673
5674 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5675 async fn failed_reload_spawn_clears_current_process_facts() {
5676 let supervisor =
5677 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5678 let mut runtime = supervisor.runtime_config();
5679 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5680 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5681 let mut child = None;
5682 let spec = ModuleSpec {
5683 module_id: "failed-reload-clears-facts".to_string(),
5684 program: PathBuf::from("/unused/failed-reload-module"),
5685 args: Vec::new(),
5686 env: Vec::new(),
5687 reserved: false,
5688 reserved_prefixes: Vec::new(),
5689 protocol: ModuleProtocol::Subc,
5690 overlap: Default::default(),
5691 };
5692
5693 let result = handle_reload_spawn_failure(
5694 &spec,
5695 &runtime,
5696 &supervisor.process_liveness,
5697 &snapshot,
5698 &mut child,
5699 "forced reload spawn failure".to_string(),
5700 )
5701 .await;
5702
5703 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5704 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5705 assert_snapshot_process_facts_cleared(&snapshot);
5706 }
5707
5708 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5709 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5710 let supervisor =
5711 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5712 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5713 let module = supervisor.supervised_module(
5714 ModuleSpec {
5715 module_id: "drop-clears-facts".to_string(),
5716 program: PathBuf::from("/unused/drop-module"),
5717 args: Vec::new(),
5718 env: Vec::new(),
5719 reserved: false,
5720 reserved_prefixes: Vec::new(),
5721 protocol: ModuleProtocol::Subc,
5722 overlap: Default::default(),
5723 },
5724 supervisor.runtime_config(),
5725 Arc::clone(&snapshot),
5726 None,
5727 );
5728 assert!(!module
5729 .inner
5730 .monitor
5731 .lock()
5732 .unwrap()
5733 .as_ref()
5734 .unwrap()
5735 .is_finished());
5736
5737 drop(module);
5738
5739 assert_eq!(
5740 lock_snapshot(&snapshot).unwrap().state,
5741 ModuleState::Stopped
5742 );
5743 assert_snapshot_process_facts_cleared(&snapshot);
5744 }
5745
5746 #[cfg(unix)]
5747 #[tokio::test]
5748 async fn rescan_preserves_running_protocol_until_respawn() {
5749 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5750 let initial = ModuleSpec {
5751 module_id: "rescan-protocol".into(),
5752 program: PathBuf::from("/bin/sleep"),
5753 args: vec!["60".into()],
5754 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5755 .into_iter()
5756 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5757 .collect(),
5758 reserved: false,
5759 reserved_prefixes: vec![],
5760 protocol: ModuleProtocol::None,
5761 overlap: Default::default(),
5762 };
5763 let supervisor =
5764 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5765 let module = supervisor.spawn(initial.clone()).unwrap();
5766 assert!(module.status().unwrap().live);
5767 let mut next = initial;
5768 next.protocol = ModuleProtocol::Subc;
5769 module
5770 .update_configuration(next.clone(), HealthConfig::default(), None)
5771 .await
5772 .unwrap();
5773 assert!(
5774 module.status().unwrap().live,
5775 "rescan must not require HELLO from the old non-wire process"
5776 );
5777 let runtime = supervisor.runtime_config();
5778 let action = on_child_exit(
5779 &next,
5780 RestartPolicy::default(),
5781 &supervisor.registry,
5782 &module.inner.snapshot,
5783 &runtime.terminal_ring,
5784 &runtime.spawn_events,
5785 &runtime.child_roster,
5786 ExitReport {
5787 kind: ExitKind::Clean,
5788 code: Some(0),
5789 signal: None,
5790 at_ms: unix_ms_now(),
5791 },
5792 )
5793 .await;
5794 assert!(
5795 matches!(action, NextAction::Restart { .. }),
5796 "the old non-wire process's clean exit must restart"
5797 );
5798 module.drain().await.unwrap();
5799 }
5800
5801 #[cfg(unix)]
5802 #[tokio::test]
5803 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
5804 use std::os::unix::fs::PermissionsExt;
5805 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
5806 let script = dir.join("module.sh");
5807 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
5808 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
5809 let record_path = dir.join("live-children.json");
5810 let supervisor =
5811 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5812 .with_live_children_record(&record_path);
5813 for (program, args) in [
5814 (PathBuf::from("sleep"), vec!["60".into()]),
5815 (script, vec![]),
5816 ] {
5817 let spec = ModuleSpec {
5818 module_id: "image-identity".into(),
5819 program: program.clone(),
5820 args,
5821 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5822 .into_iter()
5823 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
5824 .collect(),
5825 reserved: false,
5826 reserved_prefixes: vec![],
5827 protocol: ModuleProtocol::None,
5828 overlap: Default::default(),
5829 };
5830 let module = supervisor.spawn(spec).unwrap();
5831 #[cfg(target_os = "macos")]
5832 {
5833 let deadline = Instant::now() + Duration::from_secs(5);
5836 while crate::live_children::read_record(&record_path)
5837 .unwrap()
5838 .iter()
5839 .all(|entry| entry.executable.is_none())
5840 {
5841 assert!(Instant::now() < deadline, "module image was not confirmed");
5842 tokio::time::sleep(Duration::from_millis(5)).await;
5843 }
5844 }
5845 let entry = crate::live_children::read_record(&record_path)
5846 .unwrap()
5847 .pop()
5848 .unwrap();
5849 let observed = subc_os::Process::open(entry.pid)
5850 .unwrap()
5851 .unwrap()
5852 .observe()
5853 .unwrap();
5854 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
5855 module.drain().await.unwrap();
5856 assert_eq!(
5857 verdict,
5858 crate::live_children::IdentityVerdict::Matches,
5859 "program {program:?}: recorded {entry:?}, observed {observed:?}"
5860 );
5861 }
5862 }
5863
5864 #[cfg(unix)]
5865 fn http_fixture(
5866 dir: &std::path::Path,
5867 url: &str,
5868 threshold: u32,
5869 ) -> crate::daemon_config::ConfiguredModule {
5870 let path = dir.join("subc.jsonc");
5871 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
5872 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
5873 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
5874 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
5875 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
5876 }}}).to_string()).unwrap();
5877 crate::daemon_config::load(&path)
5878 .unwrap()
5879 .unwrap()
5880 .modules
5881 .pop()
5882 .unwrap()
5883 }
5884
5885 #[cfg(unix)]
5886 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
5887 timeout(Duration::from_secs(5), async {
5888 loop {
5889 if module.status().unwrap().health.status == status {
5890 break;
5891 }
5892 sleep(Duration::from_millis(5)).await;
5893 }
5894 })
5895 .await
5896 .unwrap_or_else(|_| {
5897 panic!(
5898 "expected {status:?}, got {:?}",
5899 module.status().unwrap().health
5900 )
5901 });
5902 }
5903
5904 #[cfg(unix)]
5905 #[tokio::test]
5906 async fn http_health_status_flips_ok_failing_ok() {
5907 use tokio::io::{AsyncReadExt, AsyncWriteExt};
5908 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
5909 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5910 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
5911 let serving_status = status.clone();
5912 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
5913 let server = tokio::spawn(async move {
5914 loop {
5915 let (mut stream, _) = listener.accept().await.unwrap();
5916 let mut request = [0u8; 2048];
5917 let count = stream.read(&mut request).await.unwrap();
5918 assert!(count > 0, "a probe must send an HTTP request");
5919 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
5920 let body = if code == 200 {
5921 "ready"
5922 } else {
5923 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
5924 };
5925 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
5926 let _ = stream.write_all(response.as_bytes()).await;
5927 }
5928 });
5929 let configured = http_fixture(&dir, &url, 1000);
5930 let module =
5931 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5932 .supervise_configured_with_health(
5933 configured.module_spec(),
5934 true,
5935 configured.health,
5936 configured.drain_timeout_ms,
5937 configured.restart,
5938 )
5939 .unwrap();
5940 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
5941 status.store(503, std::sync::atomic::Ordering::SeqCst);
5942 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
5943 assert!(module
5944 .status()
5945 .unwrap()
5946 .health
5947 .detail
5948 .unwrap()
5949 .contains("scratch failure"));
5950 status.store(200, std::sync::atomic::Ordering::SeqCst);
5951 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
5952 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
5953 let before = module.status().unwrap();
5954 let (spec, mut health) = module.configuration().unwrap();
5955 health.http = None;
5956 module
5957 .update_configuration(spec.clone(), health.clone(), Some(10))
5958 .await
5959 .unwrap();
5960 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
5961 health.http = Some(url);
5962 module
5963 .update_configuration(spec, health, Some(10))
5964 .await
5965 .unwrap();
5966 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
5967 assert_eq!(
5968 module.status().unwrap().pid,
5969 before.pid,
5970 "changing a probe must apply live, not restart its process"
5971 );
5972 let (spec, mut health) = module.configuration().unwrap();
5973 health.failure_threshold = 2;
5974 module
5975 .update_configuration(spec, health, Some(10))
5976 .await
5977 .unwrap();
5978 status.store(503, std::sync::atomic::Ordering::SeqCst);
5979 timeout(Duration::from_secs(5), async {
5980 while module.status().unwrap().spawn_generation == before.spawn_generation {
5981 sleep(Duration::from_millis(5)).await;
5982 }
5983 })
5984 .await
5985 .expect("sustained HTTP 503 responses must trigger the health restart policy");
5986 module.drain().await.unwrap();
5987 server.abort();
5988 }
5989
5990 #[cfg(unix)]
5991 #[tokio::test]
5992 async fn http_health_no_listener_restarts_after_consecutive_failures() {
5993 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
5994 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5995 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
5996 drop(unused);
5997 let configured = http_fixture(&dir, &url, 2);
5998 let module =
5999 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6000 .supervise_configured_with_health(
6001 configured.module_spec(),
6002 true,
6003 configured.health,
6004 configured.drain_timeout_ms,
6005 configured.restart,
6006 )
6007 .unwrap();
6008 let before = module.status().unwrap().spawn_generation;
6009 timeout(Duration::from_secs(5), async {
6010 loop {
6011 let status = module.status().unwrap();
6012 if status.spawn_generation > before {
6013 assert!(status.lifetime_restarts > 0);
6014 break;
6015 }
6016 sleep(Duration::from_millis(5)).await;
6017 }
6018 })
6019 .await
6020 .expect("sustained HTTP refusal must trigger the health restart policy");
6021 module.drain().await.unwrap();
6022 }
6023
6024 #[cfg(unix)]
6025 #[tokio::test]
6026 async fn http_health_timeout_honours_deadline() {
6027 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6028 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6029 let server = tokio::spawn(async move {
6030 let _held = listener.accept().await.unwrap();
6031 std::future::pending::<()>().await;
6032 });
6033 let error = timeout(
6034 Duration::from_secs(1),
6035 probe_http_health(&url, Duration::from_millis(10)),
6036 )
6037 .await
6038 .expect("the probe must enforce its own deadline")
6039 .unwrap_err();
6040 server.abort();
6041 assert!(error.to_string().contains("timed out"));
6042 }
6043
6044 #[cfg(unix)]
6045 #[tokio::test]
6046 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6047 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6048 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6049 let url = format!(
6050 "http://localhost:{}/healthz",
6051 listener.local_addr().unwrap().port()
6052 );
6053 let server = tokio::spawn(async move {
6054 let (mut stream, _) = listener.accept().await.unwrap();
6055 let mut request = [0u8; 2048];
6056 assert!(stream.read(&mut request).await.unwrap() > 0);
6057 stream
6058 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6059 .await
6060 .unwrap();
6061 });
6062 assert_eq!(
6067 probe_http_health(&url, Duration::from_secs(10))
6068 .await
6069 .unwrap()
6070 .status,
6071 HealthStatus::Ok
6072 );
6073 server.await.unwrap();
6074 }
6075
6076 #[cfg(unix)]
6077 #[tokio::test]
6078 async fn http_health_timeout_keeps_partial_status_and_body() {
6079 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6080 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6081 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6082 let server = tokio::spawn(async move {
6083 let (mut stream, _) = listener.accept().await.unwrap();
6084 let mut request = [0u8; 2048];
6085 assert!(stream.read(&mut request).await.unwrap() > 0);
6086 stream
6087 .write_all(
6088 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6089 )
6090 .await
6091 .unwrap();
6092 std::future::pending::<()>().await;
6093 });
6094 let error = probe_http_health(&url, Duration::from_secs(1))
6095 .await
6096 .unwrap_err()
6097 .to_string();
6098 server.abort();
6099 assert!(
6100 error.contains("timed out")
6101 && error.contains("503 Unavailable")
6102 && error.contains("partial diagnostic"),
6103 "{error}"
6104 );
6105 }
6106
6107 #[cfg(unix)]
6108 #[tokio::test]
6109 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6110 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6111 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6112 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6113 let server = tokio::spawn(async move {
6114 let (mut stream, _) = listener.accept().await.unwrap();
6115 let mut request = [0u8; 2048];
6116 assert!(stream.read(&mut request).await.unwrap() > 0);
6117 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6118 let response = format!(
6119 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6120 body.len()
6121 );
6122 stream.write_all(response.as_bytes()).await.unwrap();
6123 });
6124 let error = probe_http_health(&url, Duration::from_secs(1))
6125 .await
6126 .unwrap_err()
6127 .to_string();
6128 server.await.unwrap();
6129 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6130 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6131 assert!(!error.contains("not-in-diagnostic"));
6132 }
6133
6134 #[cfg(unix)]
6135 #[tokio::test]
6136 async fn http_health_real_nats_server_monitoring() {
6137 if std::process::Command::new("nats-server")
6138 .arg("--version")
6139 .env("XDG_DATA_HOME", std::env::temp_dir())
6140 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6141 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6142 .output()
6143 .is_err()
6144 {
6145 eprintln!(
6146 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6147 );
6148 return;
6149 }
6150 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6151 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6152 let port = monitor.local_addr().unwrap().port();
6153 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6154 let client_port = client.local_addr().unwrap().port();
6155 let config = dir.join("server.conf");
6156 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6157 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6158 configured.program = PathBuf::from("nats-server");
6159 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6160 drop(monitor);
6161 drop(client);
6162 let module =
6163 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6164 .supervise_configured_with_health(
6165 configured.module_spec(),
6166 true,
6167 configured.health,
6168 configured.drain_timeout_ms,
6169 configured.restart,
6170 )
6171 .unwrap();
6172 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6173 module.drain().await.unwrap();
6174 }
6175
6176 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6177 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6178 let supervisor =
6179 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6180 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6181 let initial = ModuleSpec {
6182 module_id: "rescan-preserves-spawn-facts".to_string(),
6183 program: PathBuf::from("/spawned/module"),
6184 args: Vec::new(),
6185 env: Vec::new(),
6186 reserved: false,
6187 reserved_prefixes: Vec::new(),
6188 protocol: ModuleProtocol::Subc,
6189 overlap: Default::default(),
6190 };
6191 let module = supervisor.supervised_module(
6192 initial.clone(),
6193 supervisor.runtime_config(),
6194 snapshot,
6195 None,
6196 );
6197 let before = module.status().unwrap();
6198 let mut replacement = initial;
6199 replacement.program = PathBuf::from("/rescanned/replacement-module");
6200
6201 module
6202 .update_configuration(replacement, HealthConfig::default(), None)
6203 .await
6204 .unwrap();
6205
6206 let after = module.status().unwrap();
6207 assert_eq!(after.pid, before.pid);
6208 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6209 assert_eq!(after.spawned_from, before.spawned_from);
6210 drop(module);
6211 }
6212}
6213
6214fn unix_ms_now() -> u64 {
6215 SystemTime::now()
6216 .duration_since(UNIX_EPOCH)
6217 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6218 .unwrap_or(0)
6219}
6220
6221async fn supervise_loop(
6222 mut spec: ModuleSpec,
6223 mut runtime: SupervisorRuntimeConfig,
6224 registry: Arc<Registry>,
6225 process_liveness: Arc<SupervisorProcessLiveness>,
6226 snapshot: SharedSnapshot,
6227 mut child: Option<SupervisedChild>,
6228 mut commands: mpsc::Receiver<SupervisorCommand>,
6229) {
6230 let mut health_probe = HealthProbeRuntime::default();
6231 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6235 Ok(changes) => Some(changes),
6236 Err(err) => {
6237 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6240 None
6241 }
6242 };
6243 let mut pending_respawn: Option<PendingRespawn> = None;
6247 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6250 #[cfg(target_os = "linux")]
6251 let mut previous_instance_recheck = tokio::time::interval_at(
6252 Instant::now() + Duration::from_secs(5),
6253 Duration::from_secs(5),
6254 );
6255 #[cfg(target_os = "linux")]
6256 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6257 loop {
6258 #[cfg(test)]
6259 {
6260 lock_snapshot(&snapshot).unwrap().actor_select = None;
6261 }
6262 #[cfg(target_os = "macos")]
6263 if let Some(active) = child.as_mut() {
6264 active.confirm_privacy_exec().await;
6265 }
6266 if let Some(scheduled) = runtime
6267 .scheduled_respawn
6268 .lock()
6269 .unwrap_or_else(|p| p.into_inner())
6270 .take()
6271 {
6272 pending_respawn = Some(scheduled);
6273 }
6274 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6275 pending_respawn = None;
6276 cancel_deferred_reload(
6277 &runtime,
6278 &spec.module_id,
6279 "respawn cancelled by a supervisor command",
6280 );
6281 }
6282 if child.is_none()
6283 && pending_respawn.is_none()
6284 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6285 {
6286 cancel_deferred_reload(
6287 &runtime,
6288 &spec.module_id,
6289 "respawn cancelled before a replacement was spawned",
6290 );
6291 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6292 state.respawn_pending = false;
6293 state.coalesced_restart_pending = false;
6294 if matches!(
6295 state.state,
6296 ModuleState::Restarting
6297 | ModuleState::Starting
6298 | ModuleState::Draining
6299 | ModuleState::Unresponsive
6300 ) {
6301 error!(module_id = %spec.module_id, state = ?state.state,
6302 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6303 state.state = ModuleState::Failed;
6304 clear_current_process_facts(state);
6305 }
6306 });
6307 }
6308 if let Some(command) = requeued.pop_front() {
6309 if !handle_supervisor_command(
6310 command,
6311 &mut spec,
6312 &mut runtime,
6313 ®istry,
6314 &process_liveness,
6315 &snapshot,
6316 &mut child,
6317 &mut commands,
6318 &mut requeued,
6319 )
6320 .await
6321 {
6322 return;
6323 }
6324 if child.is_some() || !respawn_still_pending(&snapshot) {
6325 pending_respawn = None;
6326 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6327 state.respawn_pending = false
6328 });
6329 }
6330 continue;
6331 }
6332 if child.is_some() {
6333 #[cfg(test)]
6334 {
6335 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6336 }
6337 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6338 let wake_after = health_probe.wake_after();
6339 let probe_sleep = async move {
6340 match wake_after {
6341 Some(delay) => sleep(delay).await,
6342 None => std::future::pending().await,
6343 }
6344 };
6345 tokio::pin!(probe_sleep);
6346 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6351 #[cfg(test)]
6352 {
6353 let registration_pending = wire_child
6354 && registration_changes
6355 .as_ref()
6356 .is_some_and(|changes| changes.has_changed().unwrap());
6357 let mut state = lock_snapshot(&snapshot).unwrap();
6358 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6359 generation: state.spawn_generation,
6360 turn: state.actor_turns,
6361 registered_connection: health_probe.registered_connection,
6362 next_probe_at: health_probe.next_probe_at,
6363 wake_after,
6364 });
6365 }
6366 let active_child = child.as_mut().expect("child checked above");
6367 tokio::select! {
6368 wait_result = active_child.wait() => {
6369 #[cfg(test)]
6370 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6371 let exit_report = match wait_result {
6380 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6381 Err(err) => {
6382 active_child.drain_stderr(&spec.module_id).await;
6383 fail_after_wait_error(
6384 &spec.module_id,
6385 &snapshot,
6386 &runtime.terminal_ring,
6387 &runtime.spawn_events,
6388 format!("failed to wait for supervised module: {err}"),
6389 );
6390 untrack_if_registration_released(
6391 &process_liveness,
6392 ®istry,
6393 &spec.module_id,
6394 &snapshot,
6395 );
6396 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6397 child = None;
6398 continue;
6399 }
6400 };
6401 active_child.drain_stderr(&spec.module_id).await;
6402
6403 let next = on_child_exit(
6404 &spec,
6405 runtime.restart_policy,
6406 ®istry,
6407 &snapshot,
6408 &runtime.terminal_ring,
6409 &runtime.spawn_events,
6410 &runtime.child_roster,
6411 exit_report,
6412 ).await;
6413 runtime.start_requester = None;
6414 runtime.pending_activity_restart = false;
6415 active_child.release_roster();
6418 match next {
6419 NextAction::Stop { registration_released } => {
6420 if registration_released {
6421 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6422 }
6423 child = None;
6424 }
6425 NextAction::Restart { schedule } => {
6426 let delay = schedule.map_or(
6427 runtime.restart_policy.delay_for_restart(0),
6428 |schedule| schedule.delay,
6429 );
6430 if let Some(schedule) = schedule {
6431 log_crash_respawn(&spec.module_id, schedule);
6432 }
6433 child = None;
6441 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6442 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6443 }
6444 }
6445 }
6446 command = commands.recv() => {
6447 #[cfg(test)]
6448 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6449 let Some(command) = command else {
6450 return;
6451 };
6452 if !handle_supervisor_command(
6453 command,
6454 &mut spec,
6455 &mut runtime,
6456 ®istry,
6457 &process_liveness,
6458 &snapshot,
6459 &mut child,
6460 &mut commands,
6461 &mut requeued,
6462 ).await {
6463 return;
6464 }
6465 }
6466 _ = async {
6467 match registration_changes.as_mut() {
6468 Some(changes) => { let _ = changes.changed().await; }
6469 None => std::future::pending().await,
6470 }
6471 }, if wire_child => {
6472 #[cfg(test)]
6473 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6474 }
6475 _ = &mut probe_sleep => {
6476 #[cfg(test)]
6477 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6478 if health_probe.due() {
6479 run_health_probe_cycle(
6480 &spec,
6481 &runtime,
6482 ®istry,
6483 &process_liveness,
6484 &snapshot,
6485 &mut child,
6486 ).await;
6487 if child.is_none() {
6488 runtime.start_requester = None;
6489 runtime.pending_activity_restart = false;
6490 }
6491 if child.is_some() {
6492 health_probe.schedule_next(&spec, runtime.health.cadence);
6493 }
6494 }
6495 }
6496 }
6497 } else if let Some(pending) = pending_respawn {
6498 tokio::select! {
6499 _ = sleep_until(pending.deadline) => {
6500 pending_respawn = None;
6501 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6502 if !respawn_still_pending(&snapshot) {
6506 continue;
6507 }
6508 if runtime.child_roster.is_closed() {
6513 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6514 state.state = ModuleState::Stopped;
6515 });
6516 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6517 continue;
6518 }
6519 if let Err(err) = release_dead_registration(
6520 ®istry,
6521 runtime.forwarding.as_deref(),
6522 &snapshot,
6523 &spec.module_id,
6524 ).await {
6525 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6526 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6527 continue;
6528 }
6529
6530 if matches!(pending.kind, RespawnKind::Reload) {
6531 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6532 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
6533 if result.is_ok() && runtime.pending_activity_restart {
6534 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6535 }
6536 runtime.pending_activity_restart = false;
6537 if let Some(reply) = reply { let _ = reply.send(result); }
6538 continue;
6539 }
6540 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6541 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6542 Ok(next_child) => {
6543 child = Some(next_child);
6544 if runtime.pending_activity_restart {
6545 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6546 runtime.pending_activity_restart = false;
6547 }
6548 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6549 }
6550 Err(err) => {
6551 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6552 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6553 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6554 }
6555 }
6556 }
6557 command = commands.recv() => {
6558 let Some(command) = command else {
6559 return;
6560 };
6561 if !handle_supervisor_command(
6562 command,
6563 &mut spec,
6564 &mut runtime,
6565 ®istry,
6566 &process_liveness,
6567 &snapshot,
6568 &mut child,
6569 &mut commands,
6570 &mut requeued,
6571 ).await {
6572 return;
6573 }
6574 if child.is_some() || !respawn_still_pending(&snapshot) {
6579 pending_respawn = None;
6580 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6581 }
6582 }
6583 }
6584 } else {
6585 #[cfg(target_os = "linux")]
6586 let command = if lock_snapshot(&snapshot)
6587 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6588 {
6589 tokio::select! {
6590 command = commands.recv() => command,
6591 _ = previous_instance_recheck.tick() => {
6592 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6593 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6594 if enabled && !runtime.child_roster.is_closed() {
6595 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6596 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6597 Ok(next_child) => child = Some(next_child),
6598 Err(error) => {
6599 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6600 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6601 }
6602 }
6603 } else {
6604 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6605 state.waiting_for_previous_pid = None;
6606 });
6607 }
6608 }
6609 continue;
6610 }
6611 }
6612 } else {
6613 commands.recv().await
6614 };
6615 #[cfg(not(target_os = "linux"))]
6616 let command = commands.recv().await;
6617 let Some(command) = command else {
6618 return;
6619 };
6620 if !handle_supervisor_command(
6621 command,
6622 &mut spec,
6623 &mut runtime,
6624 ®istry,
6625 &process_liveness,
6626 &snapshot,
6627 &mut child,
6628 &mut commands,
6629 &mut requeued,
6630 )
6631 .await
6632 {
6633 return;
6634 }
6635 }
6636 }
6637}
6638
6639fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6640 info!(
6641 module_id,
6642 restart_in_window = schedule.restart_in_window,
6643 delay_ms = schedule.delay.as_millis() as u64,
6644 "respawning after crash"
6645 );
6646}
6647
6648fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6654 matches!(
6655 lock_snapshot(snapshot),
6656 Ok(state) if state.enabled && state.state == ModuleState::Restarting
6657 )
6658}
6659
6660enum NextAction {
6661 Stop {
6662 registration_released: bool,
6663 },
6664 Restart {
6665 schedule: Option<CrashRestartSchedule>,
6666 },
6667}
6668
6669#[allow(clippy::too_many_arguments)]
6670async fn handle_supervisor_command(
6671 command: SupervisorCommand,
6672 spec: &mut ModuleSpec,
6673 runtime: &mut SupervisorRuntimeConfig,
6674 registry: &Arc<Registry>,
6675 process_liveness: &SupervisorProcessLiveness,
6676 snapshot: &SharedSnapshot,
6677 child: &mut Option<SupervisedChild>,
6678 commands: &mut mpsc::Receiver<SupervisorCommand>,
6679 requeued: &mut VecDeque<SupervisorCommand>,
6680) -> bool {
6681 #[cfg(target_os = "linux")]
6682 let command = if let Some(pid) = runtime.child_roster.previous_instance_pid(&spec.module_id) {
6683 match command {
6684 SupervisorCommand::Restart { reply, .. }
6685 | SupervisorCommand::Reload { reply, .. }
6686 | SupervisorCommand::Swap { reply, .. } => {
6687 let _ = reply.send(Err(previous_instance_error(spec, pid)));
6688 return true;
6689 }
6690 SupervisorCommand::SetEnabled {
6691 enabled: true,
6692 reply,
6693 ..
6694 } => {
6695 let _ = reply.send(Err(previous_instance_error(spec, pid)));
6696 return true;
6697 }
6698 other => other,
6699 }
6700 } else {
6701 command
6702 };
6703 match command {
6704 SupervisorCommand::Drain { requester, reply } => {
6705 let result = drain_optional_child(
6708 &spec.module_id,
6709 spec.protocol,
6710 StopNotice::NotSent,
6711 registry,
6712 runtime.forwarding.as_deref(),
6713 snapshot,
6714 &runtime.terminal_ring,
6715 &runtime.spawn_events,
6716 child,
6717 runtime.drain_timeout,
6718 ModuleState::Stopped,
6719 None,
6720 )
6721 .await;
6722 let registration_released = result.is_ok();
6723 if registration_released {
6724 runtime.record_activity_transition(
6725 "subc.module_stopped",
6726 &spec.module_id,
6727 requester,
6728 );
6729 }
6730 let _ = reply.send(result);
6731 if registration_released {
6732 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6733 }
6734 false
6735 }
6736 SupervisorCommand::Retire { requester, reply } => {
6737 let result = async {
6738 let stop_notice = begin_forwarding_drain_if_configured(
6739 spec,
6740 runtime,
6741 registry,
6742 snapshot,
6743 None,
6744 RouteCloseReason::Disable,
6745 )
6746 .await?;
6747 drain_optional_child(
6748 &spec.module_id,
6749 spec.protocol,
6750 stop_notice,
6751 registry,
6752 runtime.forwarding.as_deref(),
6753 snapshot,
6754 &runtime.terminal_ring,
6755 &runtime.spawn_events,
6756 child,
6757 runtime.drain_timeout,
6758 ModuleState::Stopped,
6759 None,
6760 )
6761 .await
6762 }
6763 .await;
6764 let registration_released = result.is_ok();
6765 if registration_released {
6766 runtime.record_activity_transition(
6767 "subc.module_stopped",
6768 &spec.module_id,
6769 requester,
6770 );
6771 }
6772 let _ = reply.send(result);
6773 if registration_released {
6774 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6775 }
6776 false
6777 }
6778 SupervisorCommand::Restart {
6779 requester,
6780 drain_timeout_ms,
6781 received_at_generation,
6782 queued_at,
6783 reply,
6784 } => {
6785 info!(
6789 module_id = %spec.module_id,
6790 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6791 "restart command dequeued"
6792 );
6793 let validation = match lock_snapshot(snapshot) {
6805 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
6806 module_id: spec.module_id.clone(),
6807 }),
6808 Ok(_) => Ok(()),
6809 Err(err) => Err(err),
6810 };
6811 let initiated = validation.is_ok();
6812 let _ = reply.send(validation);
6813 let satisfied_by_generation = if initiated && child.is_some() {
6824 lock_snapshot(snapshot).ok().and_then(|state| {
6825 (state.spawn_generation > received_at_generation
6826 && !state.configuration_updated_since_spawn)
6827 .then_some(state.spawn_generation)
6828 })
6829 } else {
6830 None
6831 };
6832 let satisfied_by_pending = initiated
6833 && child.is_none()
6834 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
6835 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
6836 if pending {
6837 state.coalesced_restart_pending = true;
6838 }
6839 pending
6840 });
6841 if satisfied_by_pending {
6842 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
6843 } else if let Some(generation) = satisfied_by_generation {
6844 info!(
6845 module_id = %spec.module_id,
6846 received_at_generation,
6847 "restart already satisfied by generation {generation}; not restarting again"
6848 );
6849 } else if initiated {
6850 let drain_timeout = drain_timeout_ms
6853 .map(Duration::from_millis)
6854 .unwrap_or(runtime.drain_timeout);
6855 if let Err(err) = restart_child(
6856 spec,
6857 runtime,
6858 registry,
6859 process_liveness,
6860 snapshot,
6861 child,
6862 drain_timeout,
6863 )
6864 .await
6865 {
6866 warn!(
6867 module_id = %spec.module_id,
6868 error = %err,
6869 "operator restart failed after initiation ack; module state carries the outcome"
6870 );
6871 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6872 state.state = ModuleState::Failed;
6873 clear_current_process_facts(state);
6874 });
6875 } else {
6876 runtime.start_requester = requester;
6877 runtime.pending_activity_restart = true;
6878 }
6879 }
6880 true
6881 }
6882 SupervisorCommand::Reload { requester, reply } => {
6883 let result =
6884 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
6885 if result.is_ok() {
6886 runtime.start_requester = requester;
6887 runtime.pending_activity_restart = true;
6888 }
6889 if result.is_ok()
6890 && runtime
6891 .scheduled_respawn
6892 .lock()
6893 .unwrap_or_else(|p| p.into_inner())
6894 .as_ref()
6895 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
6896 {
6897 *runtime
6898 .deferred_reload_reply
6899 .lock()
6900 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
6901 } else {
6902 let _ = reply.send(result);
6903 }
6904 true
6905 }
6906 SupervisorCommand::SetEnabled {
6907 enabled,
6908 requester,
6909 reply,
6910 } => {
6911 let previous_requester = runtime.start_requester.clone();
6912 if enabled {
6913 runtime.start_requester = requester.clone();
6914 }
6915 let result = set_child_enabled(
6916 spec,
6917 runtime,
6918 registry,
6919 process_liveness,
6920 snapshot,
6921 child,
6922 enabled,
6923 )
6924 .await;
6925 if matches!(result, Ok(true)) {
6926 runtime.pending_activity_restart = false;
6927 if enabled {
6928 runtime.start_requester = requester;
6929 } else {
6930 runtime.start_requester = None;
6931 runtime.record_activity_transition(
6932 "subc.module_disabled",
6933 &spec.module_id,
6934 requester,
6935 );
6936 }
6937 } else {
6938 runtime.start_requester = previous_requester;
6939 }
6940 let _ = reply.send(result);
6941 true
6942 }
6943 SupervisorCommand::UpdateConfiguration {
6944 spec: next_spec,
6945 health,
6946 drain_timeout_ms,
6947 reply,
6948 } => {
6949 if let Some(handle) = &runtime.supervisor_handle {
6950 handle.apply_identity_configuration(&next_spec);
6951 }
6952 *spec = next_spec;
6953 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6954 state.configuration_updated_since_spawn = true;
6955 });
6956 let health_changed = runtime.health != health;
6957 runtime.health = health;
6958 if health_changed {
6961 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6962 state.health = ModuleHealthStatus::default();
6963 });
6964 }
6965 runtime.drain_timeout = drain_timeout_ms
6966 .map(Duration::from_millis)
6967 .unwrap_or(runtime.default_drain_timeout);
6968 *runtime
6969 .effective_drain_timeout
6970 .lock()
6971 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
6972 let _ = reply.send(());
6973 true
6974 }
6975 SupervisorCommand::Swap {
6976 requester,
6977 ready_timeout,
6978 reply,
6979 } => {
6980 let mut swap_runtime = runtime.clone();
6981 swap_runtime.start_requester = requester;
6982 let before = lock_snapshot(snapshot)
6983 .ok()
6984 .map(|state| state.spawn_generation);
6985 let end = swap::run_swap(
6986 spec,
6987 &swap_runtime,
6988 registry,
6989 process_liveness,
6990 snapshot,
6991 child,
6992 commands,
6993 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
6994 reply,
6995 )
6996 .await;
6997 if before
6998 != lock_snapshot(snapshot)
6999 .ok()
7000 .map(|state| state.spawn_generation)
7001 {
7002 runtime.start_requester = swap_runtime.start_requester;
7003 }
7004 requeued.extend(end.requeue);
7005 true
7006 }
7007 }
7008}
7009
7010async fn restart_child(
7011 spec: &ModuleSpec,
7012 runtime: &SupervisorRuntimeConfig,
7013 registry: &Registry,
7014 process_liveness: &SupervisorProcessLiveness,
7015 snapshot: &SharedSnapshot,
7016 child: &mut Option<SupervisedChild>,
7017 drain_timeout: Duration,
7018) -> Result<(), SuperviseError> {
7019 if !lock_snapshot(snapshot)?.enabled {
7021 return Err(SuperviseError::Disabled {
7022 module_id: spec.module_id.clone(),
7023 });
7024 }
7025 let stop_notice = begin_forwarding_drain_with_timeout(
7026 spec,
7027 runtime,
7028 registry,
7029 snapshot,
7030 None,
7031 RouteCloseReason::Restart,
7032 drain_timeout,
7033 )
7034 .await?;
7035
7036 if child.is_some() {
7037 drain_optional_child(
7038 &spec.module_id,
7039 spec.protocol,
7040 stop_notice,
7041 registry,
7042 runtime.forwarding.as_deref(),
7043 snapshot,
7044 &runtime.terminal_ring,
7045 &runtime.spawn_events,
7046 child,
7047 drain_timeout,
7048 ModuleState::Restarting,
7049 Some(true),
7050 )
7051 .await?;
7052 } else {
7053 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7054 state.enabled = true;
7055 state.state = ModuleState::Restarting;
7056 clear_current_process_facts(state);
7057 })?;
7058 release_dead_registration(
7059 registry,
7060 runtime.forwarding.as_deref(),
7061 snapshot,
7062 &spec.module_id,
7063 )
7064 .await?;
7065 }
7066
7067 reset_restart_count(snapshot, &spec.module_id)?;
7068 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7069 schedule_respawn(
7070 runtime,
7071 snapshot,
7072 &spec.module_id,
7073 runtime.restart_policy.backoff,
7074 RespawnKind::Spawn,
7075 )
7076}
7077
7078async fn reload_child(
7079 spec: &ModuleSpec,
7080 runtime: &SupervisorRuntimeConfig,
7081 registry: &Registry,
7082 process_liveness: &SupervisorProcessLiveness,
7083 snapshot: &SharedSnapshot,
7084 child: &mut Option<SupervisedChild>,
7085) -> Result<(), SuperviseError> {
7086 if !lock_snapshot(snapshot)?.enabled {
7088 return Err(SuperviseError::Disabled {
7089 module_id: spec.module_id.clone(),
7090 });
7091 }
7092 let stop_notice = begin_forwarding_drain(
7093 spec,
7094 runtime,
7095 registry,
7096 snapshot,
7097 Some(true),
7098 RouteCloseReason::Reload,
7099 )
7100 .await?;
7101
7102 if child.is_some() {
7103 drain_optional_child(
7104 &spec.module_id,
7105 spec.protocol,
7106 stop_notice,
7107 registry,
7108 runtime.forwarding.as_deref(),
7109 snapshot,
7110 &runtime.terminal_ring,
7111 &runtime.spawn_events,
7112 child,
7113 runtime.drain_timeout,
7114 ModuleState::Restarting,
7115 Some(true),
7116 )
7117 .await?;
7118 } else {
7119 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7120 state.enabled = true;
7121 state.state = ModuleState::Restarting;
7122 clear_current_process_facts(state);
7123 })?;
7124 release_dead_registration(
7125 registry,
7126 runtime.forwarding.as_deref(),
7127 snapshot,
7128 &spec.module_id,
7129 )
7130 .await?;
7131 }
7132
7133 reset_restart_count(snapshot, &spec.module_id)?;
7134 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7135 schedule_respawn(
7136 runtime,
7137 snapshot,
7138 &spec.module_id,
7139 runtime.restart_policy.backoff,
7140 RespawnKind::Reload,
7141 )
7142}
7143
7144async fn finish_reload_child(
7145 spec: &ModuleSpec,
7146 runtime: &SupervisorRuntimeConfig,
7147 registry: &Registry,
7148 process_liveness: &SupervisorProcessLiveness,
7149 snapshot: &SharedSnapshot,
7150 child: &mut Option<SupervisedChild>,
7151) -> Result<(), SuperviseError> {
7152 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7153 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7154 Ok(next_child) => next_child,
7155 Err(err) => {
7156 return handle_reload_spawn_failure(
7157 spec,
7158 runtime,
7159 process_liveness,
7160 snapshot,
7161 child,
7162 format!("new child failed to spawn: {err}"),
7163 )
7164 .await;
7165 }
7166 };
7167 *child = Some(next_child);
7168
7169 let wait_outcome = {
7170 let active_child = child.as_mut().expect("new reload child was just stored");
7171 wait_for_registration_after_reload(
7172 registry,
7173 &spec.module_id,
7174 snapshot,
7175 active_child,
7176 REGISTRY_RELEASE_TIMEOUT,
7177 )
7178 .await?
7179 };
7180
7181 match wait_outcome {
7182 RegistrationWaitOutcome::Registered => {
7183 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7184 Ok(())
7185 }
7186 RegistrationWaitOutcome::Exited(exit_report) => {
7187 if let Some(active_child) = child.as_mut() {
7188 active_child.drain_stderr(&spec.module_id).await;
7189 }
7190 let mut exited_child = child.take().expect("exited reload child is still stored");
7193 #[cfg(test)]
7194 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7195 gate.reached.notify_one();
7196 gate.resume.notified().await;
7197 }
7198 let result = handle_reload_child_registration_failure(
7199 spec,
7200 runtime,
7201 registry,
7202 process_liveness,
7203 snapshot,
7204 child,
7205 ReloadRegistrationFailure {
7206 exit_report: registration_failure_exit_report(exit_report),
7207 reason: exited_child
7208 .spawn_failure
7209 .clone()
7210 .unwrap_or_else(|| "new child exited before registering".to_string()),
7211 },
7212 )
7213 .await;
7214 exited_child.release_roster();
7215 result
7216 }
7217 RegistrationWaitOutcome::TimedOut => {
7218 let mut timed_out_child = child
7219 .take()
7220 .expect("timed-out reload child is still running");
7221 timed_out_child
7222 .start_kill()
7223 .map_err(|source| SuperviseError::Kill {
7224 module_id: spec.module_id.clone(),
7225 source,
7226 })?;
7227 let status = timed_out_child
7228 .wait()
7229 .await
7230 .map_err(|source| SuperviseError::Wait {
7231 module_id: spec.module_id.clone(),
7232 source,
7233 })?;
7234 timed_out_child.drain_stderr(&spec.module_id).await;
7235 handle_reload_child_registration_failure(
7236 spec,
7237 runtime,
7238 registry,
7239 process_liveness,
7240 snapshot,
7241 child,
7242 ReloadRegistrationFailure {
7243 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7244 snapshot,
7245 &timed_out_child,
7246 &status,
7247 )),
7248 reason: format!(
7249 "new child did not register within {:?}",
7250 REGISTRY_RELEASE_TIMEOUT
7251 ),
7252 },
7253 )
7254 .await
7255 }
7256 }
7257}
7258
7259async fn set_child_enabled(
7260 spec: &ModuleSpec,
7261 runtime: &SupervisorRuntimeConfig,
7262 registry: &Registry,
7263 process_liveness: &SupervisorProcessLiveness,
7264 snapshot: &SharedSnapshot,
7265 child: &mut Option<SupervisedChild>,
7266 enabled: bool,
7267) -> Result<bool, SuperviseError> {
7268 let (current_enabled, current_state, respawn_pending) = {
7269 let state = lock_snapshot(snapshot)?;
7270 (state.enabled, state.state, state.respawn_pending)
7271 };
7272 let revive_terminal = enabled
7280 && current_enabled
7281 && child.is_none()
7282 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7283 || (current_state == ModuleState::Restarting && !respawn_pending));
7284 if current_enabled == enabled && !revive_terminal {
7285 return Ok(false);
7286 }
7287
7288 if enabled {
7289 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7290 state.enabled = true;
7291 state.state = ModuleState::Starting;
7292 clear_current_process_facts(state);
7293 })?;
7294 #[cfg(test)]
7295 if runtime.test_seed_stale_facts_before_enable_spawn {
7296 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7297 state.process_alive = true;
7298 state.pid = Some(41);
7299 state.spawned_at_ms = Some(42);
7300 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7301 state.spawned_file_identity = Some(SpawnedFileIdentity {
7302 device: 43,
7303 inode: 44,
7304 #[cfg(windows)]
7305 windows_image: None,
7306 });
7307 })?;
7308 }
7309 release_dead_registration(
7310 registry,
7311 runtime.forwarding.as_deref(),
7312 snapshot,
7313 &spec.module_id,
7314 )
7315 .await?;
7316 reset_restart_count(snapshot, &spec.module_id)?;
7317 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7318 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7319 Ok(next_child) => next_child,
7320 Err(err) => {
7321 if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7322 state.state = ModuleState::Failed;
7323 clear_current_process_facts(state);
7324 }) {
7325 error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7326 }
7327 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7328 return Err(err);
7329 }
7330 };
7331 *child = Some(next_child);
7332 debug!(module_id = %spec.module_id, "supervised module enabled");
7333 Ok(true)
7334 } else {
7335 let stop_notice = begin_forwarding_drain_if_configured(
7336 spec,
7337 runtime,
7338 registry,
7339 snapshot,
7340 Some(false),
7341 RouteCloseReason::Disable,
7342 )
7343 .await?;
7344 drain_optional_child(
7345 &spec.module_id,
7346 spec.protocol,
7347 stop_notice,
7348 registry,
7349 runtime.forwarding.as_deref(),
7350 snapshot,
7351 &runtime.terminal_ring,
7352 &runtime.spawn_events,
7353 child,
7354 runtime.drain_timeout,
7355 ModuleState::Disabled,
7356 Some(false),
7357 )
7358 .await?;
7359 debug!(module_id = %spec.module_id, "supervised module disabled");
7360 Ok(true)
7361 }
7362}
7363
7364#[allow(clippy::too_many_arguments)]
7365async fn on_child_exit(
7366 spec: &ModuleSpec,
7367 policy: RestartPolicy,
7368 registry: &Registry,
7369 snapshot: &SharedSnapshot,
7370 terminal_ring: &Arc<Mutex<TerminalRing>>,
7371 spawn_events: &SpawnEventFeed,
7372 roster: &ChildRoster,
7373 exit_report: ExitReport,
7374) -> NextAction {
7375 if roster.is_closed() {
7381 return on_child_exit_during_daemon_shutdown(
7382 spec,
7383 registry,
7384 snapshot,
7385 terminal_ring,
7386 spawn_events,
7387 exit_report,
7388 )
7389 .await;
7390 }
7391 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7406 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7407 match exit_report.kind {
7408 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7409 info!(
7410 module_id = %spec.module_id,
7411 exit_code = ?exit_report.code,
7412 exit_signal = ?exit_report.signal,
7413 "supervised module exited cleanly"
7414 );
7415 if let Err(err) = publish_after_terminal_record(
7416 &spec.module_id,
7417 || {
7418 record_terminal(
7419 &spec.module_id,
7420 terminal_ring,
7421 spawn_events,
7422 &exit_report,
7423 TerminalDisposition::Stopped,
7424 )
7425 },
7426 || {
7427 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7428 state.state = ModuleState::Stopped;
7429 clear_current_process_facts(state);
7430 state.last_exit = Some(exit_report.clone());
7431 })
7432 },
7433 ) {
7434 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7435 }
7436 let registration_released = match wait_for_registration_release(
7437 registry,
7438 &spec.module_id,
7439 REGISTRY_RELEASE_TIMEOUT,
7440 )
7441 .await
7442 {
7443 Ok(()) => true,
7444 Err(err) => {
7445 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7446 false
7447 }
7448 };
7449 NextAction::Stop {
7450 registration_released,
7451 }
7452 }
7453 ExitKind::Clean | ExitKind::Crash => {
7454 if unrequested_clean_exit_of_protocol_none {
7455 warn!(
7456 module_id = %spec.module_id,
7457 exit_code = ?exit_report.code,
7458 exit_signal = ?exit_report.signal,
7459 "protocol-none module exited cleanly without a stop request; handling it as a crash"
7460 );
7461 } else {
7462 warn!(
7463 module_id = %spec.module_id,
7464 exit_code = ?exit_report.code,
7465 exit_signal = ?exit_report.signal,
7466 "supervised module exited abnormally (crash)"
7467 );
7468 }
7469 let mut restart_schedule = None;
7470 let mut disposition = TerminalDisposition::Disabled;
7471 let mut final_state = ModuleState::Disabled;
7472 let mut disposition_detail = lock_snapshot(snapshot)
7476 .ok()
7477 .and_then(|mut state| state.spawn_failure.take());
7478 let now = Instant::now();
7479 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7480 if state.enabled {
7481 if let Some(schedule) = state.next_crash_restart(&policy, now) {
7482 state.respawn_pending = true;
7488 final_state = ModuleState::Restarting;
7489 restart_schedule = Some(schedule);
7490 disposition = TerminalDisposition::Restarting;
7491 } else {
7492 final_state = ModuleState::Failed;
7493 disposition = TerminalDisposition::Failed;
7494 let budget = policy.budget_exhausted_detail();
7495 disposition_detail =
7496 Some(disposition_detail.take().map_or_else(
7497 || budget.clone(),
7498 |cause| format!("{cause}; {budget}"),
7499 ));
7500 }
7501 } else {
7502 disposition = TerminalDisposition::Disabled;
7503 }
7504 }) {
7505 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7506 return NextAction::Stop {
7507 registration_released: false,
7508 };
7509 }
7510 if disposition == TerminalDisposition::Failed {
7511 error!(
7516 module_id = %spec.module_id,
7517 max_restarts = policy.max_restarts,
7518 window_secs = policy.window.as_secs(),
7519 "module stopped: {}",
7520 policy.budget_exhausted_detail()
7521 );
7522 }
7523 if let Err(err) = publish_after_terminal_record(
7524 &spec.module_id,
7525 || {
7526 record_terminal_with_detail(
7527 &spec.module_id,
7528 terminal_ring,
7529 spawn_events,
7530 &exit_report,
7531 disposition,
7532 disposition_detail,
7533 )
7534 },
7535 || {
7536 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7537 state.state = final_state;
7538 clear_current_process_facts(state);
7539 state.last_exit = Some(exit_report.clone());
7540 })
7541 },
7542 ) {
7543 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7544 }
7545
7546 if let Some(schedule) = restart_schedule {
7547 NextAction::Restart {
7548 schedule: Some(schedule),
7549 }
7550 } else {
7551 let registration_released = match wait_for_registration_release(
7552 registry,
7553 &spec.module_id,
7554 REGISTRY_RELEASE_TIMEOUT,
7555 )
7556 .await
7557 {
7558 Ok(()) => true,
7559 Err(err) => {
7560 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7561 false
7562 }
7563 };
7564 NextAction::Stop {
7565 registration_released,
7566 }
7567 }
7568 }
7569 ExitKind::DeliberateSeverance => {
7570 warn!(
7571 module_id = %spec.module_id,
7572 exit_code = ?exit_report.code,
7573 exit_signal = ?exit_report.signal,
7574 "supervised module exited after deliberate connection severance"
7575 );
7576 let should_restart = match lock_snapshot(snapshot) {
7577 Ok(state) => state.enabled,
7578 Err(err) => {
7579 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7580 return NextAction::Stop {
7581 registration_released: false,
7582 };
7583 }
7584 };
7585 let final_state = if should_restart {
7586 ModuleState::Restarting
7587 } else {
7588 ModuleState::Disabled
7589 };
7590 if let Err(err) = publish_after_terminal_record(
7591 &spec.module_id,
7592 || {
7593 record_terminal(
7594 &spec.module_id,
7595 terminal_ring,
7596 spawn_events,
7597 &exit_report,
7598 terminal_disposition(final_state),
7599 )
7600 },
7601 || {
7602 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7603 state.state = final_state;
7604 clear_current_process_facts(state);
7605 state.last_exit = Some(exit_report.clone());
7606 state.lifetime_restarts += 1;
7607 })
7608 },
7609 ) {
7610 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7611 return NextAction::Stop {
7612 registration_released: false,
7613 };
7614 }
7615
7616 if should_restart {
7617 NextAction::Restart { schedule: None }
7618 } else {
7619 let registration_released = match wait_for_registration_release(
7620 registry,
7621 &spec.module_id,
7622 REGISTRY_RELEASE_TIMEOUT,
7623 )
7624 .await
7625 {
7626 Ok(()) => true,
7627 Err(err) => {
7628 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7629 false
7630 }
7631 };
7632 NextAction::Stop {
7633 registration_released,
7634 }
7635 }
7636 }
7637 }
7638}
7639
7640async fn on_child_exit_during_daemon_shutdown(
7641 spec: &ModuleSpec,
7642 registry: &Registry,
7643 snapshot: &SharedSnapshot,
7644 terminal_ring: &Arc<Mutex<TerminalRing>>,
7645 spawn_events: &SpawnEventFeed,
7646 exit_report: ExitReport,
7647) -> NextAction {
7648 info!(
7649 module_id = %spec.module_id,
7650 exit_code = ?exit_report.code,
7651 exit_signal = ?exit_report.signal,
7652 exit_kind = ?exit_report.kind,
7653 "supervised module exited during daemon shutdown; not restarting it"
7654 );
7655 if let Err(err) = publish_after_terminal_record(
7656 &spec.module_id,
7657 || {
7658 record_terminal(
7659 &spec.module_id,
7660 terminal_ring,
7661 spawn_events,
7662 &exit_report,
7663 TerminalDisposition::DaemonShutdown,
7664 )
7665 },
7666 || {
7667 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7668 state.state = ModuleState::Stopped;
7669 clear_current_process_facts(state);
7670 state.last_exit = Some(exit_report.clone());
7671 })
7672 },
7673 ) {
7674 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7675 }
7676 let registration_released =
7677 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7678 .await
7679 .is_ok();
7680 NextAction::Stop {
7681 registration_released,
7682 }
7683}
7684
7685fn publish_after_terminal_record<T>(
7690 module_id: &str,
7691 record: impl FnOnce(),
7692 publish: impl FnOnce() -> T,
7693) -> T {
7694 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7695 let published = publish();
7696 if let Err(panic) = recorded {
7697 error!(
7698 module_id,
7699 "terminal recording panicked; published final state before resuming unwind"
7700 );
7701 std::panic::resume_unwind(panic);
7702 }
7703 published
7704}
7705
7706fn fail_after_wait_error(
7707 module_id: &str,
7708 snapshot: &SharedSnapshot,
7709 terminal_ring: &Arc<Mutex<TerminalRing>>,
7710 spawn_events: &SpawnEventFeed,
7711 detail: String,
7712) {
7713 let report = wait_error_exit_report();
7714 publish_after_terminal_record(
7715 module_id,
7716 || {
7717 record_terminal_with_detail(
7718 module_id,
7719 terminal_ring,
7720 spawn_events,
7721 &report,
7722 TerminalDisposition::Failed,
7723 Some(detail),
7724 )
7725 },
7726 || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7727 );
7728}
7729
7730#[cfg(test)]
7731fn record_wait_error_terminal(
7732 module_id: &str,
7733 terminal_ring: &Arc<Mutex<TerminalRing>>,
7734 spawn_events: &SpawnEventFeed,
7735) {
7736 record_terminal(
7737 module_id,
7738 terminal_ring,
7739 spawn_events,
7740 &wait_error_exit_report(),
7741 TerminalDisposition::Failed,
7742 );
7743}
7744
7745fn record_terminal(
7746 module_id: &str,
7747 terminal_ring: &Arc<Mutex<TerminalRing>>,
7748 spawn_events: &SpawnEventFeed,
7749 exit_report: &ExitReport,
7750 disposition: TerminalDisposition,
7751) {
7752 record_terminal_with_detail(
7753 module_id,
7754 terminal_ring,
7755 spawn_events,
7756 exit_report,
7757 disposition,
7758 None,
7759 );
7760}
7761
7762fn durable_terminal_history_of(
7766 terminal_ring: &Mutex<TerminalRing>,
7767 module_id: &str,
7768) -> subc_control::TerminalHistory {
7769 let read = terminal_ring
7770 .lock()
7771 .unwrap_or_else(|p| p.into_inner())
7772 .capture_durable_history();
7773 read.read(module_id)
7774}
7775
7776fn record_terminal_with_detail(
7777 module_id: &str,
7778 terminal_ring: &Arc<Mutex<TerminalRing>>,
7779 spawn_events: &SpawnEventFeed,
7780 exit_report: &ExitReport,
7781 disposition: TerminalDisposition,
7782 disposition_detail: Option<String>,
7783) {
7784 #[cfg(test)]
7785 terminal_history_tests::before_terminal_record(spawn_events);
7786 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7787 let record = TerminalRecord {
7788 exit_code: exit_report.code,
7789 exit_signal: exit_report.signal,
7790 at_ms: exit_report.at_ms,
7791 disposition,
7792 exit_kind: exit_report.kind.into(),
7793 disposition_detail,
7794 };
7795 terminal_ring
7796 .lock()
7797 .unwrap_or_else(|poisoned| poisoned.into_inner())
7798 .record_exit(module_id, record);
7799}
7800
7801fn untrack_if_registration_released(
7802 process_liveness: &SupervisorProcessLiveness,
7803 registry: &Registry,
7804 module_id: &str,
7805 snapshot: &SharedSnapshot,
7806) {
7807 match registry.get_module(module_id) {
7808 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
7809 Ok(Some(_)) => {}
7810 Err(err) => {
7811 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
7812 }
7813 }
7814}
7815
7816#[cfg(test)]
7830fn apply_wire_spawn_args(
7831 command: &mut Command,
7832 spec: &ModuleSpec,
7833 connection_file_path: Option<&std::path::Path>,
7834 handle: Option<&SupervisorHandle>,
7835) -> Result<Option<NonceHandoff>, SuperviseError> {
7836 apply_wire_spawn_args_for_role(
7837 command,
7838 spec,
7839 connection_file_path,
7840 handle,
7841 SpawnRole::Plain,
7842 )
7843}
7844
7845#[cfg(unix)]
7850type NonceHandoff = subc_os::LaunchNonceHandoff;
7851#[cfg(windows)]
7852type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
7853#[cfg(not(any(unix, windows)))]
7854type NonceHandoff = std::convert::Infallible;
7855
7856fn apply_wire_spawn_args_for_role(
7877 command: &mut Command,
7878 spec: &ModuleSpec,
7879 connection_file_path: Option<&std::path::Path>,
7880 handle: Option<&SupervisorHandle>,
7881 role: SpawnRole,
7882) -> Result<Option<NonceHandoff>, SuperviseError> {
7883 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
7884 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
7889 #[cfg(windows)]
7890 {
7891 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
7892 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
7893 }
7894 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
7896 if spec.protocol == ModuleProtocol::None {
7897 return Ok(None);
7898 }
7899 if let Some(connection_file_path) = connection_file_path {
7900 command.arg(SUBC_ARG).arg(connection_file_path);
7901 }
7902
7903 let nonce = generate_launch_nonce()?;
7907 if let Some(handle) = handle {
7908 match role {
7909 SpawnRole::Plain => {
7910 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
7911 if spec.reserved {
7912 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
7913 }
7914 }
7915 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
7916 }
7917 }
7918 #[cfg(unix)]
7919 let handoff = {
7920 let handoff =
7921 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
7922 program: spec.program.clone(),
7923 source,
7924 cgroup_path: None,
7925 })?;
7926 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
7927 Some(handoff)
7928 };
7929 #[cfg(windows)]
7930 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
7931 #[cfg(not(any(unix, windows)))]
7932 let handoff = None;
7933 #[cfg(not(unix))]
7937 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
7938 Ok(handoff)
7939}
7940
7941fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
7942 command.env_remove(CK_LOG_ENV);
7943 command.env_remove(SUBC_SPAWN_ROLE_ENV);
7950 for (key, value) in &spec.env {
7951 if matches!(
7955 key.as_str(),
7956 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
7957 ) || key == SUBC_SPAWN_ROLE_ENV
7958 {
7959 continue;
7960 }
7961 command.env(key, value);
7962 }
7963}
7964
7965#[cfg(windows)]
7966fn prepare_windows_nonce_pipe(
7967 command: &mut Command,
7968 spec: &ModuleSpec,
7969 nonce: &str,
7970) -> Option<NonceHandoff> {
7971 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
7972 Ok(handoff) => {
7973 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
7974 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
7979 Some(handoff)
7980 }
7981 Err(error) => {
7982 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
7983 None
7984 }
7985 }
7986}
7987
7988#[cfg(windows)]
7989fn start_windows_nonce_pipe(
7990 handoff: Option<NonceHandoff>,
7991 pid: u32,
7992 spec: &ModuleSpec,
7993) -> Option<subc_os::LaunchNoncePipeDelivery> {
7994 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
7998 Ok(delivery) => Some(delivery),
7999 Err(error) => {
8000 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8001 None
8002 }
8003 })
8004}
8005
8006fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8007 #[cfg(windows)]
8008 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8009 return Some(
8010 if snapshot
8011 .pipe_consumed
8012 .as_ref()
8013 .is_some_and(|consumed| consumed.get().is_some())
8014 {
8015 "pipe"
8016 } else {
8017 "env"
8018 }
8019 .to_owned(),
8020 );
8021 }
8022 let _ = snapshot;
8023 None
8024}
8025
8026#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8029enum SpawnRole {
8030 Plain,
8031 SwapCandidate,
8032}
8033
8034fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8037 if role == SpawnRole::SwapCandidate {
8038 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8039 }
8040}
8041
8042#[cfg(target_os = "linux")]
8043fn previous_instance_error(spec: &ModuleSpec, pid: u32) -> SuperviseError {
8044 SuperviseError::Spawn {
8045 program: spec.program.clone(),
8046 source: io::Error::new(io::ErrorKind::WouldBlock, format!(
8047 "waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified"
8048 )),
8049 cgroup_path: None,
8050 }
8051}
8052
8053fn spawn_child(
8054 spec: &ModuleSpec,
8055 connection_file_path: Option<&std::path::Path>,
8056 handle: Option<&SupervisorHandle>,
8057 ring: &Arc<Mutex<StderrRing>>,
8058 capture_logs_dir: Option<&std::path::Path>,
8059 roster: &ChildRoster,
8060 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8061) -> Result<SupervisedChild, SuperviseError> {
8062 spawn_child_in_slot(
8063 spec,
8064 connection_file_path,
8065 handle,
8066 ring,
8067 capture_logs_dir,
8068 roster,
8069 #[cfg(target_os = "linux")]
8070 cgroup_placement,
8071 SpawnRole::Plain,
8072 false,
8073 )
8074}
8075
8076#[allow(clippy::too_many_arguments)]
8089fn spawn_child_in_slot(
8090 spec: &ModuleSpec,
8091 connection_file_path: Option<&std::path::Path>,
8092 handle: Option<&SupervisorHandle>,
8093 ring: &Arc<Mutex<StderrRing>>,
8094 capture_logs_dir: Option<&std::path::Path>,
8095 roster: &ChildRoster,
8096 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8097 role: SpawnRole,
8098 alternate_slot: bool,
8099) -> Result<SupervisedChild, SuperviseError> {
8100 #[cfg(target_os = "linux")]
8101 if let Some(pid) = roster.previous_instance_pid(&spec.module_id) {
8102 return Err(previous_instance_error(spec, pid));
8103 }
8104 if roster.is_closed() {
8105 return Err(SuperviseError::Spawn {
8106 program: spec.program.clone(),
8107 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8108 cgroup_path: None,
8109 });
8110 }
8111 #[cfg(target_os = "linux")]
8112 let cgroup_name = {
8113 if cgroup_placement.is_none() {
8118 swap::cgroup_name(&spec.module_id, alternate_slot)
8119 } else {
8120 let nonce = generate_launch_nonce()?;
8121 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8122 let mut end = spec.module_id.len().min(64);
8125 while !spec.module_id.is_char_boundary(end) {
8126 end -= 1;
8127 }
8128 format!(
8129 "{}_{suffix}",
8130 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8131 )
8132 }
8133 };
8134 #[cfg(not(target_os = "linux"))]
8135 let _ = alternate_slot;
8136 #[cfg(target_os = "macos")]
8137 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8138 #[cfg(windows)]
8139 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8140 program: spec.program.clone(),
8141 source,
8142 cgroup_path: None,
8143 })?;
8144 #[cfg(windows)]
8145 let mut command = Command::new(image_capture.path());
8146 #[cfg(not(any(target_os = "macos", windows)))]
8147 let mut command = Command::new(&spec.program);
8148 command.args(&spec.args);
8149 apply_child_env(&mut command, spec);
8179 apply_spawn_role(&mut command, role);
8180 let nonce_handoff =
8181 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8182
8183 #[cfg(target_os = "linux")]
8184 let cgroup_path = cgroup_placement
8185 .map(|placement| placement.module_path(&cgroup_name))
8186 .transpose()
8187 .map_err(|source| SuperviseError::Cgroup {
8188 module_id: spec.module_id.clone(),
8189 source,
8190 })?;
8191 #[cfg(not(target_os = "linux"))]
8192 let cgroup_path: Option<PathBuf> = None;
8193 #[cfg(target_os = "linux")]
8194 if let Some(path) = &cgroup_path {
8195 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8196 if let Some(placement) = cgroup_placement {
8197 remove_module_cgroup(placement, &cgroup_name);
8198 }
8199 return Err(error);
8200 }
8201 }
8202
8203 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8204 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8205 match ChildOutputSink::open(&path, capture_retention(spec)) {
8206 Ok(sink) => sink,
8207 Err(error) => {
8208 warn!(
8209 module_id = %spec.module_id,
8210 path = %path.display(),
8211 error = %error,
8212 "could not open child output capture file; forwarding to stderr"
8213 );
8214 ChildOutputSink::Stderr
8215 }
8216 }
8217 } else {
8218 ChildOutputSink::Stderr
8219 };
8220
8221 command.stdout(Stdio::piped());
8222 command.stderr(Stdio::piped());
8223 command.kill_on_drop(true);
8224 #[cfg(unix)]
8241 command.process_group(0);
8242 command.stdin(Stdio::null());
8243 #[cfg(unix)]
8247 if let Some(handoff) = nonce_handoff {
8248 handoff.install_last(command.as_std_mut());
8249 }
8250 #[cfg(not(any(unix, windows)))]
8251 let _ = nonce_handoff;
8252
8253 #[cfg(windows)]
8258 subc_jobobject::suspend_on_create_async(&mut command);
8259 let mut child = match command.spawn() {
8260 Ok(child) => child,
8261 Err(source) => {
8262 #[cfg(target_os = "linux")]
8263 if let Some(placement) = cgroup_placement {
8264 remove_module_cgroup(placement, &cgroup_name);
8265 }
8266 return Err(SuperviseError::Spawn {
8267 program: spec.program.clone(),
8268 source,
8269 cgroup_path,
8270 });
8271 }
8272 };
8273 #[cfg(target_os = "macos")]
8278 drop(exec_ack);
8279
8280 #[cfg(windows)]
8285 let windows_image =
8286 Arc::new(
8287 image_capture
8288 .bind_tokio(&child)
8289 .map_err(|source| SuperviseError::Spawn {
8290 program: spec.program.clone(),
8291 source,
8292 cgroup_path: None,
8293 })?,
8294 );
8295 #[cfg(windows)]
8297 let job = contain_spawned_child(&child, spec)?;
8298 let spawned_at_ms = unix_ms_now();
8299 #[cfg(not(windows))]
8300 let spawned_from = spec.program.clone();
8301 #[cfg(windows)]
8302 let spawned_from = windows_image.path().to_path_buf();
8303 #[cfg(not(windows))]
8304 let spawned_file_identity = spawned_file_identity(&spawned_from);
8305 #[cfg(windows)]
8306 let spawned_file_identity = Some(SpawnedFileIdentity {
8307 device: 0,
8308 inode: 0,
8309 windows_image: Some(Arc::clone(&windows_image)),
8310 });
8311 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8312 program: spec.program.clone(),
8313 source: io::Error::other("spawned child exposed no live pid"),
8314 cgroup_path: cgroup_path.clone(),
8315 })?;
8316 #[cfg(windows)]
8317 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8318 #[cfg(not(windows))]
8319 let process_start_time = crate::provenance::process_start_time(pid);
8320 #[cfg(windows)]
8321 let process_start_time = Some(windows_image.start_time());
8322 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8323 #[cfg(all(test, target_os = "macos"))]
8324 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8325 #[cfg(not(windows))]
8330 let recorded_image = observe_spawned_image(pid);
8331 #[cfg(target_os = "macos")]
8335 let recorded_image = if privacy_exec.is_some() {
8336 None
8337 } else {
8338 recorded_image
8339 };
8340 #[cfg(target_os = "linux")]
8341 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8342 #[cfg(not(target_os = "linux"))]
8343 let recorded_cgroup_name = None;
8344 let roster_guard = roster.admit(
8345 spec.module_id.clone(),
8346 pid,
8347 spec.protocol,
8348 process_start_time,
8349 crate::child_roster::RecordedIdentity {
8350 #[cfg(not(windows))]
8351 start_time: recorded_image.map(|image| image.start_time),
8352 #[cfg(windows)]
8353 start_time: process_start_time,
8354 #[cfg(not(windows))]
8355 executable: recorded_image
8356 .and_then(|image| image.executable)
8357 .map(crate::live_children::ExecutableIdentity::from),
8358 #[cfg(windows)]
8359 executable: Some(crate::live_children::ExecutableIdentity::from(
8360 windows_image.identity(),
8361 )),
8362 cgroup_name: recorded_cgroup_name,
8363 #[cfg(target_os = "linux")]
8364 cgroup_placement: cgroup_placement.cloned(),
8365 },
8366 );
8367 if roster.is_closed() {
8376 #[cfg(target_os = "linux")]
8378 kill_module_cgroup(cgroup_placement, &cgroup_name);
8379 if let Err(error) = child.start_kill() {
8380 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8381 }
8382 #[cfg(target_os = "linux")]
8383 if let Some(placement) = cgroup_placement {
8384 while matches!(child.try_wait(), Ok(None)) {
8388 std::thread::yield_now();
8389 }
8390 if matches!(
8391 subc_cgroup::kill_module(Some(placement), &cgroup_name),
8392 subc_cgroup::KillOutcome::Killed
8393 ) {
8394 if let Ok(path) = placement.module_path(&cgroup_name) {
8395 while std::fs::read_to_string(path.join("cgroup.events"))
8396 .ok()
8397 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8398 {
8399 std::thread::yield_now();
8400 }
8401 }
8402 }
8403 remove_module_cgroup(placement, &cgroup_name);
8404 }
8405 drop(roster_guard);
8406 return Err(SuperviseError::Spawn {
8407 program: spec.program.clone(),
8408 source: io::Error::other(
8409 "the daemon began shutting down while this process was starting; ended it",
8410 ),
8411 cgroup_path,
8412 });
8413 }
8414
8415 let stdout_pump = match child.stdout.take() {
8416 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8417 None => {
8418 warn!(
8419 module_id = %spec.module_id,
8420 "spawned child exposed no stdout pipe; file capture will be incomplete"
8421 );
8422 None
8423 }
8424 };
8425 let stderr_pump = match child.stderr.take() {
8426 Some(stderr) => {
8427 let generation = ring
8428 .lock()
8429 .unwrap_or_else(|poisoned| poisoned.into_inner())
8430 .begin_process();
8431 Some(StderrPump {
8432 task: tokio::spawn(pump_stderr_to(
8433 stderr,
8434 Arc::clone(ring),
8435 generation,
8436 output_sink,
8437 )),
8438 generation,
8439 })
8440 }
8441 None => {
8442 ring.lock()
8446 .unwrap_or_else(|poisoned| poisoned.into_inner())
8447 .mark_not_captured("stderr pipe was not available on spawn");
8448 warn!(
8449 module_id = %spec.module_id,
8450 "spawned child exposed no stderr pipe; tail will be unavailable"
8451 );
8452 None
8453 }
8454 };
8455
8456 Ok(SupervisedChild {
8457 child,
8458 #[cfg(windows)]
8459 nonce_delivery,
8460 protocol: spec.protocol,
8461 #[cfg(target_os = "linux")]
8462 module_id: cgroup_name,
8463 #[cfg(target_os = "linux")]
8464 cgroup_placement: cgroup_placement.cloned(),
8465 #[cfg(windows)]
8466 job,
8467 stdout_pump,
8468 stderr_pump,
8469 stderr_ring: Arc::clone(ring),
8470 spawned_at_ms,
8471 spawned_from,
8472 spawned_file_identity,
8473 process_start_time,
8474 process_identity,
8475 pid,
8476 roster_guard: Some(roster_guard),
8477 #[cfg(target_os = "macos")]
8478 privacy_exec,
8479 #[cfg(target_os = "macos")]
8480 report_ready: Arc::new(OnceLock::new()),
8481 spawn_failure: None,
8482 })
8483}
8484
8485#[cfg(target_os = "linux")]
8486pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
8487 use subc_cgroup::KillOutcome;
8488 match subc_cgroup::kill_module(placement, module_id) {
8489 KillOutcome::Killed => {}
8490 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
8491 debug!(
8492 module_id,
8493 "cgroup tree kill unavailable; using direct-child kill"
8494 );
8495 }
8496 KillOutcome::IoError { path, error } => {
8497 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
8498 }
8499 }
8500}
8501
8502#[cfg(windows)]
8516fn contain_spawned_child(
8517 child: &Child,
8518 spec: &ModuleSpec,
8519) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
8520 let module_id = spec.module_id.as_str();
8521 let Some(pid) = child.id() else {
8522 warn!(
8525 module_id,
8526 "spawned child had already exited before containment; no job object attached"
8527 );
8528 return Ok(None);
8529 };
8530
8531 let job = match subc_jobobject::JobObject::new() {
8532 Ok(job) => job,
8533 Err(source) => {
8534 warn!(
8535 module_id,
8536 error = %source,
8537 "could not create a job object; this module's helper processes will not be \
8538 reaped on teardown"
8539 );
8540 resume_suspended_child(pid, spec)?;
8543 return Ok(None);
8544 }
8545 };
8546
8547 if let Err(source) = job.assign(child) {
8548 warn!(
8549 module_id,
8550 error = %source,
8551 "could not assign the child to its job object; this module's helper processes \
8552 will not be reaped on teardown"
8553 );
8554 resume_suspended_child(pid, spec)?;
8555 return Ok(None);
8556 }
8557
8558 resume_suspended_child(pid, spec)?;
8559 Ok(Some(job))
8560}
8561
8562#[cfg(windows)]
8567fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
8568 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
8569 let _ = std::process::Command::new("taskkill.exe")
8573 .args(["/PID", &pid.to_string(), "/T", "/F"])
8574 .stdin(Stdio::null())
8575 .stdout(Stdio::null())
8576 .stderr(Stdio::null())
8577 .status();
8578 return Err(SuperviseError::Spawn {
8579 program: spec.program.clone(),
8580 source,
8581 cgroup_path: None,
8582 });
8583 }
8584 Ok(())
8585}
8586
8587#[cfg(target_os = "linux")]
8588fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
8589 match placement.remove_module(module_id) {
8590 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
8591 Err(error) => warn!(
8592 module_id,
8593 error = %error,
8594 "could not remove module cgroup after process exit; continuing teardown"
8595 ),
8596 }
8597}
8598
8599#[cfg(target_os = "linux")]
8600async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
8601 if matches!(
8605 subc_cgroup::kill_module(Some(placement), module_id),
8606 subc_cgroup::KillOutcome::Killed
8607 ) {
8608 if let Ok(path) = placement.module_path(module_id) {
8609 while std::fs::read_to_string(path.join("cgroup.events"))
8610 .ok()
8611 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8612 {
8613 sleep(Duration::from_millis(1)).await;
8614 }
8615 }
8616 }
8617 remove_module_cgroup(placement, module_id);
8618}
8619
8620#[cfg(target_os = "linux")]
8621fn apply_cgroup_placement(
8622 command: &mut Command,
8623 spec: &ModuleSpec,
8624 path: &std::path::Path,
8625) -> Result<(), SuperviseError> {
8626 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
8627 module_id: spec.module_id.clone(),
8628 source,
8629 })
8630}
8631
8632fn capture_retention(spec: &ModuleSpec) -> Retention {
8633 let defaults = Retention::default();
8634 let value = |name: &str| {
8635 spec.env
8636 .iter()
8637 .rev()
8638 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
8639 };
8640 Retention {
8641 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
8642 .and_then(|value| value.parse().ok())
8643 .unwrap_or(defaults.max_file_mb),
8644 keep: value(CAPTURE_KEEP_ENV)
8645 .and_then(|value| value.parse().ok())
8646 .unwrap_or(defaults.keep),
8647 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
8648 .and_then(|value| value.parse().ok())
8649 .unwrap_or(defaults.max_age_days),
8650 }
8651}
8652
8653fn generate_launch_nonce() -> Result<String, SuperviseError> {
8656 let mut bytes = [0u8; 32];
8657 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
8658 reason: source.to_string(),
8659 })?;
8660 let mut hex = String::with_capacity(64);
8661 for b in bytes {
8662 use std::fmt::Write;
8663 let _ = write!(hex, "{b:02x}");
8664 }
8665 Ok(hex)
8666}
8667
8668fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
8671 if a.len() != b.len() {
8672 return false;
8673 }
8674 let mut diff = 0u8;
8675 for (x, y) in a.iter().zip(b.iter()) {
8676 diff |= x ^ y;
8677 }
8678 diff == 0
8679}
8680
8681#[cfg(not(windows))]
8685fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
8686 let process = subc_os::Process::open(pid).ok().flatten()?;
8687 #[cfg(target_os = "linux")]
8688 return sample_after_linux_exec(
8689 || {
8690 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
8691 let committed = linux_exec_committed(&stat);
8692 #[cfg(test)]
8693 spawn_image_tests::report_exec_state(committed);
8694 committed
8695 },
8696 || process.observe(),
8697 );
8698 #[cfg(not(target_os = "linux"))]
8699 process.observe()
8700}
8701
8702#[cfg(target_os = "linux")]
8706fn linux_exec_committed(stat: &str) -> Option<bool> {
8707 const PF_FORKNOEXEC: u64 = 0x40;
8708 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
8709 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
8710}
8711
8712#[cfg(target_os = "linux")]
8713fn sample_after_linux_exec(
8714 mut exec_committed: impl FnMut() -> Option<bool>,
8715 observe: impl FnOnce() -> Option<subc_os::Observation>,
8716) -> Option<subc_os::Observation> {
8717 let deadline = std::time::Instant::now() + Duration::from_secs(1);
8718 while !exec_committed()? {
8719 if std::time::Instant::now() >= deadline {
8723 return None;
8724 }
8725 std::thread::sleep(Duration::from_micros(50));
8726 }
8727 observe()
8728}
8729
8730#[cfg(all(test, target_os = "linux"))]
8731mod spawn_image_tests {
8732 use super::*;
8733 use std::cell::{Cell, RefCell};
8734
8735 enum ExecEvent {
8736 Uncommitted,
8737 Image(Option<subc_os::Observation>),
8738 }
8739
8740 thread_local! {
8741 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
8742 }
8743
8744 pub(super) fn report_exec_state(committed: Option<bool>) {
8745 if committed == Some(false) {
8746 EXEC_EVENTS.with(|events| {
8747 if let Some(events) = events.borrow_mut().take() {
8748 let _ = events.send(ExecEvent::Uncommitted);
8749 }
8750 });
8751 }
8752 }
8753
8754 #[test]
8755 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
8756 use std::io::{Read, Write};
8757 use std::os::fd::AsRawFd;
8758
8759 let descriptor = std::fs::File::open("/dev/null").unwrap();
8760 let mut command = std::process::Command::new("/bin/sleep");
8761 command.arg("30");
8762 let (mut ready, mut release) =
8763 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
8764 .unwrap();
8765 let spawn = std::thread::spawn(move || command.spawn());
8766 let mut message = [0u8; 8];
8767 ready.read_exact(&mut message).unwrap();
8768 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
8769 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
8770 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
8771 let (events, received) = std::sync::mpsc::channel();
8772 let observer = std::thread::spawn(move || {
8773 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
8774 let image = observe_spawned_image(pid);
8775 let _ = events.send(ExecEvent::Image(image));
8776 });
8777 let first = received.recv_timeout(Duration::from_secs(5));
8781 release.write_all(&[1]).unwrap();
8782 let mut child = spawn.join().unwrap().unwrap();
8783 let after = match first {
8784 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
8785 Ok(ExecEvent::Image(image)) => image,
8786 _ => None,
8787 },
8788 Ok(ExecEvent::Image(image)) => image,
8789 Err(_) => None,
8790 };
8791 observer.join().unwrap();
8792 child.kill().unwrap();
8793 child.wait().unwrap();
8794
8795 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
8796 assert_eq!(
8797 inherited.unwrap().executable,
8798 subc_os::file_identity(&std::env::current_exe().unwrap())
8799 );
8800 assert_eq!(
8801 after.unwrap().executable,
8802 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
8803 );
8804 }
8805
8806 #[test]
8807 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
8808 let committed = Cell::new(false);
8809 let inherited = subc_os::Observation {
8810 start_time: 23,
8811 executable: Some(subc_os::FileIdentity {
8812 device: 29,
8813 inode: 31,
8814 }),
8815 };
8816 let module = subc_os::Observation {
8817 start_time: 23,
8818 executable: Some(subc_os::FileIdentity {
8819 device: 29,
8820 inode: 37,
8821 }),
8822 };
8823 let polls = Cell::new(0);
8824 let image = sample_after_linux_exec(
8825 || {
8826 polls.set(polls.get() + 1);
8827 committed.set(polls.get() == 2);
8830 Some(committed.get())
8831 },
8832 || Some(if committed.get() { module } else { inherited }),
8833 );
8834 assert_eq!(image, Some(module));
8835 assert_eq!(polls.get(), 2);
8836 }
8837
8838 #[test]
8839 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
8840 assert_eq!(
8841 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
8842 Some(false)
8843 );
8844 assert_eq!(
8845 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
8846 Some(true)
8847 );
8848 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
8849 assert_eq!(linux_exec_committed("17 (a) R"), None);
8850 }
8851
8852 #[test]
8853 fn unreadable_exec_state_never_samples_an_inherited_image() {
8854 assert_eq!(
8855 sample_after_linux_exec(
8856 || None,
8857 || panic!("must not sample before exec confirmation")
8858 ),
8859 None
8860 );
8861 }
8862
8863 #[test]
8864 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
8865 assert_eq!(
8866 sample_after_linux_exec(
8867 || Some(false),
8868 || panic!("must not sample after an exec timeout")
8869 ),
8870 None
8871 );
8872 }
8873}
8874
8875#[cfg(windows)]
8878fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
8879 let mut program = spec.program.clone();
8880 if program.is_absolute() || program.components().count() > 1 {
8881 match subc_os::ExecutableCapture::open(&program) {
8884 Ok(capture) => return Ok(capture),
8885 Err(error)
8886 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
8887 Err(error) => return Err(error),
8888 }
8889 program.set_extension("exe");
8890 return subc_os::ExecutableCapture::open(&program);
8891 }
8892 if program.extension().is_none() {
8893 program.set_extension("exe");
8894 }
8895 let custom_path = spec
8896 .env
8897 .iter()
8898 .rev()
8899 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
8900 let mut directories = Vec::new();
8901 if let Some(parent) = std::env::current_exe()?.parent() {
8904 directories.push(parent.to_path_buf());
8905 }
8906 directories.push(std::env::current_dir()?);
8907 if let Some(windows) = std::env::var_os("SystemRoot") {
8908 let windows = PathBuf::from(windows);
8909 directories.push(windows.join("System32"));
8910 directories.push(windows.clone());
8911 }
8912 let path = custom_path
8913 .map(|(_, value)| std::ffi::OsString::from(value))
8914 .or_else(|| std::env::var_os("PATH"));
8915 if let Some(path) = path {
8916 directories.extend(std::env::split_paths(&path));
8917 }
8918 for directory in directories {
8919 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
8920 Ok(capture) => return Ok(capture),
8921 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
8922 Err(error) => return Err(error),
8923 }
8924 }
8925 Err(io::Error::new(
8926 io::ErrorKind::NotFound,
8927 "resolved executable not found",
8928 ))
8929}
8930
8931fn spawn_and_mark_running(
8932 spec: &ModuleSpec,
8933 runtime: &SupervisorRuntimeConfig,
8934 snapshot: &SharedSnapshot,
8935) -> Result<SupervisedChild, SuperviseError> {
8936 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
8937 let child = spawn_child(
8938 spec,
8939 runtime.connection_file_path.as_deref(),
8940 runtime.supervisor_handle.as_ref(),
8941 &runtime.stderr_ring,
8942 runtime.capture_logs_dir.as_deref(),
8943 &runtime.child_roster,
8944 #[cfg(target_os = "linux")]
8945 runtime.cgroup_placement.as_ref(),
8946 )?;
8947 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
8948 if let Some(spawn) = activity_spawn {
8949 spawn.complete(snapshot);
8950 }
8951 Ok(child)
8952}
8953
8954enum RegistrationWaitOutcome {
8955 Registered,
8956 Exited(ExitReport),
8957 TimedOut,
8958}
8959
8960struct ReloadRegistrationFailure {
8961 exit_report: ExitReport,
8962 reason: String,
8963}
8964
8965#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8966enum BusyGaugeObservation {
8967 Quiescent,
8968 Busy,
8969 Omitted,
8970}
8971
8972fn busy_gauge_observation(metrics: Option<&Value>, gauges: &[String]) -> BusyGaugeObservation {
8973 let Some(metrics) = metrics.and_then(Value::as_object) else {
8974 return BusyGaugeObservation::Omitted;
8975 };
8976 let mut sum = 0u128;
8977 for gauge in gauges {
8978 let Some(value) = metrics.get(gauge) else {
8979 return BusyGaugeObservation::Omitted;
8980 };
8981 let Some(value) = value.as_u64() else {
8982 return BusyGaugeObservation::Busy;
8983 };
8984 sum = sum.saturating_add(u128::from(value));
8985 }
8986 if sum == 0 {
8987 BusyGaugeObservation::Quiescent
8988 } else {
8989 BusyGaugeObservation::Busy
8990 }
8991}
8992
8993fn declared_busy_gauges(
8994 registry: &Registry,
8995 module_id: &str,
8996) -> Result<Vec<String>, SuperviseError> {
8997 busy_gauges_of(
8998 registry
8999 .get_module(module_id)
9000 .map_err(SuperviseError::Registry)?,
9001 )
9002}
9003
9004fn declared_busy_gauges_for_connection(
9008 registry: &Registry,
9009 connection_id: ConnectionId,
9010) -> Result<Vec<String>, SuperviseError> {
9011 busy_gauges_of(
9012 registry
9013 .get_module_by_connection(connection_id)
9014 .map_err(SuperviseError::Registry)?,
9015 )
9016}
9017
9018fn busy_gauges_of(
9019 registration: Option<crate::registry::ModuleRegistration>,
9020) -> Result<Vec<String>, SuperviseError> {
9021 let Some(registration) = registration else {
9022 return Ok(Vec::new());
9023 };
9024 let Some(self_signals) = registration.manifest.self_signals else {
9025 return Ok(Vec::new());
9026 };
9027
9028 let mut gauges = Vec::new();
9029 for declaration in self_signals {
9030 if declaration.kind != SelfSignalKind::Busy {
9031 continue;
9032 }
9033 match declaration.anchored_to {
9034 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9035 gauges.extend(declared)
9036 }
9037 _ => {
9038 gauges.push(String::new());
9041 }
9042 }
9043 }
9044 Ok(gauges)
9045}
9046
9047async fn wait_for_forwarding_quiescence(
9052 forwarding: &ForwardingTable,
9053 module_id: &str,
9054 runtime: &SupervisorRuntimeConfig,
9055 endpoint: crate::ModuleEndpointId,
9056 deadline: Instant,
9057 busy_gauges: &[String],
9058 scope: DrainScope,
9059) -> Result<bool, SuperviseError> {
9060 let mut gauges_quiescent = busy_gauges.is_empty();
9061 let mut next_probe_at = Instant::now();
9062 let mut omission_counted = false;
9063
9064 loop {
9065 let now = Instant::now();
9066 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9067 let report = match scope {
9068 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9069 DrainScope::Endpoint(endpoint) => {
9070 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9071 }
9072 };
9073 gauges_quiescent = match report {
9074 Ok(report) => match busy_gauge_observation(report.metrics.as_ref(), busy_gauges) {
9075 BusyGaugeObservation::Quiescent => true,
9076 BusyGaugeObservation::Busy => false,
9077 BusyGaugeObservation::Omitted => {
9078 if !omission_counted {
9079 forwarding
9080 .counters()
9081 .increment_drains_with_undeclared_gauge();
9082 omission_counted = true;
9083 }
9084 false
9085 }
9086 },
9087 Err(err) => {
9088 warn!(
9089 module_id,
9090 error = %err,
9091 "drain health.check did not produce declared busy gauges; treating module as busy"
9092 );
9093 false
9094 }
9095 };
9096 next_probe_at = Instant::now() + runtime.health.cadence.max(REGISTRY_RELEASE_POLL);
9097 }
9098
9099 let in_flight = forwarding
9100 .endpoint_in_flight_count(endpoint)
9101 .map_err(SuperviseError::Forwarding)?;
9102 if in_flight == 0 && gauges_quiescent {
9103 return Ok(true);
9104 }
9105
9106 let now = Instant::now();
9107 if now >= deadline {
9108 return Ok(false);
9109 }
9110 let mut wait = deadline
9111 .saturating_duration_since(now)
9112 .min(REGISTRY_RELEASE_POLL);
9113 if !busy_gauges.is_empty() {
9114 wait = wait.min(next_probe_at.saturating_duration_since(now));
9115 }
9116 sleep(wait).await;
9117 }
9118}
9119
9120fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9128 match wait_result {
9129 Ok(drained) => *drained,
9130 Err(_) => false,
9131 }
9132}
9133
9134fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9135 for released in released_routes {
9136 let frame = match Frame::build_with_version(
9137 released.negotiated_ver,
9138 FrameType::Goodbye,
9139 control_flags(),
9140 released.channel,
9141 released.epoch,
9142 0,
9143 Vec::new(),
9144 ) {
9145 Ok(frame) => frame,
9146 Err(err) => {
9147 warn!(
9148 route_channel = released.channel,
9149 error = %err,
9150 "failed to build supervisor drain route GOODBYE frame"
9151 );
9152 continue;
9153 }
9154 };
9155 if !released.close_on_delivery_failure() {
9156 crate::forwarding::send_module_route_goodbye(
9157 &forwarding.counters(),
9158 &released.sink,
9159 frame,
9160 released.module_id.as_deref(),
9161 "supervisor drain",
9162 );
9163 continue;
9164 }
9165 if let Err(err) = released.sink.try_send(frame) {
9166 warn!(
9167 target_connection_id = released.connection_id.get(),
9168 route_channel = released.channel,
9169 error = %err,
9170 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9171 );
9172 let _ = forwarding.escalate_client_delivery_failure(
9173 released.connection_id,
9174 released.channel,
9175 released.epoch,
9176 CloseReason::new(
9177 "route_goodbye_delivery_failed",
9178 format!(
9179 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9180 released.channel
9181 ),
9182 ),
9183 crate::forwarding::UndeliveredFrame {
9184 module_id: released.module_id.as_deref(),
9185 sink: &released.sink,
9186 },
9187 );
9188 }
9189 }
9190}
9191
9192fn send_module_draining(
9193 module_id: &str,
9194 reason: RouteCloseReason,
9195 deadline_ms: u64,
9196 target: &ModuleDrainTarget,
9197) {
9198 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9199 reason,
9200 deadline_ms,
9201 }) {
9202 Ok(body) => body,
9203 Err(err) => {
9204 warn!(
9205 module_id,
9206 error = %err,
9207 "failed to encode module draining command"
9208 );
9209 return;
9210 }
9211 };
9212 let frame = match Frame::build_with_version(
9213 target.negotiated_ver,
9214 FrameType::Push,
9215 control_flags(),
9216 0,
9217 0,
9218 0,
9219 body,
9220 ) {
9221 Ok(frame) => frame,
9222 Err(err) => {
9223 warn!(
9224 module_id,
9225 error = %err,
9226 "failed to build module draining command frame"
9227 );
9228 return;
9229 }
9230 };
9231 if let Err(err) = target.sink.try_send(frame) {
9232 warn!(
9233 module_id,
9234 target_connection_id = target.endpoint.connection_id.get(),
9235 error = %err,
9236 "module draining command was not delivered to peer"
9237 );
9238 }
9239}
9240
9241fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9243 match Frame::build_with_version(
9244 negotiated_ver,
9245 FrameType::Goodbye,
9246 control_flags(),
9247 0,
9248 0,
9249 0,
9250 Vec::new(),
9251 ) {
9252 Ok(frame) => Some(frame),
9253 Err(err) => {
9254 warn!(
9255 module_id,
9256 error = %err,
9257 "failed to build module GOODBYE frame"
9258 );
9259 None
9260 }
9261 }
9262}
9263
9264#[cfg(unix)]
9278async fn send_module_goodbyes_for_daemon_shutdown(
9279 forwarding: &Arc<ForwardingTable>,
9280 reason: &CloseReason,
9281 wait_for_flush: bool,
9282) {
9283 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
9284 let targets = match forwarding.module_connections() {
9285 Ok(targets) => targets,
9286 Err(err) => {
9287 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
9288 return;
9289 }
9290 };
9291 let deadline = Instant::now() + GOODBYE_BUDGET;
9292 let mut sends = tokio::task::JoinSet::new();
9293 for target in targets {
9294 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
9295 continue;
9296 };
9297 if !wait_for_flush {
9298 if let Err(err) = target.sink.try_send(frame) {
9299 debug!(
9300 module_id = %target.module_id,
9301 error = %err,
9302 "shutdown module GOODBYE was not queued"
9303 );
9304 }
9305 continue;
9306 }
9307 let forwarding = Arc::clone(forwarding);
9308 let reason = reason.clone();
9309 sends.spawn(async move {
9310 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
9311 Ok(Ok(())) => {}
9312 Ok(Err(err)) => debug!(
9313 module_id = %target.module_id,
9314 error = %err,
9315 "module connection closed before its shutdown GOODBYE was written"
9316 ),
9317 Err(_) => warn!(
9318 module_id = %target.module_id,
9319 budget = ?GOODBYE_BUDGET,
9320 "shutdown module GOODBYE was not written within its budget; closing anyway"
9321 ),
9322 }
9323 forwarding.request_connection_close(target.endpoint.connection_id, reason);
9324 });
9325 }
9326 while sends.join_next().await.is_some() {}
9328}
9329
9330fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
9331 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
9332 return;
9333 };
9334 if let Err(err) = target.sink.try_send(frame) {
9335 warn!(
9336 module_id,
9337 target_connection_id = target.endpoint.connection_id.get(),
9338 error = %err,
9339 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
9340 );
9341 forwarding.request_connection_close(
9342 target.endpoint.connection_id,
9343 CloseReason::new(
9344 "module_goodbye_delivery_failed",
9345 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
9346 ),
9347 );
9348 }
9349}
9350
9351#[derive(Clone, Copy)]
9352struct ForwardingDrainContext<'a> {
9353 spec: &'a ModuleSpec,
9354 runtime: &'a SupervisorRuntimeConfig,
9355 registry: &'a Registry,
9356 scope: DrainScope,
9357}
9358
9359#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9361enum DrainScope {
9362 Active,
9365 Endpoint(crate::ModuleEndpointId),
9370}
9371
9372#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9380enum StopNotice {
9381 SentOverConnection,
9384 NoConnection,
9388 NotSent,
9392}
9393
9394async fn begin_forwarding_drain(
9395 spec: &ModuleSpec,
9396 runtime: &SupervisorRuntimeConfig,
9397 registry: &Registry,
9398 snapshot: &SharedSnapshot,
9399 enabled: Option<bool>,
9400 reason: RouteCloseReason,
9401) -> Result<StopNotice, SuperviseError> {
9402 let Some(forwarding) = runtime.forwarding.as_ref() else {
9403 return Err(SuperviseError::ReloadUnavailable {
9404 module_id: spec.module_id.clone(),
9405 reason: "supervisor was not configured with a forwarding table".to_string(),
9406 });
9407 };
9408
9409 begin_forwarding_drain_with(
9410 forwarding,
9411 ForwardingDrainContext {
9412 spec,
9413 runtime,
9414 registry,
9415 scope: DrainScope::Active,
9416 },
9417 snapshot,
9418 enabled,
9419 reason,
9420 runtime.drain_timeout,
9421 )
9422 .await
9423}
9424
9425async fn begin_forwarding_drain_if_configured(
9426 spec: &ModuleSpec,
9427 runtime: &SupervisorRuntimeConfig,
9428 registry: &Registry,
9429 snapshot: &SharedSnapshot,
9430 enabled: Option<bool>,
9431 reason: RouteCloseReason,
9432) -> Result<StopNotice, SuperviseError> {
9433 begin_forwarding_drain_with_timeout(
9434 spec,
9435 runtime,
9436 registry,
9437 snapshot,
9438 enabled,
9439 reason,
9440 runtime.drain_timeout,
9441 )
9442 .await
9443}
9444
9445async fn begin_forwarding_drain_with_timeout(
9449 spec: &ModuleSpec,
9450 runtime: &SupervisorRuntimeConfig,
9451 registry: &Registry,
9452 snapshot: &SharedSnapshot,
9453 enabled: Option<bool>,
9454 reason: RouteCloseReason,
9455 drain_timeout: Duration,
9456) -> Result<StopNotice, SuperviseError> {
9457 let Some(forwarding) = runtime.forwarding.as_ref() else {
9458 return Ok(StopNotice::NotSent);
9459 };
9460
9461 begin_forwarding_drain_with(
9462 forwarding,
9463 ForwardingDrainContext {
9464 spec,
9465 runtime,
9466 registry,
9467 scope: DrainScope::Active,
9468 },
9469 snapshot,
9470 enabled,
9471 reason,
9472 drain_timeout,
9473 )
9474 .await
9475}
9476
9477async fn begin_forwarding_drain_with(
9478 forwarding: &ForwardingTable,
9479 context: ForwardingDrainContext<'_>,
9480 snapshot: &SharedSnapshot,
9481 enabled: Option<bool>,
9482 reason: RouteCloseReason,
9483 drain_timeout: Duration,
9484) -> Result<StopNotice, SuperviseError> {
9485 let ForwardingDrainContext {
9486 spec,
9487 runtime,
9488 registry,
9489 scope,
9490 } = context;
9491 debug_assert_ne!(reason, RouteCloseReason::Crash);
9492 let terminal = matches!(reason, RouteCloseReason::Disable);
9493 let drain_started_at = Instant::now();
9494 let drain_deadline = drain_started_at + drain_timeout;
9495 let deadline_ms =
9496 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
9497 let busy_gauges = match scope {
9498 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
9499 DrainScope::Endpoint(endpoint) => {
9500 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
9501 }
9502 };
9503
9504 let gate_started = Instant::now();
9507 let drain_target = match scope {
9508 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
9509 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
9510 }
9511 .map_err(SuperviseError::Forwarding)?;
9512 info!(
9517 module_id = %spec.module_id,
9518 ?reason,
9519 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
9520 connected = drain_target.is_some(),
9521 "module drain began; route admission closed"
9522 );
9523 if scope == DrainScope::Active {
9524 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9525 state.state = ModuleState::Draining;
9526 state.draining_to_replace =
9527 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
9528 if let Some(enabled) = enabled {
9529 state.enabled = enabled;
9530 }
9531 })?;
9532 }
9533
9534 let Some(target) = drain_target.as_ref() else {
9535 return Ok(StopNotice::NoConnection);
9539 };
9540 {
9541 send_module_draining(&spec.module_id, reason, deadline_ms, target);
9542 #[cfg(feature = "test-support")]
9543 forwarding.wait_before_route_closing_for_test().await;
9544 let routes = forwarding
9545 .endpoint_routes(target.endpoint)
9546 .map_err(SuperviseError::Forwarding)?;
9547 let routes_notified = routes.len();
9548 crate::control::send_route_control_pushes(
9549 forwarding,
9550 routes.clone(),
9551 ClientControlPush::RouteClosing {
9552 module_id: spec.module_id.clone(),
9553 channels: Vec::new(),
9554 reason,
9555 },
9556 );
9557 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
9558
9559 let wait_result = wait_for_forwarding_quiescence(
9565 forwarding,
9566 &spec.module_id,
9567 runtime,
9568 target.endpoint,
9569 drain_deadline,
9570 &busy_gauges,
9571 scope,
9572 )
9573 .await;
9574 let drained = drained_after_quiescence_wait(&wait_result);
9575 if let Err(err) = &wait_result {
9576 error!(
9577 module_id = %spec.module_id,
9578 ?reason,
9579 error = %err,
9580 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
9581 );
9582 } else if !drained {
9583 let holdouts = forwarding
9589 .endpoint_drain_holdouts(target.endpoint)
9590 .unwrap_or_default();
9591 warn!(
9592 module_id = %spec.module_id,
9593 waited = ?drain_timeout,
9594 ?reason,
9595 held_requests = holdouts.requests,
9596 held_routes = holdouts.routes,
9597 total_routes = holdouts.total_routes,
9598 top_connections = ?holdouts.top_connections,
9599 held = %holdouts
9602 .held
9603 .iter()
9604 .map(|(channel, corr)| format!("{channel}:{corr}"))
9605 .collect::<Vec<_>>()
9606 .join(","),
9607 "route drain timed out before request quiescence; forcing teardown"
9608 );
9609 }
9610 crate::control::send_route_control_pushes(
9611 forwarding,
9612 routes,
9613 ClientControlPush::RouteClosed {
9614 module_id: spec.module_id.clone(),
9615 channels: Vec::new(),
9616 reason,
9617 drained,
9618 abandoned: target.abandoned_bindings.len() as u32,
9619 excluded_subscriptions: target.excluded_subscriptions,
9620 terminal: Some(terminal),
9621 },
9622 );
9623 wait_result?;
9624
9625 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
9631 Ok(routes) => routes,
9632 Err(err) => {
9633 warn!(
9634 module_id = %spec.module_id,
9635 ?reason,
9636 error = %err,
9637 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
9638 );
9639 send_module_goodbye(&spec.module_id, forwarding, target);
9640 return Err(SuperviseError::Forwarding(err));
9641 }
9642 };
9643 let route_goodbye_count = released_routes.len();
9644 send_route_goodbyes(forwarding, released_routes);
9645 send_module_goodbye(&spec.module_id, forwarding, target);
9646
9647 info!(
9653 module_id = %spec.module_id,
9654 ?reason,
9655 routes_notified,
9656 route_goodbyes = route_goodbye_count,
9657 abandoned_reservations = target.abandoned_bindings.len(),
9658 excluded_subscriptions = target.excluded_subscriptions,
9659 drained,
9660 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
9661 );
9662 }
9663
9664 Ok(StopNotice::SentOverConnection)
9665}
9666
9667async fn wait_for_registration_after_reload(
9670 registry: &Registry,
9671 module_id: &str,
9672 snapshot: &SharedSnapshot,
9673 child: &mut SupervisedChild,
9674 wait: Duration,
9675) -> Result<RegistrationWaitOutcome, SuperviseError> {
9676 wait_for_slot_registration(
9677 registry,
9678 crate::registry::RegistrationSlot::Active(module_id),
9679 module_id,
9680 snapshot,
9681 child,
9682 wait,
9683 )
9684 .await
9685}
9686
9687async fn wait_for_slot_registration(
9695 registry: &Registry,
9696 slot: crate::registry::RegistrationSlot<'_>,
9697 module_id: &str,
9698 snapshot: &SharedSnapshot,
9699 child: &mut SupervisedChild,
9700 wait: Duration,
9701) -> Result<RegistrationWaitOutcome, SuperviseError> {
9702 let deadline = Instant::now() + wait;
9703 loop {
9704 if registry
9705 .registration(slot)
9706 .map_err(SuperviseError::Registry)?
9707 .is_some()
9708 {
9709 return Ok(RegistrationWaitOutcome::Registered);
9710 }
9711
9712 let now = Instant::now();
9713 if now >= deadline {
9714 return Ok(RegistrationWaitOutcome::TimedOut);
9715 }
9716 let remaining = deadline.saturating_duration_since(now);
9717 let poll = remaining.min(REGISTRY_RELEASE_POLL);
9718
9719 tokio::select! {
9720 wait_result = child.wait() => {
9721 let status = wait_result.map_err(|source| SuperviseError::Wait {
9722 module_id: module_id.to_string(),
9723 source,
9724 })?;
9725 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
9726 snapshot,
9727 child,
9728 &status,
9729 )));
9730 }
9731 _ = sleep(poll) => {}
9732 }
9733 }
9734}
9735
9736fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
9737 if exit_report.kind != ExitKind::DeliberateSeverance {
9740 exit_report.kind = ExitKind::Crash;
9741 }
9742 exit_report
9743}
9744
9745async fn handle_reload_child_registration_failure(
9746 spec: &ModuleSpec,
9747 runtime: &SupervisorRuntimeConfig,
9748 registry: &Registry,
9749 process_liveness: &SupervisorProcessLiveness,
9750 snapshot: &SharedSnapshot,
9751 _child: &mut Option<SupervisedChild>,
9752 failure: ReloadRegistrationFailure,
9753) -> Result<(), SuperviseError> {
9754 let ReloadRegistrationFailure {
9755 exit_report,
9756 reason,
9757 } = failure;
9758 match on_child_exit(
9759 spec,
9760 runtime.restart_policy,
9761 registry,
9762 snapshot,
9763 &runtime.terminal_ring,
9764 &runtime.spawn_events,
9765 &runtime.child_roster,
9766 exit_report,
9767 )
9768 .await
9769 {
9770 NextAction::Stop {
9771 registration_released,
9772 } => {
9773 if registration_released {
9774 process_liveness.untrack_if_current(&spec.module_id, snapshot);
9775 }
9776 }
9777 NextAction::Restart { schedule } => {
9778 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
9779 schedule.delay
9780 });
9781 if let Some(schedule) = schedule {
9782 log_crash_respawn(&spec.module_id, schedule);
9783 }
9784 schedule_respawn(
9785 runtime,
9786 snapshot,
9787 &spec.module_id,
9788 delay,
9789 RespawnKind::Spawn,
9790 )?;
9791 }
9792 }
9793 Err(SuperviseError::ReloadFailed {
9794 module_id: spec.module_id.clone(),
9795 reason,
9796 })
9797}
9798
9799async fn handle_reload_spawn_failure(
9800 spec: &ModuleSpec,
9801 runtime: &SupervisorRuntimeConfig,
9802 process_liveness: &SupervisorProcessLiveness,
9803 snapshot: &SharedSnapshot,
9804 _child: &mut Option<SupervisedChild>,
9805 reason: String,
9806) -> Result<(), SuperviseError> {
9807 let now = Instant::now();
9808 let mut schedule = None;
9809 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9810 clear_current_process_facts(state);
9811 if state.enabled {
9812 schedule = state.next_crash_restart(&runtime.restart_policy, now);
9813 state.state = if schedule.is_some() {
9814 ModuleState::Restarting
9815 } else {
9816 ModuleState::Failed
9817 };
9818 } else {
9819 state.state = ModuleState::Disabled;
9820 }
9821 })?;
9822 if let Some(schedule) = schedule {
9823 schedule_respawn(
9824 runtime,
9825 snapshot,
9826 &spec.module_id,
9827 schedule.delay,
9828 RespawnKind::Spawn,
9829 )?;
9830 } else {
9831 process_liveness.untrack_if_current(&spec.module_id, snapshot);
9832 }
9833 Err(SuperviseError::ReloadFailed {
9834 module_id: spec.module_id.clone(),
9835 reason,
9836 })
9837}
9838
9839fn control_flags() -> Flags {
9840 Flags::new(false, Priority::Passive, false)
9841}
9842
9843#[allow(clippy::too_many_arguments)]
9844async fn drain_optional_child(
9845 module_id: &str,
9846 protocol: ModuleProtocol,
9847 stop_notice: StopNotice,
9848 registry: &Registry,
9849 forwarding: Option<&ForwardingTable>,
9850 snapshot: &SharedSnapshot,
9851 terminal_ring: &Arc<Mutex<TerminalRing>>,
9852 spawn_events: &SpawnEventFeed,
9853 child: &mut Option<SupervisedChild>,
9854 drain_timeout: Duration,
9855 final_state: ModuleState,
9856 enabled: Option<bool>,
9857) -> Result<(), SuperviseError> {
9858 if let Some(child) = child.take() {
9859 drain_child_to_state(
9860 module_id,
9861 protocol,
9862 stop_notice,
9863 registry,
9864 forwarding,
9865 snapshot,
9866 terminal_ring,
9867 spawn_events,
9868 child,
9869 drain_timeout,
9870 final_state,
9871 enabled,
9872 )
9873 .await
9874 } else {
9875 update_snapshot(snapshot, Some(module_id), |state| {
9876 state.state = final_state;
9877 if let Some(enabled) = enabled {
9878 state.enabled = enabled;
9879 }
9880 clear_current_process_facts(state);
9881 })?;
9882 release_dead_registration(registry, forwarding, snapshot, module_id).await
9883 }
9884}
9885
9886#[allow(clippy::too_many_arguments)]
9887async fn drain_child_to_state(
9888 module_id: &str,
9889 _protocol: ModuleProtocol,
9890 stop_notice: StopNotice,
9891 registry: &Registry,
9892 forwarding: Option<&ForwardingTable>,
9893 snapshot: &SharedSnapshot,
9894 terminal_ring: &Arc<Mutex<TerminalRing>>,
9895 spawn_events: &SpawnEventFeed,
9896 mut child: SupervisedChild,
9897 drain_timeout: Duration,
9898 final_state: ModuleState,
9899 enabled: Option<bool>,
9900) -> Result<(), SuperviseError> {
9901 let protocol = child.protocol;
9902 update_snapshot(snapshot, Some(module_id), |state| {
9903 state.state = ModuleState::Draining;
9904 state.draining_to_replace = final_state == ModuleState::Restarting;
9905 if let Some(enabled) = enabled {
9906 state.enabled = enabled;
9907 }
9908 })?;
9909
9910 if stop_notice != StopNotice::SentOverConnection {
9921 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
9922 info!(
9923 module_id,
9924 pid = child.pid,
9925 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
9926 "module has no connection yet; requesting stop by signal"
9927 );
9928 }
9929 request_graceful_stop(module_id, &child);
9930 }
9931
9932 let exit_report = match timeout(drain_timeout, child.wait()).await {
9933 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
9934 Ok(Err(source)) => {
9935 fail_after_wait_error(
9936 module_id,
9937 snapshot,
9938 terminal_ring,
9939 spawn_events,
9940 format!("failed to wait during drain: {source}"),
9941 );
9942 return Err(SuperviseError::Wait {
9943 module_id: module_id.to_string(),
9944 source,
9945 });
9946 }
9947 Err(_) => {
9948 warn!(
9961 module_id,
9962 pid = child.pid,
9963 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
9964 reason = ?final_state,
9965 ?stop_notice,
9966 "drain budget expired before the module exited; killing it"
9967 );
9968 child.start_kill().map_err(|source| {
9969 fail_after_wait_error(
9970 module_id,
9971 snapshot,
9972 terminal_ring,
9973 spawn_events,
9974 format!("failed to kill during drain: {source}"),
9975 );
9976 SuperviseError::Kill {
9977 module_id: module_id.to_string(),
9978 source,
9979 }
9980 })?;
9981 let status = child.wait().await.map_err(|source| {
9982 fail_after_wait_error(
9983 module_id,
9984 snapshot,
9985 terminal_ring,
9986 spawn_events,
9987 format!("failed to wait after drain kill: {source}"),
9988 );
9989 SuperviseError::Wait {
9990 module_id: module_id.to_string(),
9991 source,
9992 }
9993 })?;
9994 classify_reaped_child_exit(snapshot, &child, &status)
9995 }
9996 };
9997
9998 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
9999 publish_after_terminal_record(
10000 module_id,
10001 || {
10002 record_terminal_with_detail(
10003 module_id,
10004 terminal_ring,
10005 spawn_events,
10006 &exit_report,
10007 terminal_disposition(final_state),
10008 detail,
10009 )
10010 },
10011 || {
10012 update_snapshot(snapshot, Some(module_id), |state| {
10013 state.state = final_state;
10014 if let Some(enabled) = enabled {
10015 state.enabled = enabled;
10016 }
10017 clear_current_process_facts(state);
10018 state.last_exit = Some(exit_report.clone());
10019 if exit_report.kind == ExitKind::DeliberateSeverance {
10020 state.lifetime_restarts += 1;
10021 }
10022 })
10023 },
10024 )?;
10025 child.drain_stderr(module_id).await;
10026
10027 release_dead_registration(registry, forwarding, snapshot, module_id).await
10028}
10029
10030#[cfg(unix)]
10050fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10051 let Some(pid) = child
10052 .id()
10053 .and_then(|pid| i32::try_from(pid).ok())
10054 .and_then(rustix::process::Pid::from_raw)
10055 else {
10056 debug!(
10057 module_id,
10058 "no pid to signal for teardown; falling through to the drain wait"
10059 );
10060 return;
10061 };
10062 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10063 Ok(()) => debug!(
10064 module_id,
10065 "sent SIGTERM to a module nothing else asked to stop"
10066 ),
10067 Err(err) => debug!(
10068 module_id,
10069 error = %err,
10070 "SIGTERM to module failed; the drain wait and kill still apply"
10071 ),
10072 }
10073}
10074
10075#[cfg(not(unix))]
10083fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10084 debug!(
10085 module_id,
10086 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10087 );
10088}
10089
10090fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10091 match final_state {
10092 ModuleState::Stopped => TerminalDisposition::Stopped,
10093 ModuleState::Disabled => TerminalDisposition::Disabled,
10094 ModuleState::Restarting => TerminalDisposition::Restarting,
10095 ModuleState::Failed => TerminalDisposition::Failed,
10096 ModuleState::Starting
10097 | ModuleState::Running
10098 | ModuleState::Unresponsive
10099 | ModuleState::Draining => {
10100 unreachable!("terminal exits only finish in terminal or restarting states")
10101 }
10102 }
10103}
10104
10105async fn release_dead_registration(
10113 registry: &Registry,
10114 forwarding: Option<&ForwardingTable>,
10115 snapshot: &SharedSnapshot,
10116 module_id: &str,
10117) -> Result<(), SuperviseError> {
10118 let result = async {
10119 let registration = registry
10120 .get_module(module_id)
10121 .map_err(SuperviseError::Registry)?;
10122 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10123 Ok(()) => return Ok(()),
10124 Err(SuperviseError::RegistrationStillActive { .. }) => {}
10125 Err(err) => return Err(err),
10126 }
10127 let pid = lock_snapshot(snapshot)?.reaped_pid;
10128 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10129 warn!(
10130 module_id,
10131 pid,
10132 connection_id = registration.connection_id.get(),
10133 "reaped module registration outlived release grace; closing dead connection"
10134 );
10135 forwarding.request_connection_close(
10136 registration.connection_id,
10137 CloseReason::new(
10138 "supervised_process_reaped",
10139 format!("module '{module_id}' pid {pid} exited"),
10140 ),
10141 );
10142 wait_for_slot_registration_release(
10143 registry,
10144 crate::registry::RegistrationSlot::Connection(registration.connection_id),
10145 REGISTRY_RELEASE_TIMEOUT,
10146 )
10147 .await?;
10148 }
10149 wait_for_registration_release(registry, module_id, Duration::ZERO).await
10150 }
10151 .await;
10152 if let Err(err) = &result {
10153 fail_snapshot(snapshot, Some(module_id), None);
10154 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10155 }
10156 result
10157}
10158
10159async fn wait_for_registration_release(
10162 registry: &Registry,
10163 module_id: &str,
10164 wait: Duration,
10165) -> Result<(), SuperviseError> {
10166 wait_for_slot_registration_release(
10167 registry,
10168 crate::registry::RegistrationSlot::Active(module_id),
10169 wait,
10170 )
10171 .await
10172}
10173
10174async fn wait_for_slot_registration_release(
10182 registry: &Registry,
10183 slot: crate::registry::RegistrationSlot<'_>,
10184 wait: Duration,
10185) -> Result<(), SuperviseError> {
10186 let deadline = Instant::now() + wait;
10187 let mut release_events = registration_release_events().subscribe();
10188 let still_active = |registration: &crate::registry::ModuleRegistration| {
10189 SuperviseError::RegistrationStillActive {
10190 module_id: registration.manifest.module_id.clone(),
10191 waited: wait,
10192 }
10193 };
10194 loop {
10195 let _observed_generation = *release_events.borrow_and_update();
10196 let Some(registration) = registry
10197 .registration(slot)
10198 .map_err(SuperviseError::Registry)?
10199 else {
10200 return Ok(());
10201 };
10202
10203 let now = Instant::now();
10204 if now >= deadline {
10205 return Err(still_active(®istration));
10206 }
10207
10208 let remaining = deadline.saturating_duration_since(now);
10209 match timeout(remaining, release_events.changed()).await {
10210 Ok(Ok(())) | Ok(Err(_)) => {}
10211 Err(_) => return Err(still_active(®istration)),
10212 }
10213 }
10214}
10215
10216#[cfg(test)]
10217mod slot_registration_wait_tests {
10218 use super::*;
10219 use crate::registry::{ConnectionId, RegistrationSlot};
10220 use subc_protocol::manifest::ModuleManifest;
10221
10222 #[tokio::test]
10223 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10224 let registry = Arc::new(Registry::default());
10225 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
10226 let runtime = supervisor.runtime_config();
10227 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10228 let spec = ModuleSpec {
10229 module_id: "enable-stale-registration".to_string(),
10230 program: PathBuf::from("/missing/enable-retry-test"),
10231 args: Vec::new(),
10232 env: Vec::new(),
10233 reserved: false,
10234 reserved_prefixes: Vec::new(),
10235 protocol: ModuleProtocol::Subc,
10236 overlap: Default::default(),
10237 };
10238 let connection = ConnectionId::new(90);
10239 registry
10240 .register_with_control_ops(
10241 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10242 1,
10243 connection,
10244 Vec::new(),
10245 )
10246 .unwrap();
10247 let mut child = None;
10248 let err = set_child_enabled(
10249 &spec,
10250 &runtime,
10251 ®istry,
10252 &supervisor.process_liveness,
10253 &snapshot,
10254 &mut child,
10255 true,
10256 )
10257 .await
10258 .unwrap_err();
10259 assert!(matches!(
10260 err,
10261 SuperviseError::RegistrationStillActive { .. }
10262 ));
10263 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10264 assert!(child.is_none());
10265 registry.deregister_connection(connection).unwrap();
10266 let err = set_child_enabled(
10267 &spec,
10268 &runtime,
10269 ®istry,
10270 &supervisor.process_liveness,
10271 &snapshot,
10272 &mut child,
10273 true,
10274 )
10275 .await
10276 .unwrap_err();
10277 assert!(
10278 matches!(err, SuperviseError::Spawn { .. }),
10279 "second enable must attempt a spawn: {err}"
10280 );
10281 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10282 }
10283
10284 const INCUMBENT: u64 = 1;
10285 const CANDIDATE: u64 = 2;
10286
10287 fn swapped_registry() -> Arc<Registry> {
10288 let registry = Arc::new(Registry::default());
10289 let manifest = ModuleManifest::builder("m", "0.1.0").build();
10290 registry
10291 .register_with_control_ops(
10292 manifest.clone(),
10293 1,
10294 ConnectionId::new(INCUMBENT),
10295 Vec::new(),
10296 )
10297 .unwrap();
10298 registry
10299 .register_candidate_with_control_ops(
10300 manifest,
10301 1,
10302 ConnectionId::new(CANDIDATE),
10303 Vec::new(),
10304 )
10305 .unwrap();
10306 registry
10307 }
10308
10309 #[tokio::test]
10313 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
10314 let registry = swapped_registry();
10315 registry.promote_candidate("m").unwrap().unwrap();
10316
10317 assert!(matches!(
10318 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
10319 Err(SuperviseError::RegistrationStillActive { .. })
10320 ));
10321
10322 assert!(matches!(
10324 wait_for_slot_registration_release(
10325 ®istry,
10326 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10327 Duration::from_millis(50),
10328 )
10329 .await,
10330 Err(SuperviseError::RegistrationStillActive { .. })
10331 ));
10332
10333 let releaser = Arc::clone(®istry);
10334 let release = tokio::spawn(async move {
10335 sleep(Duration::from_millis(20)).await;
10336 releaser
10337 .deregister_connection(ConnectionId::new(INCUMBENT))
10338 .unwrap();
10339 notify_registration_release();
10340 });
10341 wait_for_slot_registration_release(
10342 ®istry,
10343 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10344 Duration::from_secs(5),
10345 )
10346 .await
10347 .expect("the incumbent's own registration is released");
10348 release.await.unwrap();
10349 assert!(registry.get_module("m").unwrap().is_some());
10350 }
10351
10352 #[tokio::test]
10355 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
10356 let registry = swapped_registry();
10357 assert!(matches!(
10358 wait_for_slot_registration_release(
10359 ®istry,
10360 RegistrationSlot::Candidate("m"),
10361 Duration::from_millis(50),
10362 )
10363 .await,
10364 Err(SuperviseError::RegistrationStillActive { .. })
10365 ));
10366 registry
10367 .deregister_connection(ConnectionId::new(CANDIDATE))
10368 .unwrap();
10369 wait_for_slot_registration_release(
10370 ®istry,
10371 RegistrationSlot::Candidate("m"),
10372 Duration::from_millis(50),
10373 )
10374 .await
10375 .expect("a candidate slot with no candidate is released");
10376 assert!(registry
10377 .registration(RegistrationSlot::Active("m"))
10378 .unwrap()
10379 .is_some());
10380 }
10381}
10382
10383fn classify_exit(status: &ExitStatus) -> ExitReport {
10384 ExitReport {
10385 kind: if status.success() {
10386 ExitKind::Clean
10387 } else {
10388 ExitKind::Crash
10389 },
10390 code: status.code(),
10391 signal: exit_signal(status),
10392 at_ms: unix_ms_now(),
10393 }
10394}
10395
10396fn wait_error_exit_report() -> ExitReport {
10402 ExitReport {
10403 kind: ExitKind::Crash,
10404 code: None,
10405 signal: None,
10406 at_ms: unix_ms_now(),
10407 }
10408}
10409
10410#[cfg(unix)]
10411fn exit_signal(status: &ExitStatus) -> Option<i32> {
10412 use std::os::unix::process::ExitStatusExt;
10413
10414 status.signal()
10415}
10416
10417#[cfg(not(unix))]
10418fn exit_signal(_status: &ExitStatus) -> Option<i32> {
10419 None
10420}
10421
10422fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
10428 update_snapshot(snapshot, Some(module_id), |state| {
10429 state.clear_crash_restarts();
10430 })
10431}
10432
10433fn set_running(
10434 snapshot: &SharedSnapshot,
10435 child: &SupervisedChild,
10436 module_id: &str,
10437 spawn_events: &SpawnEventFeed,
10438) -> Result<(), SuperviseError> {
10439 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10440 module_id: Some(module_id.to_string()),
10441 })?;
10442 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
10443 if std::mem::take(&mut state.coalesced_restart_pending) {
10444 let generation = state.spawn_generation;
10445 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
10446 }
10447 state.drain_disposition_detail = None;
10448 state.spawn_failure = None;
10449 state.waiting_for_previous_pid = None;
10450 state.in_alternate_slot = false;
10453 state.configuration_updated_since_spawn = false;
10454 state.spawned_protocol = Some(child.protocol);
10455 state.state = ModuleState::Running;
10456 state.enabled = true;
10457 state.process_alive = true;
10458 state.pid = child.id();
10459 #[cfg(windows)]
10460 {
10461 state.pipe_consumed = child
10462 .nonce_delivery
10463 .as_ref()
10464 .map(|delivery| delivery.consumed());
10465 }
10466 #[cfg(target_os = "macos")]
10467 {
10468 state.report_ready = Some(Arc::clone(&child.report_ready));
10469 }
10470 state.spawned_at_ms = Some(child.spawned_at_ms);
10471 state.spawned_from = Some(child.spawned_from.clone());
10472 state.spawned_file_identity = child.spawned_file_identity.clone();
10473 state.process_start_time = child.process_start_time;
10474 Ok(())
10475}
10476
10477fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
10478 state.waiting_for_previous_pid = None;
10479 state.process_alive = false;
10480 state.spawned_protocol = None;
10481 state.pid = None;
10482 #[cfg(windows)]
10483 {
10484 state.pipe_consumed = None;
10485 }
10486 #[cfg(target_os = "macos")]
10487 {
10488 state.report_ready = None;
10489 }
10490 state.spawned_at_ms = None;
10491 state.spawned_from = None;
10492 state.spawned_file_identity = None;
10493 state.process_start_time = None;
10494 state.deliberate_severance = None;
10495}
10496
10497#[cfg(test)]
10498fn record_deliberate_severance(
10499 snapshot: &SharedSnapshot,
10500 identity: ProcessIdentity,
10501) -> Result<(), SuperviseError> {
10502 update_snapshot(snapshot, None, |state| {
10503 state.deliberate_severance = Some(identity);
10504 })
10505}
10506
10507fn apply_deliberate_severance_marker(
10508 snapshot: &SharedSnapshot,
10509 exited_identity: Option<ProcessIdentity>,
10510 mut exit_report: ExitReport,
10511) -> ExitReport {
10512 let marker = lock_snapshot(snapshot)
10513 .ok()
10514 .and_then(|mut state| state.deliberate_severance.take());
10515 if marker.is_some() && marker == exited_identity {
10516 exit_report.kind = ExitKind::DeliberateSeverance;
10517 }
10518 exit_report
10519}
10520
10521fn classify_reaped_child_exit(
10522 snapshot: &SharedSnapshot,
10523 child: &SupervisedChild,
10524 status: &ExitStatus,
10525) -> ExitReport {
10526 let _ = update_snapshot(snapshot, None, |state| {
10527 state.reaped_pid = Some(child.pid);
10528 state.spawn_failure = child.spawn_failure.clone();
10529 });
10530 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
10531}
10532
10533fn fail_snapshot(
10534 snapshot: &SharedSnapshot,
10535 module_id: Option<&str>,
10536 last_exit: Option<ExitReport>,
10537) {
10538 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
10539 state.state = ModuleState::Failed;
10540 clear_current_process_facts(state);
10541 if let Some(last_exit) = last_exit {
10542 state.last_exit = Some(last_exit);
10543 }
10544 }) {
10545 error!(error = %err, "failed to mark supervisor state failed");
10546 }
10547}
10548
10549fn update_snapshot(
10550 snapshot: &SharedSnapshot,
10551 module_id: Option<&str>,
10552 update: impl FnOnce(&mut SupervisorSnapshot),
10553) -> Result<(), SuperviseError> {
10554 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10555 module_id: module_id.map(ToOwned::to_owned),
10556 })?;
10557 update(&mut state);
10558 Ok(())
10559}
10560
10561const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
10562
10563fn lock_snapshot_for_control<'a>(
10564 snapshot: &'a SharedSnapshot,
10565 module_id: &str,
10566 caller: &'static str,
10567) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
10568 let started_at = Instant::now();
10569 let guard = lock_snapshot(snapshot)?;
10570 let waited = started_at.elapsed();
10571 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
10572 warn!(
10573 module_id = %module_id,
10574 waited_ms = waited.as_millis() as u64,
10575 caller = %caller,
10576 "slow snapshot lock"
10577 );
10578 }
10579 Ok(guard)
10580}
10581
10582fn lock_snapshot(
10583 snapshot: &SharedSnapshot,
10584) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
10585 snapshot
10586 .lock()
10587 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
10588}
10589
10590#[cfg(test)]
10591mod terminal_history_tests {
10592 use std::{
10593 path::PathBuf,
10594 sync::Arc,
10595 time::{Duration, Instant},
10596 };
10597
10598 use tokio::{sync::mpsc, time::sleep};
10599
10600 use super::{
10601 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
10602 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
10603 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
10604 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
10605 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
10606 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
10607 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
10608 SupervisorHealthStatus, SupervisorSnapshot,
10609 };
10610 use super::Instant as ClockInstant;
10615 use crate::{
10616 registry::Registry,
10617 terminal_ring::{TerminalRing, TerminalRingConfig},
10618 };
10619 use std::sync::Mutex;
10620 use subc_control::TerminalDisposition;
10621
10622 pub(super) fn fake_aft_stub_path() -> PathBuf {
10627 let mut path = std::env::current_exe().expect("current_exe available in tests");
10628 path.pop();
10629 path.pop();
10630 path.push(if cfg!(windows) {
10631 "fake-aft-stub.exe"
10632 } else {
10633 "fake-aft-stub"
10634 });
10635 assert!(
10636 path.exists(),
10637 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
10638 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
10639 path.display()
10640 );
10641 path
10642 }
10643
10644 #[test]
10645 fn reserved_never_spawned_refuses_every_hello() {
10646 let supervisor = SupervisorHandle::default();
10651 supervisor.apply_identity_configuration(&ModuleSpec {
10652 module_id: "never-spawned".to_string(),
10653 program: PathBuf::from("/usr/bin/false"),
10654 args: Vec::new(),
10655 env: Vec::new(),
10656 reserved: true,
10657 reserved_prefixes: Vec::new(),
10658 protocol: ModuleProtocol::Subc,
10659 overlap: Default::default(),
10660 });
10661 assert!(
10662 supervisor
10663 .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
10664 .is_some(),
10665 "forged nonce must refuse on a reserved never-spawned id"
10666 );
10667 assert!(
10668 supervisor
10669 .reserved_hello_rejection("never-spawned", None)
10670 .is_some(),
10671 "absent nonce must refuse on a reserved never-spawned id"
10672 );
10673 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
10675 supervisor.apply_identity_configuration(&ModuleSpec {
10676 module_id: "never-spawned".to_string(),
10677 program: PathBuf::from("/usr/bin/false"),
10678 args: Vec::new(),
10679 env: Vec::new(),
10680 reserved: true,
10681 reserved_prefixes: Vec::new(),
10682 protocol: ModuleProtocol::Subc,
10683 overlap: Default::default(),
10684 });
10685 assert!(supervisor
10686 .reserved_hello_rejection("never-spawned", Some("minted"))
10687 .is_none());
10688 assert!(supervisor
10689 .reserved_hello_rejection("never-spawned", Some("forged"))
10690 .is_some());
10691 }
10692
10693 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
10696 let now = ClockInstant::now();
10697 for _ in 0..count {
10698 state.crash_restarts.push_back(now);
10699 }
10700 }
10701
10702 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
10706 let aged = state
10707 .crash_restarts
10708 .front()
10709 .expect("a crash restart must be recorded before it can be aged")
10710 .checked_sub(window + Duration::from_secs(1))
10711 .expect("the test clock is far enough from its origin to age an instant");
10712 state.crash_restarts[0] = aged;
10713 }
10714
10715 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
10716 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
10717 seed_crash_restarts(&mut state, count);
10718 state
10719 }
10720
10721 #[test]
10722 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
10723 let policy = RestartPolicy::new(3, Duration::ZERO);
10724 let now = ClockInstant::now();
10725 assert!(daemon_will_restart(
10726 &mut snapshot_with_restarts(true, 2),
10727 &policy,
10728 now
10729 ));
10730 assert!(!daemon_will_restart(
10731 &mut snapshot_with_restarts(true, 3),
10732 &policy,
10733 now
10734 ));
10735 assert!(!daemon_will_restart(
10736 &mut snapshot_with_restarts(false, 0),
10737 &policy,
10738 now
10739 ));
10740 }
10741
10742 #[test]
10743 fn crash_restart_backoff_escalates_with_in_window_count() {
10744 let policy = RestartPolicy::new(4, Duration::from_millis(100))
10745 .with_max_backoff(Duration::from_secs(30));
10746 let now = ClockInstant::now();
10747 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
10748 let schedules = (0..4)
10749 .map(|_| {
10750 state
10751 .next_crash_restart(&policy, now)
10752 .expect("the test policy allows four crash restarts")
10753 })
10754 .collect::<Vec<_>>();
10755
10756 assert_eq!(
10757 schedules
10758 .iter()
10759 .map(|schedule| schedule.restart_in_window)
10760 .collect::<Vec<_>>(),
10761 vec![0, 1, 2, 3]
10762 );
10763 assert_eq!(
10764 schedules
10765 .iter()
10766 .map(|schedule| schedule.delay)
10767 .collect::<Vec<_>>(),
10768 vec![
10769 Duration::from_millis(100),
10770 Duration::from_secs(1),
10771 Duration::from_secs(10),
10772 Duration::from_secs(30),
10773 ]
10774 );
10775 }
10776
10777 #[test]
10778 fn crash_restart_backoff_resets_after_ring_clear() {
10779 let policy = RestartPolicy::new(3, Duration::from_millis(100));
10780 let now = ClockInstant::now();
10781 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
10782 assert_eq!(
10783 state.next_crash_restart(&policy, now).unwrap().delay,
10784 Duration::from_millis(100)
10785 );
10786 assert_eq!(
10787 state.next_crash_restart(&policy, now).unwrap().delay,
10788 Duration::from_secs(1)
10789 );
10790
10791 state.clear_crash_restarts();
10792 let schedule = state
10793 .next_crash_restart(&policy, now)
10794 .expect("a cleared ring must allow another restart");
10795 assert_eq!(schedule.restart_in_window, 0);
10796 assert_eq!(schedule.delay, Duration::from_millis(100));
10797 }
10798
10799 #[test]
10800 fn crash_restart_backoff_ignores_aged_restarts() {
10801 let policy = RestartPolicy::new(3, Duration::from_millis(100));
10802 let now = ClockInstant::now();
10803 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
10804 state
10805 .next_crash_restart(&policy, now)
10806 .expect("the first restart is allowed");
10807 state
10808 .next_crash_restart(&policy, now)
10809 .expect("the second restart is allowed");
10810 state.crash_restarts[0] = now
10811 .checked_sub(policy.window + Duration::from_secs(1))
10812 .expect("the fake clock can age a restart past the window");
10813
10814 let schedule = state
10815 .next_crash_restart(&policy, now)
10816 .expect("an aged restart must release its slot");
10817 assert_eq!(schedule.restart_in_window, 1);
10818 assert_eq!(schedule.delay, Duration::from_secs(1));
10819 assert_eq!(state.crash_restarts.len(), 2);
10820 }
10821
10822 #[test]
10826 fn a_budget_spent_before_the_window_no_longer_refuses() {
10827 let policy = RestartPolicy::new(3, Duration::ZERO);
10828 let mut state = snapshot_with_restarts(true, 3);
10829 let now = ClockInstant::now();
10830 assert!(!daemon_will_restart(&mut state, &policy, now));
10831
10832 assert!(daemon_will_restart(
10833 &mut state,
10834 &policy,
10835 now + policy.window + Duration::from_secs(1)
10836 ));
10837 assert!(
10838 state.crash_restarts.is_empty(),
10839 "reading the budget must drop the instants that left the window"
10840 );
10841 }
10842
10843 fn module_with_recovery_snapshot(
10844 state: ModuleState,
10845 enabled: bool,
10846 restart_count: u32,
10847 ) -> SupervisedModule {
10848 let registry = Arc::new(Registry::default());
10849 let supervisor =
10850 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
10851 let runtime = supervisor.runtime_config();
10852 let spec = ModuleSpec {
10853 module_id: "recovery-snapshot".to_string(),
10854 program: fake_aft_stub_path(),
10855 args: Vec::new(),
10856 env: Vec::new(),
10857 reserved: false,
10858 reserved_prefixes: Vec::new(),
10859 protocol: ModuleProtocol::Subc,
10860 overlap: Default::default(),
10861 };
10862 let mut snapshot = SupervisorSnapshot::new(state, enabled);
10863 seed_crash_restarts(&mut snapshot, restart_count);
10864 let (commands, _rx) = mpsc::channel(4);
10867 SupervisedModule {
10868 inner: Arc::new(SupervisedModuleInner {
10869 module_id: spec.module_id.clone(),
10870 registry,
10871 snapshot: Arc::new(Mutex::new(snapshot)),
10872 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
10873 spec,
10874 health: runtime.health,
10875 })),
10876 stderr_ring: runtime.stderr_ring,
10877 terminal_ring: runtime.terminal_ring,
10878 commands,
10879 monitor: Mutex::new(None),
10880 restart_policy: runtime.restart_policy,
10881 effective_drain_timeout: runtime.effective_drain_timeout,
10882 provenance_probe: supervisor.provenance_probe.clone(),
10883 }),
10884 }
10885 }
10886
10887 #[cfg(target_os = "linux")]
10888 #[tokio::test]
10889 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
10890 let supervisor =
10891 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
10892 .with_cgroup_placement(None);
10893 let result = supervisor.spawn(ModuleSpec {
10894 module_id: "no-cgroup-placement".to_string(),
10895 program: fake_aft_stub_path(),
10896 args: Vec::new(),
10897 env: Vec::new(),
10898 reserved: false,
10899 reserved_prefixes: Vec::new(),
10900 protocol: ModuleProtocol::Subc,
10901 overlap: Default::default(),
10902 });
10903
10904 assert!(
10905 result.is_ok(),
10906 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
10907 );
10908 }
10909
10910 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10911 async fn undecided_snapshot_uses_shared_restart_predicate() {
10912 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
10913 .will_recover_after_connection_loss()
10914 .unwrap());
10915 assert!(
10916 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
10917 .will_recover_after_connection_loss()
10918 .unwrap()
10919 );
10920 }
10921
10922 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10923 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
10924 assert!(
10925 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
10926 .will_recover_after_connection_loss()
10927 .unwrap()
10928 );
10929 }
10930
10931 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10932 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
10933 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
10934 .will_recover_after_connection_loss()
10935 .unwrap());
10936 assert!(
10937 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
10938 .will_recover_after_connection_loss()
10939 .unwrap()
10940 );
10941 }
10942
10943 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10944 async fn warming_snapshot_is_limited_to_startup_phases() {
10945 for state in [
10946 ModuleState::Starting,
10947 ModuleState::Running,
10948 ModuleState::Restarting,
10949 ] {
10950 assert!(
10951 module_with_recovery_snapshot(state, true, 0)
10952 .is_warming()
10953 .unwrap(),
10954 "{state:?} should be warming"
10955 );
10956 }
10957 for state in [
10958 ModuleState::Unresponsive,
10959 ModuleState::Draining,
10960 ModuleState::Stopped,
10961 ModuleState::Failed,
10962 ModuleState::Disabled,
10963 ] {
10964 assert!(
10965 !module_with_recovery_snapshot(state, true, 0)
10966 .is_warming()
10967 .unwrap(),
10968 "{state:?} should not be warming"
10969 );
10970 }
10971 }
10972
10973 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10974 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
10975 let registry = Arc::new(Registry::default());
10976 let supervisor =
10977 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
10978 let module = supervisor
10979 .spawn(ModuleSpec {
10980 module_id: "terminal-history".to_string(),
10981 program: fake_aft_stub_path(),
10982 args: Vec::new(),
10983 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
10984 reserved: false,
10985 reserved_prefixes: Vec::new(),
10986 protocol: ModuleProtocol::Subc,
10987 overlap: Default::default(),
10988 })
10989 .unwrap();
10990
10991 let deadline = Instant::now() + Duration::from_secs(5);
10992 loop {
10993 let history = module.terminal_history();
10994 if history.entries.len() == 2 {
10995 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
10996 assert_eq!(history.dropped, 0);
10997 assert_eq!(
10998 history
10999 .entries
11000 .iter()
11001 .map(|entry| entry.exit_code)
11002 .collect::<Vec<_>>(),
11003 vec![Some(23), Some(23)]
11004 );
11005 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11006 return;
11007 }
11008 assert!(
11009 Instant::now() < deadline,
11010 "module did not retain two terminal exits: {history:?}"
11011 );
11012 sleep(Duration::from_millis(10)).await;
11013 }
11014 }
11015
11016 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11020 async fn disable_during_crash_backoff_cancels_pending_respawn() {
11021 let backoff = Duration::from_secs(2);
11022 let supervisor = Supervisor::new_for_test(
11023 Arc::new(Registry::default()),
11024 RestartPolicy::new(10, backoff),
11025 );
11026 let module = supervisor
11027 .spawn(ModuleSpec {
11028 module_id: "disable-during-backoff".to_string(),
11029 program: fake_aft_stub_path(),
11030 args: Vec::new(),
11031 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11032 reserved: false,
11033 reserved_prefixes: Vec::new(),
11034 protocol: ModuleProtocol::Subc,
11035 overlap: Default::default(),
11036 })
11037 .unwrap();
11038
11039 let deadline = Instant::now() + Duration::from_secs(5);
11041 loop {
11042 if module.status().unwrap().state == ModuleState::Restarting {
11043 break;
11044 }
11045 assert!(
11046 Instant::now() < deadline,
11047 "module never entered the crash backoff"
11048 );
11049 sleep(Duration::from_millis(10)).await;
11050 }
11051
11052 let started = Instant::now();
11053 module.set_enabled(false).await.unwrap();
11054 let waited = started.elapsed();
11055
11056 assert!(
11057 waited < backoff / 2,
11058 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11059 );
11060 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11061
11062 sleep(backoff + Duration::from_millis(500)).await;
11064 let status = module.status().unwrap();
11065 assert_eq!(status.state, ModuleState::Disabled);
11066 assert_eq!(
11067 status.spawn_generation, 1,
11068 "module respawned after the operator disabled it"
11069 );
11070 }
11071
11072 #[cfg(unix)]
11075 fn protocol_none_sigterm_exits_clean_spec(
11076 module_id: &str,
11077 dir: &std::path::Path,
11078 ) -> (ModuleSpec, PathBuf, PathBuf) {
11079 let ready = dir.join("ready");
11080 let marker = dir.join("sigterm");
11081 let spec = ModuleSpec {
11082 module_id: module_id.to_string(),
11083 program: fake_aft_stub_path(),
11084 args: Vec::new(),
11085 env: vec![
11086 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11087 (
11088 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11089 marker.display().to_string(),
11090 ),
11091 (
11092 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11093 ready.display().to_string(),
11094 ),
11095 ],
11096 reserved: false,
11097 reserved_prefixes: Vec::new(),
11098 protocol: ModuleProtocol::None,
11099 overlap: Default::default(),
11100 };
11101 (spec, ready, marker)
11102 }
11103
11104 #[cfg(unix)]
11108 async fn wait_for_file(path: &std::path::Path) {
11109 let deadline = Instant::now() + Duration::from_secs(10);
11110 while !path.exists() {
11111 assert!(
11112 Instant::now() < deadline,
11113 "{} never appeared",
11114 path.display()
11115 );
11116 sleep(Duration::from_millis(10)).await;
11117 }
11118 }
11119
11120 #[cfg(unix)]
11124 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11125 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11126 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11127 let (spec, ready, marker) =
11128 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11129 let supervisor = Supervisor::new_for_test(
11130 Arc::new(Registry::default()),
11131 RestartPolicy::new(3, Duration::ZERO),
11132 );
11133 let module = supervisor.spawn(spec).unwrap();
11134 wait_for_file(&ready).await;
11135 let deadline = Instant::now() + Duration::from_secs(10);
11139 let first_pid = loop {
11140 let status = module.status().unwrap();
11141 if status.state == ModuleState::Running {
11142 if let Some(pid) = status.pid {
11143 break pid;
11144 }
11145 }
11146 assert!(
11147 Instant::now() < deadline,
11148 "a running module must report its pid after exec confirmation: {status:?}"
11149 );
11150 sleep(Duration::from_millis(10)).await;
11151 };
11152
11153 rustix::process::kill_process(
11154 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11155 rustix::process::Signal::TERM,
11156 )
11157 .unwrap();
11158
11159 let deadline = Instant::now() + Duration::from_secs(10);
11160 let respawned = loop {
11161 let status = module.status().unwrap();
11162 if status.state == ModuleState::Running
11163 && status.pid.is_some_and(|pid| pid != first_pid)
11164 {
11165 break status;
11166 }
11167 assert!(
11168 Instant::now() < deadline,
11169 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11170 );
11171 sleep(Duration::from_millis(10)).await;
11172 };
11173 assert_eq!(respawned.spawn_generation, 2);
11174 assert!(
11175 marker.exists(),
11176 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11177 );
11178
11179 let history = module.terminal_history();
11180 assert_eq!(history.entries.len(), 1, "{history:?}");
11181 let entry = &history.entries[0];
11182 assert_eq!(entry.exit_code, Some(0));
11183 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11184 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11185
11186 module.stop().await.unwrap();
11187 }
11188
11189 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11193 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11194 let supervisor = Supervisor::new_for_test(
11195 Arc::new(Registry::default()),
11196 RestartPolicy::new(1, Duration::ZERO),
11197 );
11198 let module = supervisor
11199 .spawn(ModuleSpec {
11200 module_id: "none-clean-exit-budget".to_string(),
11201 program: fake_aft_stub_path(),
11202 args: Vec::new(),
11203 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11204 reserved: false,
11205 reserved_prefixes: Vec::new(),
11206 protocol: ModuleProtocol::None,
11207 overlap: Default::default(),
11208 })
11209 .unwrap();
11210
11211 let deadline = Instant::now() + Duration::from_secs(10);
11214 loop {
11215 let status = module.status().unwrap();
11216 if status.state == ModuleState::Failed {
11217 break;
11218 }
11219 assert!(
11220 Instant::now() < deadline,
11221 "module never exhausted its budget: {status:?} {:?}",
11222 module.terminal_history()
11223 );
11224 sleep(Duration::from_millis(10)).await;
11225 }
11226 let history = module.terminal_history();
11227 assert_eq!(
11228 history
11229 .entries
11230 .iter()
11231 .map(|entry| (entry.exit_code, entry.disposition.clone()))
11232 .collect::<Vec<_>>(),
11233 vec![
11234 (Some(0), TerminalDisposition::Restarting),
11235 (Some(0), TerminalDisposition::Failed),
11236 ]
11237 );
11238 let detail = history.entries[1]
11239 .disposition_detail
11240 .as_deref()
11241 .expect("a budget failure names the budget");
11242 assert!(detail.contains("max_restarts=1"), "{detail}");
11243 assert_eq!(module.status().unwrap().spawn_generation, 2);
11244 }
11245
11246 #[test]
11247 fn restart_budget_failure_is_published_after_its_terminal_record() {
11248 use std::io::{Read, Write};
11249 let supervisor = Supervisor::new_for_test(
11250 Arc::new(Registry::default()),
11251 RestartPolicy::new(0, Duration::ZERO),
11252 );
11253 let runtime = supervisor.runtime_config();
11254 let spec = ModuleSpec {
11255 protocol: ModuleProtocol::None,
11256 ..windowed_crash_spec("budget-publication-order")
11257 };
11258 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11259 ModuleState::Running,
11260 true,
11261 )));
11262 lock_snapshot(&snapshot).unwrap().process_alive = true;
11263 let reaped_snapshot = snapshot.clone();
11264 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11265 let history = runtime.terminal_ring.clone();
11266 let reap = std::thread::spawn(move || {
11267 tokio::runtime::Builder::new_current_thread()
11268 .enable_all()
11269 .build()
11270 .unwrap()
11271 .block_on(on_child_exit(
11272 &spec,
11273 runtime.restart_policy,
11274 &supervisor.registry,
11275 &reaped_snapshot,
11276 &runtime.terminal_ring,
11277 &runtime.spawn_events,
11278 &runtime.child_roster,
11279 ExitReport {
11280 kind: ExitKind::Clean,
11281 code: Some(0),
11282 signal: None,
11283 at_ms: 1,
11284 },
11285 ))
11286 });
11287 let mut reached = [0];
11288 barrier.read_exact(&mut reached).unwrap();
11289 assert_eq!(&reached, b"R");
11290 let (before_state, before_alive) = {
11291 let state = lock_snapshot(&snapshot).unwrap();
11292 (state.state, state.process_alive)
11293 };
11294 let before_history = history.lock().unwrap().snapshot();
11295 barrier.write_all(b"X").unwrap();
11296 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
11297 assert!(before_history.entries.is_empty());
11298 assert_ne!(
11299 before_state,
11300 ModuleState::Failed,
11301 "Failed was visible before its terminal record could be written"
11302 );
11303 assert!(before_alive);
11304 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11305 let history = history.lock().unwrap().snapshot();
11306 assert_eq!(history.entries.len(), 1);
11307 assert_eq!(history.entries[0].exit_code, Some(0));
11308 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11309 }
11310
11311 #[test]
11312 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
11313 use std::io::{Read, Write};
11314
11315 let supervisor = Supervisor::new_for_test(
11316 Arc::new(Registry::default()),
11317 RestartPolicy::new(3, Duration::ZERO),
11318 );
11319 let mut runtime = supervisor.runtime_config();
11320 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
11321 let snapshot = module.inner.snapshot.clone();
11322 lock_snapshot(&snapshot).unwrap().process_alive = true;
11323 runtime.terminal_ring = module.inner.terminal_ring.clone();
11324 let spec = windowed_crash_spec(module.module_id());
11325 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11326 let reaped_snapshot = snapshot.clone();
11327 let reap = std::thread::spawn(move || {
11328 tokio::runtime::Builder::new_current_thread()
11329 .enable_all()
11330 .build()
11331 .unwrap()
11332 .block_on(on_child_exit(
11333 &spec,
11334 runtime.restart_policy,
11335 &supervisor.registry,
11336 &reaped_snapshot,
11337 &runtime.terminal_ring,
11338 &runtime.spawn_events,
11339 &runtime.child_roster,
11340 ExitReport {
11341 kind: ExitKind::Crash,
11342 code: Some(2),
11343 signal: None,
11344 at_ms: 1,
11345 },
11346 ))
11347 });
11348
11349 let mut reached = [0];
11353 barrier.read_exact(&mut reached).unwrap();
11354 assert_eq!(&reached, b"R");
11355 let before_status = module.status().unwrap();
11356 let before_history = module.terminal_history();
11357 let before_recovers = module.will_recover_after_connection_loss().unwrap();
11358 barrier.write_all(b"X").unwrap();
11359 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
11360
11361 assert_eq!(before_status.state, ModuleState::Running);
11362 assert!(before_status.process_alive);
11363 assert_eq!(before_status.restart_count, 3);
11364 assert!(before_history.entries.is_empty());
11365 assert!(
11366 before_recovers,
11367 "connection cleanup would report a terminal close for a scheduled crash restart"
11368 );
11369 let restarting = module.status().unwrap();
11370 assert_eq!(restarting.state, ModuleState::Restarting);
11371 assert_eq!(restarting.restart_count, 3);
11372 assert!(module.will_recover_after_connection_loss().unwrap());
11373 let history = module.terminal_history();
11374 assert_eq!(history.entries.len(), 1);
11375 assert_eq!(
11376 history.entries[0].disposition,
11377 TerminalDisposition::Restarting
11378 );
11379
11380 update_snapshot(&snapshot, Some(module.module_id()), |state| {
11383 state.state = ModuleState::Running;
11384 state.respawn_pending = false;
11385 })
11386 .unwrap();
11387 assert!(!module.will_recover_after_connection_loss().unwrap());
11388 update_snapshot(&snapshot, Some(module.module_id()), |state| {
11389 state.enabled = false;
11390 state.respawn_pending = true;
11391 })
11392 .unwrap();
11393 assert!(!module.will_recover_after_connection_loss().unwrap());
11394 }
11395
11396 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
11397 use std::net::{TcpListener, TcpStream};
11398 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
11399 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
11400 let (barrier, _) = listener.accept().unwrap();
11401 for stream in [&writer, &barrier] {
11402 stream
11403 .set_read_timeout(Some(Duration::from_secs(10)))
11404 .unwrap();
11405 stream
11406 .set_write_timeout(Some(Duration::from_secs(10)))
11407 .unwrap();
11408 }
11409 *events.1.lock().unwrap() = Some(writer);
11410 barrier
11411 }
11412
11413 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
11414 use std::io::{Read, Write};
11415 let barrier = events.1.lock().unwrap().take();
11416 if let Some(mut barrier) = barrier {
11417 barrier.write_all(b"R").unwrap();
11418 let mut release = [0];
11419 barrier.read_exact(&mut release).unwrap();
11420 assert_eq!(&release, b"X");
11421 }
11422 }
11423
11424 #[test]
11425 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
11426 use std::io::{Read, Write};
11427
11428 let supervisor = Supervisor::new_for_test(
11429 Arc::new(Registry::default()),
11430 RestartPolicy::new(0, Duration::ZERO),
11431 );
11432 let mut runtime = supervisor.runtime_config();
11433 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
11434 let snapshot = module.inner.snapshot.clone();
11435 runtime.terminal_ring = module.inner.terminal_ring.clone();
11436 let spec = ModuleSpec {
11437 module_id: module.module_id().into(),
11438 program: fake_aft_stub_path(),
11439 protocol: ModuleProtocol::None,
11440 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11441 ..windowed_crash_spec(module.module_id())
11442 };
11443 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11444 let reap = std::thread::spawn(move || {
11445 tokio::runtime::Builder::new_current_thread()
11446 .enable_all()
11447 .build()
11448 .unwrap()
11449 .block_on(async {
11450 let mut child =
11451 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11452 health_restart_child(
11453 &spec,
11454 &runtime,
11455 &supervisor.registry,
11456 &supervisor.process_liveness,
11457 &snapshot,
11458 &mut child,
11459 SupervisorHealthStatus::Failing,
11460 Some("health probe did not answer"),
11461 1,
11462 )
11463 .await
11464 .unwrap();
11465 assert!(child.is_none());
11466 });
11467 });
11468 let mut reached = [0];
11473 barrier.read_exact(&mut reached).unwrap();
11474 assert_eq!(&reached, b"R");
11475 let before_status = module.status().unwrap();
11476 let before_history = module.terminal_history();
11477 barrier.write_all(b"X").unwrap();
11478 reap.join().unwrap();
11479 assert!(before_history.entries.is_empty());
11480 assert_ne!(
11481 before_status.state,
11482 ModuleState::Failed,
11483 "Failed was visible before its terminal record could be written"
11484 );
11485 assert!(
11486 before_status.process_alive,
11487 "process death was visible before its terminal record could be written"
11488 );
11489 let failed = module.status().unwrap();
11490 assert_eq!(failed.state, ModuleState::Failed);
11491 assert!(!failed.process_alive);
11492 let history = module.terminal_history();
11493 assert_eq!(history.entries.len(), 1);
11494 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11495 assert_eq!(
11496 history.entries[0].disposition_detail.as_deref(),
11497 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
11498 );
11499 }
11500
11501 #[test]
11502 fn observed_exits_publish_process_death_after_terminal_history() {
11503 use std::io::{Read, Write};
11504 for (kind, enabled, shutdown, wait_error, expected_state) in [
11505 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
11506 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
11507 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
11508 (
11509 ExitKind::DeliberateSeverance,
11510 true,
11511 false,
11512 false,
11513 ModuleState::Restarting,
11514 ),
11515 #[cfg(unix)]
11516 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
11517 (ExitKind::Crash, true, false, true, ModuleState::Failed),
11518 ] {
11519 let supervisor = Supervisor::new_for_test(
11520 Arc::new(Registry::default()),
11521 RestartPolicy::new(1, Duration::ZERO),
11522 );
11523 let runtime = supervisor.runtime_config();
11524 #[cfg(unix)]
11525 if shutdown {
11526 runtime.child_roster.close();
11527 }
11528 #[cfg(not(unix))]
11529 let _ = shutdown;
11530 let spec = windowed_crash_spec("exit-publication-order");
11531 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11532 ModuleState::Running,
11533 enabled,
11534 )));
11535 lock_snapshot(&snapshot).unwrap().process_alive = true;
11536 let reaped_snapshot = snapshot.clone();
11537 let history = runtime.terminal_ring.clone();
11538 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11539 let reap = std::thread::spawn(move || {
11540 if wait_error {
11541 super::fail_after_wait_error(
11542 &spec.module_id,
11543 &reaped_snapshot,
11544 &runtime.terminal_ring,
11545 &runtime.spawn_events,
11546 "wait failed".into(),
11547 );
11548 } else {
11549 tokio::runtime::Builder::new_current_thread()
11550 .enable_all()
11551 .build()
11552 .unwrap()
11553 .block_on(on_child_exit(
11554 &spec,
11555 runtime.restart_policy,
11556 &supervisor.registry,
11557 &reaped_snapshot,
11558 &runtime.terminal_ring,
11559 &runtime.spawn_events,
11560 &runtime.child_roster,
11561 ExitReport {
11562 kind,
11563 code: Some(0),
11564 signal: None,
11565 at_ms: 1,
11566 },
11567 ));
11568 }
11569 });
11570 let mut reached = [0];
11571 barrier.read_exact(&mut reached).unwrap();
11572 assert_eq!(&reached, b"R");
11573 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
11574 let before_history = history.lock().unwrap().snapshot();
11575 barrier.write_all(b"X").unwrap();
11576 reap.join().unwrap();
11577 assert!(
11578 before_alive,
11579 "{kind:?}: death was published before recording"
11580 );
11581 assert!(before_history.entries.is_empty());
11582 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
11583 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
11584 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
11585 }
11586 }
11587
11588 #[tokio::test]
11589 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
11590 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
11591 let path = dir.join("terminals.jsonl");
11592 std::fs::create_dir(&path).unwrap();
11593 let supervisor = Supervisor::new_for_test(
11594 Arc::new(Registry::default()),
11595 RestartPolicy::new(0, Duration::ZERO),
11596 )
11597 .with_terminal_journal(path, "health-budget-journal-failure".into());
11598 let runtime = supervisor.runtime_config();
11599 let spec = ModuleSpec {
11600 program: fake_aft_stub_path(),
11601 protocol: ModuleProtocol::None,
11602 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11603 ..windowed_crash_spec("health-budget-journal-failure")
11604 };
11605 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
11606 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11607 health_restart_child(
11608 &spec,
11609 &runtime,
11610 &supervisor.registry,
11611 &supervisor.process_liveness,
11612 &snapshot,
11613 &mut child,
11614 SupervisorHealthStatus::Failing,
11615 None,
11616 1,
11617 )
11618 .await
11619 .unwrap();
11620 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11621 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
11622 let history = runtime
11623 .terminal_ring
11624 .lock()
11625 .unwrap()
11626 .durable_history(&spec.module_id);
11627 assert!(history.journal_write_failures > 0);
11628 assert_eq!(history.entries.len(), 1);
11629 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11630 }
11631
11632 #[test]
11633 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
11634 let supervisor = Supervisor::new_for_test(
11635 Arc::new(Registry::default()),
11636 RestartPolicy::new(0, Duration::ZERO),
11637 );
11638 let runtime = supervisor.runtime_config();
11639 let spec = ModuleSpec {
11640 program: fake_aft_stub_path(),
11641 protocol: ModuleProtocol::None,
11642 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11643 ..windowed_crash_spec("health-budget-recording-panic")
11644 };
11645 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
11646 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
11647 tokio::runtime::Builder::new_current_thread()
11648 .enable_all()
11649 .build()
11650 .unwrap()
11651 .block_on(async {
11652 let mut child =
11653 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11654 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
11655 health_restart_child(
11656 &spec,
11657 &runtime,
11658 &supervisor.registry,
11659 &supervisor.process_liveness,
11660 &snapshot,
11661 &mut child,
11662 SupervisorHealthStatus::Failing,
11663 None,
11664 1,
11665 )
11666 .await
11667 .unwrap();
11668 });
11669 }));
11670 let panic = result.expect_err("recording must still unwind");
11671 assert_eq!(
11672 panic.downcast_ref::<String>().map(String::as_str),
11673 Some("spawn event sequence exhausted")
11674 );
11675 let state = lock_snapshot(&snapshot).unwrap();
11676 assert_eq!(state.state, ModuleState::Failed);
11677 assert!(!state.process_alive);
11678 assert!(state.last_exit.is_some());
11679 }
11680
11681 #[tokio::test]
11682 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
11683 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
11684 let path = dir.join("terminals.jsonl");
11685 std::fs::create_dir(&path).unwrap();
11686 let supervisor = Supervisor::new_for_test(
11687 Arc::new(Registry::default()),
11688 RestartPolicy::new(0, Duration::ZERO),
11689 )
11690 .with_terminal_journal(path, "budget-journal-failure".into());
11691 let runtime = supervisor.runtime_config();
11692 let spec = windowed_crash_spec("budget-journal-failure");
11693 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
11694 assert!(matches!(
11695 on_child_exit(
11696 &spec,
11697 runtime.restart_policy,
11698 &supervisor.registry,
11699 &snapshot,
11700 &runtime.terminal_ring,
11701 &runtime.spawn_events,
11702 &runtime.child_roster,
11703 crash_exit_report(1),
11704 )
11705 .await,
11706 NextAction::Stop { .. }
11707 ));
11708 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11709 let history = runtime
11710 .terminal_ring
11711 .lock()
11712 .unwrap()
11713 .durable_history(&spec.module_id);
11714 assert!(history.journal_write_failures > 0);
11715 assert_eq!(history.entries.len(), 1);
11716 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11717 }
11718
11719 #[test]
11720 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
11721 let supervisor = Supervisor::new_for_test(
11722 Arc::new(Registry::default()),
11723 RestartPolicy::new(0, Duration::ZERO),
11724 );
11725 let runtime = supervisor.runtime_config();
11726 let spec = windowed_crash_spec("budget-recording-panic");
11727 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
11728 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
11729 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
11732 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
11733 tokio::runtime::Builder::new_current_thread()
11734 .enable_all()
11735 .build()
11736 .unwrap()
11737 .block_on(on_child_exit(
11738 &spec,
11739 runtime.restart_policy,
11740 &supervisor.registry,
11741 &snapshot,
11742 &runtime.terminal_ring,
11743 &runtime.spawn_events,
11744 &runtime.child_roster,
11745 crash_exit_report(1),
11746 ))
11747 }));
11748 let panic = result.err().expect("recording must still unwind");
11749 assert_eq!(
11750 panic.downcast_ref::<String>().map(String::as_str),
11751 Some("spawn event sequence exhausted")
11752 );
11753 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11754 }
11755
11756 #[cfg(unix)]
11759 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11760 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
11761 for disable in [false, true] {
11762 let label = if disable {
11763 "none-requested-disable"
11764 } else {
11765 "none-requested-stop"
11766 };
11767 let dir = cortexkit_test_support::ScratchDir::new(label);
11768 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
11769 let supervisor = Supervisor::new_for_test(
11770 Arc::new(Registry::default()),
11771 RestartPolicy::new(3, Duration::ZERO),
11772 );
11773 let module = supervisor.spawn(spec).unwrap();
11774 wait_for_file(&ready).await;
11775
11776 if disable {
11777 module.set_enabled(false).await.unwrap();
11778 } else {
11779 module.stop().await.unwrap();
11780 }
11781 assert!(
11782 marker.exists(),
11783 "{label}: the child must have left through its SIGTERM handler with exit 0"
11784 );
11785
11786 sleep(Duration::from_millis(500)).await;
11789 let status = module.status().unwrap();
11790 let expected = if disable {
11791 ModuleState::Disabled
11792 } else {
11793 ModuleState::Stopped
11794 };
11795 assert_eq!(status.state, expected, "{label}");
11796 assert_eq!(
11797 status.spawn_generation, 1,
11798 "{label}: respawned after a requested stop"
11799 );
11800 let history = module.terminal_history();
11801 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
11802 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
11803 assert_ne!(
11804 history.entries[0].disposition,
11805 TerminalDisposition::Restarting,
11806 "{label}"
11807 );
11808 }
11809 }
11810
11811 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11814 async fn subc_wire_clean_exit_is_still_a_stop() {
11815 let supervisor = Supervisor::new_for_test(
11816 Arc::new(Registry::default()),
11817 RestartPolicy::new(3, Duration::ZERO),
11818 );
11819 let module = supervisor
11820 .spawn(ModuleSpec {
11821 module_id: "wire-clean-exit".to_string(),
11822 program: fake_aft_stub_path(),
11823 args: Vec::new(),
11824 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11825 reserved: false,
11826 reserved_prefixes: Vec::new(),
11827 protocol: ModuleProtocol::Subc,
11828 overlap: Default::default(),
11829 })
11830 .unwrap();
11831
11832 let deadline = Instant::now() + Duration::from_secs(10);
11833 while module.terminal_history().entries.is_empty() {
11834 assert!(Instant::now() < deadline, "module never exited");
11835 sleep(Duration::from_millis(10)).await;
11836 }
11837 sleep(Duration::from_millis(500)).await;
11839 let status = module.status().unwrap();
11840 assert_eq!(status.state, ModuleState::Stopped);
11841 assert_eq!(status.spawn_generation, 1);
11842 let history = module.terminal_history();
11843 assert_eq!(history.entries.len(), 1, "{history:?}");
11844 assert_eq!(history.entries[0].exit_code, Some(0));
11845 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
11846 }
11847
11848 #[cfg(unix)]
11849 #[tokio::test]
11850 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
11851 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
11852 let record = dir.join("live-children.json");
11853 let supervisor = Supervisor::new_for_test(
11854 Arc::new(Registry::default()),
11855 RestartPolicy::new(0, Duration::ZERO),
11856 );
11857 let mut runtime = supervisor.runtime_config();
11858 runtime.child_roster.record_to(record.clone());
11859 let gate = Arc::new(super::ReloadExitRecordGate::default());
11860 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
11861 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
11862 let spec = ModuleSpec {
11863 module_id: "reload-exit-roster".into(),
11864 program: fake_aft_stub_path(),
11865 args: Vec::new(),
11866 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
11867 reserved: false,
11868 reserved_prefixes: Vec::new(),
11869 protocol: ModuleProtocol::Subc,
11870 overlap: Default::default(),
11871 };
11872 let mut child = None;
11873 let reload = super::finish_reload_child(
11874 &spec,
11875 &runtime,
11876 &supervisor.registry,
11877 &supervisor.process_liveness,
11878 &snapshot,
11879 &mut child,
11880 );
11881 tokio::pin!(reload);
11882 tokio::select! {
11883 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
11884 _ = gate.reached.notified() => {}
11885 }
11886 assert!(runtime
11887 .terminal_ring
11888 .lock()
11889 .unwrap()
11890 .snapshot()
11891 .entries
11892 .is_empty());
11893 assert_eq!(
11894 crate::live_children::read_record(&record).unwrap().len(),
11895 1,
11896 "shutdown must still wait for the reaped child until its terminal record exists"
11897 );
11898 runtime.child_roster.close();
11899 gate.resume.notify_one();
11900 assert!(reload.await.is_err());
11901 assert!(crate::live_children::read_record(&record)
11902 .unwrap()
11903 .is_empty());
11904 let history = runtime.terminal_ring.lock().unwrap().snapshot();
11905 assert_eq!(history.entries.len(), 1);
11906 assert_eq!(
11907 history.entries[0].disposition,
11908 TerminalDisposition::DaemonShutdown
11909 );
11910 }
11911
11912 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11916 async fn every_restart_increment_path_advances_lifetime_count() {
11917 let supervisor = Supervisor::new_for_test(
11918 Arc::new(Registry::default()),
11919 RestartPolicy::new(1, Duration::ZERO),
11920 );
11921 let runtime = supervisor.runtime_config();
11922 let spec = ModuleSpec {
11923 module_id: "lifetime-increment-path".to_string(),
11924 program: PathBuf::from("/unused/lifetime-increment-path"),
11925 args: Vec::new(),
11926 env: Vec::new(),
11927 reserved: false,
11928 reserved_prefixes: Vec::new(),
11929 protocol: ModuleProtocol::Subc,
11930 overlap: Default::default(),
11931 };
11932
11933 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
11934 assert!(matches!(
11935 on_child_exit(
11936 &spec,
11937 runtime.restart_policy,
11938 &supervisor.registry,
11939 &crash_snapshot,
11940 &runtime.terminal_ring,
11941 &runtime.spawn_events,
11942 &runtime.child_roster,
11943 ExitReport {
11944 kind: ExitKind::Crash,
11945 code: Some(1),
11946 signal: None,
11947 at_ms: 1,
11948 },
11949 )
11950 .await,
11951 NextAction::Restart { schedule: _ }
11952 ));
11953 let (crash_restarts, crash_lifetime) = {
11954 let state = lock_snapshot(&crash_snapshot).unwrap();
11955 (state.crash_restarts.len(), state.lifetime_restarts)
11956 };
11957 assert_eq!(crash_restarts, 1);
11958 assert_eq!(crash_lifetime, 1);
11959
11960 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
11961 let mut health_child = None;
11962 assert!(matches!(
11963 health_restart_child(
11964 &spec,
11965 &runtime,
11966 &supervisor.registry,
11967 &supervisor.process_liveness,
11968 &health_snapshot,
11969 &mut health_child,
11970 SupervisorHealthStatus::Failing,
11971 None,
11972 2,
11973 )
11974 .await,
11975 Ok(())
11976 ));
11977 assert!(health_child.is_none());
11978 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
11979 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
11980 let (health_restarts, health_lifetime) = {
11981 let state = lock_snapshot(&health_snapshot).unwrap();
11982 (state.crash_restarts.len(), state.lifetime_restarts)
11983 };
11984 assert_eq!(health_restarts, 1);
11985 assert_eq!(health_lifetime, 1);
11986
11987 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
11988 let mut reload_child = None;
11989 assert!(matches!(
11990 handle_reload_spawn_failure(
11991 &spec,
11992 &runtime,
11993 &supervisor.process_liveness,
11994 &reload_snapshot,
11995 &mut reload_child,
11996 "forced reload spawn failure".to_string(),
11997 )
11998 .await,
11999 Err(SuperviseError::ReloadFailed { .. })
12000 ));
12001 let (reload_restarts, reload_lifetime) = {
12002 let state = lock_snapshot(&reload_snapshot).unwrap();
12003 (state.crash_restarts.len(), state.lifetime_restarts)
12004 };
12005 assert_eq!(reload_restarts, 1);
12006 assert_eq!(reload_lifetime, 1);
12007 }
12008
12009 #[tokio::test]
12010 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12011 let supervisor = Supervisor::new_for_test(
12012 Arc::new(Registry::default()),
12013 RestartPolicy::new(3, Duration::ZERO),
12014 );
12015 let runtime = supervisor.runtime_config();
12016 let spec = ModuleSpec {
12017 module_id: "deliberately-severed".to_string(),
12018 program: PathBuf::from("/unused/deliberately-severed"),
12019 args: Vec::new(),
12020 env: Vec::new(),
12021 reserved: false,
12022 reserved_prefixes: Vec::new(),
12023 protocol: ModuleProtocol::Subc,
12024 overlap: Default::default(),
12025 };
12026 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12027 let process = ProcessIdentity {
12028 pid: 41,
12029 start_time: 101,
12030 };
12031 record_deliberate_severance(&snapshot, process).unwrap();
12032 let exit_report = apply_deliberate_severance_marker(
12033 &snapshot,
12034 Some(process),
12035 ExitReport {
12036 kind: ExitKind::Crash,
12037 code: Some(1),
12038 signal: None,
12039 at_ms: 1,
12040 },
12041 );
12042 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12043
12044 assert!(matches!(
12045 on_child_exit(
12046 &spec,
12047 runtime.restart_policy,
12048 &supervisor.registry,
12049 &snapshot,
12050 &runtime.terminal_ring,
12051 &runtime.spawn_events,
12052 &runtime.child_roster,
12053 exit_report,
12054 )
12055 .await,
12056 NextAction::Restart { schedule: _ }
12057 ));
12058 let state = lock_snapshot(&snapshot).unwrap();
12059 assert_eq!(state.lifetime_restarts, 1);
12060 assert_eq!(state.crash_restarts.len(), 0);
12061 }
12062
12063 #[tokio::test]
12064 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12065 let supervisor = Supervisor::new_for_test(
12066 Arc::new(Registry::default()),
12067 RestartPolicy::new(3, Duration::ZERO),
12068 );
12069 let runtime = supervisor.runtime_config();
12070 let spec = ModuleSpec {
12071 module_id: "genuine-crash".to_string(),
12072 program: PathBuf::from("/unused/genuine-crash"),
12073 args: Vec::new(),
12074 env: Vec::new(),
12075 reserved: false,
12076 reserved_prefixes: Vec::new(),
12077 protocol: ModuleProtocol::Subc,
12078 overlap: Default::default(),
12079 };
12080 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12081
12082 assert!(matches!(
12083 on_child_exit(
12084 &spec,
12085 runtime.restart_policy,
12086 &supervisor.registry,
12087 &snapshot,
12088 &runtime.terminal_ring,
12089 &runtime.spawn_events,
12090 &runtime.child_roster,
12091 ExitReport {
12092 kind: ExitKind::Crash,
12093 code: Some(1),
12094 signal: None,
12095 at_ms: 1,
12096 },
12097 )
12098 .await,
12099 NextAction::Restart { schedule: _ }
12100 ));
12101 let state = lock_snapshot(&snapshot).unwrap();
12102 assert_eq!(state.lifetime_restarts, 1);
12103 assert_eq!(state.crash_restarts.len(), 1);
12104 }
12105
12106 fn crash_exit_report(at_ms: u64) -> ExitReport {
12107 ExitReport {
12108 kind: ExitKind::Crash,
12109 code: Some(1),
12110 signal: None,
12111 at_ms,
12112 }
12113 }
12114
12115 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12116 ModuleSpec {
12117 module_id: module_id.to_string(),
12118 program: PathBuf::from("/unused").join(module_id),
12119 args: Vec::new(),
12120 env: Vec::new(),
12121 reserved: false,
12122 reserved_prefixes: Vec::new(),
12123 protocol: ModuleProtocol::Subc,
12124 overlap: Default::default(),
12125 }
12126 }
12127
12128 #[tokio::test]
12134 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12135 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12136 let supervisor = Supervisor::new_for_test(
12137 Arc::new(Registry::default()),
12138 RestartPolicy::new(2, Duration::ZERO),
12139 );
12140 let runtime = supervisor.runtime_config();
12141 let spec = windowed_crash_spec("crash-loop-in-window");
12142 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12143
12144 for attempt in 1..=2 {
12145 assert!(
12146 matches!(
12147 on_child_exit(
12148 &spec,
12149 runtime.restart_policy,
12150 &supervisor.registry,
12151 &snapshot,
12152 &runtime.terminal_ring,
12153 &runtime.spawn_events,
12154 &runtime.child_roster,
12155 crash_exit_report(attempt),
12156 )
12157 .await,
12158 NextAction::Restart { schedule: _ }
12159 ),
12160 "crash {attempt} is inside the budget and must respawn"
12161 );
12162 }
12163
12164 assert!(matches!(
12165 on_child_exit(
12166 &spec,
12167 runtime.restart_policy,
12168 &supervisor.registry,
12169 &snapshot,
12170 &runtime.terminal_ring,
12171 &runtime.spawn_events,
12172 &runtime.child_roster,
12173 crash_exit_report(3),
12174 )
12175 .await,
12176 NextAction::Stop { .. }
12177 ));
12178
12179 {
12180 let state = lock_snapshot(&snapshot).unwrap();
12181 assert_eq!(state.state, ModuleState::Failed);
12182 assert_eq!(state.crash_restarts.len(), 2);
12183 assert_eq!(state.lifetime_restarts, 2);
12184 }
12185
12186 let history = runtime
12187 .terminal_ring
12188 .lock()
12189 .expect("terminal ring is not poisoned")
12190 .snapshot();
12191 let last = history
12192 .entries
12193 .last()
12194 .expect("the refused crash is retained");
12195 assert_eq!(last.disposition, TerminalDisposition::Failed);
12196 assert_eq!(
12197 last.disposition_detail.as_deref(),
12198 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12199 );
12200
12201 let captured = crate::router::test_log::captured_logs(&logs);
12202 assert!(
12203 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12204 "the stop must be logged with its window: {captured}"
12205 );
12206 }
12207
12208 #[tokio::test]
12216 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12217 let supervisor = Supervisor::new_for_test(
12218 Arc::new(Registry::default()),
12219 RestartPolicy::new(2, Duration::ZERO),
12220 );
12221 let runtime = supervisor.runtime_config();
12222 let spec = windowed_crash_spec("crash-across-windows");
12223 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12224
12225 for attempt in 1..=2 {
12226 assert!(matches!(
12227 on_child_exit(
12228 &spec,
12229 runtime.restart_policy,
12230 &supervisor.registry,
12231 &snapshot,
12232 &runtime.terminal_ring,
12233 &runtime.spawn_events,
12234 &runtime.child_roster,
12235 crash_exit_report(attempt),
12236 )
12237 .await,
12238 NextAction::Restart { schedule: _ }
12239 ));
12240 }
12241
12242 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12245 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12246 })
12247 .unwrap();
12248
12249 assert!(
12250 matches!(
12251 on_child_exit(
12252 &spec,
12253 runtime.restart_policy,
12254 &supervisor.registry,
12255 &snapshot,
12256 &runtime.terminal_ring,
12257 &runtime.spawn_events,
12258 &runtime.child_roster,
12259 crash_exit_report(3),
12260 )
12261 .await,
12262 NextAction::Restart { schedule: _ }
12263 ),
12264 "a crash older than the window must not hold a budget slot"
12265 );
12266
12267 let state = lock_snapshot(&snapshot).unwrap();
12268 assert_eq!(state.state, ModuleState::Restarting);
12269 assert_eq!(
12270 state.crash_restarts.len(),
12271 2,
12272 "the aged instant is dropped and the new one takes its place"
12273 );
12274 assert_eq!(
12275 state.lifetime_restarts, 3,
12276 "the ledger counts every restart, including the ones the window forgot"
12277 );
12278 }
12279
12280 #[tokio::test]
12285 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
12286 let supervisor = Supervisor::new_for_test(
12287 Arc::new(Registry::default()),
12288 RestartPolicy::new(2, Duration::ZERO),
12289 );
12290 let runtime = supervisor.runtime_config();
12291 let spec = windowed_crash_spec("operator-cleared-budget");
12292 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12293
12294 for attempt in 1..=2 {
12295 assert!(matches!(
12296 on_child_exit(
12297 &spec,
12298 runtime.restart_policy,
12299 &supervisor.registry,
12300 &snapshot,
12301 &runtime.terminal_ring,
12302 &runtime.spawn_events,
12303 &runtime.child_roster,
12304 crash_exit_report(attempt),
12305 )
12306 .await,
12307 NextAction::Restart { schedule: _ }
12308 ));
12309 }
12310
12311 reset_restart_count(&snapshot, &spec.module_id).unwrap();
12312 {
12313 let state = lock_snapshot(&snapshot).unwrap();
12314 assert!(
12315 state.crash_restarts.is_empty(),
12316 "an operator restart returns the full budget"
12317 );
12318 assert_eq!(
12319 state.lifetime_restarts, 2,
12320 "clearing the budget must not unmake the crashes"
12321 );
12322 }
12323
12324 assert!(
12325 matches!(
12326 on_child_exit(
12327 &spec,
12328 runtime.restart_policy,
12329 &supervisor.registry,
12330 &snapshot,
12331 &runtime.terminal_ring,
12332 &runtime.spawn_events,
12333 &runtime.child_roster,
12334 crash_exit_report(3),
12335 )
12336 .await,
12337 NextAction::Restart { schedule: _ }
12338 ),
12339 "the cleared budget must be spendable again"
12340 );
12341 let state = lock_snapshot(&snapshot).unwrap();
12342 assert_eq!(state.crash_restarts.len(), 1);
12343 assert_eq!(state.lifetime_restarts, 3);
12344 }
12345
12346 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12347 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
12348 let severed = ProcessIdentity {
12349 pid: 41,
12350 start_time: 101,
12351 };
12352 let successor = ProcessIdentity {
12353 pid: 41,
12354 start_time: 202,
12355 };
12356 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12357 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
12358 state.pid = Some(successor.pid);
12359 state.process_start_time = Some(successor.start_time);
12360 })
12361 .unwrap();
12362 assert!(!module.record_deliberate_severance(severed).unwrap());
12363
12364 let exit_report = apply_deliberate_severance_marker(
12365 &module.inner.snapshot,
12366 Some(successor),
12367 ExitReport {
12368 kind: ExitKind::Crash,
12369 code: Some(1),
12370 signal: None,
12371 at_ms: 1,
12372 },
12373 );
12374
12375 assert_eq!(exit_report.kind, ExitKind::Crash);
12376 }
12377
12378 #[tokio::test]
12379 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
12380 let registry = Registry::default();
12381 let supervisor = Supervisor::new_for_test(
12382 Arc::new(Registry::default()),
12383 RestartPolicy::new(3, Duration::ZERO),
12384 );
12385 let runtime = supervisor.runtime_config();
12386 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12387 let spec = ModuleSpec {
12388 module_id: "drain-deliberate-severance".to_string(),
12389 program: fake_aft_stub_path(),
12390 args: Vec::new(),
12391 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12392 reserved: false,
12393 reserved_prefixes: Vec::new(),
12394 protocol: ModuleProtocol::Subc,
12395 overlap: Default::default(),
12396 };
12397 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12398 let process = ProcessIdentity {
12399 pid: 41,
12400 start_time: 101,
12401 };
12402 child.process_identity = Some(process);
12403 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12404 state.pid = Some(process.pid);
12405 state.process_start_time = Some(process.start_time);
12406 })
12407 .unwrap();
12408 record_deliberate_severance(&snapshot, process).unwrap();
12409
12410 drain_child_to_state(
12411 &spec.module_id,
12412 spec.protocol,
12413 StopNotice::SentOverConnection,
12416 ®istry,
12417 None,
12418 &snapshot,
12419 &runtime.terminal_ring,
12420 &runtime.spawn_events,
12421 child,
12422 Duration::from_secs(1),
12423 ModuleState::Stopped,
12424 Some(false),
12425 )
12426 .await
12427 .unwrap();
12428
12429 let state = lock_snapshot(&snapshot).unwrap();
12430 assert_eq!(
12431 state.last_exit.as_ref().map(|exit| exit.kind),
12432 Some(ExitKind::DeliberateSeverance)
12433 );
12434 assert_eq!(state.lifetime_restarts, 1);
12435 assert_eq!(state.crash_restarts.len(), 0);
12436 drop(state);
12437 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12438 assert_eq!(
12439 history.entries[0].exit_kind,
12440 subc_control::TerminalExitKind::DeliberateSeverance
12441 );
12442 }
12443
12444 #[tokio::test]
12445 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
12446 let registry = Registry::default();
12447 let supervisor = Supervisor::new_for_test(
12448 Arc::new(Registry::default()),
12449 RestartPolicy::new(3, Duration::ZERO),
12450 );
12451 let runtime = supervisor.runtime_config();
12452 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12453 let spec = ModuleSpec {
12454 module_id: "ordinary-drain".to_string(),
12455 program: fake_aft_stub_path(),
12456 args: Vec::new(),
12457 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12458 reserved: false,
12459 reserved_prefixes: Vec::new(),
12460 protocol: ModuleProtocol::Subc,
12461 overlap: Default::default(),
12462 };
12463 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12464
12465 drain_child_to_state(
12466 &spec.module_id,
12467 spec.protocol,
12468 StopNotice::SentOverConnection,
12471 ®istry,
12472 None,
12473 &snapshot,
12474 &runtime.terminal_ring,
12475 &runtime.spawn_events,
12476 child,
12477 Duration::from_secs(1),
12478 ModuleState::Stopped,
12479 Some(false),
12480 )
12481 .await
12482 .unwrap();
12483
12484 let state = lock_snapshot(&snapshot).unwrap();
12485 assert_eq!(
12486 state.last_exit.as_ref().map(|exit| exit.kind),
12487 Some(ExitKind::Crash)
12488 );
12489 assert_eq!(state.lifetime_restarts, 0);
12490 assert_eq!(state.crash_restarts.len(), 0);
12491 }
12492
12493 #[test]
12494 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
12495 assert!(!include_str!("server.rs")
12501 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
12502 }
12503
12504 #[test]
12511 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
12512 assert!(drained_after_quiescence_wait(&Ok(true)));
12513 assert!(!drained_after_quiescence_wait(&Ok(false)));
12514 assert!(!drained_after_quiescence_wait(&Err(
12515 SuperviseError::StatePoisoned { module_id: None }
12516 )));
12517 }
12518
12519 #[test]
12528 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
12529 let ring = Arc::new(Mutex::new(TerminalRing::new(
12530 TerminalRingConfig::default(),
12531 0,
12532 )));
12533 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
12534
12535 let snapshot = ring.lock().unwrap().snapshot();
12536 assert_eq!(snapshot.entries.len(), 1);
12537 let entry = &snapshot.entries[0];
12538 assert_eq!(entry.exit_code, None);
12539 assert_eq!(entry.exit_signal, None);
12540 assert_eq!(entry.disposition, TerminalDisposition::Failed);
12541 }
12542
12543 #[test]
12544 fn wait_error_exit_path_preserves_spawn_event_density() {
12545 let feed = super::SpawnEventFeed::default();
12546 feed.configure_incarnation("wait-error-density".to_string());
12547 feed.emit_spawned("wait-error", 41, 1);
12548 let ring = Arc::new(Mutex::new(TerminalRing::new(
12549 TerminalRingConfig::default(),
12550 0,
12551 )));
12552
12553 record_wait_error_terminal("wait-error", &ring, &feed);
12554 feed.emit_spawned("after-wait-error", 42, 2);
12555
12556 let state = feed.0.lock().unwrap();
12557 let sequences = state
12558 .events
12559 .iter()
12560 .map(|event| event.cursor.seq)
12561 .collect::<Vec<_>>();
12562 assert_eq!(sequences, vec![1, 2, 3]);
12563 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
12564 assert_eq!(state.events[1].exit_code, None);
12565 assert_eq!(state.events[1].exit_signal, None);
12566 }
12567
12568 #[test]
12572 fn wait_error_exit_report_is_classified_as_a_crash() {
12573 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
12574 }
12575}
12576
12577#[cfg(all(test, unix))]
12578mod health_event_tests {
12579 use super::*;
12580 use cortexkit_test_support::ScratchDir;
12581 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
12582
12583 struct Actor {
12584 module: SupervisedModule,
12585 registry: Arc<Registry>,
12586 forwarding: Arc<ForwardingTable>,
12587 spec: ModuleSpec,
12588 health: HealthConfig,
12589 rx: mpsc::Receiver<crate::router::OutboundFrame>,
12590 _home: ScratchDir,
12591 }
12592
12593 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
12594 let deadline = std::time::Instant::now() + Duration::from_secs(10);
12598 let tick = Instant::now();
12599 loop {
12600 if let Some(value) = observe() {
12601 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
12602 return value;
12603 }
12604 assert!(
12605 std::time::Instant::now() < deadline,
12606 "timed out waiting for {reason}"
12607 );
12608 tokio::task::yield_now().await;
12609 }
12610 }
12611
12612 impl Actor {
12613 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
12614 let home = ScratchDir::new("health-events");
12615 let registry = Arc::new(Registry::default());
12616 let forwarding = Arc::new(ForwardingTable::default());
12617 let health = HealthConfig {
12618 cadence,
12619 deadline: Duration::from_secs(3600),
12620 ..HealthConfig::default()
12621 };
12622 let supervisor =
12623 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
12624 .with_forwarding(forwarding.clone())
12625 .with_health_config(health.clone());
12626 let spec = ModuleSpec {
12627 module_id: "event-health-child".into(),
12628 program: PathBuf::from("/bin/sh"),
12629 args: vec!["-c".into(), "exec sleep 600".into()],
12632 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
12633 .into_iter()
12634 .map(|key| (key.into(), home.path().display().to_string()))
12635 .collect(),
12636 reserved: false,
12637 reserved_prefixes: vec![],
12638 protocol,
12639 overlap: Default::default(),
12640 };
12641 let module = supervisor.spawn(spec.clone()).unwrap();
12642 let (_, rx) = mpsc::channel(8);
12643 let actor = Self {
12644 module,
12645 registry,
12646 forwarding,
12647 spec,
12648 health,
12649 rx,
12650 _home: home,
12651 };
12652 actor
12653 .wait_for_select(|checkpoint| checkpoint.generation > 0)
12654 .await;
12655 actor
12656 }
12657
12658 fn turns(&self) -> u64 {
12659 lock_snapshot(&self.module.inner.snapshot)
12660 .unwrap()
12661 .actor_turns
12662 }
12663
12664 async fn wait_for_select(
12665 &self,
12666 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
12667 ) -> ActorSelectCheckpoint {
12668 wait_for("actor select checkpoint", || {
12669 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
12670 state.actor_select.filter(|checkpoint| {
12671 checkpoint.generation == state.spawn_generation
12672 && state.state == ModuleState::Running
12673 && state.process_alive
12674 && state.reported_pid().is_some()
12675 && expected(checkpoint)
12676 })
12677 })
12678 .await
12679 }
12680
12681 fn hello(&mut self, connection: u64, health: bool) {
12682 let manifest =
12683 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
12684 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
12685 .build();
12686 let connection = ConnectionId::new(connection);
12687 let (tx, rx) = mpsc::channel(8);
12688 self.rx = rx;
12689 self.forwarding
12690 .register_module_connection(
12691 connection,
12692 self.spec.module_id.clone(),
12693 subc_protocol::PROTOCOL_VERSION,
12694 Concurrency::ModuleManaged,
12695 FrameSink::new(tx),
12696 )
12697 .unwrap();
12698 self.registry
12699 .register_with_control_ops(
12700 manifest,
12701 subc_protocol::PROTOCOL_VERSION,
12702 connection,
12703 if health {
12704 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
12705 } else {
12706 vec![]
12707 },
12708 )
12709 .unwrap();
12710 }
12711
12712 async fn probe(&mut self) -> crate::router::OutboundFrame {
12713 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
12714 Ok(frame) => Some(frame),
12715 Err(mpsc::error::TryRecvError::Empty) => None,
12716 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
12717 })
12718 .await;
12719 let body: Value = serde_json::from_slice(&frame.body).unwrap();
12720 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
12721 frame
12722 }
12723
12724 fn no_probe(&mut self) {
12725 assert!(matches!(
12726 self.rx.try_recv(),
12727 Err(mpsc::error::TryRecvError::Empty)
12728 ));
12729 }
12730
12731 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
12732 self.forwarding
12733 .complete_module_control_rpc(
12734 ConnectionId::new(connection),
12735 frame.header.corr,
12736 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
12737 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
12738 status: HealthStatus::Ok,
12739 detail: None,
12740 metrics: None,
12741 }),
12742 )
12743 .unwrap();
12744 }
12745 }
12746
12747 #[tokio::test(start_paused = true)]
12748 async fn no_health_children_park_for_sixty_seconds() {
12749 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
12750 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
12751 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
12752 unadvertised.hello(1, false);
12753 for _ in 0..6000 {
12758 tokio::time::advance(Duration::from_millis(10)).await;
12759 tokio::task::yield_now().await;
12760 }
12761 for actor in [&none, &unregistered, &unadvertised] {
12762 assert!(
12763 actor.turns() <= 5,
12764 "no-health actor woke {} times",
12765 actor.turns()
12766 );
12767 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
12768 }
12769 }
12770
12771 #[tokio::test(start_paused = true)]
12772 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
12773 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
12777 let tick = Instant::now();
12778 actor.hello(1, true);
12779 actor.probe().await;
12780 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
12781 }
12782
12783 #[tokio::test(start_paused = true)]
12784 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
12785 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
12786 let hello_tick = Instant::now();
12787 actor.hello(1, true);
12788 let armed = actor
12789 .wait_for_select(|checkpoint| {
12790 checkpoint.registered_connection == Some(ConnectionId::new(1))
12791 && checkpoint.next_probe_at.is_some()
12792 })
12793 .await;
12794 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
12795 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
12796 let first_deadline = hello_tick + delay;
12797 assert_eq!(armed.next_probe_at, Some(first_deadline));
12798 tokio::time::advance(Duration::from_secs(29)).await;
12799 let turns = actor.turns();
12800 actor
12801 .registry
12802 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
12803 .unwrap();
12804 let updated = actor
12805 .wait_for_select(|checkpoint| checkpoint.turn > turns)
12806 .await;
12807 assert_eq!(
12808 updated.next_probe_at,
12809 Some(first_deadline),
12810 "catalog update must not reset cadence"
12811 );
12812 actor.no_probe();
12813 tokio::time::advance(first_deadline - Instant::now()).await;
12814 let frame = actor.probe().await;
12815 actor.answer(1, frame);
12816 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
12817 let next_deadline = Instant::now() + next;
12818 let rearmed = actor
12819 .wait_for_select(|checkpoint| {
12820 checkpoint
12821 .next_probe_at
12822 .is_some_and(|deadline| deadline > first_deadline)
12823 })
12824 .await;
12825 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
12826 tokio::time::advance(next - Duration::from_nanos(1)).await;
12827 actor.no_probe();
12828 tokio::time::advance(Duration::from_nanos(1)).await;
12829 actor.probe().await;
12830 }
12831
12832 #[tokio::test(start_paused = true)]
12833 async fn disconnect_disarms_and_reconnect_rearms_health() {
12834 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
12835 actor.hello(1, true);
12836 actor
12837 .wait_for_select(|checkpoint| {
12838 checkpoint.registered_connection == Some(ConnectionId::new(1))
12839 && checkpoint.next_probe_at.is_some()
12840 })
12841 .await;
12842 let turns = actor.turns();
12843 let tick = Instant::now();
12844 actor
12845 .registry
12846 .deregister_connection(ConnectionId::new(1))
12847 .unwrap();
12848 let disarmed = actor
12849 .wait_for_select(|checkpoint| {
12850 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
12851 })
12852 .await;
12853 assert_eq!(disarmed.next_probe_at, None);
12854 assert_eq!(disarmed.wake_after, None);
12855 assert_eq!(
12856 Instant::now(),
12857 tick,
12858 "disconnect must disarm in the notification tick"
12859 );
12860 tokio::time::advance(Duration::from_secs(60)).await;
12861 actor.no_probe();
12862 let reconnect_tick = Instant::now();
12863 actor.hello(2, true);
12864 let rearmed = actor
12865 .wait_for_select(|checkpoint| {
12866 checkpoint.registered_connection == Some(ConnectionId::new(2))
12867 && checkpoint.next_probe_at.is_some()
12868 })
12869 .await;
12870 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
12871 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
12872 tokio::time::advance(delay).await;
12873 actor.probe().await;
12874 }
12875
12876 #[tokio::test(start_paused = true)]
12877 async fn reregistration_without_health_disarms_the_old_schedule() {
12878 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
12879 actor.hello(1, true);
12880 actor
12881 .wait_for_select(|checkpoint| {
12882 checkpoint.registered_connection == Some(ConnectionId::new(1))
12883 && checkpoint.next_probe_at.is_some()
12884 })
12885 .await;
12886 actor
12887 .registry
12888 .deregister_connection(ConnectionId::new(1))
12889 .unwrap();
12890 actor.hello(2, false);
12891 let disarmed = actor
12892 .wait_for_select(|checkpoint| {
12893 checkpoint.registered_connection == Some(ConnectionId::new(2))
12894 })
12895 .await;
12896 assert_eq!(disarmed.next_probe_at, None);
12897 assert_eq!(disarmed.wake_after, None);
12898 tokio::time::advance(Duration::from_secs(60)).await;
12899 actor.no_probe();
12900 }
12901
12902 #[tokio::test(start_paused = true)]
12903 async fn restart_waits_for_its_own_hello_before_rearming_health() {
12904 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
12905 actor.hello(1, true);
12906 let armed = actor
12907 .wait_for_select(|checkpoint| {
12908 checkpoint.registered_connection == Some(ConnectionId::new(1))
12909 && checkpoint.next_probe_at.is_some()
12910 })
12911 .await;
12912 actor
12916 .registry
12917 .deregister_connection(ConnectionId::new(1))
12918 .unwrap();
12919 actor.module.restart(Some(0)).await.unwrap();
12920 let parked = actor
12921 .wait_for_select(|checkpoint| {
12922 checkpoint.generation > armed.generation
12923 && checkpoint.registered_connection.is_none()
12924 && checkpoint.next_probe_at.is_none()
12925 })
12926 .await;
12927 assert_eq!(parked.wake_after, None);
12928 actor.rx.close();
12929 while actor.rx.try_recv().is_ok() {}
12932 let turns = actor.turns();
12933 tokio::time::advance(Duration::from_secs(60)).await;
12934 let after = actor.turns();
12935 eprintln!(
12936 "restart park turns: before={turns} after={after} delta={}",
12937 after - turns
12938 );
12939 assert_eq!(after, turns, "replacement must park before HELLO");
12940 let hello_tick = Instant::now();
12941 actor.hello(2, true);
12942 let rearmed = actor
12943 .wait_for_select(|checkpoint| {
12944 checkpoint.registered_connection == Some(ConnectionId::new(2))
12945 && checkpoint.next_probe_at.is_some()
12946 })
12947 .await;
12948 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
12949 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
12950 tokio::time::advance(delay).await;
12951 actor.probe().await;
12952 }
12953
12954 #[tokio::test(start_paused = true)]
12955 async fn rescan_adds_and_removes_http_health_without_polling() {
12956 use tokio::{
12957 io::{AsyncReadExt, AsyncWriteExt},
12958 net::TcpListener,
12959 };
12960 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
12961 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
12962 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
12963 actor.health.cadence = Duration::from_millis(1);
12965 let (started_tx, mut started_rx) = mpsc::channel(8);
12966 let peer = tokio::spawn(async move {
12967 loop {
12968 let (mut socket, _) = listener.accept().await.unwrap();
12969 let mut request = [0; 1024];
12970 assert!(socket.read(&mut request).await.unwrap() > 0);
12971 started_tx.send(()).await.unwrap();
12972 socket
12973 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
12974 .await
12975 .unwrap();
12976 }
12977 });
12978 let tick = Instant::now();
12979 let turns = actor.turns();
12980 actor
12981 .module
12982 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
12983 .await
12984 .unwrap();
12985 let armed = actor
12986 .wait_for_select(|checkpoint| {
12987 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
12988 })
12989 .await;
12990 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
12991 assert_eq!(Instant::now(), tick);
12992 tokio::time::advance(Duration::from_millis(1)).await;
12993 let mut started = false;
12994 wait_for("successful HTTP probe", || {
12995 started |= started_rx.try_recv().is_ok();
12996 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
12997 .then_some(())
12998 })
12999 .await;
13000 assert!(started, "added HTTP check must start at its first deadline");
13001 assert_eq!(
13002 actor.module.status().unwrap().health.status,
13003 SupervisorHealthStatus::Ok
13004 );
13005 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13006 actor.health.http = None;
13007 let turns = actor.turns();
13008 actor
13009 .module
13010 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13011 .await
13012 .unwrap();
13013 let parked = actor
13014 .wait_for_select(|checkpoint| {
13015 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13016 })
13017 .await;
13018 assert_eq!(parked.wake_after, None);
13019 let turns = actor.turns();
13020 tokio::time::advance(Duration::from_secs(60)).await;
13021 assert_eq!(
13022 actor.turns(),
13023 turns,
13024 "removed HTTP check must park the actor"
13025 );
13026 assert!(
13027 started_rx.try_recv().is_err(),
13028 "removed HTTP check must stay disarmed"
13029 );
13030 assert_eq!(
13031 actor.module.status().unwrap().health,
13032 ModuleHealthStatus::default()
13033 );
13034 peer.abort();
13035 }
13036}
13037
13038#[cfg(test)]
13039mod health_evidence_tests {
13040 use super::{HealthProbeError, HealthProbeEvidence};
13041 use std::collections::HashSet;
13042
13043 #[test]
13051 fn only_a_dead_lane_is_proof_of_death() {
13052 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
13053 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
13057 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
13058 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
13059 }
13060
13061 #[test]
13067 fn every_evidence_class_has_a_distinct_label() {
13068 let labels = [
13069 HealthProbeError::lane_dead("").label(),
13070 HealthProbeError::no_answer("").label(),
13071 HealthProbeError::bad_answer("").label(),
13072 HealthProbeError::misconfigured("").label(),
13073 ];
13074 let unique: HashSet<_> = labels.iter().collect();
13075 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
13076 }
13077
13078 #[test]
13084 fn classification_preserves_the_original_message() {
13085 let err = HealthProbeError::no_answer("module did not answer within 5s");
13086 assert_eq!(err.to_string(), "module did not answer within 5s");
13087 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13088 }
13089}
13090
13091#[cfg(test)]
13092mod health_tombstone_tests {
13093 use std::{path::PathBuf, sync::Arc, time::Duration};
13094
13095 use subc_protocol::{
13096 manifest::Concurrency,
13097 session::{HealthStatus, ModuleControlResponse},
13098 };
13099 use tokio::sync::mpsc;
13100
13101 use super::{
13102 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
13103 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
13104 };
13105 use crate::{
13106 control::ControlHandler,
13107 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
13108 registry::{ConnectionId, Registry},
13109 router::FrameSink,
13110 };
13111
13112 struct ProbeHarness {
13113 spec: ModuleSpec,
13114 runtime: SupervisorRuntimeConfig,
13115 forwarding: Arc<ForwardingTable>,
13116 module_connection: ConnectionId,
13117 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13118 handler: ControlHandler,
13119 module: super::SupervisedModule,
13120 }
13121
13122 fn probe_harness() -> ProbeHarness {
13123 let registry = Arc::new(Registry::default());
13124 let forwarding = Arc::new(ForwardingTable::default());
13125 let supervisor_handle = super::SupervisorHandle::new();
13126 let health = HealthConfig {
13127 http: None,
13128 cadence: Duration::from_secs(30),
13129 deadline: Duration::from_secs(5),
13130 failure_threshold: 3,
13131 on_degraded: HealthAction::Report,
13132 on_failing: HealthAction::Report,
13133 critical: false,
13134 };
13135 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13136 .with_forwarding(Arc::clone(&forwarding))
13137 .with_handle(supervisor_handle.clone())
13138 .with_health_config(health);
13139 let spec = ModuleSpec {
13140 module_id: "late-health-module".to_string(),
13141 program: PathBuf::from("disabled-module"),
13142 args: Vec::new(),
13143 env: Vec::new(),
13144 reserved: false,
13145 reserved_prefixes: Vec::new(),
13146 protocol: ModuleProtocol::Subc,
13147 overlap: Default::default(),
13148 };
13149 let module = supervisor
13150 .supervise_configured(spec.clone(), false)
13151 .unwrap();
13152 let runtime = supervisor.runtime_config();
13153 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
13154 .with_supervisor(supervisor_handle);
13155 let module_connection = ConnectionId::new(700);
13156 let (module_tx, module_rx) = mpsc::channel(8);
13157 forwarding
13158 .register_module_connection(
13159 module_connection,
13160 spec.module_id.clone(),
13161 subc_protocol::PROTOCOL_VERSION,
13162 Concurrency::ModuleManaged,
13163 FrameSink::new(module_tx),
13164 )
13165 .unwrap();
13166
13167 ProbeHarness {
13168 spec,
13169 runtime,
13170 forwarding,
13171 module_connection,
13172 module_rx,
13173 handler,
13174 module,
13175 }
13176 }
13177
13178 async fn finish_after(
13179 harness: &mut ProbeHarness,
13180 stall: Duration,
13181 ) -> ModuleControlRpcCompletion {
13182 assert!(stall > harness.runtime.health.deadline);
13183 let deadline = harness.runtime.health.deadline;
13184 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13185 let answer = async {
13186 let frame = harness.module_rx.recv().await.expect("health.check frame");
13187 tokio::time::advance(deadline).await;
13188 tokio::task::yield_now().await;
13189 tokio::time::advance(stall - deadline).await;
13190 harness
13191 .forwarding
13192 .complete_module_control_rpc(
13193 harness.module_connection,
13194 frame.header.corr,
13195 Some("health.check"),
13196 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13197 status: HealthStatus::Ok,
13198 detail: None,
13199 metrics: None,
13200 }),
13201 )
13202 .unwrap()
13203 };
13204 let (probe_result, completion) = tokio::join!(probe, answer);
13205 let err = probe_result.expect_err("probe must miss its deadline");
13206 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13207 completion
13208 }
13209
13210 async fn time_out_without_answer(harness: &mut ProbeHarness) {
13211 let deadline = harness.runtime.health.deadline;
13212 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13213 let exhaust_deadline = async {
13214 let _frame = harness.module_rx.recv().await.expect("health.check frame");
13215 tokio::time::advance(deadline).await;
13216 tokio::task::yield_now().await;
13217 };
13218 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
13219 let err = probe_result.expect_err("probe must miss its deadline");
13220 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13221 }
13222
13223 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
13224 let registry = Arc::clone(&harness.module.inner.registry);
13225 let snapshot = Arc::clone(&harness.module.inner.snapshot);
13226 let process_liveness = super::SupervisorProcessLiveness::default();
13227 let mut child = None;
13228 let cycle = super::run_health_probe_cycle(
13229 &harness.spec,
13230 &harness.runtime,
13231 ®istry,
13232 &process_liveness,
13233 &snapshot,
13234 &mut child,
13235 );
13236 let peer = async {
13237 let frame = harness.module_rx.recv().await.expect("health.check frame");
13238 if answer {
13239 harness
13240 .forwarding
13241 .complete_module_control_rpc(
13242 harness.module_connection,
13243 frame.header.corr,
13244 Some("health.check"),
13245 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13246 status: HealthStatus::Ok,
13247 detail: None,
13248 metrics: Some(serde_json::json!({"ready": true})),
13249 }),
13250 )
13251 .unwrap();
13252 } else {
13253 tokio::time::advance(harness.runtime.health.deadline).await;
13254 tokio::task::yield_now().await;
13255 }
13256 };
13257 tokio::join!(cycle, peer);
13258 }
13259
13260 #[tokio::test(start_paused = true)]
13261 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
13262 let mut harness = probe_harness();
13263 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
13267 monitor.abort();
13268 let _ = monitor.await;
13269 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
13270 state.enabled = true;
13271 state.state = super::ModuleState::Running;
13272 state.process_alive = true;
13273 })
13274 .unwrap();
13275
13276 run_probe_cycle(&mut harness, true).await;
13277 assert_eq!(
13278 harness.module.status().unwrap().health.status,
13279 super::SupervisorHealthStatus::Ok
13280 );
13281
13282 for failures in 1..harness.runtime.health.failure_threshold {
13283 run_probe_cycle(&mut harness, false).await;
13284 let status = harness.module.status().unwrap();
13285 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
13286 assert_eq!(status.health.consecutive_failures, failures);
13287 assert!(status.health.last_probe_ms.is_some());
13288 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
13289 assert!(status.health.metrics.is_none());
13290 assert_eq!(status.state, super::ModuleState::Running);
13291 assert!(status.process_alive);
13292 assert_eq!(status.restart_count, 0);
13293 assert_eq!(status.lifetime_restarts, 0);
13294 assert!(status.health.last_action.is_none());
13295 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
13296 }
13297
13298 run_probe_cycle(&mut harness, true).await;
13299 let recovered = harness.module.status().unwrap();
13300 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
13301 assert_eq!(recovered.health.consecutive_failures, 0);
13302 assert!(recovered.health.detail.is_none());
13303 assert_eq!(
13304 recovered.health.metrics,
13305 Some(serde_json::json!({"ready": true}))
13306 );
13307 assert_eq!(recovered.lifetime_restarts, 0);
13308
13309 for failures in 1..=harness.runtime.health.failure_threshold {
13310 run_probe_cycle(&mut harness, false).await;
13311 let status = harness.module.status().unwrap();
13312 assert_eq!(status.health.consecutive_failures, failures);
13313 if failures < harness.runtime.health.failure_threshold {
13314 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
13315 assert_eq!(status.state, super::ModuleState::Running);
13316 assert_eq!(status.lifetime_restarts, 0);
13317 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
13318 } else {
13319 assert_eq!(
13320 status.health.status,
13321 super::SupervisorHealthStatus::Unresponsive
13322 );
13323 assert_eq!(status.state, super::ModuleState::Restarting);
13324 assert_eq!(status.restart_count, 1);
13325 assert_eq!(status.lifetime_restarts, 1);
13326 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
13327 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
13328 }
13329 }
13330 }
13331
13332 #[tokio::test(start_paused = true)]
13333 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
13334 let mut harness = probe_harness();
13335
13336 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
13337 let first_latency = match &first {
13338 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
13339 other => panic!("late answer was not retained: {other:?}"),
13340 };
13341 assert!(harness.handler.observe_module_control_completion(first));
13342
13343 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
13344 let second_latency = match &second {
13345 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
13346 other => panic!("late answer was not retained: {other:?}"),
13347 };
13348 assert!(harness.handler.observe_module_control_completion(second));
13349
13350 assert_eq!(first_latency, Duration::from_secs(8));
13351 assert_eq!(
13352 second_latency - first_latency,
13353 Duration::from_secs(3),
13354 "latency must grow linearly with the additional stall"
13355 );
13356 let health = harness.module.status().unwrap().health;
13357 assert_eq!(health.late_answer_count, 2);
13358 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
13359 }
13360
13361 #[tokio::test(start_paused = true)]
13369 async fn late_answer_clears_the_consecutive_failure_streak() {
13370 let mut harness = probe_harness();
13371
13372 time_out_without_answer(&mut harness).await;
13374 harness
13375 .module
13376 .record_health_probe_failure_for_test("[no-answer] test miss")
13377 .unwrap();
13378 assert_eq!(
13379 harness.module.status().unwrap().health.consecutive_failures,
13380 1,
13381 "precondition: the miss must be on the streak before the late answer"
13382 );
13383
13384 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
13386 assert!(matches!(
13387 late,
13388 ModuleControlRpcCompletion::LateHealthAnswer { .. }
13389 ));
13390 assert!(harness.handler.observe_module_control_completion(late));
13391
13392 let health = harness.module.status().unwrap().health;
13393 assert_eq!(
13394 health.consecutive_failures, 0,
13395 "a late answer is an answer: the streak must reset"
13396 );
13397 assert_eq!(health.late_answer_count, 1);
13398 }
13399
13400 #[tokio::test(start_paused = true)]
13401 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
13402 let mut harness = probe_harness();
13403
13404 for _ in 0..20 {
13405 time_out_without_answer(&mut harness).await;
13406 assert_eq!(
13407 harness.forwarding.health_probe_tombstone_count().unwrap(),
13408 1
13409 );
13410 }
13411 }
13412}
13413
13414#[cfg(test)]
13415mod child_env_tests {
13416 #[cfg(windows)]
13417 use super::{
13418 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
13419 };
13420 #[cfg(windows)]
13421 use std::sync::{Arc, OnceLock};
13422 #[cfg(windows)]
13423 #[test]
13424 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
13425 let spec = spec(Vec::new());
13426 let mut command = Command::new("unused");
13427 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
13428 .unwrap()
13429 .unwrap();
13430 let name = command
13431 .as_std()
13432 .get_envs()
13433 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
13434 .and_then(|(_, value)| value)
13435 .unwrap();
13436 assert_eq!(name, OsStr::new(handoff.name()));
13437 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
13438 assert!(command.as_std().get_envs().any(|(key, value)| key
13439 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
13440 && value == Some(OsStr::new("env"))));
13441 assert!(command
13442 .as_std()
13443 .get_envs()
13444 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
13445 }
13446
13447 #[cfg(windows)]
13448 #[test]
13449 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
13450 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
13451 snapshot.process_alive = true;
13452 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
13453 assert_eq!(
13454 observed_launch_nonce_source(&snapshot).as_deref(),
13455 Some("env")
13456 );
13457 let consumed = Arc::new(OnceLock::new());
13458 snapshot.pipe_consumed = Some(consumed.clone());
13459 assert_eq!(
13460 observed_launch_nonce_source(&snapshot).as_deref(),
13461 Some("env")
13462 );
13463 consumed.set(()).unwrap();
13464 assert_eq!(
13465 observed_launch_nonce_source(&snapshot).as_deref(),
13466 Some("pipe")
13467 );
13468 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
13469 assert_eq!(
13470 observed_launch_nonce_source(&snapshot).as_deref(),
13471 Some("env")
13472 );
13473 clear_current_process_facts(&mut snapshot);
13474 assert_eq!(observed_launch_nonce_source(&snapshot), None);
13475 }
13476 use super::{
13477 apply_child_env, apply_spawn_role, apply_wire_spawn_args, ModuleProtocol, ModuleSpec,
13478 SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG, SUBC_LAUNCH_NONCE_ENV,
13479 SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
13480 };
13481 use std::{ffi::OsStr, path::PathBuf};
13482 use tokio::process::Command;
13483
13484 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
13485 ModuleSpec {
13486 module_id: "env-plan".to_string(),
13487 program: PathBuf::from("/nonexistent"),
13488 args: Vec::new(),
13489 env,
13490 reserved: false,
13491 reserved_prefixes: Vec::new(),
13492 protocol: ModuleProtocol::Subc,
13493 overlap: Default::default(),
13494 }
13495 }
13496
13497 #[test]
13511 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
13512 let mut command = Command::new("/nonexistent");
13513 apply_child_env(&mut command, &spec(Vec::new()));
13514 let removed = command
13515 .as_std()
13516 .get_envs()
13517 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
13518 assert!(
13519 removed,
13520 "ambient CK_LOG must be explicitly removed for an unconfigured module"
13521 );
13522
13523 let mut configured = Command::new("/nonexistent");
13524 apply_child_env(
13525 &mut configured,
13526 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
13527 );
13528 let effective = configured
13529 .as_std()
13530 .get_envs()
13531 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
13532 .last()
13533 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
13534 assert_eq!(
13535 effective,
13536 Some(Some("debug".to_string())),
13537 "a module's configured CK_LOG must survive the ambient removal"
13538 );
13539 }
13540
13541 #[test]
13550 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
13551 let connection_file = std::path::Path::new("/run/subc-connection.json");
13552 let handle = SupervisorHandle::new();
13553
13554 let mut none_spec = spec(Vec::new());
13555 none_spec.protocol = ModuleProtocol::None;
13556 let mut none = Command::new("/nonexistent");
13557 let none_handoff =
13558 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
13559 .expect("protocol-none spawn args apply");
13560 assert!(
13561 none_handoff.is_none(),
13562 "protocol:none spawn must not receive a nonce descriptor"
13563 );
13564 #[cfg(windows)]
13565 assert!(!none.as_std().get_envs().any(|(key, value)| key
13566 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
13567 && value.is_some()));
13568 assert!(
13569 !none.as_std().get_envs().any(|(key, value)| key
13570 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
13571 && value.is_some()),
13572 "protocol:none spawn must not name a nonce descriptor"
13573 );
13574 let none_args: Vec<String> = none
13575 .as_std()
13576 .get_args()
13577 .map(|a| a.to_string_lossy().into_owned())
13578 .collect();
13579 assert!(
13580 !none_args.iter().any(|a| a == SUBC_ARG),
13581 "protocol:none argv must not carry --subc; got {none_args:?}"
13582 );
13583 let none_has_nonce = none
13584 .as_std()
13585 .get_envs()
13586 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
13587 assert!(
13588 !none_has_nonce,
13589 "protocol:none spawn must not receive a launch nonce"
13590 );
13591 let none_has_module_id = none
13592 .as_std()
13593 .get_envs()
13594 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
13595 assert!(
13596 none_has_module_id,
13597 "SUBC_MODULE_ID is inert and stays on every path"
13598 );
13599 assert!(
13600 handle.spawn_nonce(&none_spec.module_id).is_none(),
13601 "no nonce record for a process that will never present one"
13602 );
13603
13604 let wire_spec = spec(Vec::new());
13606 let mut wire = Command::new("/nonexistent");
13607 let wire_handoff =
13608 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
13609 .expect("subc-wire spawn args apply");
13610 let wire_fd_env = wire
13611 .as_std()
13612 .get_envs()
13613 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
13614 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
13615 #[cfg(unix)]
13616 assert_eq!(
13617 wire_fd_env,
13618 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
13619 "a subc-wire spawn names the pipe it will receive at descriptor 3"
13620 );
13621 #[cfg(windows)]
13622 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
13623 #[cfg(not(any(unix, windows)))]
13624 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
13625 let wire_args: Vec<String> = wire
13626 .as_std()
13627 .get_args()
13628 .map(|a| a.to_string_lossy().into_owned())
13629 .collect();
13630 assert_eq!(
13631 wire_args,
13632 vec![
13633 SUBC_ARG.to_string(),
13634 connection_file.to_string_lossy().into_owned()
13635 ],
13636 "a subc-wire spawn still carries --subc <path>"
13637 );
13638 assert_eq!(
13639 wire.as_std()
13640 .get_envs()
13641 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
13642 !cfg!(unix),
13643 "only Windows supplies the environment nonce"
13644 );
13645 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
13646 }
13647
13648 #[test]
13658 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
13659 let role = |command: &Command| {
13660 command
13661 .as_std()
13662 .get_envs()
13663 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
13664 .last()
13665 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
13666 };
13667 let forged = spec(vec![(
13668 SUBC_SPAWN_ROLE_ENV.to_string(),
13669 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
13670 )]);
13671
13672 let mut plain = Command::new("/nonexistent");
13673 apply_child_env(&mut plain, &forged);
13674 apply_spawn_role(&mut plain, SpawnRole::Plain);
13675 assert_eq!(
13676 role(&plain),
13677 Some(None),
13678 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
13679 );
13680
13681 let mut candidate = Command::new("/nonexistent");
13682 apply_child_env(&mut candidate, &spec(Vec::new()));
13683 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
13684 assert_eq!(
13685 role(&candidate),
13686 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
13687 );
13688 }
13689
13690 #[test]
13696 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
13697 let mut command = Command::new("/nonexistent");
13698 apply_child_env(
13699 &mut command,
13700 &spec(vec![
13701 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
13702 ("KEPT".to_string(), "yes".to_string()),
13703 ]),
13704 );
13705 let keys: Vec<String> = command
13706 .as_std()
13707 .get_envs()
13708 .filter(|(_, value)| value.is_some())
13709 .map(|(key, _)| key.to_string_lossy().into_owned())
13710 .collect();
13711 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
13712 assert!(
13713 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
13714 "daemon-private capture key leaked to the child: {keys:?}"
13715 );
13716 }
13717}
13718
13719#[cfg(test)]
13720mod jitter_tests {
13721 use super::jittered_health_delay;
13722 use std::{collections::HashSet, time::Duration};
13723
13724 const FLEET: [&str; 14] = [
13733 "aft",
13734 "alfonso-core",
13735 "magic-context",
13736 "broca",
13737 "thalamus",
13738 "quota",
13739 "engram",
13740 "plexus",
13741 "cerebellum",
13742 "astrocyte",
13743 "synapse",
13744 "subc-mcp",
13745 "cortexkit-credentials",
13746 "subc-federation",
13747 ];
13748
13749 #[test]
13757 fn probe_delays_disperse_across_the_fleet() {
13758 let cadence = Duration::from_secs(30);
13759 let delays: HashSet<Duration> = FLEET
13760 .iter()
13761 .map(|id| jittered_health_delay(id, 0, cadence))
13762 .collect();
13763 assert_eq!(
13764 delays.len(),
13765 FLEET.len(),
13766 "every supervised module must land on its own probe offset"
13767 );
13768 }
13769
13770 #[test]
13776 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
13777 let cadence = Duration::from_secs(30);
13778 let span = cadence / 10;
13779 for id in FLEET {
13780 for probe_index in 0..8 {
13781 let delay = jittered_health_delay(id, probe_index, cadence);
13782 assert!(
13783 delay >= cadence,
13784 "{id}#{probe_index}: jitter must not shorten the cadence"
13785 );
13786 assert!(
13787 delay < cadence + span,
13788 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
13789 );
13790 }
13791 }
13792 }
13793
13794 #[test]
13800 fn a_module_offset_is_stable_across_restarts() {
13801 let cadence = Duration::from_secs(30);
13802 for id in FLEET {
13803 assert_eq!(
13804 jittered_health_delay(id, 0, cadence),
13805 jittered_health_delay(id, 0, cadence),
13806 "{id}: the same module and probe index must produce the same offset"
13807 );
13808 }
13809 }
13810
13811 #[test]
13813 fn zero_cadence_yields_zero_delay() {
13814 assert_eq!(
13815 jittered_health_delay("aft", 0, Duration::ZERO),
13816 Duration::ZERO
13817 );
13818 }
13819}
13820
13821#[cfg(all(test, target_os = "linux"))]
13822mod cgroup_placement_tests {
13823 use super::{
13824 apply_cgroup_placement, remove_module_cgroup, ModuleProtocol, ModuleSpec, SuperviseError,
13825 SupervisedChild,
13826 };
13827 use crate::stderr_tail::{StderrRing, StderrTailConfig};
13828 use cortexkit_test_support::ScratchDir;
13829 use std::{
13830 fs, io,
13831 path::{Path, PathBuf},
13832 sync::{Arc, Mutex},
13833 };
13834 use tokio::process::Command;
13835
13836 #[tokio::test]
13837 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
13838 use super::*;
13839 let dir = ScratchDir::new("unique-spawn-cgroups");
13840 let root = PathBuf::from(format!(
13841 "/sys/fs/cgroup/subc-unique-test-{}-{}",
13842 std::process::id(),
13843 unix_ms_now()
13844 ));
13845 if let Err(error) = fs::create_dir(&root) {
13846 assert!(
13847 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
13848 "required cgroup test cannot execute: {error}"
13849 );
13850 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
13851 return;
13852 }
13853 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
13854 let group_count = || {
13855 fs::read_dir(root.join("subc-modules"))
13856 .unwrap()
13857 .map(|entry| entry.unwrap().file_type().unwrap())
13858 .filter(|kind| kind.is_dir())
13859 .count()
13860 };
13861 let supervisor =
13862 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
13863 .with_cgroup_placement(Some(placement.clone()));
13864 let runtime = supervisor.runtime_config();
13865 let mut spec = ModuleSpec {
13866 module_id: "unique-spawn".into(),
13867 program: PathBuf::from("/bin/sleep"),
13868 args: vec!["60".into()],
13869 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13870 .into_iter()
13871 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
13872 .collect(),
13873 reserved: false,
13874 reserved_prefixes: vec![],
13875 protocol: ModuleProtocol::None,
13876 overlap: Default::default(),
13877 };
13878 let spawn = |spec: &ModuleSpec| {
13879 spawn_child(
13880 spec,
13881 None,
13882 None,
13883 &runtime.stderr_ring,
13884 None,
13885 &runtime.child_roster,
13886 Some(&placement),
13887 )
13888 .unwrap()
13889 };
13890 let mut live = spawn(&spec);
13891 for _ in 0..3 {
13892 let next = spawn(&spec);
13894 assert_ne!(live.module_id, next.module_id);
13895 live.start_kill().unwrap();
13896 live.wait().await.unwrap();
13897 live = next;
13898 assert!(
13899 live.child.try_wait().unwrap().is_none(),
13900 "retiring the old slot must not kill the replacement"
13901 );
13902 assert_eq!(
13903 group_count(),
13904 1,
13905 "only the live spawn's cgroup should remain"
13906 );
13907 }
13908 supervisor.begin_daemon_shutdown();
13909 let reap = tokio::spawn(async move {
13910 live.wait().await.unwrap();
13911 });
13912 supervisor
13913 .end_children_for_daemon_shutdown(false, std::future::pending())
13914 .await;
13915 reap.await.unwrap();
13916 assert_eq!(group_count(), 0);
13917 spec.program = PathBuf::from("/bin/true");
13919 spec.args.clear();
13920 let fresh_roster = ChildRoster::default();
13921 let mut short = spawn_child(
13922 &spec,
13923 None,
13924 None,
13925 &runtime.stderr_ring,
13926 None,
13927 &fresh_roster,
13928 Some(&placement),
13929 )
13930 .unwrap();
13931 short.wait().await.unwrap();
13932 assert_eq!(group_count(), 0);
13933 spec.module_id = "_".repeat(255);
13934 let mut long_id = spawn_child(
13935 &spec,
13936 None,
13937 None,
13938 &runtime.stderr_ring,
13939 None,
13940 &fresh_roster,
13941 Some(&placement),
13942 )
13943 .unwrap();
13944 long_id.wait().await.unwrap();
13945 assert_eq!(
13946 group_count(),
13947 0,
13948 "valid long module IDs must not exceed cgroup NAME_MAX"
13949 );
13950 fs::remove_dir(root.join("subc-modules")).unwrap();
13951 fs::remove_dir(root).unwrap();
13952 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
13953 }
13954
13955 #[test]
13956 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
13957 let path = Path::new("/definitely-missing-subc-cgroup");
13958 let mut command = Command::new("true");
13959 let error = apply_cgroup_placement(
13960 &mut command,
13961 &ModuleSpec {
13962 module_id: "broken-cgroup".to_string(),
13963 program: PathBuf::from("true"),
13964 args: Vec::new(),
13965 env: Vec::new(),
13966 reserved: false,
13967 reserved_prefixes: Vec::new(),
13968 protocol: ModuleProtocol::Subc,
13969 overlap: Default::default(),
13970 },
13971 path,
13972 )
13973 .expect_err("a parent cgroup open failure must reject the supervised spawn");
13974 let reason = error.to_string();
13975
13976 assert!(
13977 matches!(error, SuperviseError::Cgroup { .. }),
13978 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
13979 );
13980 assert!(
13981 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
13982 "parent cgroup open failure must name cgroup.procs: {reason}"
13983 );
13984 }
13985
13986 #[tokio::test]
13987 async fn reaping_a_child_removes_its_empty_module_cgroup() {
13988 let root = ScratchDir::new("supervisor-reap-cgroup");
13989 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
13990 let placement = subc_cgroup::prepare_at(&root)
13991 .expect("prepare scratch cgroup root")
13992 .expect("scratch root has a cgroup.procs marker");
13993 let module_id = "reaped-module";
13994 let module = placement
13995 .module_path(module_id)
13996 .expect("create scratch module cgroup");
13997 let child = Command::new("true")
13998 .env("XDG_DATA_HOME", root.path())
13999 .env("XDG_RUNTIME_DIR", root.path())
14000 .env("XDG_CONFIG_HOME", root.path())
14001 .spawn()
14002 .expect("spawn short-lived child");
14003 let pid = child.id().expect("spawned child has pid");
14004 let mut child = SupervisedChild {
14005 child,
14006 protocol: ModuleProtocol::Subc,
14007 module_id: module_id.to_string(),
14008 cgroup_placement: Some(placement),
14009 stdout_pump: None,
14010 stderr_pump: None,
14011 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
14012 spawned_at_ms: 0,
14013 spawned_from: PathBuf::from("true"),
14014 spawned_file_identity: None,
14015 process_start_time: None,
14016 process_identity: None,
14017 pid,
14018 roster_guard: None,
14019 #[cfg(target_os = "macos")]
14020 privacy_exec: None,
14021 spawn_failure: None,
14022 };
14023
14024 child.wait().await.expect("reap short-lived child");
14025
14026 assert!(
14027 !module.exists(),
14028 "reaping the supervised child must remove its empty cgroup"
14029 );
14030 }
14031
14032 #[test]
14033 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
14034 let root = ScratchDir::new("supervisor-non-empty-cgroup");
14035 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14036 let placement = subc_cgroup::prepare_at(&root)
14037 .expect("prepare scratch cgroup root")
14038 .expect("scratch root has a cgroup.procs marker");
14039 let module = placement
14040 .module_path("surviving-module")
14041 .expect("create scratch module cgroup");
14042 fs::write(module.join("surviving-process"), b"still present")
14043 .expect("make scratch cgroup non-empty");
14044 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
14045
14046 remove_module_cgroup(&placement, "surviving-module");
14047
14048 let logs = crate::router::test_log::captured_logs(&logs);
14049 assert!(
14050 module.exists(),
14051 "failed removal must leave the cgroup intact"
14052 );
14053 assert!(
14054 logs.contains("could not remove module cgroup after process exit; continuing teardown")
14055 && logs.contains("surviving-module"),
14056 "best-effort removal must report the failure without returning it: {logs}"
14057 );
14058 }
14059
14060 #[test]
14061 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
14062 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
14063 let reason = SuperviseError::Spawn {
14064 program: PathBuf::from("/bin/true"),
14065 source: io::Error::from_raw_os_error(13),
14066 cgroup_path: Some(cgroup_path.clone()),
14067 }
14068 .to_string();
14069
14070 assert!(
14071 reason.contains(&cgroup_path.display().to_string()),
14072 "a pre_exec spawn failure must name the cgroup path: {reason}"
14073 );
14074 }
14075}
14076
14077#[cfg(test)]
14078mod spawn_subscriber_lag_tests {
14079 use super::*;
14080
14081 #[tokio::test]
14086 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
14087 let feed = SpawnEventFeed::default();
14088 feed.configure_incarnation("lag-incarnation".to_string());
14089 let (tx, mut rx) = mpsc::channel(1);
14092 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
14093 .expect("subscribe");
14094 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
14095 for index in 0..emitted {
14096 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
14097 tokio::task::yield_now().await;
14100 }
14101 assert_eq!(
14102 feed.subscriber_count(),
14103 0,
14104 "the lagged subscriber must be removed"
14105 );
14106
14107 let mut data = Vec::new();
14108 let mut last = None;
14109 loop {
14110 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14111 .await
14112 .expect("the forwarder must finish once the subscriber is dropped");
14113 let Some(outbound) = next else { break };
14114 let frame = outbound.frame;
14115 if frame.header.ty == FrameType::StreamData {
14116 assert!(last.is_none(), "no data may follow the terminal frame");
14117 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
14118 data.push(event.cursor.seq);
14119 } else {
14120 assert!(last.is_none(), "exactly one terminal frame");
14121 last = Some(frame);
14122 }
14123 }
14124 assert!(!data.is_empty(), "queued frames drain before the terminal");
14125 for pair in data.windows(2) {
14126 assert_eq!(
14127 pair[1],
14128 pair[0] + 1,
14129 "queued frames arrive dense and in order"
14130 );
14131 }
14132 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
14133 assert_eq!(terminal.header.ty, FrameType::Error);
14134 assert_eq!(terminal.header.corr, 7);
14135 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
14136 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
14137 let detail = body.detail.expect("lagged error carries detail");
14138 assert_eq!(
14139 detail["first_undelivered_cursor"]["seq"],
14140 data.last().unwrap() + 1,
14141 "the named cursor is the first event the subscriber did not receive"
14142 );
14143 assert_eq!(
14144 detail["first_undelivered_cursor"]["daemon_incarnation"],
14145 "lag-incarnation"
14146 );
14147 }
14148}
14149
14150#[cfg(test)]
14151mod terminal_history_read_concurrency_tests {
14152 use super::*;
14153 use crate::terminal_journal::read_pause;
14154 use cortexkit_test_support::ScratchDir;
14155 use std::sync::mpsc as std_mpsc;
14156
14157 fn journaled_ring(
14158 journal: &Arc<crate::terminal_journal::TerminalJournal>,
14159 ) -> Arc<Mutex<TerminalRing>> {
14160 Arc::new(Mutex::new(
14161 TerminalRing::new(TerminalRingConfig::default(), 1)
14162 .with_journal(Some(Arc::clone(journal))),
14163 ))
14164 }
14165
14166 fn crash(at_ms: u64) -> ExitReport {
14167 ExitReport {
14168 kind: ExitKind::Crash,
14169 code: Some(1),
14170 signal: None,
14171 at_ms,
14172 }
14173 }
14174
14175 fn record_within(
14178 module_id: &'static str,
14179 ring: &Arc<Mutex<TerminalRing>>,
14180 at_ms: u64,
14181 bound: Duration,
14182 ) -> bool {
14183 let ring = Arc::clone(ring);
14184 let (done, done_rx) = std_mpsc::channel();
14185 std::thread::spawn(move || {
14186 record_terminal(
14187 module_id,
14188 &ring,
14189 &SpawnEventFeed::default(),
14190 &crash(at_ms),
14191 TerminalDisposition::Restarting,
14192 );
14193 let _ = done.send(());
14194 });
14195 done_rx.recv_timeout(bound).is_ok()
14196 }
14197
14198 #[test]
14203 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
14204 let dir = ScratchDir::new("terminal-history-concurrent-read");
14205 let path = dir.join("terminals.jsonl");
14206 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
14207 path.clone(),
14208 "daemon".into(),
14209 ));
14210 let reader_ring = journaled_ring(&journal);
14211 let other_ring = journaled_ring(&journal);
14212 assert!(record_within(
14213 "reader-module",
14214 &reader_ring,
14215 10,
14216 Duration::from_secs(5)
14217 ));
14218
14219 let (started, release) = read_pause::install(&path);
14220 let reading = {
14221 let ring = Arc::clone(&reader_ring);
14222 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
14223 };
14224 started
14225 .recv_timeout(Duration::from_secs(5))
14226 .expect("the history read reached its pause");
14227
14228 let bound = Duration::from_secs(1);
14229 assert!(
14230 record_within("other-module", &other_ring, 20, bound),
14231 "another module's exit waited on a history read (journal writer held)"
14232 );
14233 assert!(
14234 record_within("reader-module", &reader_ring, 30, bound),
14235 "the read module's own exit waited on its history read (ring held)"
14236 );
14237
14238 drop(release);
14239 let paused = reading.join().unwrap();
14240 assert_eq!(
14241 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14242 vec![10],
14243 "an exit recorded after the read began lands in neither half of it"
14244 );
14245 assert_eq!(paused.journal_skipped_lines, 0);
14246 assert_eq!(paused.journal_read_errors, 0);
14247
14248 let after = durable_terminal_history_of(&reader_ring, "reader-module");
14249 assert_eq!(
14250 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14251 vec![10, 30],
14252 "the next read merges ring and journal with no duplicate"
14253 );
14254 assert_eq!(after.journal_skipped_lines, 0);
14255 }
14256}
14257
14258#[cfg(test)]
14263mod stderr_settle_tests {
14264 use std::{
14265 future::Future,
14266 io,
14267 pin::Pin,
14268 sync::{Arc, Mutex},
14269 task::{Context, Poll},
14270 time::Duration,
14271 };
14272
14273 use tokio::{
14274 io::{AsyncRead, ReadBuf},
14275 sync::oneshot,
14276 time::Instant,
14277 };
14278
14279 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
14280 use crate::stderr_tail::{
14281 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
14282 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
14283 };
14284
14285 const BOUND: Duration = Duration::from_millis(250);
14286
14287 struct HeldReader {
14291 before: Option<Vec<u8>>,
14292 gate: Option<oneshot::Receiver<()>>,
14293 after: io::Cursor<Vec<u8>>,
14294 }
14295
14296 impl AsyncRead for HeldReader {
14297 fn poll_read(
14298 mut self: Pin<&mut Self>,
14299 cx: &mut Context<'_>,
14300 buf: &mut ReadBuf<'_>,
14301 ) -> Poll<io::Result<()>> {
14302 if let Some(bytes) = self.before.take() {
14303 buf.put_slice(&bytes);
14304 return Poll::Ready(Ok(()));
14305 }
14306 if let Some(gate) = self.gate.as_mut() {
14307 match Pin::new(gate).poll(cx) {
14308 Poll::Pending => return Poll::Pending,
14309 Poll::Ready(_) => self.gate = None,
14310 }
14311 }
14312 Pin::new(&mut self.after).poll_read(cx, buf)
14313 }
14314 }
14315
14316 struct DiscardSink;
14317
14318 impl OutputSink for DiscardSink {
14319 fn write_line(&mut self, _line: &[u8]) {}
14320 }
14321
14322 fn line(text: &str) -> TailEntry {
14323 TailEntry::Line {
14324 text: text.to_string(),
14325 truncated: false,
14326 at_ms: None,
14327 }
14328 }
14329
14330 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
14331 ring.lock().unwrap()
14332 }
14333
14334 fn held_pump(
14338 ring: &Arc<Mutex<StderrRing>>,
14339 before: &str,
14340 after: &str,
14341 ) -> (StderrPump, oneshot::Sender<()>) {
14342 let generation = lock(ring).begin_process();
14343 let (release, gate) = oneshot::channel();
14344 let reader = HeldReader {
14345 before: Some(before.as_bytes().to_vec()),
14346 gate: Some(gate),
14347 after: io::Cursor::new(after.as_bytes().to_vec()),
14348 };
14349 let task = tokio::spawn(pump_stderr_to(
14350 reader,
14351 Arc::clone(ring),
14352 generation,
14353 DiscardSink,
14354 ));
14355 (StderrPump { task, generation }, release)
14356 }
14357
14358 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
14359 for _ in 0..1000 {
14360 if done(&lock(ring)) {
14361 return;
14362 }
14363 tokio::time::sleep(Duration::from_millis(1)).await;
14364 }
14365 panic!(
14366 "ring never reached the expected state: {:?}",
14367 lock(ring).snapshot(None, None)
14368 );
14369 }
14370
14371 #[tokio::test(start_paused = true)]
14372 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
14373 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14374 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
14375
14376 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
14377 let before_release = lock(&ring).snapshot(None, None);
14378 assert!(
14379 matches!(before_release.capture, CaptureState::Incomplete { .. }),
14380 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
14381 );
14382
14383 let next = lock(&ring).begin_process();
14386 lock(&ring).push_line_from(next, "next process booting");
14387 release.send(()).unwrap();
14388 wait_until(&ring, |ring| {
14389 ring.snapshot(None, None).capture == CaptureState::Captured
14390 })
14391 .await;
14392
14393 assert_eq!(
14394 untimed(lock(&ring).snapshot(None, None).entries),
14395 vec![
14396 line("booting"),
14397 line("config error: missing storage"),
14398 TailEntry::ProcessStart,
14399 line("next process booting"),
14400 ],
14401 "the crash's last line must survive a slow reader and stay in the crashed process's section"
14402 );
14403 }
14404
14405 #[tokio::test(start_paused = true)]
14406 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
14407 ) {
14408 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14409 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
14412
14413 let started = Instant::now();
14414 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
14415 assert_eq!(
14416 started.elapsed(),
14417 BOUND,
14418 "the restart must wait exactly the bound for a pipe that stays open, no longer"
14419 );
14420
14421 let next = lock(&ring).begin_process();
14422 lock(&ring).push_line_from(next, "next process booting");
14423 tokio::time::sleep(Duration::from_secs(60)).await;
14424
14425 let snapshot = lock(&ring).snapshot(None, None);
14426 match &snapshot.capture {
14427 CaptureState::Incomplete { reason } => assert!(
14428 reason.contains("had not reached EOF") && reason.contains("250ms"),
14429 "the reason must say what is missing and after how long: {reason}"
14430 ),
14431 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
14432 }
14433 assert_eq!(
14434 untimed(snapshot.entries),
14435 vec![
14436 line("parent exiting"),
14437 TailEntry::ProcessStart,
14438 line("next process booting"),
14439 ]
14440 );
14441 }
14442
14443 #[tokio::test(start_paused = true)]
14444 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
14445 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14446 let (pump, release) = held_pump(&ring, "one\n", "two\n");
14447 release.send(()).unwrap();
14448
14449 settle_stderr_pump("clean", &ring, pump, BOUND).await;
14450
14451 let snapshot = lock(&ring).snapshot(None, None);
14452 assert_eq!(snapshot.capture, CaptureState::Captured);
14453 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
14454 }
14455
14456 #[tokio::test(start_paused = true)]
14457 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
14458 const BURST: usize = 150;
14459 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
14460 let path = capture.join("two-pipe.stderr.log");
14461 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
14462 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14463 let generation = lock(&ring).begin_process();
14464 let stdout = (0..BURST)
14465 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
14466 .collect::<String>();
14467 let stderr = (0..BURST)
14468 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
14469 .collect::<String>();
14470 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
14475 let (release, gate) = oneshot::channel();
14476 let reader = HeldReader {
14477 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
14478 gate: Some(gate),
14479 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
14480 };
14481 let (finished, completion) = oneshot::channel();
14482 let stdout_sink = sink.clone();
14483 let pump = tokio::spawn(async move {
14484 pump_stdout_to(reader, stdout_sink).await;
14485 finished.send(()).unwrap();
14486 });
14487 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
14488
14489 let started = Instant::now();
14490 settle_stdout_pump("two-pipe", pump, BOUND).await;
14491 assert_eq!(
14492 started.elapsed(),
14493 BOUND,
14494 "a late reader must not hold up restart"
14495 );
14496 assert_eq!(
14497 std::fs::read_to_string(&path).unwrap().lines().count(),
14498 BURST + 1
14499 );
14500
14501 release
14502 .send(())
14503 .expect("stdout reader must survive the exit drain bound");
14504 tokio::time::timeout(Duration::from_secs(1), completion)
14505 .await
14506 .expect("released stdout reader must reach EOF")
14507 .expect("stdout capture must finish at EOF, not be cancelled");
14508 let contents = std::fs::read_to_string(&path).unwrap();
14509 let mut lines = contents
14510 .lines()
14511 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
14512 .collect::<Vec<_>>();
14513 let mut expected = (0..BURST)
14514 .flat_map(|i| {
14515 [
14516 format!("out-{i:04}-{}", "o".repeat(64)),
14517 format!("err-{i:04}-{}", "e".repeat(64)),
14518 ]
14519 })
14520 .collect::<Vec<_>>();
14521 lines.sort_unstable();
14522 expected.sort_unstable();
14523 assert_eq!(
14524 lines, expected,
14525 "both pipes must retain every complete line exactly once"
14526 );
14527 }
14528}
14529
14530#[cfg(all(test, windows))]
14544mod job_containment_tests {
14545 use super::*;
14546 use cortexkit_test_support::ScratchDir;
14547 use std::{
14548 path::{Path, PathBuf},
14549 sync::{Arc, Mutex},
14550 time::{Duration, Instant},
14551 };
14552
14553 fn stub_path() -> PathBuf {
14559 let mut path = std::env::current_exe().expect("current_exe available in tests");
14560 path.pop();
14561 path.pop();
14562 path.push("fake-aft-stub.exe");
14563 assert!(
14564 path.exists(),
14565 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
14566 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
14567 path.display()
14568 );
14569 path
14570 }
14571
14572 fn read_grandchild_pid(path: &Path) -> u32 {
14574 let deadline = Instant::now() + Duration::from_secs(10);
14575 loop {
14576 if let Ok(contents) = std::fs::read_to_string(path) {
14577 if let Ok(pid) = contents.trim().parse() {
14578 return pid;
14579 }
14580 }
14581 assert!(
14582 Instant::now() < deadline,
14583 "the stub never recorded a grandchild pid at {}",
14584 path.display()
14585 );
14586 std::thread::sleep(Duration::from_millis(10));
14587 }
14588 }
14589
14590 struct Fixture {
14593 _dir: ScratchDir,
14594 module_id: String,
14595 grandchild: u32,
14596 child: Option<SupervisedChild>,
14597 registry: Arc<Registry>,
14598 snapshot: Arc<Mutex<SupervisorSnapshot>>,
14599 terminal_ring: Arc<Mutex<TerminalRing>>,
14600 spawn_events: SpawnEventFeed,
14601 }
14602
14603 fn fixture(label: &str, module_id: &str) -> Fixture {
14604 let dir = ScratchDir::new(label);
14605 let pid_file = dir.join("grandchild.pid");
14606 let supervisor = Supervisor::new_for_test(
14607 Arc::new(Registry::default()),
14608 RestartPolicy::new(3, Duration::ZERO),
14609 );
14610 let runtime = supervisor.runtime_config();
14611 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14612 let spec = ModuleSpec {
14613 module_id: module_id.to_string(),
14614 program: stub_path(),
14615 args: Vec::new(),
14619 env: vec![
14620 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
14621 (
14622 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
14623 pid_file.display().to_string(),
14624 ),
14625 ],
14626 reserved: false,
14627 reserved_prefixes: Vec::new(),
14628 protocol: ModuleProtocol::Subc,
14629 overlap: Default::default(),
14630 };
14631 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
14632 .expect("spawn the supervised fixture");
14633 let grandchild = read_grandchild_pid(&pid_file);
14634 Fixture {
14635 _dir: dir,
14636 module_id: module_id.to_string(),
14637 grandchild,
14638 child: Some(child),
14639 registry: Arc::new(Registry::default()),
14640 snapshot,
14641 terminal_ring: Arc::clone(&runtime.terminal_ring),
14642 spawn_events: SpawnEventFeed::default(),
14643 }
14644 }
14645
14646 impl Fixture {
14647 async fn drain(&mut self) {
14649 let child = self
14650 .child
14651 .take()
14652 .expect("the fixture child is still present");
14653 drain_child_to_state(
14654 &self.module_id,
14655 ModuleProtocol::Subc,
14656 StopNotice::NotSent,
14659 &self.registry,
14660 None,
14661 &self.snapshot,
14662 &self.terminal_ring,
14663 &self.spawn_events,
14664 child,
14665 Duration::from_millis(500),
14666 ModuleState::Stopped,
14667 Some(false),
14668 )
14669 .await
14670 .expect("drain the supervised fixture");
14671 }
14672 }
14673
14674 #[tokio::test]
14680 async fn teardown_reaps_the_grandchild() {
14681 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
14682 let grandchild = fixture.grandchild;
14683
14684 assert!(
14685 subc_jobobject::process_exists(grandchild),
14686 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
14687 );
14688
14689 fixture.drain().await;
14690
14691 assert!(
14692 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
14693 "grandchild {grandchild} outlived module teardown: the tree was not contained"
14694 );
14695 }
14696
14697 #[test]
14710 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
14711 let dir = ScratchDir::new("teardown-uncontained");
14712 let pid_file = dir.join("grandchild.pid");
14713 let mut child = std::process::Command::new(stub_path())
14714 .env("FAKE_AFT_NEVER_CONNECT", "1")
14715 .env(
14716 "FAKE_AFT_GRANDCHILD_PID_FILE",
14717 pid_file.display().to_string(),
14718 )
14719 .stdin(std::process::Stdio::null())
14720 .stdout(std::process::Stdio::null())
14721 .stderr(std::process::Stdio::null())
14722 .spawn()
14723 .expect("spawn the uncontained fixture");
14724 let grandchild = read_grandchild_pid(&pid_file);
14725
14726 child.kill().expect("kill the direct child");
14728 let _ = child.wait();
14729
14730 assert!(
14731 subc_jobobject::process_exists(grandchild),
14732 "grandchild {grandchild} died with the direct child, so this control no longer \
14733 distinguishes contained from uncontained teardown and the regression test is \
14734 passing vacuously"
14735 );
14736
14737 kill_tree(grandchild);
14740 }
14741
14742 #[tokio::test]
14751 async fn dropping_containment_reaps_the_grandchild() {
14752 let mut fixture = fixture("drop-containment", "tree-drop");
14753 let grandchild = fixture.grandchild;
14754
14755 assert!(subc_jobobject::process_exists(grandchild));
14756
14757 fixture.child.as_mut().expect("child present").job = None;
14759
14760 assert!(
14761 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
14762 "grandchild {grandchild} survived the containment handle closing, so a daemon \
14763 crash would leave the tree behind"
14764 );
14765 }
14766
14767 fn kill_tree(pid: u32) {
14769 let _ = std::process::Command::new("taskkill.exe")
14770 .args(["/PID", &pid.to_string(), "/T", "/F"])
14771 .stdin(std::process::Stdio::null())
14772 .stdout(std::process::Stdio::null())
14773 .stderr(std::process::Stdio::null())
14774 .status();
14775 assert!(
14776 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
14777 "could not clean up grandchild {pid}"
14778 );
14779 }
14780}
14781
14782#[cfg(test)]
14783mod privacy_trampoline_configuration_tests {
14784 #[tokio::test]
14785 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
14786 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
14787 #[cfg(target_os = "macos")]
14788 {
14789 let supervisor = super::Supervisor::new(
14790 std::sync::Arc::new(crate::Registry::default()),
14791 super::RestartPolicy::default(),
14792 );
14793 let error = supervisor.spawn(spec()).unwrap_err();
14794 assert!(
14795 error
14796 .to_string()
14797 .contains("no privacy trampoline configured"),
14798 "{error}"
14799 );
14800 }
14801 }
14802
14803 #[tokio::test]
14804 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
14805 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
14806 #[cfg(target_os = "macos")]
14807 {
14808 let supervisor = super::Supervisor::new(
14809 std::sync::Arc::new(crate::Registry::default()),
14810 super::RestartPolicy::default(),
14811 )
14812 .with_privacy_trampoline(std::env::current_exe().unwrap());
14813 let error = supervisor.spawn(spec()).unwrap_err();
14814 assert!(
14815 error
14816 .to_string()
14817 .contains("binary does not implement the privacy trampoline protocol"),
14818 "{error}"
14819 );
14820 }
14821 }
14822
14823 #[cfg(target_os = "macos")]
14824 fn spec() -> super::ModuleSpec {
14825 super::ModuleSpec {
14826 module_id: "privacy-configuration".into(),
14827 program: "/bin/sleep".into(),
14828 args: vec!["30".into()],
14829 env: vec![],
14830 reserved: false,
14831 reserved_prefixes: vec![],
14832 protocol: subc_control::ModuleProtocol::None,
14833 overlap: super::ModuleOverlap::Exclusive,
14834 }
14835 }
14836}
14837
14838#[cfg(test)]
14839mod privacy_exec_boundary_tests {
14840 #[cfg(target_os = "macos")]
14841 use super::*;
14842 #[cfg(target_os = "macos")]
14843 use std::{
14844 io::{Read, Write},
14845 net::{TcpListener, TcpStream},
14846 };
14847
14848 #[cfg(target_os = "macos")]
14851 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
14852 if let Some((_, path)) = spec
14853 .env
14854 .iter()
14855 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
14856 {
14857 let mut barrier = TcpStream::connect(path).unwrap();
14858 barrier
14859 .set_read_timeout(Some(Duration::from_secs(30)))
14860 .unwrap();
14861 barrier.write_all(&pid.to_ne_bytes()).unwrap();
14862 let mut release = [0];
14863 barrier.read_exact(&mut release).unwrap();
14864 assert_eq!(&release, b"X");
14865 }
14866 }
14867
14868 #[cfg(target_os = "macos")]
14869 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
14870 ModuleSpec {
14871 module_id: "privacy-boundary".into(),
14872 program: program.into(),
14873 args: args.iter().map(|arg| (*arg).into()).collect(),
14874 env: vec![],
14875 reserved: false,
14876 reserved_prefixes: vec![],
14877 protocol: ModuleProtocol::None,
14878 overlap: ModuleOverlap::Exclusive,
14879 }
14880 }
14881
14882 #[cfg(target_os = "macos")]
14883 async fn accept(listener: TcpListener) -> TcpStream {
14884 let listener = tokio::net::TcpListener::from_std({
14886 listener.set_nonblocking(true).unwrap();
14887 listener
14888 })
14889 .unwrap();
14890 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
14891 .await
14892 .unwrap()
14893 .unwrap();
14894 let stream = stream.into_std().unwrap();
14895 stream.set_nonblocking(false).unwrap();
14896 stream
14897 .set_read_timeout(Some(Duration::from_secs(30)))
14898 .unwrap();
14899 stream
14900 }
14901
14902 #[tokio::test]
14903 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
14904 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
14905 #[cfg(target_os = "macos")]
14906 {
14907 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
14908 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
14911 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
14912 let record = root.join("live-children.json");
14913 let supervisor =
14914 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
14915 .with_live_children_record(&record);
14916 let runtime = supervisor.runtime_config();
14917 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14918 let mut spec = spec("/bin/sleep", &["30"]);
14919 spec.env = vec![
14920 (
14921 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
14922 exec_listener.local_addr().unwrap().to_string(),
14923 ),
14924 (
14925 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
14926 sample_listener.local_addr().unwrap().to_string(),
14927 ),
14928 ];
14929 let spawn = tokio::task::spawn_blocking(move || {
14930 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
14931 });
14932 let mut sample = accept(sample_listener).await;
14933 let mut pid = [0; 4];
14934 sample.read_exact(&mut pid).unwrap();
14935 let pid = u32::from_ne_bytes(pid);
14936 let mut exec = accept(exec_listener).await;
14937 let mut ready = [0];
14938 exec.read_exact(&mut ready).unwrap();
14939 assert_eq!(&ready, b"R");
14940 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
14943 assert_eq!(
14944 observe_spawned_image(pid).unwrap().executable,
14945 Some(trampoline)
14946 );
14947 sample.write_all(b"X").unwrap();
14948 let mut child = spawn.await.unwrap();
14949 let early = crate::live_children::read_record(&record).unwrap();
14950 assert_eq!(early.len(), 1);
14951 assert_eq!(early[0].pid, pid);
14952 assert_eq!(
14953 early[0].executable, None,
14954 "unconfirmed trampoline image entered the roster"
14955 );
14956 assert!(child.report_ready.get().is_none());
14957 child.privacy_exec.as_mut().unwrap().deadline =
14960 tokio::time::Instant::now() + Duration::from_secs(30);
14961 exec.write_all(b"X").unwrap();
14962 child.confirm_privacy_exec().await;
14963 assert_eq!(child.spawn_failure, None);
14964 assert!(child.report_ready.get().is_some());
14965 let confirmed = crate::live_children::read_record(&record).unwrap();
14966 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
14967 assert_ne!(module, trampoline);
14968 assert_eq!(confirmed[0].executable, Some(module.into()));
14969 child.start_kill().unwrap();
14970 child.wait().await.unwrap();
14971 child.release_roster();
14972 }
14973 }
14974
14975 #[tokio::test]
14976 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
14977 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
14978 #[cfg(target_os = "macos")]
14979 {
14980 let registry = Arc::new(Registry::default());
14981 let policy = RestartPolicy::new(0, Duration::ZERO);
14982 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
14983 let runtime = supervisor.runtime_config();
14984 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14985 let spec = spec("/bin/sh", &["-c", "exit 121"]);
14986 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14990 let pid = child.pid;
14991 tokio::task::spawn_blocking(move || {
14992 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
14993 })
14994 .await
14995 .unwrap()
14996 .unwrap();
14997 child.privacy_exec.as_mut().unwrap().deadline =
14998 tokio::time::Instant::now() + Duration::from_secs(30);
14999 let status = child.wait().await.unwrap();
15000 assert_eq!(status.code(), Some(121));
15001 assert!(child.privacy_exec.is_none());
15002 assert!(
15003 child.report_ready.get().is_none(),
15004 "an exited module must not publish a live pid"
15005 );
15006 let report = classify_reaped_child_exit(&snapshot, &child, &status);
15007 on_child_exit(
15008 &spec,
15009 policy,
15010 ®istry,
15011 &snapshot,
15012 &runtime.terminal_ring,
15013 &runtime.spawn_events,
15014 &runtime.child_roster,
15015 report,
15016 )
15017 .await;
15018 let state = lock_snapshot(&snapshot).unwrap();
15019 assert_eq!(state.state, ModuleState::Failed);
15020 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
15021 assert_eq!(state.reported_pid(), None);
15022 drop(state);
15023 let history = runtime.terminal_ring.lock().unwrap().snapshot();
15024 assert_eq!(history.entries.len(), 1);
15025 let terminal = &history.entries[0];
15026 assert_eq!(terminal.exit_code, Some(121));
15027 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
15028 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
15029 assert_eq!(
15030 terminal.disposition_detail.as_deref(),
15031 Some(policy.budget_exhausted_detail().as_str()),
15032 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
15033 );
15034 assert_eq!(child.spawn_failure, None);
15035 child.release_roster();
15036 }
15037 }
15038}
15039
15040#[cfg(all(test, unix))]
15044mod launch_nonce_descriptor_tests {
15045 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
15046 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15047 use cortexkit_test_support::ScratchDir;
15048 use std::{
15049 path::PathBuf,
15050 sync::{Arc, Mutex},
15051 time::{Duration, Instant},
15052 };
15053
15054 async fn probe(role: super::SpawnRole) {
15055 let scratch = ScratchDir::new("launch-nonce-descriptor");
15056 let fd_copy = scratch.join("from-descriptor");
15057 let env_copy = scratch.join("environment");
15058 let script = format!(
15059 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
15060 fd = fd_copy.display(), env = env_copy.display(),
15061 );
15062 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
15063 let spec = ModuleSpec {
15064 module_id: "nonce-descriptor-probe".to_string(),
15065 program: PathBuf::from("/bin/sh"),
15066 args: vec!["-c".to_string(), script],
15067 env: vec![
15068 xdg("XDG_DATA_HOME"),
15069 xdg("XDG_RUNTIME_DIR"),
15070 xdg("XDG_CONFIG_HOME"),
15071 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
15072 ],
15073 reserved: true,
15074 reserved_prefixes: Vec::new(),
15075 protocol: ModuleProtocol::Subc,
15076 overlap: Default::default(),
15077 };
15078 let handle = SupervisorHandle::new();
15079 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15080 let roster = ChildRoster::default();
15081 #[cfg(target_os = "macos")]
15082 {
15083 let path = super::test_privacy_trampoline();
15084 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
15085 }
15086 let child = super::spawn_child_in_slot(
15087 &spec,
15088 None,
15089 Some(&handle),
15090 &ring,
15091 None,
15092 &roster,
15093 #[cfg(target_os = "linux")]
15094 None,
15095 role,
15096 matches!(role, super::SpawnRole::SwapCandidate),
15097 )
15098 .expect("spawn probe");
15099 let deadline = Instant::now() + Duration::from_secs(10);
15100 while !(fd_copy.exists() && env_copy.exists()) {
15101 assert!(Instant::now() < deadline, "probe never wrote its copies");
15102 tokio::time::sleep(Duration::from_millis(20)).await;
15103 }
15104 let nonce = std::fs::read_to_string(fd_copy).unwrap();
15105 assert!(!nonce.is_empty());
15106 let environment = std::fs::read_to_string(env_copy).unwrap();
15107 assert!(environment
15108 .lines()
15109 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
15110 let copy = environment
15111 .lines()
15112 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
15113 assert_eq!(
15114 copy, None,
15115 "Unix children must never receive the environment nonce"
15116 );
15117 if matches!(role, super::SpawnRole::Plain) {
15118 assert_eq!(
15119 handle.spawn_nonce(&spec.module_id).as_deref(),
15120 Some(nonce.as_str())
15121 );
15122 }
15123 drop(child);
15124 }
15125
15126 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15127 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
15128 probe(super::SpawnRole::Plain).await;
15129 }
15130
15131 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15132 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
15133 probe(super::SpawnRole::SwapCandidate).await;
15134 }
15135}
15136
15137#[cfg(all(test, target_os = "linux"))]
15138mod cgroup_containment_tests {
15139 use super::*;
15140 use cortexkit_test_support::ScratchDir;
15141
15142 fn running(pid: u32) -> bool {
15143 std::fs::read_to_string(format!("/proc/{pid}/stat"))
15145 .ok()
15146 .and_then(|stat| {
15147 stat.rsplit_once(") ")
15148 .map(|(_, rest)| rest.starts_with('Z'))
15149 })
15150 .is_some_and(|zombie| !zombie)
15151 }
15152
15153 #[tokio::test]
15154 async fn linux_teardown_reaps_the_grandchild() {
15155 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
15156 }
15157
15158 #[tokio::test]
15159 async fn linux_shutdown_straggler_reaps_the_grandchild() {
15160 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
15161 }
15162
15163 async fn teardown_tree(test_name: &str, shutdown: bool) {
15164 let dir = ScratchDir::new(test_name);
15165 let root = PathBuf::from(format!(
15166 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
15167 std::process::id(),
15168 unix_ms_now()
15169 ));
15170 if let Err(error) = std::fs::create_dir(&root) {
15171 assert!(
15172 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15173 "required cgroup test cannot execute: {error}"
15174 );
15175 eprintln!(
15176 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
15177 root.display()
15178 );
15179 return;
15180 }
15181 let placement = subc_cgroup::prepare_at(&root)
15182 .expect("prepare isolated kernel cgroup")
15183 .expect("isolated cgroup is delegated");
15184 let module_id = "tree-teardown";
15185 let module = placement
15186 .module_path(module_id)
15187 .expect("create isolated module cgroup");
15188 if !module.join("cgroup.kill").exists() {
15189 std::fs::remove_dir(&module).unwrap();
15190 std::fs::remove_dir(root.join("subc-modules")).unwrap();
15191 std::fs::remove_dir(&root).unwrap();
15192 assert!(
15193 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15194 "required cgroup.kill interface unavailable"
15195 );
15196 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
15197 return;
15198 }
15199 let supervisor = Supervisor::new_for_test(
15200 Arc::new(Registry::default()),
15201 RestartPolicy::new(3, Duration::ZERO),
15202 )
15203 .with_cgroup_placement(Some(placement));
15204 let mut runtime = supervisor.runtime_config();
15205 runtime.child_roster = runtime
15206 .child_roster
15207 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
15208 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15209 let pid_file = dir.join("grandchild.pid");
15210 let spec = ModuleSpec {
15211 module_id: module_id.to_string(),
15212 program: PathBuf::from("/bin/sh"),
15213 args: vec![
15214 "-c".into(),
15215 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
15216 "fixture".into(),
15217 pid_file.display().to_string(),
15218 ],
15219 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15220 .into_iter()
15221 .map(|key| (key.to_string(), dir.display().to_string()))
15222 .collect(),
15223 reserved: false,
15224 reserved_prefixes: Vec::new(),
15225 protocol: ModuleProtocol::None,
15226 overlap: Default::default(),
15227 };
15228 let child =
15229 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
15230 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
15231 let grandchild: u32 = loop {
15232 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
15233 if let Ok(pid) = contents.trim().parse() {
15234 break pid;
15235 }
15236 }
15237 assert!(
15238 tokio::time::Instant::now() < deadline,
15239 "grandchild pid was not recorded"
15240 );
15241 tokio::time::sleep(Duration::from_millis(10)).await;
15242 };
15243 assert!(
15244 running(grandchild),
15245 "grandchild must be alive before teardown"
15246 );
15247 if shutdown {
15248 let mut child = child;
15249 crate::child_roster::end_children_for_daemon_shutdown(
15250 &runtime.child_roster,
15251 false,
15252 std::future::pending(),
15253 )
15254 .await;
15255 child.wait().await.expect("reap shutdown straggler");
15256 } else {
15257 drain_child_to_state(
15258 module_id,
15259 ModuleProtocol::None,
15260 StopNotice::NotSent,
15261 &Registry::default(),
15262 None,
15263 &snapshot,
15264 &runtime.terminal_ring,
15265 &SpawnEventFeed::default(),
15266 child,
15267 Duration::from_millis(100),
15268 ModuleState::Stopped,
15269 Some(false),
15270 )
15271 .await
15272 .expect("real supervisor teardown");
15273 }
15274 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
15275 while running(grandchild) && tokio::time::Instant::now() < deadline {
15276 tokio::time::sleep(Duration::from_millis(10)).await;
15277 }
15278 let survived = running(grandchild);
15279 if survived {
15281 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
15282 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
15283 tokio::time::sleep(Duration::from_millis(100)).await;
15284 }
15285 if module.exists() {
15286 std::fs::remove_dir(&module).expect("remove empty module cgroup");
15287 }
15288 std::fs::remove_dir(root.join("subc-modules")).unwrap();
15289 std::fs::remove_dir(&root).unwrap();
15290 assert!(
15291 !survived,
15292 "grandchild {grandchild} outlived module teardown"
15293 );
15294 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
15295 }
15296}