1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use subc_control::{
14 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
15 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
16};
17use subc_protocol::{
18 manifest::{SelfSignalKind, SignalAnchor},
19 session::{
20 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
21 MODULE_CONTROL_OP_HEALTH_CHECK,
22 },
23 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
24};
25use tokio::{
26 process::{Child, Command},
27 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
28 task::JoinHandle,
29 time::{sleep, sleep_until, timeout, timeout_at, Instant},
30};
31use tracing::{debug, error, info, warn};
32
33use crate::{
34 child_roster::ChildRoster,
35 daemon_config::{
36 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
37 },
38 forwarding::{
39 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
40 ModuleDrainTarget, PendingModuleControlRpc,
41 },
42 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
43 registry::{ConnectionId, RegistrationEndReason, RegistryError},
44 stderr_tail::{
45 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
46 StderrTailSnapshot,
47 },
48 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
49 Frame, FrameSink, Registry,
50};
51
52#[cfg(not(windows))]
53use crate::provenance::spawned_file_identity;
54
55#[path = "supervise_swap.rs"]
56mod swap;
57
58#[path = "supervise_activity.rs"]
59mod activity;
60
61pub const SUBC_ARG: &str = "--subc";
67
68const DEFAULT_MAX_RESTARTS: u32 = 3;
69const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
70const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
71const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
75pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
86const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
87const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
88const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
100const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
123pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
125const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
126pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
131
132#[cfg(target_os = "macos")]
133struct PrivacyExec {
134 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
135 deadline: tokio::time::Instant,
136 expected: Option<subc_os::FileIdentity>,
137 trampoline: Option<subc_os::FileIdentity>,
138 script: bool,
139 module_id: String,
140 #[cfg(test)]
141 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
142}
143
144#[cfg(all(test, target_os = "macos"))]
145pub(crate) fn test_privacy_trampoline() -> PathBuf {
146 let path = std::env::current_exe()
149 .unwrap()
150 .parent()
151 .unwrap()
152 .parent()
153 .unwrap()
154 .join("privacy-trampoline-fixture");
155 assert!(
159 path.exists(),
160 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
161 --bins --features test-support` or `cargo test -p subc-daemon` first",
162 path.display()
163 );
164 path
165}
166
167#[cfg(target_os = "macos")]
168fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
169 use std::io::Read;
170 let mut probe = std::process::Command::new(path)
171 .args(["__disclaim-exec", "--probe"])
172 .stdin(Stdio::null())
173 .stdout(Stdio::piped())
174 .stderr(Stdio::piped())
175 .spawn()
176 .map_err(|error| {
177 format!(
178 "privacy trampoline probe failed for {}: {error}",
179 path.display()
180 )
181 })?;
182 let deadline = std::time::Instant::now() + Duration::from_secs(5);
183 let status = loop {
184 match probe.try_wait() {
185 Ok(Some(status)) => break status,
186 Ok(None) if std::time::Instant::now() < deadline => {
187 std::thread::sleep(Duration::from_millis(5))
188 }
189 result => {
190 let _ = probe.kill();
191 let _ = probe.wait();
192 return Err(format!(
193 "privacy trampoline probe failed or timed out for {}: {result:?}",
194 path.display()
195 ));
196 }
197 }
198 };
199 let mut answer = String::new();
200 if let Some(stdout) = probe.stdout.take() {
201 let _ = stdout.take(256).read_to_string(&mut answer);
202 }
203 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
204 return Ok(());
205 }
206 let mut diagnostic = String::new();
207 if let Some(stderr) = probe.stderr.take() {
208 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
209 }
210 let cause = diagnostic
211 .trim()
212 .strip_prefix("ck-subc: own privacy identity refused: ")
213 .unwrap_or("binary does not implement the privacy trampoline protocol");
214 Err(format!(
215 "{cause}: probe of {} exited {status}",
216 path.display()
217 ))
218}
219
220#[cfg(target_os = "macos")]
221fn privacy_command(
222 spec: &ModuleSpec,
223 roster: &ChildRoster,
224) -> Result<
225 (
226 Command,
227 Option<PrivacyExec>,
228 subc_os::privacy_identity::ExecAcknowledgement,
229 ),
230 SuperviseError,
231> {
232 let failure = |cause: String| {
233 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
234 SuperviseError::Spawn {
235 program: spec.program.clone(),
236 source: io::Error::other(cause),
237 cgroup_path: None,
238 }
239 };
240 let trampoline = roster.privacy_trampoline().map_err(failure)?;
241 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
246 let path = spec
247 .env
248 .iter()
249 .find(|(key, _)| key == "PATH")
250 .map(|(_, value)| std::ffi::OsString::from(value))
251 .or_else(|| std::env::var_os("PATH"))
252 .unwrap_or_else(|| "/usr/bin:/bin".into());
253 std::env::split_paths(&path)
254 .map(|dir| dir.join(&spec.program))
255 .find(|path| path.is_file())
256 .unwrap_or_else(|| spec.program.clone())
257 } else {
258 spec.program.clone()
259 };
260 let expected = subc_os::file_identity(&program);
261 let trampoline_image = subc_os::file_identity(&trampoline);
262 let script = {
263 use std::io::Read;
264 let mut prefix = [0u8; 2];
265 std::fs::File::open(&program)
266 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
267 };
268 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
269 return Err(failure(
270 "privacy identity module executable is missing or is the trampoline itself".to_string(),
271 ));
272 }
273 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
274 .map_err(|error| failure(error.to_string()))?;
275 let reader =
276 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
277 let mut command = Command::new(&trampoline);
278 command
279 .arg("__disclaim-exec")
280 .arg(ack.fd().to_string())
281 .arg(&program);
282 ack.install(command.as_std_mut());
283 Ok((
284 command,
285 Some(PrivacyExec {
286 reader,
287 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
288 expected,
289 trampoline: trampoline_image,
290 script,
291 module_id: spec.module_id.clone(),
292 #[cfg(test)]
293 confirmation_pause: spec
294 .env
295 .iter()
296 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
297 .map(|(_, address)| (address.clone(), None)),
298 }),
299 ack,
300 ))
301}
302
303struct SupervisedChild {
304 child: Child,
305 exec: crate::activity::ExecOwner,
306 #[cfg(windows)]
307 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
308 #[cfg(target_os = "macos")]
309 privacy_exec: Option<PrivacyExec>,
310 #[cfg(target_os = "macos")]
317 report_ready: Arc<OnceLock<()>>,
318 spawn_failure: Option<String>,
320 protocol: ModuleProtocol,
325 #[cfg(target_os = "linux")]
331 module_id: String,
332 #[cfg(target_os = "linux")]
333 cgroup_placement: Option<subc_cgroup::Placement>,
334 #[cfg(windows)]
356 job: Option<subc_jobobject::JobObject>,
357 stdout_pump: Option<JoinHandle<()>>,
358 stderr_pump: Option<StderrPump>,
359 stderr_ring: Arc<Mutex<StderrRing>>,
360 spawned_at_ms: u64,
361 spawned_from: PathBuf,
362 spawned_file_identity: Option<SpawnedFileIdentity>,
363 process_start_time: Option<u64>,
364 process_identity: Option<ProcessIdentity>,
365 pid: u32,
366 roster_guard: Option<crate::child_roster::RosterGuard>,
369}
370
371impl SupervisedChild {
372 fn id(&self) -> Option<u32> {
373 Some(self.pid)
374 }
375
376 fn process_identity(&self) -> Option<ProcessIdentity> {
377 self.process_identity
378 }
379
380 async fn wait(&mut self) -> io::Result<ExitStatus> {
381 #[cfg(target_os = "macos")]
382 self.confirm_privacy_exec().await;
383 let result = self.child.wait().await;
391 #[cfg(target_os = "linux")]
392 if result.is_ok() {
393 if let Some(placement) = self.cgroup_placement.as_ref() {
394 cleanup_reaped_cgroup(placement, &self.module_id).await;
395 self.cgroup_placement = None;
398 }
399 }
400 result
401 }
402
403 #[cfg(target_os = "macos")]
404 async fn confirm_privacy_exec(&mut self) {
405 let Some(pending) = &mut self.privacy_exec else {
406 return;
407 };
408 let result = tokio::time::timeout_at(pending.deadline, async {
409 let mut record = Vec::new();
410 loop {
411 let mut ready = pending.reader.readable().await?;
412 let read = ready.try_io(|reader| {
413 use std::io::Read;
414 let mut reader = reader.get_ref();
415 let mut buffer = [0u8; 256];
416 reader.read(&mut buffer).map(|count| (count, buffer))
417 });
418 match read {
419 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
420 Ok(Ok((count, buffer))) => {
421 if record.len() + count > 1024 {
422 return Err(io::Error::other(
423 "privacy exec refusal record is too long",
424 ));
425 }
426 record.extend_from_slice(&buffer[..count]);
427 }
428 Ok(Err(error)) => return Err(error),
429 Err(_) => continue,
430 }
431 }
432 })
433 .await;
434 #[cfg(test)]
435 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
436 use tokio::io::{AsyncReadExt, AsyncWriteExt};
437 if stream.is_none() {
438 *stream = Some(
439 tokio::net::TcpStream::connect(address.as_str())
440 .await
441 .unwrap(),
442 );
443 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
444 stream
445 .as_mut()
446 .unwrap()
447 .write_all(&self.pid.to_ne_bytes())
448 .await
449 .unwrap();
450 }
451 let mut reply = [0];
452 stream
453 .as_mut()
454 .unwrap()
455 .read_exact(&mut reply)
456 .await
457 .unwrap();
458 pending.confirmation_pause = None;
459 if &reply == b"F" {
460 Ok(Err(io::Error::other("injected late exec refusal")))
461 } else {
462 assert_eq!(&reply, b"X");
463 result
464 }
465 } else {
466 result
467 };
468 let pending = self.privacy_exec.as_ref().expect("pending exec");
471 let cause = match result {
472 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
473 Ok(Err(error)) => Some(format!(
474 "privacy identity exec acknowledgement failed: {error}"
475 )),
476 Ok(Ok(record)) if !record.is_empty() => Some(
477 std::str::from_utf8(&record)
478 .ok()
479 .and_then(|record| {
480 record
481 .trim()
482 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
483 })
484 .filter(|cause| !cause.is_empty())
485 .unwrap_or("invalid privacy exec refusal record")
486 .to_string(),
487 ),
488 Ok(Ok(_)) => match self.child.try_wait() {
489 Ok(Some(_status)) => None,
493 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
494 Ok(None) => {
495 let image = observe_spawned_image(self.pid);
496 if let Some(image) = image.filter(|image| {
497 image.executable.is_some()
498 && image.executable != pending.trampoline
499 && (image.executable == pending.expected || pending.script)
500 }) {
501 if let Err(error) = self
502 .roster_guard
503 .as_ref()
504 .map(|guard| guard.confirm_executable(image))
505 .transpose()
506 {
507 Some(format!(
508 "could not durably record confirmed module image: {error}"
509 ))
510 } else {
511 let _ = self.report_ready.set(());
512 info!(module_id = %pending.module_id, pid = self.pid,
513 "module spawned with own privacy identity (responsibility disclaimed)");
514 None
515 }
516 } else if image.is_none()
517 || image.is_some_and(|image| image.executable.is_none())
518 {
519 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
526 Ok(Ok(_status)) => None,
527 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
528 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
529 }
530 } else {
531 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
532 }
533 }
534 },
535 };
536 let pending = self.privacy_exec.take().expect("pending exec");
537 self.exec.0.finish(if cause.is_some() {
540 crate::activity::ExecState::Refused
541 } else {
542 crate::activity::ExecState::CommittedExec
543 });
544 if let Some(cause) = cause {
545 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
546 self.spawn_failure = Some(cause);
547 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
550 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
551 }
552 let _ = self.child.start_kill();
553 }
554 }
555
556 fn release_roster(&mut self) {
560 self.roster_guard = None;
561 }
562
563 fn start_kill(&mut self) -> io::Result<()> {
576 #[cfg(windows)]
577 if let Some(job) = &self.job {
578 if let Err(error) = job.terminate() {
579 debug!(
580 error = %error,
581 "job termination failed; the direct-child kill still owns the outcome"
582 );
583 }
584 }
585 #[cfg(target_os = "linux")]
586 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
587 self.child.start_kill()
588 }
589
590 async fn drain_stderr(&mut self, module_id: &str) {
591 if let Some(pump) = self.stdout_pump.take() {
592 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
593 }
594
595 let Some(pump) = self.stderr_pump.take() else {
596 return;
597 };
598 settle_stderr_pump(
599 module_id,
600 &self.stderr_ring,
601 pump,
602 STDERR_PUMP_DRAIN_TIMEOUT,
603 )
604 .await;
605 }
606}
607
608async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
612 match timeout(bound, &mut pump).await {
613 Ok(Ok(())) => {}
614 Ok(Err(error)) => {
615 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
616 }
617 Err(_) => {
618 drop(pump);
619 warn!(
620 module_id,
621 waited = ?bound,
622 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
623 );
624 }
625 }
626}
627
628struct StderrPump {
631 task: JoinHandle<()>,
632 generation: u64,
633}
634
635async fn settle_stderr_pump(
641 module_id: &str,
642 ring: &Arc<Mutex<StderrRing>>,
643 pump: StderrPump,
644 bound: Duration,
645) {
646 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
647 let StderrPump {
648 mut task,
649 generation,
650 } = pump;
651 lock().retire_pump(generation);
652 match timeout(bound, &mut task).await {
653 Ok(Ok(())) => {}
654 Ok(Err(err)) => {
655 let mut ring = lock();
656 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
657 ring.finish_pump(generation);
658 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
659 }
660 Err(_) => {
661 drop(task);
663 lock().mark_pump_late(
664 generation,
665 format!(
666 "stderr of the exited process had not reached EOF {bound:?} after it was \
667 retired (a descendant may still hold the pipe open); lines it still \
668 writes are kept in that process's section"
669 ),
670 );
671 warn!(
672 module_id,
673 waited = ?bound,
674 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
675 );
676 }
677 }
678}
679
680fn registration_release_events() -> &'static watch::Sender<u64> {
681 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
682 EVENTS.get_or_init(|| {
683 let (sender, _receiver) = watch::channel(0);
684 sender
685 })
686}
687
688pub(crate) fn notify_registration_release() {
689 let events = registration_release_events();
690 let next_generation = (*events.borrow()).wrapping_add(1);
691 events.send_replace(next_generation);
692}
693
694#[derive(Debug, Clone, PartialEq, Eq)]
696pub struct ModuleSpec {
697 pub module_id: String,
698 pub program: PathBuf,
699 pub args: Vec<String>,
700 pub env: Vec<(String, String)>,
701 pub reserved: bool,
706 pub reserved_prefixes: Vec<String>,
711 pub protocol: ModuleProtocol,
730 pub overlap: ModuleOverlap,
735}
736
737#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
744pub enum ModuleOverlap {
745 #[default]
747 Exclusive,
748 Safe,
760}
761
762impl ModuleOverlap {
763 pub fn as_str(self) -> &'static str {
764 match self {
765 Self::Exclusive => "exclusive",
766 Self::Safe => "safe",
767 }
768 }
769}
770
771pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
781pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
783pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
788
789#[derive(Debug, Clone, Copy, PartialEq, Eq)]
807pub struct RestartPolicy {
808 pub max_restarts: u32,
809 pub backoff: Duration,
812 pub max_backoff: Duration,
814 pub window: Duration,
818}
819
820impl RestartPolicy {
821 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
825 Self {
826 max_restarts,
827 backoff,
828 max_backoff: DEFAULT_MAX_BACKOFF,
829 window: DEFAULT_RESTART_WINDOW,
830 }
831 }
832
833 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
834 self.max_backoff = max_backoff;
835 self
836 }
837
838 pub fn with_window(mut self, window: Duration) -> Self {
839 self.window = window;
840 self
841 }
842
843 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
848 if self.backoff.is_zero() || self.max_backoff.is_zero() {
849 return Duration::ZERO;
850 }
851
852 let mut delay = self.backoff;
853 for _ in 0..restart_in_window {
854 if delay >= self.max_backoff {
855 return self.max_backoff;
856 }
857 delay = delay
858 .checked_mul(10)
859 .unwrap_or(self.max_backoff)
860 .min(self.max_backoff);
861 }
862 delay.min(self.max_backoff)
863 }
864
865 fn budget_exhausted_detail(&self) -> String {
870 format!(
871 "crash budget exhausted: max_restarts={} within window_secs={}",
872 self.max_restarts,
873 self.window.as_secs()
874 )
875 }
876}
877
878impl Default for RestartPolicy {
879 fn default() -> Self {
880 Self {
881 max_restarts: DEFAULT_MAX_RESTARTS,
882 backoff: DEFAULT_BACKOFF,
883 max_backoff: DEFAULT_MAX_BACKOFF,
884 window: DEFAULT_RESTART_WINDOW,
885 }
886 }
887}
888
889#[derive(Debug, Clone, Copy, PartialEq, Eq)]
890struct CrashRestartSchedule {
891 restart_in_window: u32,
892 delay: Duration,
893}
894
895fn daemon_will_restart(
902 state: &mut SupervisorSnapshot,
903 policy: &RestartPolicy,
904 now: Instant,
905) -> bool {
906 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
907}
908
909const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
910const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
911const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
912const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
913
914#[derive(Debug, Clone, Copy, PartialEq, Eq)]
915pub enum HealthAction {
916 Report,
917 Restart,
918 Alert,
919}
920
921impl fmt::Display for HealthAction {
922 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
923 f.write_str(match self {
924 Self::Report => "report",
925 Self::Restart => "restart",
926 Self::Alert => "alert",
927 })
928 }
929}
930
931#[derive(Debug, Clone, PartialEq, Eq)]
932pub struct HealthConfig {
933 pub http: Option<String>,
937 pub cadence: Duration,
938 pub deadline: Duration,
939 pub failure_threshold: u32,
940 pub on_degraded: HealthAction,
941 pub on_failing: HealthAction,
942 pub critical: bool,
943}
944
945impl Default for HealthConfig {
946 fn default() -> Self {
947 Self {
948 http: None,
949 cadence: DEFAULT_HEALTH_CADENCE,
950 deadline: DEFAULT_HEALTH_DEADLINE,
951 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
952 on_degraded: HealthAction::Report,
953 on_failing: HealthAction::Report,
954 critical: false,
955 }
956 }
957}
958
959#[derive(Debug, Clone, PartialEq)]
977pub struct ModuleHealthStatus {
978 pub status: SupervisorHealthStatus,
979 pub last_probe_ms: Option<u64>,
980 pub detail: Option<String>,
981 pub metrics: Option<Value>,
982 pub consecutive_failures: u32,
983 pub late_answer_count: u64,
986 pub last_late_answer_latency_ms: Option<u64>,
988 pub last_action: Option<String>,
989 pub last_action_ms: Option<u64>,
993}
994
995impl Default for ModuleHealthStatus {
996 fn default() -> Self {
997 Self {
998 status: SupervisorHealthStatus::Unknown,
999 last_probe_ms: None,
1000 detail: None,
1001 metrics: None,
1002 consecutive_failures: 0,
1003 late_answer_count: 0,
1004 last_late_answer_latency_ms: None,
1005 last_action: None,
1006 last_action_ms: None,
1007 }
1008 }
1009}
1010
1011#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1013pub enum ModuleState {
1014 Starting,
1015 Running,
1016 Unresponsive,
1017 Restarting,
1018 Draining,
1019 Stopped,
1020 Failed,
1021 Disabled,
1022}
1023
1024impl fmt::Display for ModuleState {
1025 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1026 f.write_str(match self {
1027 Self::Starting => "starting",
1028 Self::Running => "running",
1029 Self::Unresponsive => "unresponsive",
1030 Self::Restarting => "restarting",
1031 Self::Draining => "draining",
1032 Self::Stopped => "stopped",
1033 Self::Failed => "failed",
1034 Self::Disabled => "disabled",
1035 })
1036 }
1037}
1038
1039#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1041pub enum ExitKind {
1042 Clean,
1043 Crash,
1044 DeliberateSeverance,
1045}
1046
1047impl From<ExitKind> for TerminalExitKind {
1048 fn from(kind: ExitKind) -> Self {
1049 match kind {
1050 ExitKind::Clean => Self::Clean,
1051 ExitKind::Crash => Self::Crash,
1052 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1053 }
1054 }
1055}
1056
1057#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1060pub(crate) struct ProcessIdentity {
1061 pub(crate) pid: u32,
1062 pub(crate) start_time: u64,
1063}
1064
1065#[derive(Debug, Clone, PartialEq, Eq)]
1067pub struct ExitReport {
1068 pub kind: ExitKind,
1069 pub code: Option<i32>,
1070 pub signal: Option<i32>,
1071 pub at_ms: u64,
1072}
1073
1074#[derive(Debug, Clone, PartialEq)]
1077pub struct ModuleStatus {
1078 pub module_id: String,
1079 pub waiting_for_previous_pid: Option<u32>,
1081 pub waiting_for_previous_detail: Option<String>,
1083 pub launch_nonce_source: Option<String>,
1088 pub state: ModuleState,
1089 pub enabled: bool,
1090 pub process_alive: bool,
1091 pub registration_active: bool,
1092 pub protocol: ModuleProtocol,
1097 pub live: bool,
1108 pub restart_count: u32,
1112 pub lifetime_restarts: u32,
1116 pub spawn_generation: u64,
1117 pub max_restarts: u32,
1122 pub restart_window: Duration,
1126 pub drain_timeout: Duration,
1130 pub restart_backoff: Duration,
1131 pub restart_max_backoff: Duration,
1132 pub pid: Option<u32>,
1137 pub spawned_at_ms: Option<u64>,
1138 pub spawned_from: Option<PathBuf>,
1139 pub process_start_time: Option<u64>,
1140 pub last_exit: Option<ExitReport>,
1141 pub health: ModuleHealthStatus,
1142}
1143
1144#[cfg(test)]
1145#[derive(Debug, Clone, Copy, PartialEq)]
1146struct ActorSelectCheckpoint {
1147 generation: u64,
1148 turn: u64,
1149 registered_connection: Option<ConnectionId>,
1150 next_probe_at: Option<Instant>,
1151 wake_after: Option<Duration>,
1152}
1153
1154#[derive(Debug, Clone, PartialEq)]
1155struct SupervisorSnapshot {
1156 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1157 #[cfg(windows)]
1158 pipe_consumed: Option<Arc<OnceLock<()>>>,
1159 #[cfg(test)]
1161 actor_turns: u64,
1162 #[cfg(test)]
1165 actor_select: Option<ActorSelectCheckpoint>,
1166 state: ModuleState,
1167 enabled: bool,
1168 process_alive: bool,
1169 spawned_protocol: Option<ModuleProtocol>,
1170 spawn_failure: Option<String>,
1171 waiting_for_previous_pid: Option<u32>,
1172 waiting_for_previous_detail: Option<String>,
1173 crash_restarts: VecDeque<Instant>,
1179 lifetime_restarts: u32,
1180 spawn_generation: u64,
1189 pid: Option<u32>,
1190 #[cfg(target_os = "macos")]
1191 report_ready: Option<Arc<OnceLock<()>>>,
1192 reaped_pid: Option<u32>,
1194 respawn_pending: bool,
1196 coalesced_restart_pending: bool,
1198 spawned_at_ms: Option<u64>,
1199 spawned_from: Option<PathBuf>,
1200 spawned_file_identity: Option<SpawnedFileIdentity>,
1201 process_start_time: Option<u64>,
1202 deliberate_severance: Option<ProcessIdentity>,
1203 last_exit: Option<ExitReport>,
1204 drain_disposition_detail: Option<String>,
1206 health: ModuleHealthStatus,
1207 in_alternate_slot: bool,
1212 draining_to_replace: bool,
1219 configuration_updated_since_spawn: bool,
1225}
1226
1227impl SupervisorSnapshot {
1228 fn reported_pid(&self) -> Option<u32> {
1233 #[cfg(target_os = "macos")]
1234 if self
1235 .report_ready
1236 .as_ref()
1237 .is_some_and(|ready| ready.get().is_none())
1238 {
1239 return None;
1240 }
1241 self.pid
1242 }
1243
1244 fn starting() -> Self {
1245 Self::new(ModuleState::Starting, true)
1246 }
1247
1248 fn disabled() -> Self {
1249 Self::new(ModuleState::Disabled, false)
1250 }
1251
1252 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1256 while let Some(oldest) = self.crash_restarts.front() {
1257 if now.duration_since(*oldest) > window {
1258 self.crash_restarts.pop_front();
1259 } else {
1260 break;
1261 }
1262 }
1263 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1264 }
1265
1266 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1272 self.crash_restarts.push_back(now);
1273 while self.crash_restarts.len() > policy.max_restarts as usize {
1274 self.crash_restarts.pop_front();
1275 }
1276 self.lifetime_restarts += 1;
1277 }
1278
1279 fn next_crash_restart(
1283 &mut self,
1284 policy: &RestartPolicy,
1285 now: Instant,
1286 ) -> Option<CrashRestartSchedule> {
1287 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1288 if restart_in_window >= policy.max_restarts {
1289 return None;
1290 }
1291 self.record_crash_restart(policy, now);
1292 Some(CrashRestartSchedule {
1293 restart_in_window,
1294 delay: policy.delay_for_restart(restart_in_window),
1295 })
1296 }
1297
1298 fn clear_crash_restarts(&mut self) {
1303 self.crash_restarts.clear();
1304 }
1305
1306 fn new(state: ModuleState, enabled: bool) -> Self {
1307 Self {
1308 exec_outcome: None,
1309 #[cfg(test)]
1310 actor_turns: 0,
1311 #[cfg(test)]
1312 actor_select: None,
1313 state,
1314 enabled,
1315 process_alive: false,
1316 #[cfg(windows)]
1317 pipe_consumed: None,
1318 spawned_protocol: None,
1319 spawn_failure: None,
1320 waiting_for_previous_pid: None,
1321 waiting_for_previous_detail: None,
1322 crash_restarts: VecDeque::new(),
1323 lifetime_restarts: 0,
1324 spawn_generation: 0,
1325 pid: None,
1326 #[cfg(target_os = "macos")]
1327 report_ready: None,
1328 reaped_pid: None,
1329 respawn_pending: false,
1330 coalesced_restart_pending: false,
1331 spawned_at_ms: None,
1332 spawned_from: None,
1333 spawned_file_identity: None,
1334 process_start_time: None,
1335 deliberate_severance: None,
1336 last_exit: None,
1337 drain_disposition_detail: None,
1338 health: ModuleHealthStatus::default(),
1339 in_alternate_slot: false,
1340 draining_to_replace: false,
1341 configuration_updated_since_spawn: false,
1342 }
1343 }
1344}
1345
1346type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1347
1348type SpawnSubscriberKey = (ConnectionId, u64);
1349
1350#[derive(Debug)]
1351struct SpawnSubscriber {
1352 version: u8,
1353 frames: mpsc::Sender<Frame>,
1354 lagged: Option<oneshot::Sender<SpawnCursor>>,
1358}
1359
1360#[derive(Debug)]
1361struct SpawnEventState {
1362 daemon_incarnation: String,
1363 seq: u64,
1364 capacity: usize,
1365 live: HashMap<String, LiveSpawn>,
1366 generations: HashMap<String, u64>,
1367 events: VecDeque<SpawnEvent>,
1368 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1369}
1370
1371impl Default for SpawnEventState {
1372 fn default() -> Self {
1373 Self {
1374 daemon_incarnation: "unconfigured".to_string(),
1375 seq: 0,
1376 capacity: SPAWN_EVENT_RING_CAPACITY,
1377 live: HashMap::new(),
1378 generations: HashMap::new(),
1379 events: VecDeque::new(),
1380 subscribers: HashMap::new(),
1381 }
1382 }
1383}
1384
1385#[derive(Debug, Clone, Default)]
1386struct SpawnEventFeed(
1387 Arc<Mutex<SpawnEventState>>,
1388 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1389);
1390
1391#[derive(Debug, Clone, PartialEq, Eq)]
1392pub(crate) enum SpawnSubscribeRefusal {
1393 ForeignIncarnation { current: String },
1394 TooOld { oldest: SpawnCursor },
1395 Frame(String),
1396}
1397
1398impl SpawnEventFeed {
1399 fn configure_incarnation(&self, daemon_incarnation: String) {
1400 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1401 state.daemon_incarnation = daemon_incarnation;
1402 state.seq = 0;
1403 state.live.clear();
1404 state.generations.clear();
1405 state.events.clear();
1406 state.subscribers.clear();
1407 }
1408
1409 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1410 SpawnCursor {
1411 daemon_incarnation: state.daemon_incarnation.clone(),
1412 seq: state.seq,
1413 }
1414 }
1415
1416 fn snapshot(&self) -> SpawnSnapshot {
1417 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1418 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1419 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1420 SpawnSnapshot {
1421 cursor: Self::cursor(&state),
1422 ring_bound: state.capacity as u64,
1423 live,
1424 }
1425 }
1426
1427 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1428 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1429 let generation = state
1430 .generations
1431 .get(module_id)
1432 .copied()
1433 .unwrap_or(0)
1434 .checked_add(1)
1435 .expect("spawn generation exhausted");
1436 state.generations.insert(module_id.to_string(), generation);
1437 let live = LiveSpawn {
1438 module_id: module_id.to_string(),
1439 spawn_generation: generation,
1440 pid,
1441 spawned_at_ms,
1442 };
1443 state.live.insert(module_id.to_string(), live);
1444 Self::emit_locked(
1445 &mut state,
1446 SpawnEventKind::Spawned,
1447 module_id.to_string(),
1448 generation,
1449 pid,
1450 None,
1451 None,
1452 );
1453 generation
1454 }
1455
1456 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1457 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1458 let Some(live) = state.live.remove(module_id) else {
1459 warn!(
1460 module_id,
1461 "terminal record had no live spawn event identity"
1462 );
1463 return;
1464 };
1465 Self::emit_locked(
1466 &mut state,
1467 SpawnEventKind::Exited,
1468 module_id.to_string(),
1469 live.spawn_generation,
1470 live.pid,
1471 exit_code,
1472 exit_signal,
1473 );
1474 }
1475
1476 fn emit_superseded_exited(
1483 &self,
1484 module_id: &str,
1485 spawn_generation: u64,
1486 pid: u32,
1487 exit_code: Option<i32>,
1488 exit_signal: Option<i32>,
1489 ) {
1490 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1491 if state
1492 .live
1493 .get(module_id)
1494 .is_some_and(|live| live.spawn_generation == spawn_generation)
1495 {
1496 state.live.remove(module_id);
1497 }
1498 Self::emit_locked(
1499 &mut state,
1500 SpawnEventKind::Exited,
1501 module_id.to_string(),
1502 spawn_generation,
1503 pid,
1504 exit_code,
1505 exit_signal,
1506 );
1507 }
1508
1509 #[allow(clippy::too_many_arguments)]
1510 fn emit_locked(
1511 state: &mut SpawnEventState,
1512 kind: SpawnEventKind,
1513 module_id: String,
1514 spawn_generation: u64,
1515 pid: u32,
1516 exit_code: Option<i32>,
1517 exit_signal: Option<i32>,
1518 ) {
1519 state.seq = state
1520 .seq
1521 .checked_add(1)
1522 .expect("spawn event sequence exhausted");
1523 let event = SpawnEvent {
1524 cursor: Self::cursor(state),
1525 kind,
1526 module_id,
1527 spawn_generation,
1528 pid,
1529 exit_code,
1530 exit_signal,
1531 };
1532 state.events.push_back(event.clone());
1533 while state.events.len() > state.capacity {
1534 state.events.pop_front();
1535 }
1536 let body = match serde_json::to_vec(&event) {
1537 Ok(body) => body,
1538 Err(error) => {
1539 error!(%error, "failed to serialize supervisor spawn event");
1540 return;
1541 }
1542 };
1543 state.subscribers.retain(|(connection_id, corr), subscriber| {
1544 let frame = Frame::build_with_version(
1545 subscriber.version,
1546 FrameType::StreamData,
1547 control_flags(),
1548 0,
1549 0,
1550 *corr,
1551 body.clone(),
1552 );
1553 match frame {
1554 Ok(frame) => {
1555 if subscriber.frames.try_send(frame).is_ok() {
1556 true
1557 } else {
1558 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1559 if let Some(lagged) = subscriber.lagged.take() {
1560 let _ = lagged.send(event.cursor.clone());
1561 }
1562 false
1563 }
1564 }
1565 Err(error) => {
1566 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1567 false
1568 }
1569 }
1570 });
1571 }
1572
1573 fn subscribe(
1574 &self,
1575 connection_id: ConnectionId,
1576 corr: u64,
1577 version: u8,
1578 since: Option<SpawnCursor>,
1579 sink: FrameSink,
1580 ) -> Result<(), SpawnSubscribeRefusal> {
1581 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1582 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1583 {
1584 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1585 let replay = if let Some(since) = since {
1586 if since.daemon_incarnation != state.daemon_incarnation {
1587 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1588 current: state.daemon_incarnation.clone(),
1589 });
1590 }
1591 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1592 if since.seq < oldest.seq.saturating_sub(1) {
1593 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1594 }
1595 }
1596 state
1597 .events
1598 .iter()
1599 .filter(|event| event.cursor.seq > since.seq)
1600 .cloned()
1601 .collect::<Vec<_>>()
1602 } else {
1603 Vec::new()
1604 };
1605 for event in replay {
1606 let body = serde_json::to_vec(&event)
1607 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1608 let frame = Frame::build_with_version(
1609 version,
1610 FrameType::StreamData,
1611 control_flags(),
1612 0,
1613 0,
1614 corr,
1615 body,
1616 )
1617 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1618 frames
1619 .try_send(frame)
1620 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1621 }
1622 state.subscribers.insert(
1623 (connection_id, corr),
1624 SpawnSubscriber {
1625 version,
1626 frames,
1627 lagged: Some(lagged),
1628 },
1629 );
1630 }
1631 tokio::spawn(async move {
1642 while let Some(frame) = receiver.recv().await {
1643 if sink.send(frame).await.is_err() {
1644 return;
1645 }
1646 }
1647 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1648 return;
1649 };
1650 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1651 Ok(frame) => {
1652 let _ = sink.send(frame).await;
1653 }
1654 Err(error) => {
1655 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1656 }
1657 }
1658 });
1659 Ok(())
1660 }
1661
1662 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1663 let Some(subscriber) = self
1664 .0
1665 .lock()
1666 .unwrap_or_else(|p| p.into_inner())
1667 .subscribers
1668 .remove(&(connection_id, corr))
1669 else {
1670 return false;
1671 };
1672 if let Ok(frame) = Frame::build_with_version(
1673 subscriber.version,
1674 FrameType::StreamEnd,
1675 control_flags(),
1676 0,
1677 0,
1678 corr,
1679 Vec::new(),
1680 ) {
1681 tokio::spawn(async move {
1682 let _ = subscriber.frames.send(frame).await;
1683 });
1684 }
1685 true
1686 }
1687
1688 fn remove_connection(&self, connection_id: ConnectionId) {
1689 self.0
1690 .lock()
1691 .unwrap_or_else(|p| p.into_inner())
1692 .subscribers
1693 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1694 }
1695
1696 #[cfg(any(test, feature = "test-support"))]
1697 fn set_capacity(&self, capacity: usize) {
1698 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1699 }
1700
1701 #[cfg(any(test, feature = "test-support"))]
1702 fn subscriber_count(&self) -> usize {
1703 self.0
1704 .lock()
1705 .unwrap_or_else(|p| p.into_inner())
1706 .subscribers
1707 .len()
1708 }
1709}
1710
1711fn spawn_subscriber_lagged_frame(
1714 version: u8,
1715 corr: u64,
1716 first_undelivered: SpawnCursor,
1717) -> Result<Frame, String> {
1718 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1719 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1720 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1721 .to_string(),
1722 detail: Some(serde_json::json!({
1723 "first_undelivered_cursor": first_undelivered
1724 })),
1725 })
1726 .map_err(|error| error.to_string())?;
1727 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1728 .map_err(|error| error.to_string())
1729}
1730
1731pub trait ModuleProcessLiveness: Send + Sync {
1732 fn process_live(&self, module_id: &str) -> Option<bool>;
1733
1734 fn process_replacing(&self, _module_id: &str) -> bool {
1740 false
1741 }
1742}
1743
1744#[derive(Debug, Clone, Default)]
1746pub struct SupervisorProcessLiveness {
1747 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1748}
1749
1750impl SupervisorProcessLiveness {
1751 pub fn new() -> Self {
1752 Self::default()
1753 }
1754
1755 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1756 let mut snapshots = self
1757 .snapshots
1758 .lock()
1759 .unwrap_or_else(|poisoned| poisoned.into_inner());
1760 snapshots.insert(module_id, snapshot);
1761 }
1762
1763 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1764 let mut snapshots = self
1765 .snapshots
1766 .lock()
1767 .unwrap_or_else(|poisoned| poisoned.into_inner());
1768 let is_current = snapshots
1769 .get(module_id)
1770 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1771 .unwrap_or(false);
1772 if is_current {
1773 snapshots.remove(module_id);
1774 }
1775 }
1776}
1777
1778impl ModuleProcessLiveness for SupervisorProcessLiveness {
1779 fn process_live(&self, module_id: &str) -> Option<bool> {
1780 let snapshot = {
1781 let snapshots = self
1782 .snapshots
1783 .lock()
1784 .unwrap_or_else(|poisoned| poisoned.into_inner());
1785 snapshots.get(module_id).cloned()
1786 }?;
1787 let snapshot = snapshot
1788 .lock()
1789 .unwrap_or_else(|poisoned| poisoned.into_inner());
1790 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1791 }
1792
1793 fn process_replacing(&self, module_id: &str) -> bool {
1794 let Some(snapshot) = self
1795 .snapshots
1796 .lock()
1797 .unwrap_or_else(|poisoned| poisoned.into_inner())
1798 .get(module_id)
1799 .cloned()
1800 else {
1801 return false;
1802 };
1803 let snapshot = snapshot
1804 .lock()
1805 .unwrap_or_else(|poisoned| poisoned.into_inner());
1806 snapshot.enabled
1807 && match snapshot.state {
1808 ModuleState::Restarting => true,
1809 ModuleState::Draining => snapshot.draining_to_replace,
1810 ModuleState::Starting
1811 | ModuleState::Running
1812 | ModuleState::Unresponsive
1813 | ModuleState::Stopped
1814 | ModuleState::Failed
1815 | ModuleState::Disabled => false,
1816 }
1817 }
1818}
1819
1820#[cfg(test)]
1821#[derive(Debug, Default)]
1822struct ReloadExitRecordGate {
1823 reached: tokio::sync::Notify,
1824 resume: tokio::sync::Notify,
1825}
1826
1827#[derive(Debug, Clone, Copy)]
1828enum RespawnKind {
1829 Spawn,
1830 Reload,
1831}
1832
1833#[derive(Debug, Clone, Copy)]
1834struct PendingRespawn {
1835 deadline: Instant,
1836 kind: RespawnKind,
1837}
1838
1839type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1840
1841#[derive(Debug, Clone)]
1842struct SupervisorRuntimeConfig {
1843 start_requester: Option<Principal>,
1844 pending_activity_restart: bool,
1845 provenance_probe: ExecutableIdentityProbe,
1846 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1848 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1850 restart_policy: RestartPolicy,
1851 drain_timeout: Duration,
1854 effective_drain_timeout: Arc<Mutex<Duration>>,
1857 default_drain_timeout: Duration,
1860 health: HealthConfig,
1861 connection_file_path: Option<PathBuf>,
1862 capture_logs_dir: Option<PathBuf>,
1863 forwarding: Option<Arc<ForwardingTable>>,
1864 supervisor_handle: Option<SupervisorHandle>,
1867 stderr_ring: Arc<Mutex<StderrRing>>,
1874 terminal_ring: Arc<Mutex<TerminalRing>>,
1875 spawn_events: SpawnEventFeed,
1876 child_roster: ChildRoster,
1877 #[cfg(target_os = "linux")]
1878 cgroup_placement: Option<subc_cgroup::Placement>,
1879 #[cfg(test)]
1880 test_seed_stale_facts_before_enable_spawn: bool,
1881 #[cfg(test)]
1882 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1883}
1884
1885#[derive(Debug, Clone, PartialEq, Eq)]
1886struct SupervisedConfiguration {
1887 spec: ModuleSpec,
1888 health: HealthConfig,
1889}
1890
1891#[derive(Debug, Clone, Default)]
1897pub struct SupervisorHandle {
1898 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1899 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1900 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1901 configured_ids: Arc<Mutex<HashSet<String>>>,
1913 spawn_events: SpawnEventFeed,
1914 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
1925 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
1931 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
1935 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
1943 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
1949 promotion_observer: PromotionObserverSlot,
1951 operation_lock: Arc<AsyncMutex<()>>,
1955}
1956
1957pub(crate) trait SwapPromotionObserver: Send + Sync {
1966 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
1967}
1968
1969#[derive(Clone, Default)]
1973struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
1974
1975impl fmt::Debug for PromotionObserverSlot {
1976 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1977 f.write_str("PromotionObserverSlot")
1978 }
1979}
1980
1981#[derive(Debug, Clone)]
1983struct OpenSwap {
1984 candidate_nonce: String,
1987 incumbent_nonce: Option<String>,
1992 candidate_admitted: bool,
1996}
1997
1998#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2001pub(crate) enum SwapHelloAdmission {
2002 NotSwapping,
2005 Candidate,
2007 Refused,
2010}
2011
2012#[derive(Debug, Clone, PartialEq, Eq)]
2013pub(crate) enum ReservedHelloRejection {
2014 Exact {
2015 module_id: String,
2016 },
2017 Prefix {
2018 prefix: String,
2019 owner_module_id: String,
2020 },
2021}
2022
2023impl SupervisorHandle {
2024 pub fn new() -> Self {
2025 Self::default()
2026 }
2027
2028 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2029 self.spawn_events.snapshot()
2030 }
2031
2032 pub(crate) fn subscribe_spawns(
2033 &self,
2034 connection_id: ConnectionId,
2035 corr: u64,
2036 version: u8,
2037 since: Option<SpawnCursor>,
2038 sink: FrameSink,
2039 ) -> Result<(), SpawnSubscribeRefusal> {
2040 self.spawn_events
2041 .subscribe(connection_id, corr, version, since, sink)
2042 }
2043
2044 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2045 self.spawn_events.cancel(connection_id, corr)
2046 }
2047
2048 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2049 self.spawn_events.remove_connection(connection_id);
2050 }
2051
2052 #[cfg(any(test, feature = "test-support"))]
2053 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2054 assert!(capacity > 0, "spawn event capacity must be non-zero");
2055 self.spawn_events.set_capacity(capacity);
2056 }
2057
2058 #[cfg(any(test, feature = "test-support"))]
2059 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2060 self.spawn_events.subscriber_count()
2061 }
2062
2063 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2066 self.spawn_nonces
2067 .lock()
2068 .unwrap_or_else(|poisoned| poisoned.into_inner())
2069 .insert(module_id.to_string(), nonce);
2070 }
2071
2072 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2075 self.reserved_nonces
2076 .lock()
2077 .unwrap_or_else(|poisoned| poisoned.into_inner())
2078 .insert(module_id.to_string(), Some(nonce));
2079 }
2080
2081 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2083 let mut owners = self
2084 .reserved_prefix_owners
2085 .lock()
2086 .unwrap_or_else(|poisoned| poisoned.into_inner());
2087 owners.retain(|_, owner| owner != owner_module_id);
2088 for prefix in prefixes {
2089 owners.insert(prefix.clone(), owner_module_id.to_string());
2090 }
2091 }
2092
2093 #[cfg(test)]
2095 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2096 self.spawn_nonces
2097 .lock()
2098 .unwrap_or_else(|poisoned| poisoned.into_inner())
2099 .get(module_id)
2100 .cloned()
2101 }
2102
2103 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2104 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2105 let spawn_nonce = self
2106 .spawn_nonces
2107 .lock()
2108 .unwrap_or_else(|poisoned| poisoned.into_inner())
2109 .get(&spec.module_id)
2110 .cloned();
2111 let mut reserved_nonces = self
2112 .reserved_nonces
2113 .lock()
2114 .unwrap_or_else(|poisoned| poisoned.into_inner());
2115 if spec.reserved {
2116 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2121 }
2122 drop(reserved_nonces);
2123 self.removal_tombstones
2127 .lock()
2128 .unwrap_or_else(|poisoned| poisoned.into_inner())
2129 .remove(&spec.module_id);
2130 }
2131
2132 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2137 self.reserved_hello_rejection(module_id, presented)
2138 .is_none()
2139 }
2140
2141 pub(crate) fn reserved_hello_rejection(
2142 &self,
2143 module_id: &str,
2144 presented: Option<&str>,
2145 ) -> Option<ReservedHelloRejection> {
2146 let nonces = self
2147 .reserved_nonces
2148 .lock()
2149 .unwrap_or_else(|poisoned| poisoned.into_inner());
2150 if let Some(expected) = nonces.get(module_id) {
2151 let authorized = match expected {
2155 Some(expected) => {
2156 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2157 }
2158 None => false,
2159 };
2160 if authorized {
2161 return None;
2162 }
2163 return Some(ReservedHelloRejection::Exact {
2164 module_id: module_id.to_string(),
2165 });
2166 }
2167 drop(nonces);
2168
2169 let matched_prefix = self
2170 .reserved_prefix_owners
2171 .lock()
2172 .unwrap_or_else(|poisoned| poisoned.into_inner())
2173 .iter()
2174 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2175 .max_by_key(|(prefix, _)| prefix.len())
2176 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2177 let (prefix, owner_module_id) = matched_prefix?;
2178
2179 let authorized = presented.is_some_and(|presented| {
2180 self.spawn_nonces
2181 .lock()
2182 .unwrap_or_else(|poisoned| poisoned.into_inner())
2183 .get(&owner_module_id)
2184 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2185 || self.swap_nonce_matches(&owner_module_id, presented)
2188 });
2189 if authorized {
2190 None
2191 } else {
2192 Some(ReservedHelloRejection::Prefix {
2193 prefix,
2194 owner_module_id,
2195 })
2196 }
2197 }
2198
2199 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2204 if presented.is_empty() {
2205 return false;
2206 }
2207 let nonces = self
2208 .spawn_nonces
2209 .lock()
2210 .unwrap_or_else(|poisoned| poisoned.into_inner());
2211 let current = nonces
2212 .get(module_id)
2213 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2214 drop(nonces);
2215 current || self.swap_nonce_matches(module_id, presented)
2220 }
2221
2222 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2224 let swaps = self
2225 .swaps
2226 .lock()
2227 .unwrap_or_else(|poisoned| poisoned.into_inner());
2228 swaps.get(module_id).is_some_and(|swap| {
2229 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2230 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2231 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2232 })
2233 })
2234 }
2235
2236 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2239 let incumbent_nonce = self
2240 .spawn_nonces
2241 .lock()
2242 .unwrap_or_else(|poisoned| poisoned.into_inner())
2243 .get(module_id)
2244 .cloned();
2245 self.swaps
2246 .lock()
2247 .unwrap_or_else(|poisoned| poisoned.into_inner())
2248 .insert(
2249 module_id.to_string(),
2250 OpenSwap {
2251 candidate_nonce,
2252 incumbent_nonce,
2253 candidate_admitted: false,
2254 },
2255 );
2256 }
2257
2258 pub(crate) fn close_swap(&self, module_id: &str) {
2261 self.swaps
2262 .lock()
2263 .unwrap_or_else(|poisoned| poisoned.into_inner())
2264 .remove(module_id);
2265 }
2266
2267 pub(crate) fn set_swap_promotion_observer(
2270 &self,
2271 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2272 ) {
2273 *self
2274 .promotion_observer
2275 .0
2276 .lock()
2277 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2278 }
2279
2280 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2283 let observer = self
2284 .promotion_observer
2285 .0
2286 .lock()
2287 .unwrap_or_else(|poisoned| poisoned.into_inner())
2288 .as_ref()
2289 .and_then(std::sync::Weak::upgrade);
2290 if let Some(observer) = observer {
2291 observer.swap_promoted(registration);
2292 }
2293 }
2294
2295 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2297 self.swaps
2298 .lock()
2299 .unwrap_or_else(|poisoned| poisoned.into_inner())
2300 .contains_key(module_id)
2301 }
2302
2303 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2308 let candidate_nonce = self
2309 .swaps
2310 .lock()
2311 .unwrap_or_else(|poisoned| poisoned.into_inner())
2312 .get(module_id)
2313 .map(|swap| swap.candidate_nonce.clone());
2314 let Some(nonce) = candidate_nonce else {
2315 return;
2316 };
2317 self.set_spawn_nonce(module_id, nonce.clone());
2318 if reserved {
2319 self.set_reserved_nonce(module_id, nonce);
2320 }
2321 }
2322
2323 pub(crate) fn swap_hello_admission(
2338 &self,
2339 module_id: &str,
2340 presented: Option<&str>,
2341 ) -> SwapHelloAdmission {
2342 let swaps = self
2343 .swaps
2344 .lock()
2345 .unwrap_or_else(|poisoned| poisoned.into_inner());
2346 let Some(swap) = swaps.get(module_id) else {
2347 return SwapHelloAdmission::NotSwapping;
2348 };
2349 let Some(presented) = presented else {
2350 return SwapHelloAdmission::Refused;
2351 };
2352 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2353 return if swap.candidate_admitted {
2354 SwapHelloAdmission::Refused
2355 } else {
2356 SwapHelloAdmission::Candidate
2357 };
2358 }
2359 if swap
2360 .incumbent_nonce
2361 .as_deref()
2362 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2363 {
2364 return SwapHelloAdmission::NotSwapping;
2365 }
2366 SwapHelloAdmission::Refused
2367 }
2368
2369 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2372 if let Some(swap) = self
2373 .swaps
2374 .lock()
2375 .unwrap_or_else(|poisoned| poisoned.into_inner())
2376 .get_mut(module_id)
2377 {
2378 swap.candidate_admitted = true;
2379 }
2380 }
2381
2382 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2384 self.spawn_nonces
2385 .lock()
2386 .unwrap_or_else(|poisoned| poisoned.into_inner())
2387 .get(module_id)
2388 .cloned()
2389 }
2390
2391 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2393 self.reserved_nonces
2394 .lock()
2395 .unwrap_or_else(|poisoned| poisoned.into_inner())
2396 .get(module_id)
2397 .cloned()
2398 .flatten()
2399 }
2400
2401 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2402 self.mark_configured(module.module_id());
2406 let mut modules = self
2407 .modules
2408 .lock()
2409 .unwrap_or_else(|poisoned| poisoned.into_inner());
2410 modules.insert(module.module_id().to_string(), module)
2411 }
2412
2413 fn mark_configured(&self, module_id: &str) {
2417 self.configured_ids
2418 .lock()
2419 .unwrap_or_else(|poisoned| poisoned.into_inner())
2420 .insert(module_id.to_string());
2421 }
2422
2423 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2427 let modules = self
2428 .modules
2429 .lock()
2430 .unwrap_or_else(|poisoned| poisoned.into_inner());
2431 if !modules.contains_key(module_id) {
2432 self.configured_ids
2433 .lock()
2434 .unwrap_or_else(|poisoned| poisoned.into_inner())
2435 .remove(module_id);
2436 }
2437 }
2438
2439 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2447 self.configured_ids
2448 .lock()
2449 .unwrap_or_else(|poisoned| poisoned.into_inner())
2450 .contains(module_id)
2451 }
2452
2453 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2454 let modules = self
2455 .modules
2456 .lock()
2457 .unwrap_or_else(|poisoned| poisoned.into_inner());
2458 modules.get(module_id).cloned()
2459 }
2460
2461 pub(crate) fn record_late_health_answer(
2462 &self,
2463 module_id: &str,
2464 latency_ms: u64,
2465 ) -> Result<bool, SuperviseError> {
2466 let Some(module) = self.get(module_id) else {
2467 return Ok(false);
2468 };
2469 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2470 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2471 state.health.last_late_answer_latency_ms = Some(latency_ms);
2472 state.health.consecutive_failures = 0;
2480 })?;
2481 Ok(true)
2482 }
2483
2484 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2490 let Some(module) = self.get(module_id) else {
2491 return Ok(false);
2492 };
2493 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2494 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2495 return Ok(false);
2496 };
2497 drop(snapshot);
2498 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2499 }
2500
2501 pub fn list(&self) -> Vec<SupervisedModule> {
2502 let modules = self
2503 .modules
2504 .lock()
2505 .unwrap_or_else(|poisoned| poisoned.into_inner());
2506 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2507 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2508 modules
2509 }
2510
2511 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2512 self.activity_images
2513 .lock()
2514 .unwrap_or_else(|p| p.into_inner())
2515 .remove(module_id);
2516 self.spawn_nonces
2517 .lock()
2518 .unwrap_or_else(|poisoned| poisoned.into_inner())
2519 .remove(module_id);
2520 self.close_swap(module_id);
2521 let mut reserved_nonces = self
2522 .reserved_nonces
2523 .lock()
2524 .unwrap_or_else(|poisoned| poisoned.into_inner());
2525 if reserved_nonces.contains_key(module_id) {
2526 reserved_nonces.insert(module_id.to_string(), None);
2529 }
2530 drop(reserved_nonces);
2531 self.reserved_prefix_owners
2532 .lock()
2533 .unwrap_or_else(|poisoned| poisoned.into_inner())
2534 .retain(|_, owner| owner != module_id);
2535 let removed = self
2536 .modules
2537 .lock()
2538 .unwrap_or_else(|poisoned| poisoned.into_inner())
2539 .remove(module_id);
2540 self.configured_ids
2541 .lock()
2542 .unwrap_or_else(|poisoned| poisoned.into_inner())
2543 .remove(module_id);
2544 removed
2545 }
2546
2547 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2550 self.removal_tombstones
2551 .lock()
2552 .unwrap_or_else(|poisoned| poisoned.into_inner())
2553 .insert(module_id.to_string(), unix_ms_now());
2554 }
2555
2556 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2558 self.removal_tombstones
2559 .lock()
2560 .unwrap_or_else(|poisoned| poisoned.into_inner())
2561 .get(module_id)
2562 .copied()
2563 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2564 }
2565
2566 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2571 if self.get(module_id).is_some() {
2572 return false;
2573 }
2574 let mut reserved_nonces = self
2575 .reserved_nonces
2576 .lock()
2577 .unwrap_or_else(|poisoned| poisoned.into_inner());
2578 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2579 return false;
2580 }
2581 reserved_nonces.remove(module_id);
2582 true
2583 }
2584
2585 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2586 Arc::clone(&self.operation_lock)
2587 }
2588}
2589
2590#[derive(Debug, Clone)]
2592pub struct Supervisor {
2593 registry: Arc<Registry>,
2594 restart_policy: RestartPolicy,
2595 drain_timeout: Duration,
2596 connection_file_path: Option<PathBuf>,
2597 capture_logs_dir: Option<PathBuf>,
2598 forwarding: Option<Arc<ForwardingTable>>,
2599 process_liveness: Arc<SupervisorProcessLiveness>,
2600 supervisor_handle: Option<SupervisorHandle>,
2601 health: HealthConfig,
2602 daemon_start_clock: crate::clock::StartClock,
2603 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2604 spawn_events: SpawnEventFeed,
2605 provenance_probe: ExecutableIdentityProbe,
2606 child_roster: ChildRoster,
2609 #[cfg(target_os = "linux")]
2610 cgroup_placement: Option<subc_cgroup::Placement>,
2611 #[cfg(test)]
2612 test_after_first_spawn: AfterFirstSpawnHook,
2613}
2614
2615pub(crate) struct RegisteredConfiguredModule {
2619 spec: ModuleSpec,
2620 runtime: SupervisorRuntimeConfig,
2621 module: SupervisedModule,
2622 commands: mpsc::Receiver<SupervisorCommand>,
2623}
2624
2625#[cfg(test)]
2631#[derive(Clone, Default)]
2632struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2633
2634#[cfg(test)]
2635type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2636
2637#[cfg(test)]
2638impl fmt::Debug for AfterFirstSpawnHook {
2639 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2640 f.write_str("AfterFirstSpawnHook")
2641 }
2642}
2643
2644#[cfg(test)]
2645impl AfterFirstSpawnHook {
2646 fn run(&self, module_id: &str) {
2647 if let Some(hook) = &self.0 {
2648 hook(module_id);
2649 }
2650 }
2651}
2652
2653impl Supervisor {
2654 #[cfg(test)]
2655 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2656 let supervisor = Self::new(registry, policy);
2657 #[cfg(target_os = "macos")]
2658 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2659 supervisor
2660 }
2661 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2666 let path = path.into();
2667 #[cfg(target_os = "macos")]
2668 {
2669 let result = probe_privacy_trampoline(&path).map(|()| path);
2670 if let Err(cause) = &result {
2671 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2672 }
2673 self.child_roster.set_privacy_trampoline(result);
2674 }
2675 #[cfg(not(target_os = "macos"))]
2676 let _ = path;
2677 self
2678 }
2679 #[cfg(unix)]
2690 pub(crate) fn begin_daemon_shutdown(&self) {
2691 self.child_roster.close();
2692 if let Some(journal) = &self.terminal_journal {
2693 journal.stamp_shutdown();
2694 }
2695 }
2696
2697 #[cfg(unix)]
2701 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2702 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2703 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2704 let Some(forwarding) = &self.forwarding else {
2705 return Ok(());
2706 };
2707 let modules = forwarding
2708 .begin_daemon_drain()
2709 .map_err(SuperviseError::Forwarding)?;
2710 let deadline_ms =
2711 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2712 let mut notices = tokio::task::JoinSet::new();
2713 let mut drains = Vec::new();
2714 for (module_id, endpoint) in modules {
2715 let Some(target) = forwarding
2716 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2717 .map_err(SuperviseError::Forwarding)?
2718 else {
2719 continue;
2720 };
2721 let routes = forwarding
2722 .endpoint_routes(target.endpoint)
2723 .map_err(SuperviseError::Forwarding)?;
2724 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2730 reason: RouteCloseReason::Restart,
2731 deadline_ms,
2732 })
2733 .expect("module draining serializes");
2734 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2735 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2736 for route in routes {
2737 let client = route.goodbye_target;
2738 if let Some((_, channels)) = clients
2739 .iter_mut()
2740 .find(|(existing, _)| existing.connection_id == client.connection_id)
2741 {
2742 channels.push(client.channel);
2743 } else {
2744 let channel = client.channel;
2745 clients.push((client, vec![channel]));
2746 }
2747 }
2748 for (client, mut channels) in clients {
2749 channels.sort_unstable();
2750 channels.dedup();
2751 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2752 module_id: module_id.clone(),
2753 channels,
2754 reason: RouteCloseReason::Restart,
2755 })
2756 .expect("route closing serializes");
2757 recipients.push((client.sink, client.negotiated_ver, closing));
2758 }
2759 for (sink, version, body) in recipients {
2760 notices.spawn(async move {
2761 let frame = Frame::build_with_version(
2762 version,
2763 FrameType::Push,
2764 control_flags(),
2765 0,
2766 0,
2767 0,
2768 body,
2769 )
2770 .expect("bounded lifecycle notice frame builds");
2771 sink.send_flushed(frame).await
2772 });
2773 }
2774 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2775 drains.push((module_id, target.endpoint, gauges));
2776 }
2777 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2780 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2781 if !matches!(result, Ok(Ok(()))) {
2782 warn!(?result, "daemon shutdown notice delivery failed");
2783 }
2784 }
2785 notices.abort_all();
2786 let deadline = Instant::now() + DRAIN_BUDGET;
2787 let mut waits = tokio::task::JoinSet::new();
2788 for (module_id, endpoint, gauges) in drains {
2789 let forwarding = Arc::clone(forwarding);
2790 let mut runtime = self.runtime_config();
2791 runtime.health.cadence = Duration::from_millis(100);
2792 waits.spawn(async move {
2793 let wait_result = wait_for_forwarding_quiescence(
2794 &forwarding,
2795 &module_id,
2796 &runtime,
2797 endpoint,
2798 deadline,
2799 &gauges,
2800 DrainScope::Endpoint(endpoint),
2801 )
2802 .await;
2803 forwarding
2807 .record_daemon_drain_result(
2808 endpoint,
2809 drained_after_quiescence_wait(&wait_result),
2810 )
2811 .map_err(SuperviseError::Forwarding)?;
2812 wait_result
2813 });
2814 }
2815 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2816 if !matches!(result, Ok(Ok(true))) {
2817 warn!(?result, "daemon shutdown drain did not reach quiescence");
2818 }
2819 }
2820 Ok(())
2821 }
2822
2823 #[cfg(unix)]
2835 pub(crate) async fn end_children_for_daemon_shutdown(
2836 &self,
2837 already_escalated: bool,
2838 escalate: impl std::future::Future<Output = ()>,
2839 ) {
2840 tokio::pin!(escalate);
2841 let mut escalated = already_escalated;
2842 if let Some(forwarding) = &self.forwarding {
2843 let reason = CloseReason::new(
2844 "daemon_shutdown",
2845 "the daemon is exiting after its shutdown notice and drain",
2846 );
2847 if escalated {
2848 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2851 } else {
2852 tokio::select! {
2853 biased;
2854 _ = escalate.as_mut() => {
2855 info!("second SIGTERM: abandoning module GOODBYE delivery");
2856 escalated = true;
2857 }
2858 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2859 }
2860 }
2861 let closed = forwarding.close_all_connections(&reason);
2862 debug!(closed, "closed established connections for daemon shutdown");
2863 }
2864 let escalated_here = escalated && !already_escalated;
2868 let remaining_escalate = async move {
2869 if escalated_here {
2870 std::future::pending::<()>().await;
2871 } else {
2872 escalate.await;
2873 }
2874 };
2875 crate::child_roster::end_children_for_daemon_shutdown(
2876 &self.child_roster,
2877 escalated,
2878 remaining_escalate,
2879 )
2880 .await;
2881 }
2882
2883 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2884 Self {
2885 registry,
2886 restart_policy,
2887 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2888 connection_file_path: None,
2889 capture_logs_dir: None,
2890 forwarding: None,
2891 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2892 supervisor_handle: None,
2893 health: HealthConfig::default(),
2894 daemon_start_clock: crate::clock::StartClock::capture(),
2895 terminal_journal: None,
2896 spawn_events: SpawnEventFeed::default(),
2897 provenance_probe: ExecutableIdentityProbe::default(),
2898 child_roster: ChildRoster::default(),
2899 #[cfg(target_os = "linux")]
2900 cgroup_placement: None,
2901 #[cfg(test)]
2902 test_after_first_spawn: AfterFirstSpawnHook::default(),
2903 }
2904 }
2905
2906 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2907 self.drain_timeout = drain_timeout;
2908 self
2909 }
2910
2911 pub fn with_process_liveness(
2912 mut self,
2913 process_liveness: Arc<SupervisorProcessLiveness>,
2914 ) -> Self {
2915 self.process_liveness = process_liveness;
2916 self
2917 }
2918
2919 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2920 self.connection_file_path = Some(connection_file_path.into());
2921 self
2922 }
2923
2924 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
2926 self.capture_logs_dir = Some(logs_dir.into());
2927 self
2928 }
2929
2930 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
2933 self.spawn_events.configure_incarnation(daemon_incarnation);
2937 self
2938 }
2939
2940 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
2943 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
2944 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
2945 path,
2946 daemon_incarnation,
2947 )));
2948 this
2949 }
2950
2951 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
2952 self.forwarding = Some(forwarding);
2953 self
2954 }
2955
2956 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
2957 self.spawn_events = supervisor_handle.spawn_events.clone();
2958 self.supervisor_handle = Some(supervisor_handle);
2959 self
2960 }
2961
2962 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
2963 self.health = health;
2964 self
2965 }
2966
2967 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
2971 self.child_roster.record_to(path.into());
2972 self
2973 }
2974
2975 #[cfg(target_os = "linux")]
2976 pub fn with_cgroup_placement(
2977 mut self,
2978 cgroup_placement: Option<subc_cgroup::Placement>,
2979 ) -> Self {
2980 self.cgroup_placement = cgroup_placement;
2981 self
2982 }
2983
2984 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
2990 validate_spec(&spec)?;
2991 self.establish_identity(&spec);
2992
2993 let runtime = self.runtime_config();
2994 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
2995 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
2996 let spawned = spawn_child(
2997 &spec,
2998 runtime.connection_file_path.as_deref(),
2999 self.supervisor_handle.as_ref(),
3000 &runtime.stderr_ring,
3001 runtime.capture_logs_dir.as_deref(),
3002 &runtime.child_roster,
3003 #[cfg(target_os = "linux")]
3004 runtime.cgroup_placement.as_ref(),
3005 );
3006 #[cfg(test)]
3007 self.test_after_first_spawn.run(&spec.module_id);
3008 let child = match spawned {
3009 Ok(child) => child,
3010 Err(err) => {
3011 self.abandon_unrostered(&spec.module_id);
3014 return Err(err);
3015 }
3016 };
3017 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3018 if let Some(spawn) = activity_spawn {
3019 spawn.complete(&snapshot);
3020 }
3021
3022 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3023 }
3024
3025 fn establish_identity(&self, spec: &ModuleSpec) {
3038 if let Some(supervisor_handle) = &self.supervisor_handle {
3039 supervisor_handle.apply_identity_configuration(spec);
3040 supervisor_handle.mark_configured(&spec.module_id);
3041 }
3042 }
3043
3044 fn abandon_unrostered(&self, module_id: &str) {
3047 if let Some(supervisor_handle) = &self.supervisor_handle {
3048 supervisor_handle.unmark_configured_unless_rostered(module_id);
3049 }
3050 }
3051
3052 fn mark_first_process_running(
3055 &self,
3056 spec: &ModuleSpec,
3057 runtime: &SupervisorRuntimeConfig,
3058 snapshot: &SharedSnapshot,
3059 child: &SupervisedChild,
3060 ) -> Result<(), SuperviseError> {
3061 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3062 self.abandon_unrostered(&spec.module_id);
3063 return Err(err);
3064 }
3065 self.process_liveness
3066 .track(spec.module_id.clone(), Arc::clone(snapshot));
3067 Ok(())
3068 }
3069
3070 pub fn supervise_configured(
3076 &self,
3077 spec: ModuleSpec,
3078 enabled: bool,
3079 ) -> Result<SupervisedModule, SuperviseError> {
3080 self.supervise_configured_requested(
3081 spec,
3082 enabled,
3083 self.health.clone(),
3084 None,
3085 self.restart_policy,
3086 None,
3087 )
3088 }
3089
3090 pub fn supervise_configured_with_health(
3096 &self,
3097 spec: ModuleSpec,
3098 enabled: bool,
3099 health: HealthConfig,
3100 drain_timeout_ms: Option<u64>,
3101 restart_policy: RestartPolicy,
3102 ) -> Result<SupervisedModule, SuperviseError> {
3103 self.supervise_configured_requested(
3104 spec,
3105 enabled,
3106 health,
3107 drain_timeout_ms,
3108 restart_policy,
3109 None,
3110 )
3111 }
3112
3113 pub(crate) fn supervise_configured_requested(
3114 &self,
3115 spec: ModuleSpec,
3116 enabled: bool,
3117 health: HealthConfig,
3118 drain_timeout_ms: Option<u64>,
3119 restart_policy: RestartPolicy,
3120 requester: Option<Principal>,
3121 ) -> Result<SupervisedModule, SuperviseError> {
3122 let registered = self.register_configured_with_health(
3123 spec,
3124 enabled,
3125 health,
3126 drain_timeout_ms,
3127 restart_policy,
3128 )?;
3129 self.start_registered_configured(registered, requester)
3130 }
3131
3132 pub(crate) fn register_configured_with_health(
3137 &self,
3138 spec: ModuleSpec,
3139 enabled: bool,
3140 health: HealthConfig,
3141 drain_timeout_ms: Option<u64>,
3142 restart_policy: RestartPolicy,
3143 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3144 validate_spec(&spec)?;
3145 self.establish_identity(&spec);
3146
3147 let mut runtime = self.runtime_config();
3148 runtime.health = health;
3149 runtime.restart_policy = restart_policy;
3150 if let Some(ms) = drain_timeout_ms {
3151 runtime.drain_timeout = Duration::from_millis(ms);
3152 *runtime
3153 .effective_drain_timeout
3154 .lock()
3155 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3156 }
3157 let snapshot = if enabled {
3158 SupervisorSnapshot::starting()
3159 } else {
3160 SupervisorSnapshot::disabled()
3161 };
3162 let snapshot = {
3163 let mut snapshot = snapshot;
3164 if enabled {
3165 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3166 {
3167 snapshot.waiting_for_previous_pid = Some(pid);
3168 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3169 }
3170 }
3171 snapshot
3172 };
3173 let (module, commands) =
3174 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3175 Ok(RegisteredConfiguredModule {
3176 spec,
3177 runtime,
3178 module,
3179 commands,
3180 })
3181 }
3182
3183 pub(crate) fn start_registered_configured(
3186 &self,
3187 registered: RegisteredConfiguredModule,
3188 requester: Option<Principal>,
3189 ) -> Result<SupervisedModule, SuperviseError> {
3190 let RegisteredConfiguredModule {
3191 spec,
3192 mut runtime,
3193 module,
3194 commands,
3195 } = registered;
3196 runtime.start_requester = requester;
3197 let snapshot = &module.inner.snapshot;
3198 let should_spawn = {
3199 let state = lock_snapshot(snapshot)?;
3200 state.enabled && state.waiting_for_previous_pid.is_none()
3201 };
3202 let child = if should_spawn {
3203 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3204 let spawned = spawn_child(
3205 &spec,
3206 runtime.connection_file_path.as_deref(),
3207 self.supervisor_handle.as_ref(),
3208 &runtime.stderr_ring,
3209 runtime.capture_logs_dir.as_deref(),
3210 &runtime.child_roster,
3211 #[cfg(target_os = "linux")]
3212 runtime.cgroup_placement.as_ref(),
3213 );
3214 #[cfg(test)]
3215 self.test_after_first_spawn.run(&spec.module_id);
3216 match spawned {
3217 Ok(child) => {
3218 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3219 if let Some(spawn) = activity_spawn {
3220 spawn.complete(snapshot);
3221 }
3222 Some(child)
3223 }
3224 Err(err) => {
3225 if runtime.health.critical {
3226 error!(
3227 module_id = %spec.module_id,
3228 program = %spec.program.display(),
3229 error = %err,
3230 "critical configured module failed to spawn; marking failed and alerting"
3231 );
3232 } else {
3233 error!(
3234 module_id = %spec.module_id,
3235 program = %spec.program.display(),
3236 error = %err,
3237 "configured module failed to spawn; marking failed and continuing"
3238 );
3239 }
3240 update_snapshot(snapshot, Some(&spec.module_id), |state| {
3241 state.state = ModuleState::Failed;
3242 clear_current_process_facts(state);
3243 })?;
3244 None
3245 }
3246 }
3247 } else {
3248 None
3249 };
3250 self.start_monitor(spec, runtime, &module, child, commands);
3251 Ok(module)
3252 }
3253
3254 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3255 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3256 SupervisorRuntimeConfig {
3257 start_requester: None,
3258 pending_activity_restart: false,
3259 provenance_probe: self.provenance_probe.clone(),
3260 scheduled_respawn: Arc::default(),
3261 deferred_reload_reply: Arc::default(),
3262 restart_policy: self.restart_policy,
3263 drain_timeout: self.drain_timeout,
3264 child_roster: self
3267 .child_roster
3268 .for_module(Arc::clone(&effective_drain_timeout)),
3269 effective_drain_timeout,
3270 default_drain_timeout: self.drain_timeout,
3271 health: self.health.clone(),
3272 connection_file_path: self.connection_file_path.clone(),
3273 capture_logs_dir: self.capture_logs_dir.clone(),
3274 forwarding: self.forwarding.clone(),
3275 supervisor_handle: self.supervisor_handle.clone(),
3276 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3277 terminal_ring: Arc::new(Mutex::new(
3278 TerminalRing::new(
3279 TerminalRingConfig::default(),
3280 self.daemon_start_clock.started_at_ms(),
3281 )
3282 .with_start_clock(self.daemon_start_clock)
3283 .with_journal(self.terminal_journal.clone())
3284 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3285 )),
3286 spawn_events: self.spawn_events.clone(),
3287 #[cfg(target_os = "linux")]
3288 cgroup_placement: self.cgroup_placement.clone(),
3289 #[cfg(test)]
3290 test_seed_stale_facts_before_enable_spawn: false,
3291 #[cfg(test)]
3292 test_reload_exit_record_gate: None,
3293 }
3294 }
3295
3296 fn supervised_module(
3297 &self,
3298 spec: ModuleSpec,
3299 runtime: SupervisorRuntimeConfig,
3300 snapshot: SharedSnapshot,
3301 child: Option<SupervisedChild>,
3302 ) -> SupervisedModule {
3303 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3304 self.start_monitor(spec, runtime, &module, child, commands);
3305 module
3306 }
3307
3308 fn register_module(
3309 &self,
3310 spec: &ModuleSpec,
3311 runtime: &SupervisorRuntimeConfig,
3312 snapshot: SharedSnapshot,
3313 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3314 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3315 spec: spec.clone(),
3316 health: runtime.health.clone(),
3317 }));
3318 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3319 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3320 let restart_policy = runtime.restart_policy;
3324 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3325 let (tx, rx) = mpsc::channel(4);
3326 let module_id = spec.module_id.clone();
3327 let module = SupervisedModule {
3328 inner: Arc::new(SupervisedModuleInner {
3329 module_id: module_id.clone(),
3330 registry: Arc::clone(&self.registry),
3331 snapshot,
3332 configuration,
3333 stderr_ring,
3334 terminal_ring,
3335 commands: tx,
3336 monitor: Mutex::new(None),
3337 restart_policy,
3338 effective_drain_timeout,
3339 provenance_probe: self.provenance_probe.clone(),
3340 }),
3341 };
3342 if let Some(supervisor_handle) = &self.supervisor_handle {
3345 supervisor_handle.insert(module.clone());
3346 }
3347 (module, rx)
3348 }
3349
3350 fn start_monitor(
3351 &self,
3352 spec: ModuleSpec,
3353 runtime: SupervisorRuntimeConfig,
3354 module: &SupervisedModule,
3355 child: Option<SupervisedChild>,
3356 commands: mpsc::Receiver<SupervisorCommand>,
3357 ) {
3358 let monitor = tokio::spawn(supervise_loop(
3359 spec,
3360 runtime,
3361 Arc::clone(&self.registry),
3362 Arc::clone(&self.process_liveness),
3363 Arc::clone(&module.inner.snapshot),
3364 child,
3365 commands,
3366 ));
3367 *module
3368 .inner
3369 .monitor
3370 .lock()
3371 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3372 }
3373}
3374
3375impl Default for Supervisor {
3376 fn default() -> Self {
3377 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3378 }
3379}
3380
3381#[derive(Clone)]
3383pub struct SupervisedModule {
3384 inner: Arc<SupervisedModuleInner>,
3385}
3386
3387struct SupervisedModuleInner {
3388 module_id: String,
3389 registry: Arc<Registry>,
3390 snapshot: SharedSnapshot,
3391 configuration: Arc<Mutex<SupervisedConfiguration>>,
3392 stderr_ring: Arc<Mutex<StderrRing>>,
3393 terminal_ring: Arc<Mutex<TerminalRing>>,
3394 commands: mpsc::Sender<SupervisorCommand>,
3395 monitor: Mutex<Option<JoinHandle<()>>>,
3396 restart_policy: RestartPolicy,
3400 effective_drain_timeout: Arc<Mutex<Duration>>,
3401 provenance_probe: ExecutableIdentityProbe,
3402}
3403
3404impl fmt::Debug for SupervisedModule {
3405 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3406 f.debug_struct("SupervisedModule")
3407 .field("module_id", &self.inner.module_id)
3408 .field("status", &self.status())
3409 .finish_non_exhaustive()
3410 }
3411}
3412
3413impl SupervisedModule {
3414 pub fn module_id(&self) -> &str {
3415 &self.inner.module_id
3416 }
3417
3418 #[cfg(test)]
3422 pub(crate) fn record_health_probe_failure_for_test(
3423 &self,
3424 detail: &str,
3425 ) -> Result<(), SuperviseError> {
3426 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3427 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3428 state.health.detail = Some(detail.to_string());
3429 })
3430 }
3431
3432 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3433 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3434 }
3435
3436 pub(crate) fn registration_end_reason(
3437 &self,
3438 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3439 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3440 Ok(match snapshot.state {
3441 ModuleState::Draining if snapshot.draining_to_replace => {
3442 Some(RegistrationEndReason::SupervisorRestart)
3443 }
3444 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3445 ModuleState::Restarting if snapshot.draining_to_replace => {
3446 Some(RegistrationEndReason::SupervisorRestart)
3447 }
3448 _ => None,
3449 })
3450 }
3451
3452 pub fn stderr_tail(
3459 &self,
3460 max_lines: Option<usize>,
3461 max_bytes: Option<usize>,
3462 ) -> StderrTailSnapshot {
3463 self.inner
3464 .stderr_ring
3465 .lock()
3466 .unwrap_or_else(|poisoned| poisoned.into_inner())
3467 .snapshot(max_lines, max_bytes)
3468 }
3469
3470 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3475 self.inner
3476 .terminal_ring
3477 .lock()
3478 .unwrap_or_else(|poisoned| poisoned.into_inner())
3479 .snapshot()
3480 }
3481
3482 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3487 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3488 }
3489
3490 pub(crate) async fn read_durable_terminal_history(
3495 &self,
3496 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3497 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3498 let module_id = self.inner.module_id.clone();
3499 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3500 .await
3501 }
3502
3503 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3504 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3505 .map(|(status, _)| status)
3506 }
3507
3508 pub(crate) fn record_deliberate_severance(
3509 &self,
3510 identity: ProcessIdentity,
3511 ) -> Result<bool, SuperviseError> {
3512 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3513 if snapshot.pid != Some(identity.pid)
3514 || snapshot.process_start_time != Some(identity.start_time)
3515 {
3516 return Ok(false);
3517 }
3518 snapshot.deliberate_severance = Some(identity);
3519 Ok(true)
3520 }
3521
3522 pub(crate) fn status_for_control(
3527 &self,
3528 caller: &'static str,
3529 ) -> Result<ModuleStatus, SuperviseError> {
3530 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3531 .map(|(status, _)| status)
3532 }
3533
3534 fn status_with_snapshot_lock(
3535 &self,
3536 snapshot: &SharedSnapshot,
3537 caller: Option<&'static str>,
3538 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3539 let mut guard = match caller {
3540 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3541 None => lock_snapshot(snapshot)?,
3542 };
3543 let restart_count =
3546 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3547 let snapshot = guard.clone();
3548 drop(guard);
3549 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3550 SuperviseError::StatePoisoned {
3551 module_id: Some(self.inner.module_id.clone()),
3552 }
3553 })?;
3554 let registration_active = self
3555 .inner
3556 .registry
3557 .get_module(&self.inner.module_id)
3558 .map_err(SuperviseError::Registry)?
3559 .is_some();
3560 let protocol = snapshot
3561 .spawned_protocol
3562 .unwrap_or(self.declared_protocol()?);
3563 let running_process =
3564 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3565 let live = match protocol {
3571 ModuleProtocol::Subc => running_process && registration_active,
3572 ModuleProtocol::None => running_process,
3573 };
3574
3575 Ok((
3576 ModuleStatus {
3577 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3578 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3579 module_id: self.inner.module_id.clone(),
3580 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3581 state: snapshot.state,
3582 enabled: snapshot.enabled,
3583 process_alive: snapshot.process_alive,
3584 registration_active,
3585 protocol,
3586 live,
3587 restart_count,
3588 lifetime_restarts: snapshot.lifetime_restarts,
3589 spawn_generation: snapshot.spawn_generation,
3590 max_restarts: self.inner.restart_policy.max_restarts,
3591 restart_window: self.inner.restart_policy.window,
3592 drain_timeout,
3593 restart_backoff: self.inner.restart_policy.backoff,
3594 restart_max_backoff: self.inner.restart_policy.max_backoff,
3595 pid: snapshot.reported_pid(),
3596 spawned_at_ms: snapshot.spawned_at_ms,
3597 spawned_from: snapshot.spawned_from,
3598 process_start_time: snapshot.process_start_time,
3599 last_exit: snapshot.last_exit,
3600 health: snapshot.health,
3601 },
3602 snapshot.spawned_file_identity.clone(),
3603 ))
3604 }
3605
3606 #[cfg(test)]
3607 pub(crate) fn hold_snapshot_for_test(
3608 &self,
3609 acquired: std::sync::mpsc::Sender<()>,
3610 hold: Duration,
3611 ) -> std::thread::JoinHandle<()> {
3612 let snapshot = Arc::clone(&self.inner.snapshot);
3613 std::thread::spawn(move || {
3614 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3615 acquired
3616 .send(())
3617 .expect("test receiver waits for snapshot lock");
3618 std::thread::sleep(hold);
3619 })
3620 }
3621
3622 pub(crate) async fn status_and_running_image_agreement(
3628 &self,
3629 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3630 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3631 let image = self
3632 .inner
3633 .provenance_probe
3634 .observe(
3635 status.pid,
3636 status.spawned_from.as_deref(),
3637 identity,
3638 status.process_start_time,
3639 )
3640 .await;
3641 Ok((status, image))
3642 }
3643
3644 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3645 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3646 Ok(snapshot) => snapshot.clone(),
3647 Err(_) => {
3648 return subc_control::RunningImageAgreement::Unavailable {
3649 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3650 };
3651 }
3652 };
3653 self.inner
3654 .provenance_probe
3655 .observe(
3656 snapshot.reported_pid(),
3657 snapshot.spawned_from.as_deref(),
3658 snapshot.spawned_file_identity.clone(),
3659 snapshot.process_start_time,
3660 )
3661 .await
3662 }
3663
3664 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3667 #[cfg(windows)]
3668 {
3669 let image = lock_snapshot(&self.inner.snapshot)
3670 .ok()
3671 .and_then(|snapshot| {
3672 snapshot
3673 .spawned_file_identity
3674 .as_ref()
3675 .and_then(|identity| identity.windows_image.clone())
3676 });
3677 match image {
3678 Some(image) => {
3679 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3680 }
3681 None => subc_control::ChildResourceUsage::Unavailable {
3682 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3683 },
3684 }
3685 }
3686 #[cfg(not(windows))]
3687 {
3688 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3689 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3690 Err(_) => {
3691 return subc_control::ChildResourceUsage::Unavailable {
3692 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3693 }
3694 }
3695 };
3696 crate::child_resources::read(pid, start_time)
3697 }
3698 }
3699
3700 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3701 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3702 Ok(match snapshot.state {
3703 ModuleState::Restarting => true,
3704 ModuleState::Failed | ModuleState::Disabled => false,
3705 _ if snapshot.enabled && snapshot.respawn_pending => true,
3706 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3707 })
3708 }
3709
3710 #[cfg(test)]
3711 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3712 self.is_warming_with_snapshot_lock(None)
3713 }
3714
3715 pub(crate) fn is_warming_for_control(
3716 &self,
3717 caller: &'static str,
3718 ) -> Result<bool, SuperviseError> {
3719 self.is_warming_with_snapshot_lock(Some(caller))
3720 }
3721
3722 fn is_warming_with_snapshot_lock(
3723 &self,
3724 caller: Option<&'static str>,
3725 ) -> Result<bool, SuperviseError> {
3726 let snapshot = match caller {
3727 Some(caller) => {
3728 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3729 }
3730 None => lock_snapshot(&self.inner.snapshot)?,
3731 }
3732 .clone();
3733 Ok(matches!(
3734 snapshot.state,
3735 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3736 ))
3737 }
3738
3739 pub async fn drain(&self) -> Result<(), SuperviseError> {
3741 self.stop().await
3742 }
3743
3744 pub(crate) async fn retire_requested(
3745 &self,
3746 requester: Option<Principal>,
3747 ) -> Result<(), SuperviseError> {
3748 match self.state()? {
3749 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3750 ModuleState::Starting
3751 | ModuleState::Running
3752 | ModuleState::Unresponsive
3753 | ModuleState::Restarting
3754 | ModuleState::Draining
3755 | ModuleState::Disabled => {}
3756 }
3757
3758 let (reply_tx, reply_rx) = oneshot::channel();
3759 self.inner
3760 .commands
3761 .send(SupervisorCommand::Retire {
3762 requester,
3763 reply: reply_tx,
3764 })
3765 .await
3766 .map_err(|_| SuperviseError::CommandClosed {
3767 module_id: self.inner.module_id.clone(),
3768 })?;
3769 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3770 module_id: self.inner.module_id.clone(),
3771 })?
3772 }
3773
3774 pub async fn stop(&self) -> Result<(), SuperviseError> {
3775 self.stop_requested(None).await
3776 }
3777
3778 pub(crate) async fn stop_requested(
3779 &self,
3780 requester: Option<Principal>,
3781 ) -> Result<(), SuperviseError> {
3782 match self.state()? {
3783 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3784 ModuleState::Starting
3785 | ModuleState::Running
3786 | ModuleState::Unresponsive
3787 | ModuleState::Restarting
3788 | ModuleState::Draining
3789 | ModuleState::Disabled => {}
3790 }
3791
3792 let (reply_tx, reply_rx) = oneshot::channel();
3793 self.inner
3794 .commands
3795 .send(SupervisorCommand::Drain {
3796 requester,
3797 reply: reply_tx,
3798 })
3799 .await
3800 .map_err(|_| SuperviseError::CommandClosed {
3801 module_id: self.inner.module_id.clone(),
3802 })?;
3803 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3804 module_id: self.inner.module_id.clone(),
3805 })?
3806 }
3807
3808 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3809 self.restart_requested(drain_timeout_ms, None).await
3810 }
3811
3812 pub(crate) async fn restart_requested(
3813 &self,
3814 drain_timeout_ms: Option<u64>,
3815 requester: Option<Principal>,
3816 ) -> Result<(), SuperviseError> {
3817 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3818 let (reply_tx, reply_rx) = oneshot::channel();
3819 self.inner
3820 .commands
3821 .send(SupervisorCommand::Restart {
3822 requester,
3823 drain_timeout_ms,
3824 received_at_generation,
3825 queued_at: Instant::now(),
3826 reply: reply_tx,
3827 })
3828 .await
3829 .map_err(|_| SuperviseError::CommandClosed {
3830 module_id: self.inner.module_id.clone(),
3831 })?;
3832 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3833 module_id: self.inner.module_id.clone(),
3834 })?
3835 }
3836
3837 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3842 self.swap_requested(ready_timeout, None).await
3843 }
3844
3845 pub(crate) async fn swap_requested(
3846 &self,
3847 ready_timeout: Option<Duration>,
3848 requester: Option<Principal>,
3849 ) -> Result<(), SuperviseError> {
3850 let (reply_tx, reply_rx) = oneshot::channel();
3851 self.inner
3852 .commands
3853 .send(SupervisorCommand::Swap {
3854 requester,
3855 ready_timeout,
3856 reply: reply_tx,
3857 })
3858 .await
3859 .map_err(|_| SuperviseError::CommandClosed {
3860 module_id: self.inner.module_id.clone(),
3861 })?;
3862 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3863 module_id: self.inner.module_id.clone(),
3864 })?
3865 }
3866
3867 pub async fn reload(&self) -> Result<(), SuperviseError> {
3868 self.reload_requested(None).await
3869 }
3870
3871 pub(crate) async fn reload_requested(
3872 &self,
3873 requester: Option<Principal>,
3874 ) -> Result<(), SuperviseError> {
3875 let (reply_tx, reply_rx) = oneshot::channel();
3876 self.inner
3877 .commands
3878 .send(SupervisorCommand::Reload {
3879 requester,
3880 reply: reply_tx,
3881 })
3882 .await
3883 .map_err(|_| SuperviseError::CommandClosed {
3884 module_id: self.inner.module_id.clone(),
3885 })?;
3886 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3887 module_id: self.inner.module_id.clone(),
3888 })?
3889 }
3890
3891 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3892 self.set_enabled_requested(enabled, None).await
3893 }
3894
3895 pub(crate) async fn set_enabled_requested(
3896 &self,
3897 enabled: bool,
3898 requester: Option<Principal>,
3899 ) -> Result<bool, SuperviseError> {
3900 let (reply_tx, reply_rx) = oneshot::channel();
3901 self.inner
3902 .commands
3903 .send(SupervisorCommand::SetEnabled {
3904 requester,
3905 enabled,
3906 reply: reply_tx,
3907 })
3908 .await
3909 .map_err(|_| SuperviseError::CommandClosed {
3910 module_id: self.inner.module_id.clone(),
3911 })?;
3912 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3913 module_id: self.inner.module_id.clone(),
3914 })?
3915 }
3916
3917 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3921 let configured = self
3922 .inner
3923 .configuration
3924 .lock()
3925 .map_err(|_| SuperviseError::StatePoisoned {
3926 module_id: Some(self.inner.module_id.clone()),
3927 })?
3928 .spec
3929 .protocol;
3930 let state = lock_snapshot(&self.inner.snapshot)?;
3931 Ok(state.spawned_protocol.unwrap_or(configured))
3932 }
3933
3934 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
3935 let configuration =
3936 self.inner
3937 .configuration
3938 .lock()
3939 .map_err(|_| SuperviseError::StatePoisoned {
3940 module_id: Some(self.inner.module_id.clone()),
3941 })?;
3942 Ok((configuration.spec.clone(), configuration.health.clone()))
3943 }
3944
3945 #[cfg(any(test, feature = "test-support"))]
3949 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
3950 let (_, health) = self.configuration()?;
3951 let drain_timeout_ms = u64::try_from(
3952 self.inner
3953 .effective_drain_timeout
3954 .lock()
3955 .unwrap_or_else(|poisoned| poisoned.into_inner())
3956 .as_millis(),
3957 )
3958 .ok();
3959 self.update_configuration(spec, health, drain_timeout_ms)
3960 .await
3961 }
3962
3963 pub(crate) async fn update_configuration(
3964 &self,
3965 spec: ModuleSpec,
3966 health: HealthConfig,
3967 drain_timeout_ms: Option<u64>,
3968 ) -> Result<(), SuperviseError> {
3969 if spec.module_id != self.inner.module_id {
3970 return Err(SuperviseError::InvalidSpec {
3971 reason: "a supervised module's module_id cannot be changed".to_string(),
3972 });
3973 }
3974 validate_spec(&spec)?;
3975 let (reply_tx, reply_rx) = oneshot::channel();
3976 self.inner
3977 .commands
3978 .send(SupervisorCommand::UpdateConfiguration {
3979 spec: spec.clone(),
3980 health: health.clone(),
3981 drain_timeout_ms,
3982 reply: reply_tx,
3983 })
3984 .await
3985 .map_err(|_| SuperviseError::CommandClosed {
3986 module_id: self.inner.module_id.clone(),
3987 })?;
3988 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3989 module_id: self.inner.module_id.clone(),
3990 })?;
3991 let mut configuration =
3992 self.inner
3993 .configuration
3994 .lock()
3995 .map_err(|_| SuperviseError::StatePoisoned {
3996 module_id: Some(self.inner.module_id.clone()),
3997 })?;
3998 configuration.spec = spec;
3999 configuration.health = health;
4000 Ok(())
4001 }
4002}
4003
4004impl Drop for SupervisedModuleInner {
4005 fn drop(&mut self) {
4006 let Ok(mut monitor) = self.monitor.lock() else {
4007 return;
4008 };
4009 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4010 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4011 state.state = ModuleState::Stopped;
4012 clear_current_process_facts(state);
4013 });
4014 monitor.abort();
4015 }
4016 let _ = monitor.take();
4017 }
4018}
4019
4020#[derive(Debug)]
4021enum SupervisorCommand {
4022 Drain {
4023 requester: Option<Principal>,
4024 reply: oneshot::Sender<Result<(), SuperviseError>>,
4025 },
4026 Retire {
4027 requester: Option<Principal>,
4028 reply: oneshot::Sender<Result<(), SuperviseError>>,
4029 },
4030 Restart {
4031 requester: Option<Principal>,
4032 drain_timeout_ms: Option<u64>,
4037 received_at_generation: u64,
4041 queued_at: Instant,
4044 reply: oneshot::Sender<Result<(), SuperviseError>>,
4045 },
4046 Reload {
4047 requester: Option<Principal>,
4048 reply: oneshot::Sender<Result<(), SuperviseError>>,
4049 },
4050 SetEnabled {
4051 requester: Option<Principal>,
4052 enabled: bool,
4053 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4054 },
4055 UpdateConfiguration {
4056 spec: ModuleSpec,
4057 health: HealthConfig,
4058 drain_timeout_ms: Option<u64>,
4061 reply: oneshot::Sender<()>,
4062 },
4063 Swap {
4064 requester: Option<Principal>,
4065 ready_timeout: Option<Duration>,
4068 reply: oneshot::Sender<Result<(), SuperviseError>>,
4070 },
4071}
4072
4073#[derive(Debug)]
4074pub enum SuperviseError {
4075 InvalidSpec {
4076 reason: String,
4077 },
4078 Spawn {
4079 program: PathBuf,
4080 source: io::Error,
4081 cgroup_path: Option<PathBuf>,
4082 },
4083 Cgroup {
4084 module_id: String,
4085 source: io::Error,
4086 },
4087 LaunchNonce {
4090 reason: String,
4091 },
4092 Wait {
4093 module_id: String,
4094 source: io::Error,
4095 },
4096 Kill {
4097 module_id: String,
4098 source: io::Error,
4099 },
4100 Forwarding(ForwardingError),
4101 Registry(RegistryError),
4102 ReloadUnavailable {
4103 module_id: String,
4104 reason: String,
4105 },
4106 Disabled {
4111 module_id: String,
4112 },
4113 ReloadFailed {
4114 module_id: String,
4115 reason: String,
4116 },
4117 RegistrationStillActive {
4118 module_id: String,
4119 waited: Duration,
4120 },
4121 StatePoisoned {
4122 module_id: Option<String>,
4123 },
4124 CommandClosed {
4125 module_id: String,
4126 },
4127 SwapInProgress {
4131 module_id: String,
4132 },
4133 SwapRefused {
4135 module_id: String,
4136 reason: SwapRefusal,
4137 },
4138 SwapFailed {
4142 module_id: String,
4143 arm: SwapFailureArm,
4144 detail: String,
4145 candidate_exit: Option<ExitReport>,
4148 },
4149}
4150
4151#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4153pub enum SwapRefusal {
4154 OverlapExclusive,
4156 NotRegistered,
4159 ProtocolNone,
4162 NotConfigured,
4165 AlreadySwapping,
4167}
4168
4169impl SwapRefusal {
4170 pub fn as_str(self) -> &'static str {
4171 match self {
4172 Self::OverlapExclusive => "overlap_exclusive",
4173 Self::NotRegistered => "not_registered",
4174 Self::ProtocolNone => "protocol_none",
4175 Self::NotConfigured => "not_configured",
4176 Self::AlreadySwapping => "already_swapping",
4177 }
4178 }
4179}
4180
4181#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4184pub enum SwapFailureArm {
4185 SpawnFailed,
4187 NeverRegistered,
4189 NeverReady,
4191 CandidateExited,
4193 CandidateUnhealthy,
4195 Interrupted,
4199 CutoverLost,
4204}
4205
4206impl SwapFailureArm {
4207 pub fn as_str(self) -> &'static str {
4208 match self {
4209 Self::SpawnFailed => "spawn_failed",
4210 Self::NeverRegistered => "never_registered",
4211 Self::NeverReady => "never_ready",
4212 Self::CandidateExited => "candidate_exited",
4213 Self::CandidateUnhealthy => "candidate_unhealthy",
4214 Self::Interrupted => "interrupted",
4215 Self::CutoverLost => "cutover_lost",
4216 }
4217 }
4218}
4219
4220impl fmt::Display for SuperviseError {
4221 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4222 match self {
4223 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4224 Self::Spawn {
4225 program,
4226 source,
4227 cgroup_path: Some(cgroup_path),
4228 } => write!(
4229 f,
4230 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4231 cgroup_path.display(),
4232 program.display()
4233 ),
4234 Self::Spawn {
4235 program,
4236 source,
4237 cgroup_path: None,
4238 } => write!(
4239 f,
4240 "failed to spawn module '{}': {source}",
4241 program.display()
4242 ),
4243 Self::Cgroup { module_id, source } => {
4244 write!(
4245 f,
4246 "failed to prepare cgroup for module '{module_id}': {source}"
4247 )
4248 }
4249 Self::LaunchNonce { reason } => {
4250 write!(
4251 f,
4252 "failed to generate reserved-module launch nonce: {reason}"
4253 )
4254 }
4255 Self::Wait { module_id, source } => {
4256 write!(f, "failed to wait for module '{module_id}': {source}")
4257 }
4258 Self::Kill { module_id, source } => {
4259 write!(f, "failed to kill module '{module_id}': {source}")
4260 }
4261 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4262 Self::Registry(err) => write!(f, "registry error: {err}"),
4263 Self::ReloadUnavailable { module_id, reason } => {
4264 write!(f, "reload unavailable for module '{module_id}': {reason}")
4265 }
4266 Self::Disabled { module_id } => {
4267 write!(
4268 f,
4269 "module '{module_id}' is disabled; enable it before restart or reload"
4270 )
4271 }
4272 Self::ReloadFailed { module_id, reason } => {
4273 write!(f, "reload failed for module '{module_id}': {reason}")
4274 }
4275 Self::RegistrationStillActive { module_id, waited } => write!(
4276 f,
4277 "module '{module_id}' registration remained active after waiting {waited:?}"
4278 ),
4279 Self::StatePoisoned { module_id } => match module_id {
4280 Some(module_id) => {
4281 write!(f, "supervisor state for module '{module_id}' was poisoned")
4282 }
4283 None => write!(f, "supervisor state was poisoned"),
4284 },
4285 Self::CommandClosed { module_id } => {
4286 write!(
4287 f,
4288 "supervisor command channel for module '{module_id}' is closed"
4289 )
4290 }
4291 Self::SwapInProgress { module_id } => write!(
4292 f,
4293 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4294 ),
4295 Self::SwapRefused { module_id, reason } => match reason {
4296 SwapRefusal::OverlapExclusive => write!(
4297 f,
4298 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4299 ),
4300 SwapRefusal::NotRegistered => write!(
4301 f,
4302 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4303 ),
4304 SwapRefusal::ProtocolNone => write!(
4305 f,
4306 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4307 ),
4308 SwapRefusal::NotConfigured => write!(
4309 f,
4310 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4311 ),
4312 SwapRefusal::AlreadySwapping => {
4313 write!(f, "module '{module_id}' is already being swapped")
4314 }
4315 },
4316 Self::SwapFailed {
4317 module_id,
4318 arm,
4319 detail,
4320 ..
4321 } => write!(
4322 f,
4323 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4324 arm.as_str()
4325 ),
4326 }
4327 }
4328}
4329
4330impl Error for SuperviseError {
4331 fn source(&self) -> Option<&(dyn Error + 'static)> {
4332 match self {
4333 Self::Spawn { source, .. }
4334 | Self::Cgroup { source, .. }
4335 | Self::Wait { source, .. }
4336 | Self::Kill { source, .. } => Some(source),
4337 Self::Forwarding(err) => Some(err),
4338 Self::Registry(err) => Some(err),
4339 Self::LaunchNonce { .. }
4340 | Self::InvalidSpec { .. }
4341 | Self::ReloadUnavailable { .. }
4342 | Self::Disabled { .. }
4343 | Self::ReloadFailed { .. }
4344 | Self::RegistrationStillActive { .. }
4345 | Self::StatePoisoned { .. }
4346 | Self::CommandClosed { .. }
4347 | Self::SwapInProgress { .. }
4348 | Self::SwapRefused { .. }
4349 | Self::SwapFailed { .. } => None,
4350 }
4351 }
4352}
4353
4354pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4355 if spec.module_id.trim().is_empty() {
4356 return Err(SuperviseError::InvalidSpec {
4357 reason: "module_id must not be empty".to_string(),
4358 });
4359 }
4360
4361 Ok(())
4362}
4363
4364#[derive(Debug, Default)]
4365struct HealthProbeRuntime {
4366 configured_health: Option<HealthConfig>,
4367 registered_connection: Option<crate::ConnectionId>,
4368 advertised: bool,
4369 next_probe_at: Option<Instant>,
4370 probe_index: u64,
4371}
4372
4373fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4374 lock_snapshot(snapshot)
4375 .ok()
4376 .and_then(|state| state.spawned_protocol)
4377 .unwrap_or(spec.protocol)
4378}
4379
4380impl HealthProbeRuntime {
4381 fn refresh_registration(
4382 &mut self,
4383 spec: &ModuleSpec,
4384 runtime: &SupervisorRuntimeConfig,
4385 registry: &Registry,
4386 snapshot: &SharedSnapshot,
4387 ) {
4388 if self.configured_health.as_ref() != Some(&runtime.health) {
4389 self.configured_health = Some(runtime.health.clone());
4390 self.next_probe_at = None;
4391 self.registered_connection = None;
4392 self.probe_index = 0;
4393 }
4394 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4398 self.registered_connection = None;
4399 self.advertised = runtime.health.http.is_some();
4400 if !self.advertised {
4401 self.next_probe_at = None;
4402 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4403 let unknown = ModuleHealthStatus::default();
4404 if state.health != unknown {
4405 state.health = unknown;
4406 }
4407 });
4408 } else if self.next_probe_at.is_none() {
4409 self.next_probe_at = Some(
4410 Instant::now()
4411 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4412 );
4413 }
4414 return;
4415 }
4416
4417 let registration = match registry.get_module(&spec.module_id) {
4418 Ok(registration) => registration,
4419 Err(err) => {
4420 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4421 self.advertised = false;
4422 self.next_probe_at = None;
4423 return;
4424 }
4425 };
4426
4427 let Some(registration) = registration else {
4428 self.registered_connection = None;
4429 self.advertised = false;
4430 self.next_probe_at = None;
4431 return;
4432 };
4433
4434 let advertised = registration
4435 .control_ops
4436 .iter()
4437 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4438 if !advertised {
4439 self.registered_connection = Some(registration.connection_id);
4440 self.advertised = false;
4441 self.next_probe_at = None;
4442 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4443 if state.health.status != SupervisorHealthStatus::Unknown
4444 || state.health.consecutive_failures != 0
4445 || state.health.last_probe_ms.is_some()
4446 || state.health.detail.is_some()
4447 || state.health.metrics.is_some()
4448 {
4449 state.health.status = SupervisorHealthStatus::Unknown;
4450 state.health.consecutive_failures = 0;
4451 state.health.last_probe_ms = None;
4452 state.health.detail = None;
4453 state.health.metrics = None;
4454 }
4455 });
4456 return;
4457 }
4458
4459 let reregistered = self.registered_connection != Some(registration.connection_id);
4460 self.registered_connection = Some(registration.connection_id);
4461 self.advertised = true;
4462 if reregistered || self.next_probe_at.is_none() {
4463 self.probe_index = 0;
4464 self.next_probe_at = Some(
4465 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4466 );
4467 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4468 state.health.status = SupervisorHealthStatus::Unknown;
4469 state.health.consecutive_failures = 0;
4470 state.health.detail = None;
4471 state.health.metrics = None;
4472 });
4473 }
4474 }
4475
4476 fn wake_after(&self) -> Option<Duration> {
4477 if !self.advertised {
4478 return None;
4479 }
4480 self.next_probe_at
4481 .map(|next| next.saturating_duration_since(Instant::now()))
4482 }
4483
4484 fn due(&self) -> bool {
4485 self.advertised
4486 && self
4487 .next_probe_at
4488 .is_some_and(|next| Instant::now() >= next)
4489 }
4490
4491 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4492 self.probe_index = self.probe_index.wrapping_add(1);
4493 self.next_probe_at = Some(
4494 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4495 );
4496 }
4497}
4498
4499#[derive(Debug)]
4534enum HealthProbeEvidence {
4535 LaneDead,
4537 NoAnswer,
4539 BadAnswer,
4541 Misconfigured,
4543}
4544
4545#[derive(Debug)]
4546struct HealthProbeError {
4547 evidence: HealthProbeEvidence,
4548 message: String,
4549}
4550
4551impl HealthProbeError {
4552 fn lane_dead(message: impl Into<String>) -> Self {
4553 Self::with(HealthProbeEvidence::LaneDead, message)
4554 }
4555
4556 fn no_answer(message: impl Into<String>) -> Self {
4557 Self::with(HealthProbeEvidence::NoAnswer, message)
4558 }
4559
4560 fn bad_answer(message: impl Into<String>) -> Self {
4561 Self::with(HealthProbeEvidence::BadAnswer, message)
4562 }
4563
4564 fn misconfigured(message: impl Into<String>) -> Self {
4565 Self::with(HealthProbeEvidence::Misconfigured, message)
4566 }
4567
4568 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4569 Self {
4570 evidence,
4571 message: message.into(),
4572 }
4573 }
4574
4575 #[allow(dead_code)]
4589 fn is_proof_of_death(&self) -> bool {
4590 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4591 }
4592
4593 fn label(&self) -> &'static str {
4601 match self.evidence {
4602 HealthProbeEvidence::LaneDead => "lane-dead",
4603 HealthProbeEvidence::NoAnswer => "no-answer",
4604 HealthProbeEvidence::BadAnswer => "bad-answer",
4605 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4606 }
4607 }
4608}
4609
4610impl fmt::Display for HealthProbeError {
4611 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4612 f.write_str(&self.message)
4613 }
4614}
4615
4616async fn run_health_probe_cycle(
4617 spec: &ModuleSpec,
4618 runtime: &SupervisorRuntimeConfig,
4619 registry: &Registry,
4620 process_liveness: &SupervisorProcessLiveness,
4621 snapshot: &SharedSnapshot,
4622 child: &mut Option<SupervisedChild>,
4623) {
4624 let now_ms = unix_ms_now();
4625 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4626 .then_some(runtime.health.http.as_deref())
4627 .flatten();
4628 let result = match http {
4629 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4630 None => probe_module_health(&spec.module_id, runtime, None).await,
4631 };
4632 match result {
4633 Ok(report) => {
4634 handle_health_report(
4635 spec,
4636 runtime,
4637 registry,
4638 process_liveness,
4639 snapshot,
4640 child,
4641 report,
4642 now_ms,
4643 )
4644 .await;
4645 }
4646 Err(err) => {
4647 if http.is_some() {
4648 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4649 state.health.status = SupervisorHealthStatus::Failing;
4650 });
4651 }
4652 handle_health_probe_failure(
4653 spec,
4654 runtime,
4655 registry,
4656 process_liveness,
4657 snapshot,
4658 child,
4659 err,
4660 now_ms,
4661 )
4662 .await;
4663 }
4664 }
4665}
4666
4667pub(crate) struct HttpProbeTarget<'a> {
4668 address: std::net::SocketAddr,
4669 localhost: bool,
4670 authority: &'a str,
4671 path: String,
4672}
4673
4674pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4677 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4678 return Err("must not contain whitespace, controls, or a fragment".into());
4679 }
4680 let rest = url
4681 .strip_prefix("http://")
4682 .ok_or("must use plain http://")?;
4683 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4684 let (authority, suffix) = rest.split_at(split);
4685 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4686 ("::1", rest)
4687 } else {
4688 let split = authority.find(':').unwrap_or(authority.len());
4689 authority.split_at(split)
4690 };
4691 let ip: std::net::IpAddr = match host {
4692 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4693 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4694 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4695 };
4696 let port = if port.is_empty() {
4697 80
4698 } else {
4699 port.strip_prefix(':')
4700 .and_then(|p| p.parse::<u16>().ok())
4701 .filter(|p| *p > 0)
4702 .ok_or("must have a valid nonzero TCP port")?
4703 };
4704 let path = if suffix.is_empty() {
4705 "/".into()
4706 } else if suffix.starts_with('?') {
4707 format!("/{suffix}")
4708 } else {
4709 suffix.into()
4710 };
4711 Ok(HttpProbeTarget {
4712 address: std::net::SocketAddr::new(ip, port),
4713 localhost: host == "localhost",
4714 authority,
4715 path,
4716 })
4717}
4718
4719async fn probe_http_health(
4720 url: &str,
4721 deadline: Duration,
4722) -> Result<HealthReport, HealthProbeError> {
4723 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4724 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4725 let mut response_status = String::new();
4728 let mut body = Vec::new();
4729 let probe = async {
4730 let connection = match tokio::net::TcpStream::connect(target.address).await {
4733 Err(_) if target.localhost => {
4734 tokio::net::TcpStream::connect((
4735 std::net::Ipv6Addr::LOCALHOST,
4736 target.address.port(),
4737 ))
4738 .await
4739 }
4740 result => result,
4741 };
4742 let mut stream = connection.map_err(|error| {
4743 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4744 })?;
4745 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4746 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4747 let mut reader = BufReader::new(stream);
4748 let mut budget = 16 * 1024;
4749 let status = http_line(&mut reader, &mut budget).await?;
4750 let mut words = status.split_ascii_whitespace();
4751 let version = words.next();
4752 let code = words
4753 .next()
4754 .filter(|word| word.len() == 3)
4755 .and_then(|word| word.parse::<u16>().ok());
4756 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4757 || !code.is_some_and(|code| (100..600).contains(&code))
4758 {
4759 return Err(HealthProbeError::bad_answer(format!(
4760 "invalid HTTP status: {status}"
4761 )));
4762 }
4763 let code = code.expect("validated status code");
4764 response_status = status.clone();
4765 let mut length = None;
4766 let mut chunked = false;
4767 loop {
4768 let line = http_line(&mut reader, &mut budget).await?;
4769 if line.is_empty() {
4770 break;
4771 }
4772 if let Some((name, value)) = line.split_once(':') {
4773 if name.eq_ignore_ascii_case("content-length") {
4774 length = Some(value.trim().parse::<u64>().map_err(|_| {
4775 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4776 })?);
4777 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4778 chunked = value.trim().eq_ignore_ascii_case("chunked");
4779 }
4780 }
4781 }
4782 if chunked {
4783 while body.len() < 200 {
4784 let line = http_line(&mut reader, &mut budget).await?;
4785 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4786 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4787 if size == 0 {
4788 break;
4789 }
4790 let count = size.min((200 - body.len()) as u64) as usize;
4791 let start = body.len();
4792 (&mut reader)
4793 .take(count as u64)
4794 .read_to_end(&mut body)
4795 .await
4796 .map_err(|error| {
4797 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4798 })?;
4799 if body.len() - start != count {
4800 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4801 }
4802 if size > count as u64 || body.len() == 200 {
4803 break;
4804 }
4805 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4806 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4807 }
4808 }
4809 } else {
4810 reader
4811 .take(length.unwrap_or(200).min(200))
4812 .read_to_end(&mut body)
4813 .await
4814 .map_err(|error| {
4815 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4816 })?;
4817 }
4818 if (200..300).contains(&code) {
4819 Ok(HealthReport::ok())
4820 } else {
4821 Err(HealthProbeError::bad_answer(
4822 "HTTP health endpoint returned non-2xx",
4823 ))
4824 }
4825 };
4826 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4827 Err(HealthProbeError::no_answer(format!(
4828 "HTTP probe timed out after {deadline:?}"
4829 )))
4830 });
4831 if let Err(error) = &mut result {
4832 if !response_status.is_empty() {
4833 error.message = format!(
4834 "{}; {response_status}: {}",
4835 error.message,
4836 String::from_utf8_lossy(&body)
4837 );
4838 }
4839 }
4840 result
4841}
4842
4843async fn http_line(
4844 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4845 remaining: &mut usize,
4846) -> Result<String, HealthProbeError> {
4847 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4848 let mut line = Vec::new();
4849 (&mut *reader)
4850 .take(*remaining as u64)
4851 .read_until(b'\n', &mut line)
4852 .await
4853 .map_err(|error| {
4854 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4855 })?;
4856 *remaining -= line.len();
4857 if !line.ends_with(b"\r\n") {
4858 return Err(HealthProbeError::bad_answer(
4859 "HTTP headers are incomplete or exceed 16 KiB",
4860 ));
4861 }
4862 line.truncate(line.len() - 2);
4863 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4864}
4865
4866async fn probe_module_health(
4867 module_id: &str,
4868 runtime: &SupervisorRuntimeConfig,
4869 drain_deadline: Option<Instant>,
4870) -> Result<HealthReport, HealthProbeError> {
4871 let Some(forwarding) = runtime.forwarding.as_ref() else {
4872 return Err(HealthProbeError::misconfigured(
4873 "supervisor was not configured with a forwarding table",
4874 ));
4875 };
4876 let probe_started_at = Instant::now();
4877 let mut deadline = probe_started_at + runtime.health.deadline;
4878 if let Some(drain_deadline) = drain_deadline {
4879 deadline = deadline.min(drain_deadline);
4880 }
4881 let pending = if drain_deadline.is_some() {
4882 forwarding.begin_drain_health_probe_rpc_for(
4883 module_id,
4884 MODULE_CONTROL_OP_HEALTH_CHECK,
4885 probe_started_at,
4886 deadline,
4887 )
4888 } else {
4889 forwarding.begin_health_probe_rpc_for(
4890 module_id,
4891 MODULE_CONTROL_OP_HEALTH_CHECK,
4892 probe_started_at,
4893 deadline,
4894 )
4895 }
4896 .map_err(|err| {
4897 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4900 })?;
4901 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4902}
4903
4904async fn probe_endpoint_health(
4911 endpoint: crate::ModuleEndpointId,
4912 runtime: &SupervisorRuntimeConfig,
4913 deadline_cap: Option<Instant>,
4914) -> Result<HealthReport, HealthProbeError> {
4915 let Some(forwarding) = runtime.forwarding.as_ref() else {
4916 return Err(HealthProbeError::misconfigured(
4917 "supervisor was not configured with a forwarding table",
4918 ));
4919 };
4920 let probe_started_at = Instant::now();
4921 let mut deadline = probe_started_at + runtime.health.deadline;
4922 if let Some(cap) = deadline_cap {
4923 deadline = deadline.min(cap);
4924 }
4925 let pending = forwarding
4926 .begin_endpoint_health_probe_rpc_for(
4927 endpoint,
4928 MODULE_CONTROL_OP_HEALTH_CHECK,
4929 probe_started_at,
4930 deadline,
4931 )
4932 .map_err(|err| {
4933 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4934 })?;
4935 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4936}
4937
4938async fn await_health_probe(
4940 forwarding: &ForwardingTable,
4941 pending: PendingModuleControlRpc,
4942 deadline: Instant,
4943 probe_budget: Duration,
4944) -> Result<HealthReport, HealthProbeError> {
4945 let PendingModuleControlRpc {
4946 endpoint,
4947 module_sink,
4948 negotiated_ver,
4949 corr,
4950 receiver,
4951 } = pending;
4952 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4953 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
4954 })?;
4955 let frame = Frame::build_with_version(
4956 negotiated_ver,
4957 FrameType::Request,
4958 control_flags(),
4959 0,
4960 0,
4961 corr,
4962 body,
4963 )
4964 .map_err(|err| {
4965 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
4966 })?;
4967
4968 match timeout_at(deadline, module_sink.send(frame)).await {
4974 Ok(Ok(())) => {}
4975 Ok(Err(err)) => {
4976 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
4977 return Err(HealthProbeError::lane_dead(format!(
4980 "failed to send health.check: {err}"
4981 )));
4982 }
4983 Err(_elapsed) => {
4984 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
4985 return Err(HealthProbeError::no_answer(
4989 "health.check send timed out before enqueue (module egress full)",
4990 ));
4991 }
4992 }
4993
4994 match timeout_at(deadline, receiver).await {
4995 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4999 response.health_report().ok_or_else(|| {
5000 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5001 })
5002 }
5003 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5004 format!("health.check rejected: {}", body.message),
5005 )),
5006 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5007 Err(HealthProbeError::lane_dead(message))
5008 }
5009 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5010 Err(HealthProbeError::bad_answer(message))
5011 }
5012 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5013 Err(HealthProbeError::bad_answer(format!(
5014 "expected module-control op '{expected}', got '{actual}'"
5015 )))
5016 }
5017 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5021 "module answered health.check after its daemon deadline",
5022 )),
5023 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5024 "health.check waiter was canceled before the module responded",
5025 )),
5026 Err(_) => {
5027 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5028 Err(HealthProbeError::no_answer(format!(
5029 "module did not answer health.check within {probe_budget:?}"
5030 )))
5031 }
5032 }
5033}
5034
5035#[allow(clippy::too_many_arguments)]
5036async fn handle_health_report(
5037 spec: &ModuleSpec,
5038 runtime: &SupervisorRuntimeConfig,
5039 registry: &Registry,
5040 process_liveness: &SupervisorProcessLiveness,
5041 snapshot: &SharedSnapshot,
5042 child: &mut Option<SupervisedChild>,
5043 report: HealthReport,
5044 now_ms: u64,
5045) {
5046 let status = supervisor_health_status(report.status);
5047 let detail = report.detail.clone();
5048 let metrics = truncate_health_metrics(report.metrics);
5049 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5050 state.health.status = status;
5051 state.health.last_probe_ms = Some(now_ms);
5052 state.health.detail = detail.clone();
5053 state.health.metrics = metrics.clone();
5054 state.health.consecutive_failures = 0;
5055 });
5056
5057 let action = match report.status {
5058 HealthStatus::Ok => return,
5059 HealthStatus::Degraded => runtime.health.on_degraded,
5060 HealthStatus::Failing => runtime.health.on_failing,
5061 };
5062 apply_l3_health_action(
5063 spec,
5064 runtime,
5065 registry,
5066 process_liveness,
5067 snapshot,
5068 child,
5069 status,
5070 detail.as_deref(),
5071 action,
5072 now_ms,
5073 )
5074 .await;
5075}
5076
5077#[allow(clippy::too_many_arguments)]
5078async fn handle_health_probe_failure(
5079 spec: &ModuleSpec,
5080 runtime: &SupervisorRuntimeConfig,
5081 registry: &Registry,
5082 process_liveness: &SupervisorProcessLiveness,
5083 snapshot: &SharedSnapshot,
5084 child: &mut Option<SupervisedChild>,
5085 err: HealthProbeError,
5086 now_ms: u64,
5087) {
5088 let threshold = runtime.health.failure_threshold.max(1);
5089 let mut failures = 0;
5090 let detail = format!("[{}] {err}", err.label());
5095 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5096 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5099 state.health.status = SupervisorHealthStatus::Unknown;
5100 }
5101 state.health.last_probe_ms = Some(now_ms);
5102 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5103 state.health.detail = Some(detail.clone());
5104 state.health.metrics = None;
5105 failures = state.health.consecutive_failures;
5106 });
5107
5108 if failures < threshold {
5109 warn!(
5110 module_id = %spec.module_id,
5111 consecutive_failures = failures,
5112 threshold,
5113 evidence = err.label(),
5114 detail = %detail,
5115 "health.check probe failed"
5116 );
5117 return;
5118 }
5119
5120 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5121 state.state = ModuleState::Unresponsive;
5122 state.health.status = SupervisorHealthStatus::Unresponsive;
5123 });
5124 if runtime.health.critical {
5128 error!(
5129 module_id = %spec.module_id,
5130 status = "unresponsive",
5131 evidence = err.label(),
5132 detail = %detail,
5133 "critical module health alert"
5134 );
5135 } else {
5136 warn!(
5137 module_id = %spec.module_id,
5138 status = "unresponsive",
5139 evidence = err.label(),
5140 detail = %detail,
5141 "module health threshold breached"
5142 );
5143 }
5144 if let Err(err) = health_restart_child(
5145 spec,
5146 runtime,
5147 registry,
5148 process_liveness,
5149 snapshot,
5150 child,
5151 SupervisorHealthStatus::Unresponsive,
5152 Some(&detail),
5153 now_ms,
5154 )
5155 .await
5156 {
5157 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5158 }
5159}
5160
5161#[allow(clippy::too_many_arguments)]
5162async fn apply_l3_health_action(
5163 spec: &ModuleSpec,
5164 runtime: &SupervisorRuntimeConfig,
5165 registry: &Registry,
5166 process_liveness: &SupervisorProcessLiveness,
5167 snapshot: &SharedSnapshot,
5168 child: &mut Option<SupervisedChild>,
5169 status: SupervisorHealthStatus,
5170 detail: Option<&str>,
5171 action: HealthAction,
5172 now_ms: u64,
5173) {
5174 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5175 match action {
5176 HealthAction::Report => {
5177 info!(
5178 module_id = %spec.module_id,
5179 status = ?status,
5180 detail,
5181 "module reported non-ok health"
5182 );
5183 }
5184 HealthAction::Alert => {
5185 error!(
5186 module_id = %spec.module_id,
5187 status = ?status,
5188 detail,
5189 "module health alert"
5190 );
5191 }
5192 HealthAction::Restart => {
5193 if let Err(err) = health_restart_child(
5194 spec,
5195 runtime,
5196 registry,
5197 process_liveness,
5198 snapshot,
5199 child,
5200 status,
5201 detail,
5202 now_ms,
5203 )
5204 .await
5205 {
5206 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5207 }
5208 }
5209 }
5210}
5211
5212#[allow(clippy::too_many_arguments)]
5213async fn health_restart_child(
5214 spec: &ModuleSpec,
5215 runtime: &SupervisorRuntimeConfig,
5216 registry: &Registry,
5217 process_liveness: &SupervisorProcessLiveness,
5218 snapshot: &SharedSnapshot,
5219 child: &mut Option<SupervisedChild>,
5220 status: SupervisorHealthStatus,
5221 detail: Option<&str>,
5222 now_ms: u64,
5223) -> Result<(), SuperviseError> {
5224 let (enabled, schedule) = {
5225 let mut state = lock_snapshot(snapshot)?;
5226 let enabled = state.enabled;
5227 let schedule = if enabled {
5228 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5229 } else {
5230 None
5231 };
5232 (enabled, schedule)
5233 };
5234
5235 if !enabled {
5236 return Err(SuperviseError::Disabled {
5237 module_id: spec.module_id.clone(),
5238 });
5239 }
5240
5241 if schedule.is_none() {
5242 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5243 error!(
5244 module_id = %spec.module_id,
5245 status = ?status,
5246 detail,
5247 max_restarts = runtime.restart_policy.max_restarts,
5248 window_secs = runtime.restart_policy.window.as_secs(),
5249 reason = %runtime.restart_policy.budget_exhausted_detail(),
5250 "health restart budget exhausted; marking module failed"
5251 );
5252 let stop_notice = begin_forwarding_drain_if_configured(
5253 spec,
5254 runtime,
5255 registry,
5256 snapshot,
5257 Some(true),
5258 RouteCloseReason::Disable,
5259 )
5260 .await?;
5261 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5262 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5263 })?;
5264 drain_optional_child(
5265 &spec.module_id,
5266 spec.protocol,
5267 stop_notice,
5268 registry,
5269 runtime.forwarding.as_deref(),
5270 snapshot,
5271 &runtime.terminal_ring,
5272 &runtime.spawn_events,
5273 child,
5274 runtime.drain_timeout,
5275 ModuleState::Failed,
5276 Some(true),
5277 )
5278 .await?;
5279 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5280 return Ok(());
5281 }
5282
5283 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5284 let mut restart_count = 0;
5285 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5286 restart_count = state.crash_restarts.len();
5287 state.state = ModuleState::Unresponsive;
5288 state.health.status = status;
5289 state.health.last_action = Some(HealthAction::Restart.to_string());
5290 state.health.last_action_ms = Some(now_ms);
5291 })?;
5292 warn!(
5293 module_id = %spec.module_id,
5294 status = ?status,
5295 detail,
5296 restart_count,
5297 restart_in_window = schedule.restart_in_window,
5298 delay_ms = schedule.delay.as_millis() as u64,
5299 "health-triggered module restart"
5300 );
5301
5302 let stop_notice = begin_forwarding_drain_if_configured(
5303 spec,
5304 runtime,
5305 registry,
5306 snapshot,
5307 Some(true),
5308 RouteCloseReason::Restart,
5309 )
5310 .await?;
5311 drain_optional_child(
5312 &spec.module_id,
5313 spec.protocol,
5314 stop_notice,
5315 registry,
5316 runtime.forwarding.as_deref(),
5317 snapshot,
5318 &runtime.terminal_ring,
5319 &runtime.spawn_events,
5320 child,
5321 runtime.drain_timeout,
5322 ModuleState::Restarting,
5323 Some(true),
5324 )
5325 .await?;
5326 schedule_respawn(
5327 runtime,
5328 snapshot,
5329 &spec.module_id,
5330 schedule.delay,
5331 RespawnKind::Spawn,
5332 )
5333}
5334
5335fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5336 if let Some(reply) = runtime
5337 .deferred_reload_reply
5338 .lock()
5339 .unwrap_or_else(|p| p.into_inner())
5340 .take()
5341 {
5342 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5343 module_id: module_id.to_string(),
5344 reason: reason.to_string(),
5345 }));
5346 }
5347}
5348
5349fn schedule_respawn(
5350 runtime: &SupervisorRuntimeConfig,
5351 snapshot: &SharedSnapshot,
5352 module_id: &str,
5353 delay: Duration,
5354 kind: RespawnKind,
5355) -> Result<(), SuperviseError> {
5356 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5357 update_snapshot(snapshot, Some(module_id), |state| {
5358 state.respawn_pending = true
5359 })?;
5360 *runtime
5361 .scheduled_respawn
5362 .lock()
5363 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5364 deadline: Instant::now() + delay,
5365 kind,
5366 });
5367 Ok(())
5368}
5369
5370fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5371 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5372 state.health.last_action = Some(action);
5373 state.health.last_action_ms = Some(now_ms);
5374 });
5375}
5376
5377fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5378 match status {
5379 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5380 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5381 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5382 }
5383}
5384
5385fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5397 let metrics = metrics?;
5398 match serde_json::to_vec(&metrics) {
5399 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5400 "truncated": true,
5401 "original_bytes": encoded.len(),
5402 })),
5403 Ok(_) | Err(_) => Some(metrics),
5404 }
5405}
5406
5407fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5413 if cadence.is_zero() {
5414 return Duration::ZERO;
5415 }
5416 let cadence_ms = cadence.as_millis() as u64;
5417 if cadence_ms == 0 {
5433 return cadence;
5434 }
5435 let jitter_span = (cadence_ms / 10).max(1);
5450 let hash = module_id.as_bytes().iter().fold(
5451 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5452 |acc, byte| {
5453 acc.wrapping_mul(1099511628211)
5454 .wrapping_add(u64::from(*byte))
5455 },
5456 );
5457 cadence + Duration::from_millis(hash % jitter_span)
5458}
5459
5460#[cfg(test)]
5461mod tests {
5462 use super::*;
5463
5464 #[test]
5465 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5466 let handle = SupervisorHandle::new();
5467 let module_id = "readded-tombstone";
5468 handle.record_rescan_removal(module_id);
5469 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5470
5471 handle.apply_identity_configuration(&ModuleSpec {
5472 module_id: module_id.to_string(),
5473 program: PathBuf::from("/test/module"),
5474 args: Vec::new(),
5475 env: Vec::new(),
5476 reserved: false,
5477 reserved_prefixes: Vec::new(),
5478 protocol: ModuleProtocol::Subc,
5479 overlap: Default::default(),
5480 });
5481
5482 assert!(
5483 handle.removal_tombstone_age_ms(module_id).is_none(),
5484 "a re-added module must not retain a stale removal tombstone"
5485 );
5486 }
5487
5488 #[derive(Debug, PartialEq, Eq)]
5491 struct OwnerInSpawnWindow {
5492 module_id: String,
5493 configured: bool,
5494 on_roster: bool,
5495 admission_refusal: Option<&'static str>,
5496 }
5497
5498 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5509 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5510 use crate::scopes::ScopeTable;
5511 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5512
5513 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5514 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5515 module_id: module_id.to_string(),
5516 program,
5517 args: Vec::new(),
5518 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5519 .into_iter()
5520 .map(|key| (key.to_string(), dir.path().display().to_string()))
5521 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5522 .collect(),
5523 reserved: false,
5524 reserved_prefixes: Vec::new(),
5525 protocol: ModuleProtocol::Subc,
5526 overlap: Default::default(),
5527 };
5528 let live = super::terminal_history_tests::fake_aft_stub_path();
5529 let missing = dir.path().join("definitely-missing-module");
5530
5531 let handle = SupervisorHandle::new();
5532 let mut supervisor =
5533 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5534 .with_handle(handle.clone());
5535 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5536 let hook_handle = handle.clone();
5537 let hook_observed = Arc::clone(&observed);
5538 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5539 let configured = hook_handle.is_configured(module_id);
5542 let selector = ScopeSelector::new(
5543 Principal::Reserved {
5544 module_id: module_id.to_string(),
5545 },
5546 "s".to_string(),
5547 Some(1),
5548 );
5549 let carrier = Principal::Reserved {
5550 module_id: "carrier".to_string(),
5551 };
5552 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5553 .admit(&carrier, module_id, &selector, configured)
5554 {
5555 Ok(_) => None,
5556 Err(refusal) => Some(refusal.code),
5557 };
5558 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5559 module_id: module_id.to_string(),
5560 configured,
5561 on_roster: hook_handle.get(module_id).is_some(),
5562 admission_refusal,
5563 });
5564 })));
5565
5566 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5567 let configured = supervisor
5568 .supervise_configured(stub("configured", live.clone()), true)
5569 .unwrap();
5570 let with_health = supervisor
5571 .supervise_configured_with_health(
5572 stub("with-health", live.clone()),
5573 true,
5574 HealthConfig::default(),
5575 None,
5576 RestartPolicy::default(),
5577 )
5578 .unwrap();
5579 let failed = supervisor
5582 .supervise_configured_with_health(
5583 stub("failed-spawn", missing.clone()),
5584 true,
5585 HealthConfig::default(),
5586 None,
5587 RestartPolicy::default(),
5588 )
5589 .unwrap();
5590 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5593
5594 let expected = [
5595 "plain",
5596 "configured",
5597 "with-health",
5598 "failed-spawn",
5599 "spawn-error",
5600 ]
5601 .into_iter()
5602 .map(|module_id| OwnerInSpawnWindow {
5603 module_id: module_id.to_string(),
5604 configured: true,
5605 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5606 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5607 })
5608 .collect::<Vec<_>>();
5609 assert_eq!(*observed.lock().unwrap(), expected);
5610
5611 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5612 assert!(
5613 handle.get(module_id).is_some(),
5614 "{module_id} is on the roster"
5615 );
5616 assert!(
5617 handle.is_configured(module_id),
5618 "{module_id} stays configured"
5619 );
5620 }
5621 assert!(handle.get("spawn-error").is_none());
5622 assert!(
5623 !handle.is_configured("spawn-error"),
5624 "a plain spawn that failed must not leave its module marked configured"
5625 );
5626
5627 handle.retire("failed-spawn");
5629 assert!(!handle.is_configured("failed-spawn"));
5630
5631 for module in [plain, configured, with_health] {
5632 module.stop().await.unwrap();
5633 }
5634 drop(failed);
5635 }
5636
5637 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5638 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5639 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5640 snapshot.process_alive = true;
5641 snapshot.pid = Some(41);
5642 snapshot.spawned_at_ms = Some(42);
5643 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5644 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5645 device: 43,
5646 inode: 44,
5647 #[cfg(windows)]
5648 windows_image: None,
5649 });
5650 })
5651 .unwrap();
5652 snapshot
5653 }
5654
5655 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5656 let snapshot = lock_snapshot(snapshot).unwrap();
5657 assert!(!snapshot.process_alive);
5658 assert_eq!(snapshot.pid, None);
5659 assert_eq!(snapshot.spawned_at_ms, None);
5660 assert_eq!(snapshot.spawned_from, None);
5661 assert_eq!(snapshot.spawned_file_identity, None);
5662 }
5663
5664 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5665 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5666 let supervisor =
5667 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5668 let mut runtime = supervisor.runtime_config();
5669 runtime.test_seed_stale_facts_before_enable_spawn = true;
5670 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5671 let mut child = None;
5672 let spec = ModuleSpec {
5673 module_id: "failed-enable-clears-facts".to_string(),
5674 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5675 args: Vec::new(),
5676 env: Vec::new(),
5677 reserved: false,
5678 reserved_prefixes: Vec::new(),
5679 protocol: ModuleProtocol::Subc,
5680 overlap: Default::default(),
5681 };
5682
5683 let result = set_child_enabled(
5684 &spec,
5685 &runtime,
5686 &supervisor.registry,
5687 &supervisor.process_liveness,
5688 &snapshot,
5689 &mut child,
5690 true,
5691 )
5692 .await;
5693
5694 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5695 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5696 assert_snapshot_process_facts_cleared(&snapshot);
5697 }
5698
5699 #[tokio::test]
5700 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5701 let supervisor =
5702 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5703 let runtime = supervisor.runtime_config();
5704 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5705 ModuleState::Restarting,
5706 true,
5707 )));
5708 let spec = ModuleSpec {
5709 module_id: "start-stranded-restarting".to_string(),
5710 program: super::terminal_history_tests::fake_aft_stub_path(),
5711 args: Vec::new(),
5712 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5713 reserved: false,
5714 reserved_prefixes: Vec::new(),
5715 protocol: ModuleProtocol::None,
5716 overlap: Default::default(),
5717 };
5718 let mut child = None;
5719 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5720 assert!(!super::set_child_enabled(
5721 &spec,
5722 &runtime,
5723 &Registry::default(),
5724 &supervisor.process_liveness,
5725 &snapshot,
5726 &mut child,
5727 true
5728 )
5729 .await
5730 .unwrap());
5731 assert!(child.is_none());
5732 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5733 assert!(super::set_child_enabled(
5734 &spec,
5735 &runtime,
5736 &Registry::default(),
5737 &supervisor.process_liveness,
5738 &snapshot,
5739 &mut child,
5740 true
5741 )
5742 .await
5743 .unwrap());
5744 assert_eq!(
5745 lock_snapshot(&snapshot).unwrap().state,
5746 ModuleState::Running
5747 );
5748 let mut child = child.unwrap();
5749 child.start_kill().unwrap();
5750 child.wait().await.unwrap();
5751 }
5752
5753 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5754 async fn failed_reload_spawn_clears_current_process_facts() {
5755 let supervisor =
5756 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5757 let mut runtime = supervisor.runtime_config();
5758 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5759 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5760 let mut child = None;
5761 let spec = ModuleSpec {
5762 module_id: "failed-reload-clears-facts".to_string(),
5763 program: PathBuf::from("/unused/failed-reload-module"),
5764 args: Vec::new(),
5765 env: Vec::new(),
5766 reserved: false,
5767 reserved_prefixes: Vec::new(),
5768 protocol: ModuleProtocol::Subc,
5769 overlap: Default::default(),
5770 };
5771
5772 let result = handle_reload_spawn_failure(
5773 &spec,
5774 &runtime,
5775 &supervisor.process_liveness,
5776 &snapshot,
5777 &mut child,
5778 "forced reload spawn failure".to_string(),
5779 )
5780 .await;
5781
5782 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5783 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5784 assert_snapshot_process_facts_cleared(&snapshot);
5785 }
5786
5787 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5788 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5789 let supervisor =
5790 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5791 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5792 let module = supervisor.supervised_module(
5793 ModuleSpec {
5794 module_id: "drop-clears-facts".to_string(),
5795 program: PathBuf::from("/unused/drop-module"),
5796 args: Vec::new(),
5797 env: Vec::new(),
5798 reserved: false,
5799 reserved_prefixes: Vec::new(),
5800 protocol: ModuleProtocol::Subc,
5801 overlap: Default::default(),
5802 },
5803 supervisor.runtime_config(),
5804 Arc::clone(&snapshot),
5805 None,
5806 );
5807 assert!(!module
5808 .inner
5809 .monitor
5810 .lock()
5811 .unwrap()
5812 .as_ref()
5813 .unwrap()
5814 .is_finished());
5815
5816 drop(module);
5817
5818 assert_eq!(
5819 lock_snapshot(&snapshot).unwrap().state,
5820 ModuleState::Stopped
5821 );
5822 assert_snapshot_process_facts_cleared(&snapshot);
5823 }
5824
5825 #[cfg(unix)]
5826 #[tokio::test]
5827 async fn rescan_preserves_running_protocol_until_respawn() {
5828 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5829 let initial = ModuleSpec {
5830 module_id: "rescan-protocol".into(),
5831 program: PathBuf::from("/bin/sleep"),
5832 args: vec!["60".into()],
5833 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5834 .into_iter()
5835 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5836 .collect(),
5837 reserved: false,
5838 reserved_prefixes: vec![],
5839 protocol: ModuleProtocol::None,
5840 overlap: Default::default(),
5841 };
5842 let supervisor =
5843 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5844 let module = supervisor.spawn(initial.clone()).unwrap();
5845 assert!(module.status().unwrap().live);
5846 let mut next = initial;
5847 next.protocol = ModuleProtocol::Subc;
5848 module
5849 .update_configuration(next.clone(), HealthConfig::default(), None)
5850 .await
5851 .unwrap();
5852 assert!(
5853 module.status().unwrap().live,
5854 "rescan must not require HELLO from the old non-wire process"
5855 );
5856 let runtime = supervisor.runtime_config();
5857 let action = on_child_exit(
5858 &next,
5859 RestartPolicy::default(),
5860 &supervisor.registry,
5861 &module.inner.snapshot,
5862 &runtime.terminal_ring,
5863 &runtime.spawn_events,
5864 &runtime.child_roster,
5865 ExitReport {
5866 kind: ExitKind::Clean,
5867 code: Some(0),
5868 signal: None,
5869 at_ms: unix_ms_now(),
5870 },
5871 )
5872 .await;
5873 assert!(
5874 matches!(action, NextAction::Restart { .. }),
5875 "the old non-wire process's clean exit must restart"
5876 );
5877 module.drain().await.unwrap();
5878 }
5879
5880 #[cfg(unix)]
5881 #[tokio::test]
5882 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
5883 use std::os::unix::fs::PermissionsExt;
5884 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
5885 let script = dir.join("module.sh");
5886 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
5887 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
5888 let record_path = dir.join("live-children.json");
5889 let supervisor =
5890 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5891 .with_live_children_record(&record_path);
5892 for (program, args) in [
5893 (PathBuf::from("sleep"), vec!["60".into()]),
5894 (script, vec![]),
5895 ] {
5896 let spec = ModuleSpec {
5897 module_id: "image-identity".into(),
5898 program: program.clone(),
5899 args,
5900 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5901 .into_iter()
5902 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
5903 .collect(),
5904 reserved: false,
5905 reserved_prefixes: vec![],
5906 protocol: ModuleProtocol::None,
5907 overlap: Default::default(),
5908 };
5909 let module = supervisor.spawn(spec).unwrap();
5910 #[cfg(target_os = "macos")]
5911 {
5912 let deadline = Instant::now() + Duration::from_secs(5);
5915 while crate::live_children::read_record(&record_path)
5916 .unwrap()
5917 .iter()
5918 .all(|entry| entry.executable.is_none())
5919 {
5920 assert!(Instant::now() < deadline, "module image was not confirmed");
5921 tokio::time::sleep(Duration::from_millis(5)).await;
5922 }
5923 }
5924 let entry = crate::live_children::read_record(&record_path)
5925 .unwrap()
5926 .pop()
5927 .unwrap();
5928 let observed = subc_os::Process::open(entry.pid)
5929 .unwrap()
5930 .unwrap()
5931 .observe()
5932 .unwrap();
5933 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
5934 module.drain().await.unwrap();
5935 assert_eq!(
5936 verdict,
5937 crate::live_children::IdentityVerdict::Matches,
5938 "program {program:?}: recorded {entry:?}, observed {observed:?}"
5939 );
5940 }
5941 }
5942
5943 #[cfg(unix)]
5944 fn http_fixture(
5945 dir: &std::path::Path,
5946 url: &str,
5947 threshold: u32,
5948 ) -> crate::daemon_config::ConfiguredModule {
5949 let path = dir.join("subc.jsonc");
5950 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
5951 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
5952 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
5953 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
5954 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
5955 }}}).to_string()).unwrap();
5956 crate::daemon_config::load(&path)
5957 .unwrap()
5958 .unwrap()
5959 .modules
5960 .pop()
5961 .unwrap()
5962 }
5963
5964 #[cfg(unix)]
5965 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
5966 timeout(Duration::from_secs(5), async {
5967 loop {
5968 if module.status().unwrap().health.status == status {
5969 break;
5970 }
5971 sleep(Duration::from_millis(5)).await;
5972 }
5973 })
5974 .await
5975 .unwrap_or_else(|_| {
5976 panic!(
5977 "expected {status:?}, got {:?}",
5978 module.status().unwrap().health
5979 )
5980 });
5981 }
5982
5983 #[cfg(unix)]
5984 #[tokio::test]
5985 async fn http_health_status_flips_ok_failing_ok() {
5986 use tokio::io::{AsyncReadExt, AsyncWriteExt};
5987 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
5988 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5989 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
5990 let serving_status = status.clone();
5991 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
5992 let server = tokio::spawn(async move {
5993 loop {
5994 let (mut stream, _) = listener.accept().await.unwrap();
5995 let mut request = [0u8; 2048];
5996 let count = stream.read(&mut request).await.unwrap();
5997 assert!(count > 0, "a probe must send an HTTP request");
5998 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
5999 let body = if code == 200 {
6000 "ready"
6001 } else {
6002 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6003 };
6004 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6005 let _ = stream.write_all(response.as_bytes()).await;
6006 }
6007 });
6008 let configured = http_fixture(&dir, &url, 1000);
6009 let module =
6010 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6011 .supervise_configured_with_health(
6012 configured.module_spec(),
6013 true,
6014 configured.health,
6015 configured.drain_timeout_ms,
6016 configured.restart,
6017 )
6018 .unwrap();
6019 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6020 status.store(503, std::sync::atomic::Ordering::SeqCst);
6021 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6022 assert!(module
6023 .status()
6024 .unwrap()
6025 .health
6026 .detail
6027 .unwrap()
6028 .contains("scratch failure"));
6029 status.store(200, std::sync::atomic::Ordering::SeqCst);
6030 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6031 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6032 let before = module.status().unwrap();
6033 let (spec, mut health) = module.configuration().unwrap();
6034 health.http = None;
6035 module
6036 .update_configuration(spec.clone(), health.clone(), Some(10))
6037 .await
6038 .unwrap();
6039 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6040 health.http = Some(url);
6041 module
6042 .update_configuration(spec, health, Some(10))
6043 .await
6044 .unwrap();
6045 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6046 assert_eq!(
6047 module.status().unwrap().pid,
6048 before.pid,
6049 "changing a probe must apply live, not restart its process"
6050 );
6051 let (spec, mut health) = module.configuration().unwrap();
6052 health.failure_threshold = 2;
6053 module
6054 .update_configuration(spec, health, Some(10))
6055 .await
6056 .unwrap();
6057 status.store(503, std::sync::atomic::Ordering::SeqCst);
6058 timeout(Duration::from_secs(5), async {
6059 while module.status().unwrap().spawn_generation == before.spawn_generation {
6060 sleep(Duration::from_millis(5)).await;
6061 }
6062 })
6063 .await
6064 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6065 module.drain().await.unwrap();
6066 server.abort();
6067 }
6068
6069 #[cfg(unix)]
6070 #[tokio::test]
6071 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6072 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6073 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6074 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6075 drop(unused);
6076 let configured = http_fixture(&dir, &url, 2);
6077 let module =
6078 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6079 .supervise_configured_with_health(
6080 configured.module_spec(),
6081 true,
6082 configured.health,
6083 configured.drain_timeout_ms,
6084 configured.restart,
6085 )
6086 .unwrap();
6087 let before = module.status().unwrap().spawn_generation;
6088 timeout(Duration::from_secs(5), async {
6089 loop {
6090 let status = module.status().unwrap();
6091 if status.spawn_generation > before {
6092 assert!(status.lifetime_restarts > 0);
6093 break;
6094 }
6095 sleep(Duration::from_millis(5)).await;
6096 }
6097 })
6098 .await
6099 .expect("sustained HTTP refusal must trigger the health restart policy");
6100 module.drain().await.unwrap();
6101 }
6102
6103 #[cfg(unix)]
6104 #[tokio::test]
6105 async fn http_health_timeout_honours_deadline() {
6106 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6107 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6108 let server = tokio::spawn(async move {
6109 let _held = listener.accept().await.unwrap();
6110 std::future::pending::<()>().await;
6111 });
6112 let error = timeout(
6113 Duration::from_secs(1),
6114 probe_http_health(&url, Duration::from_millis(10)),
6115 )
6116 .await
6117 .expect("the probe must enforce its own deadline")
6118 .unwrap_err();
6119 server.abort();
6120 assert!(error.to_string().contains("timed out"));
6121 }
6122
6123 #[cfg(unix)]
6124 #[tokio::test]
6125 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6126 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6127 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6128 let url = format!(
6129 "http://localhost:{}/healthz",
6130 listener.local_addr().unwrap().port()
6131 );
6132 let server = tokio::spawn(async move {
6133 let (mut stream, _) = listener.accept().await.unwrap();
6134 let mut request = [0u8; 2048];
6135 assert!(stream.read(&mut request).await.unwrap() > 0);
6136 stream
6137 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6138 .await
6139 .unwrap();
6140 });
6141 assert_eq!(
6146 probe_http_health(&url, Duration::from_secs(10))
6147 .await
6148 .unwrap()
6149 .status,
6150 HealthStatus::Ok
6151 );
6152 server.await.unwrap();
6153 }
6154
6155 #[cfg(unix)]
6156 #[tokio::test]
6157 async fn http_health_timeout_keeps_partial_status_and_body() {
6158 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6159 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6160 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6161 let server = tokio::spawn(async move {
6162 let (mut stream, _) = listener.accept().await.unwrap();
6163 let mut request = [0u8; 2048];
6164 assert!(stream.read(&mut request).await.unwrap() > 0);
6165 stream
6166 .write_all(
6167 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6168 )
6169 .await
6170 .unwrap();
6171 std::future::pending::<()>().await;
6172 });
6173 let error = probe_http_health(&url, Duration::from_secs(1))
6174 .await
6175 .unwrap_err()
6176 .to_string();
6177 server.abort();
6178 assert!(
6179 error.contains("timed out")
6180 && error.contains("503 Unavailable")
6181 && error.contains("partial diagnostic"),
6182 "{error}"
6183 );
6184 }
6185
6186 #[cfg(unix)]
6187 #[tokio::test]
6188 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6189 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6190 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6191 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6192 let server = tokio::spawn(async move {
6193 let (mut stream, _) = listener.accept().await.unwrap();
6194 let mut request = [0u8; 2048];
6195 assert!(stream.read(&mut request).await.unwrap() > 0);
6196 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6197 let response = format!(
6198 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6199 body.len()
6200 );
6201 stream.write_all(response.as_bytes()).await.unwrap();
6202 });
6203 let error = probe_http_health(&url, Duration::from_secs(1))
6204 .await
6205 .unwrap_err()
6206 .to_string();
6207 server.await.unwrap();
6208 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6209 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6210 assert!(!error.contains("not-in-diagnostic"));
6211 }
6212
6213 #[cfg(unix)]
6214 #[tokio::test]
6215 async fn http_health_real_nats_server_monitoring() {
6216 if std::process::Command::new("nats-server")
6217 .arg("--version")
6218 .env("XDG_DATA_HOME", std::env::temp_dir())
6219 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6220 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6221 .output()
6222 .is_err()
6223 {
6224 eprintln!(
6225 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6226 );
6227 return;
6228 }
6229 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6230 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6231 let port = monitor.local_addr().unwrap().port();
6232 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6233 let client_port = client.local_addr().unwrap().port();
6234 let config = dir.join("server.conf");
6235 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6236 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6237 configured.program = PathBuf::from("nats-server");
6238 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6239 drop(monitor);
6240 drop(client);
6241 let module =
6242 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6243 .supervise_configured_with_health(
6244 configured.module_spec(),
6245 true,
6246 configured.health,
6247 configured.drain_timeout_ms,
6248 configured.restart,
6249 )
6250 .unwrap();
6251 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6252 module.drain().await.unwrap();
6253 }
6254
6255 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6256 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6257 let supervisor =
6258 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6259 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6260 let initial = ModuleSpec {
6261 module_id: "rescan-preserves-spawn-facts".to_string(),
6262 program: PathBuf::from("/spawned/module"),
6263 args: Vec::new(),
6264 env: Vec::new(),
6265 reserved: false,
6266 reserved_prefixes: Vec::new(),
6267 protocol: ModuleProtocol::Subc,
6268 overlap: Default::default(),
6269 };
6270 let module = supervisor.supervised_module(
6271 initial.clone(),
6272 supervisor.runtime_config(),
6273 snapshot,
6274 None,
6275 );
6276 let before = module.status().unwrap();
6277 let mut replacement = initial;
6278 replacement.program = PathBuf::from("/rescanned/replacement-module");
6279
6280 module
6281 .update_configuration(replacement, HealthConfig::default(), None)
6282 .await
6283 .unwrap();
6284
6285 let after = module.status().unwrap();
6286 assert_eq!(after.pid, before.pid);
6287 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6288 assert_eq!(after.spawned_from, before.spawned_from);
6289 drop(module);
6290 }
6291}
6292
6293fn unix_ms_now() -> u64 {
6294 SystemTime::now()
6295 .duration_since(UNIX_EPOCH)
6296 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6297 .unwrap_or(0)
6298}
6299
6300async fn supervise_loop(
6301 mut spec: ModuleSpec,
6302 mut runtime: SupervisorRuntimeConfig,
6303 registry: Arc<Registry>,
6304 process_liveness: Arc<SupervisorProcessLiveness>,
6305 snapshot: SharedSnapshot,
6306 mut child: Option<SupervisedChild>,
6307 mut commands: mpsc::Receiver<SupervisorCommand>,
6308) {
6309 let mut health_probe = HealthProbeRuntime::default();
6310 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6314 Ok(changes) => Some(changes),
6315 Err(err) => {
6316 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6319 None
6320 }
6321 };
6322 let mut pending_respawn: Option<PendingRespawn> = None;
6326 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6329 let mut previous_instance_recheck = tokio::time::interval_at(
6330 Instant::now() + Duration::from_secs(5),
6331 Duration::from_secs(5),
6332 );
6333 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6334 loop {
6335 #[cfg(test)]
6336 {
6337 lock_snapshot(&snapshot).unwrap().actor_select = None;
6338 }
6339 #[cfg(target_os = "macos")]
6340 if let Some(active) = child.as_mut() {
6341 active.confirm_privacy_exec().await;
6342 }
6343 if let Some(scheduled) = runtime
6344 .scheduled_respawn
6345 .lock()
6346 .unwrap_or_else(|p| p.into_inner())
6347 .take()
6348 {
6349 pending_respawn = Some(scheduled);
6350 }
6351 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6352 pending_respawn = None;
6353 cancel_deferred_reload(
6354 &runtime,
6355 &spec.module_id,
6356 "respawn cancelled by a supervisor command",
6357 );
6358 }
6359 if child.is_none()
6360 && pending_respawn.is_none()
6361 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6362 {
6363 cancel_deferred_reload(
6364 &runtime,
6365 &spec.module_id,
6366 "respawn cancelled before a replacement was spawned",
6367 );
6368 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6369 state.respawn_pending = false;
6370 state.coalesced_restart_pending = false;
6371 if matches!(
6372 state.state,
6373 ModuleState::Restarting
6374 | ModuleState::Starting
6375 | ModuleState::Draining
6376 | ModuleState::Unresponsive
6377 ) {
6378 error!(module_id = %spec.module_id, state = ?state.state,
6379 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6380 state.state = ModuleState::Failed;
6381 clear_current_process_facts(state);
6382 }
6383 });
6384 }
6385 if let Some(command) = requeued.pop_front() {
6386 if !handle_supervisor_command(
6387 command,
6388 &mut spec,
6389 &mut runtime,
6390 ®istry,
6391 &process_liveness,
6392 &snapshot,
6393 &mut child,
6394 &mut commands,
6395 &mut requeued,
6396 )
6397 .await
6398 {
6399 return;
6400 }
6401 if child.is_some() || !respawn_still_pending(&snapshot) {
6402 pending_respawn = None;
6403 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6404 state.respawn_pending = false
6405 });
6406 }
6407 continue;
6408 }
6409 if child.is_some() {
6410 #[cfg(test)]
6411 {
6412 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6413 }
6414 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6415 let wake_after = health_probe.wake_after();
6416 let probe_sleep = async move {
6417 match wake_after {
6418 Some(delay) => sleep(delay).await,
6419 None => std::future::pending().await,
6420 }
6421 };
6422 tokio::pin!(probe_sleep);
6423 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6428 #[cfg(test)]
6429 {
6430 let registration_pending = wire_child
6431 && registration_changes
6432 .as_ref()
6433 .is_some_and(|changes| changes.has_changed().unwrap());
6434 let mut state = lock_snapshot(&snapshot).unwrap();
6435 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6436 generation: state.spawn_generation,
6437 turn: state.actor_turns,
6438 registered_connection: health_probe.registered_connection,
6439 next_probe_at: health_probe.next_probe_at,
6440 wake_after,
6441 });
6442 }
6443 let active_child = child.as_mut().expect("child checked above");
6444 tokio::select! {
6445 wait_result = active_child.wait() => {
6446 #[cfg(test)]
6447 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6448 let exit_report = match wait_result {
6457 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6458 Err(err) => {
6459 active_child.drain_stderr(&spec.module_id).await;
6460 fail_after_wait_error(
6461 &spec.module_id,
6462 &snapshot,
6463 &runtime.terminal_ring,
6464 &runtime.spawn_events,
6465 format!("failed to wait for supervised module: {err}"),
6466 );
6467 untrack_if_registration_released(
6468 &process_liveness,
6469 ®istry,
6470 &spec.module_id,
6471 &snapshot,
6472 );
6473 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6474 child = None;
6475 continue;
6476 }
6477 };
6478 active_child.drain_stderr(&spec.module_id).await;
6479
6480 let next = on_child_exit(
6481 &spec,
6482 runtime.restart_policy,
6483 ®istry,
6484 &snapshot,
6485 &runtime.terminal_ring,
6486 &runtime.spawn_events,
6487 &runtime.child_roster,
6488 exit_report,
6489 ).await;
6490 runtime.start_requester = None;
6491 runtime.pending_activity_restart = false;
6492 active_child.release_roster();
6495 match next {
6496 NextAction::Stop { registration_released } => {
6497 if registration_released {
6498 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6499 }
6500 child = None;
6501 }
6502 NextAction::Restart { schedule } => {
6503 let delay = schedule.map_or(
6504 runtime.restart_policy.delay_for_restart(0),
6505 |schedule| schedule.delay,
6506 );
6507 if let Some(schedule) = schedule {
6508 log_crash_respawn(&spec.module_id, schedule);
6509 }
6510 child = None;
6518 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6519 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6520 }
6521 }
6522 }
6523 command = commands.recv() => {
6524 #[cfg(test)]
6525 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6526 let Some(command) = command else {
6527 return;
6528 };
6529 if !handle_supervisor_command(
6530 command,
6531 &mut spec,
6532 &mut runtime,
6533 ®istry,
6534 &process_liveness,
6535 &snapshot,
6536 &mut child,
6537 &mut commands,
6538 &mut requeued,
6539 ).await {
6540 return;
6541 }
6542 }
6543 _ = async {
6544 match registration_changes.as_mut() {
6545 Some(changes) => { let _ = changes.changed().await; }
6546 None => std::future::pending().await,
6547 }
6548 }, if wire_child => {
6549 #[cfg(test)]
6550 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6551 }
6552 _ = &mut probe_sleep => {
6553 #[cfg(test)]
6554 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6555 if health_probe.due() {
6556 run_health_probe_cycle(
6557 &spec,
6558 &runtime,
6559 ®istry,
6560 &process_liveness,
6561 &snapshot,
6562 &mut child,
6563 ).await;
6564 if child.is_none() {
6565 runtime.start_requester = None;
6566 runtime.pending_activity_restart = false;
6567 }
6568 if child.is_some() {
6569 health_probe.schedule_next(&spec, runtime.health.cadence);
6570 }
6571 }
6572 }
6573 }
6574 } else if let Some(pending) = pending_respawn {
6575 tokio::select! {
6576 _ = sleep_until(pending.deadline) => {
6577 pending_respawn = None;
6578 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6579 if !respawn_still_pending(&snapshot) {
6583 continue;
6584 }
6585 if runtime.child_roster.is_closed() {
6590 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6591 state.state = ModuleState::Stopped;
6592 });
6593 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6594 continue;
6595 }
6596 if let Err(err) = release_dead_registration(
6597 ®istry,
6598 runtime.forwarding.as_deref(),
6599 &snapshot,
6600 &spec.module_id,
6601 ).await {
6602 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6603 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6604 continue;
6605 }
6606
6607 if matches!(pending.kind, RespawnKind::Reload) {
6608 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6609 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
6610 if result.is_ok() && runtime.pending_activity_restart {
6611 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6612 }
6613 runtime.pending_activity_restart = false;
6614 if let Some(reply) = reply { let _ = reply.send(result); }
6615 continue;
6616 }
6617 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6618 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6619 Ok(next_child) => {
6620 child = Some(next_child);
6621 if runtime.pending_activity_restart {
6622 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6623 runtime.pending_activity_restart = false;
6624 }
6625 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6626 }
6627 Err(err) => {
6628 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6629 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6630 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6631 }
6632 }
6633 }
6634 command = commands.recv() => {
6635 let Some(command) = command else {
6636 return;
6637 };
6638 if !handle_supervisor_command(
6639 command,
6640 &mut spec,
6641 &mut runtime,
6642 ®istry,
6643 &process_liveness,
6644 &snapshot,
6645 &mut child,
6646 &mut commands,
6647 &mut requeued,
6648 ).await {
6649 return;
6650 }
6651 if child.is_some() || !respawn_still_pending(&snapshot) {
6656 pending_respawn = None;
6657 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6658 }
6659 }
6660 }
6661 } else {
6662 let command = if lock_snapshot(&snapshot)
6663 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6664 {
6665 tokio::select! {
6666 command = commands.recv() => command,
6667 _ = previous_instance_recheck.tick() => {
6668 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6669 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6670 if enabled && !runtime.child_roster.is_closed() {
6671 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6672 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6673 Ok(next_child) => child = Some(next_child),
6674 Err(error) => {
6675 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6676 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6677 }
6678 }
6679 } else {
6680 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6681 state.waiting_for_previous_pid = None;
6682 state.waiting_for_previous_detail = None;
6683 });
6684 }
6685 }
6686 continue;
6687 }
6688 }
6689 } else {
6690 commands.recv().await
6691 };
6692 let Some(command) = command else {
6693 return;
6694 };
6695 if !handle_supervisor_command(
6696 command,
6697 &mut spec,
6698 &mut runtime,
6699 ®istry,
6700 &process_liveness,
6701 &snapshot,
6702 &mut child,
6703 &mut commands,
6704 &mut requeued,
6705 )
6706 .await
6707 {
6708 return;
6709 }
6710 }
6711 }
6712}
6713
6714fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6715 info!(
6716 module_id,
6717 restart_in_window = schedule.restart_in_window,
6718 delay_ms = schedule.delay.as_millis() as u64,
6719 "respawning after crash"
6720 );
6721}
6722
6723fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6729 matches!(
6730 lock_snapshot(snapshot),
6731 Ok(state) if state.enabled && state.state == ModuleState::Restarting
6732 )
6733}
6734
6735enum NextAction {
6736 Stop {
6737 registration_released: bool,
6738 },
6739 Restart {
6740 schedule: Option<CrashRestartSchedule>,
6741 },
6742}
6743
6744#[allow(clippy::too_many_arguments)]
6745async fn handle_supervisor_command(
6746 command: SupervisorCommand,
6747 spec: &mut ModuleSpec,
6748 runtime: &mut SupervisorRuntimeConfig,
6749 registry: &Arc<Registry>,
6750 process_liveness: &SupervisorProcessLiveness,
6751 snapshot: &SharedSnapshot,
6752 child: &mut Option<SupervisedChild>,
6753 commands: &mut mpsc::Receiver<SupervisorCommand>,
6754 requeued: &mut VecDeque<SupervisorCommand>,
6755) -> bool {
6756 let command =
6757 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6758 match command {
6759 SupervisorCommand::Restart { reply, .. }
6760 | SupervisorCommand::Reload { reply, .. }
6761 | SupervisorCommand::Swap { reply, .. } => {
6762 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6763 return true;
6764 }
6765 SupervisorCommand::SetEnabled {
6766 enabled: true,
6767 reply,
6768 ..
6769 } => {
6770 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6771 return true;
6772 }
6773 other => other,
6774 }
6775 } else {
6776 command
6777 };
6778 match command {
6779 SupervisorCommand::Drain { requester, reply } => {
6780 let result = drain_optional_child(
6783 &spec.module_id,
6784 spec.protocol,
6785 StopNotice::NotSent,
6786 registry,
6787 runtime.forwarding.as_deref(),
6788 snapshot,
6789 &runtime.terminal_ring,
6790 &runtime.spawn_events,
6791 child,
6792 runtime.drain_timeout,
6793 ModuleState::Stopped,
6794 None,
6795 )
6796 .await;
6797 let registration_released = result.is_ok();
6798 if registration_released {
6799 runtime.record_activity_transition(
6800 "subc.module_stopped",
6801 &spec.module_id,
6802 requester,
6803 );
6804 }
6805 let _ = reply.send(result);
6806 if registration_released {
6807 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6808 }
6809 false
6810 }
6811 SupervisorCommand::Retire { requester, reply } => {
6812 let result = async {
6813 let stop_notice = begin_forwarding_drain_if_configured(
6814 spec,
6815 runtime,
6816 registry,
6817 snapshot,
6818 None,
6819 RouteCloseReason::Disable,
6820 )
6821 .await?;
6822 drain_optional_child(
6823 &spec.module_id,
6824 spec.protocol,
6825 stop_notice,
6826 registry,
6827 runtime.forwarding.as_deref(),
6828 snapshot,
6829 &runtime.terminal_ring,
6830 &runtime.spawn_events,
6831 child,
6832 runtime.drain_timeout,
6833 ModuleState::Stopped,
6834 None,
6835 )
6836 .await
6837 }
6838 .await;
6839 let registration_released = result.is_ok();
6840 if registration_released {
6841 runtime.record_activity_transition(
6842 "subc.module_stopped",
6843 &spec.module_id,
6844 requester,
6845 );
6846 }
6847 let _ = reply.send(result);
6848 if registration_released {
6849 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6850 }
6851 false
6852 }
6853 SupervisorCommand::Restart {
6854 requester,
6855 drain_timeout_ms,
6856 received_at_generation,
6857 queued_at,
6858 reply,
6859 } => {
6860 info!(
6864 module_id = %spec.module_id,
6865 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6866 "restart command dequeued"
6867 );
6868 let validation = match lock_snapshot(snapshot) {
6880 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
6881 module_id: spec.module_id.clone(),
6882 }),
6883 Ok(_) => Ok(()),
6884 Err(err) => Err(err),
6885 };
6886 let initiated = validation.is_ok();
6887 let _ = reply.send(validation);
6888 let satisfied_by_generation = if initiated && child.is_some() {
6899 lock_snapshot(snapshot).ok().and_then(|state| {
6900 (state.spawn_generation > received_at_generation
6901 && !state.configuration_updated_since_spawn)
6902 .then_some(state.spawn_generation)
6903 })
6904 } else {
6905 None
6906 };
6907 let satisfied_by_pending = initiated
6908 && child.is_none()
6909 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
6910 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
6911 if pending {
6912 state.coalesced_restart_pending = true;
6913 }
6914 pending
6915 });
6916 if satisfied_by_pending {
6917 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
6918 } else if let Some(generation) = satisfied_by_generation {
6919 info!(
6920 module_id = %spec.module_id,
6921 received_at_generation,
6922 "restart already satisfied by generation {generation}; not restarting again"
6923 );
6924 } else if initiated {
6925 let drain_timeout = drain_timeout_ms
6928 .map(Duration::from_millis)
6929 .unwrap_or(runtime.drain_timeout);
6930 if let Err(err) = restart_child(
6931 spec,
6932 runtime,
6933 registry,
6934 process_liveness,
6935 snapshot,
6936 child,
6937 drain_timeout,
6938 )
6939 .await
6940 {
6941 warn!(
6942 module_id = %spec.module_id,
6943 error = %err,
6944 "operator restart failed after initiation ack; module state carries the outcome"
6945 );
6946 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6947 state.state = ModuleState::Failed;
6948 clear_current_process_facts(state);
6949 });
6950 } else {
6951 runtime.start_requester = requester;
6952 runtime.pending_activity_restart = true;
6953 }
6954 }
6955 true
6956 }
6957 SupervisorCommand::Reload { requester, reply } => {
6958 let result =
6959 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
6960 if result.is_ok() {
6961 runtime.start_requester = requester;
6962 runtime.pending_activity_restart = true;
6963 }
6964 if result.is_ok()
6965 && runtime
6966 .scheduled_respawn
6967 .lock()
6968 .unwrap_or_else(|p| p.into_inner())
6969 .as_ref()
6970 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
6971 {
6972 *runtime
6973 .deferred_reload_reply
6974 .lock()
6975 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
6976 } else {
6977 let _ = reply.send(result);
6978 }
6979 true
6980 }
6981 SupervisorCommand::SetEnabled {
6982 enabled,
6983 requester,
6984 reply,
6985 } => {
6986 let previous_requester = runtime.start_requester.clone();
6987 if enabled {
6988 runtime.start_requester = requester.clone();
6989 }
6990 let result = set_child_enabled(
6991 spec,
6992 runtime,
6993 registry,
6994 process_liveness,
6995 snapshot,
6996 child,
6997 enabled,
6998 )
6999 .await;
7000 if matches!(result, Ok(true)) {
7001 runtime.pending_activity_restart = false;
7002 if enabled {
7003 runtime.start_requester = requester;
7004 } else {
7005 runtime.start_requester = None;
7006 runtime.record_activity_transition(
7007 "subc.module_disabled",
7008 &spec.module_id,
7009 requester,
7010 );
7011 }
7012 } else {
7013 runtime.start_requester = previous_requester;
7014 }
7015 let _ = reply.send(result);
7016 true
7017 }
7018 SupervisorCommand::UpdateConfiguration {
7019 spec: next_spec,
7020 health,
7021 drain_timeout_ms,
7022 reply,
7023 } => {
7024 if let Some(handle) = &runtime.supervisor_handle {
7025 handle.apply_identity_configuration(&next_spec);
7026 }
7027 *spec = next_spec;
7028 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7029 state.configuration_updated_since_spawn = true;
7030 });
7031 let health_changed = runtime.health != health;
7032 runtime.health = health;
7033 if health_changed {
7036 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7037 state.health = ModuleHealthStatus::default();
7038 });
7039 }
7040 runtime.drain_timeout = drain_timeout_ms
7041 .map(Duration::from_millis)
7042 .unwrap_or(runtime.default_drain_timeout);
7043 *runtime
7044 .effective_drain_timeout
7045 .lock()
7046 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7047 let _ = reply.send(());
7048 true
7049 }
7050 SupervisorCommand::Swap {
7051 requester,
7052 ready_timeout,
7053 reply,
7054 } => {
7055 let mut swap_runtime = runtime.clone();
7056 swap_runtime.start_requester = requester;
7057 let before = lock_snapshot(snapshot)
7058 .ok()
7059 .map(|state| state.spawn_generation);
7060 let end = swap::run_swap(
7061 spec,
7062 &swap_runtime,
7063 registry,
7064 process_liveness,
7065 snapshot,
7066 child,
7067 commands,
7068 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7069 reply,
7070 )
7071 .await;
7072 if before
7073 != lock_snapshot(snapshot)
7074 .ok()
7075 .map(|state| state.spawn_generation)
7076 {
7077 runtime.start_requester = swap_runtime.start_requester;
7078 }
7079 requeued.extend(end.requeue);
7080 true
7081 }
7082 }
7083}
7084
7085async fn restart_child(
7086 spec: &ModuleSpec,
7087 runtime: &SupervisorRuntimeConfig,
7088 registry: &Registry,
7089 process_liveness: &SupervisorProcessLiveness,
7090 snapshot: &SharedSnapshot,
7091 child: &mut Option<SupervisedChild>,
7092 drain_timeout: Duration,
7093) -> Result<(), SuperviseError> {
7094 if !lock_snapshot(snapshot)?.enabled {
7096 return Err(SuperviseError::Disabled {
7097 module_id: spec.module_id.clone(),
7098 });
7099 }
7100 let stop_notice = begin_forwarding_drain_with_timeout(
7101 spec,
7102 runtime,
7103 registry,
7104 snapshot,
7105 None,
7106 RouteCloseReason::Restart,
7107 drain_timeout,
7108 )
7109 .await?;
7110
7111 if child.is_some() {
7112 drain_optional_child(
7113 &spec.module_id,
7114 spec.protocol,
7115 stop_notice,
7116 registry,
7117 runtime.forwarding.as_deref(),
7118 snapshot,
7119 &runtime.terminal_ring,
7120 &runtime.spawn_events,
7121 child,
7122 drain_timeout,
7123 ModuleState::Restarting,
7124 Some(true),
7125 )
7126 .await?;
7127 } else {
7128 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7129 state.enabled = true;
7130 state.state = ModuleState::Restarting;
7131 clear_current_process_facts(state);
7132 })?;
7133 release_dead_registration(
7134 registry,
7135 runtime.forwarding.as_deref(),
7136 snapshot,
7137 &spec.module_id,
7138 )
7139 .await?;
7140 }
7141
7142 reset_restart_count(snapshot, &spec.module_id)?;
7143 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7144 schedule_respawn(
7145 runtime,
7146 snapshot,
7147 &spec.module_id,
7148 runtime.restart_policy.backoff,
7149 RespawnKind::Spawn,
7150 )
7151}
7152
7153async fn reload_child(
7154 spec: &ModuleSpec,
7155 runtime: &SupervisorRuntimeConfig,
7156 registry: &Registry,
7157 process_liveness: &SupervisorProcessLiveness,
7158 snapshot: &SharedSnapshot,
7159 child: &mut Option<SupervisedChild>,
7160) -> Result<(), SuperviseError> {
7161 if !lock_snapshot(snapshot)?.enabled {
7163 return Err(SuperviseError::Disabled {
7164 module_id: spec.module_id.clone(),
7165 });
7166 }
7167 let stop_notice = begin_forwarding_drain(
7168 spec,
7169 runtime,
7170 registry,
7171 snapshot,
7172 Some(true),
7173 RouteCloseReason::Reload,
7174 )
7175 .await?;
7176
7177 if child.is_some() {
7178 drain_optional_child(
7179 &spec.module_id,
7180 spec.protocol,
7181 stop_notice,
7182 registry,
7183 runtime.forwarding.as_deref(),
7184 snapshot,
7185 &runtime.terminal_ring,
7186 &runtime.spawn_events,
7187 child,
7188 runtime.drain_timeout,
7189 ModuleState::Restarting,
7190 Some(true),
7191 )
7192 .await?;
7193 } else {
7194 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7195 state.enabled = true;
7196 state.state = ModuleState::Restarting;
7197 clear_current_process_facts(state);
7198 })?;
7199 release_dead_registration(
7200 registry,
7201 runtime.forwarding.as_deref(),
7202 snapshot,
7203 &spec.module_id,
7204 )
7205 .await?;
7206 }
7207
7208 reset_restart_count(snapshot, &spec.module_id)?;
7209 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7210 schedule_respawn(
7211 runtime,
7212 snapshot,
7213 &spec.module_id,
7214 runtime.restart_policy.backoff,
7215 RespawnKind::Reload,
7216 )
7217}
7218
7219async fn finish_reload_child(
7220 spec: &ModuleSpec,
7221 runtime: &SupervisorRuntimeConfig,
7222 registry: &Registry,
7223 process_liveness: &SupervisorProcessLiveness,
7224 snapshot: &SharedSnapshot,
7225 child: &mut Option<SupervisedChild>,
7226) -> Result<(), SuperviseError> {
7227 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7228 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7229 Ok(next_child) => next_child,
7230 Err(err) => {
7231 return handle_reload_spawn_failure(
7232 spec,
7233 runtime,
7234 process_liveness,
7235 snapshot,
7236 child,
7237 format!("new child failed to spawn: {err}"),
7238 )
7239 .await;
7240 }
7241 };
7242 *child = Some(next_child);
7243
7244 let wait_outcome = {
7245 let active_child = child.as_mut().expect("new reload child was just stored");
7246 wait_for_registration_after_reload(
7247 registry,
7248 &spec.module_id,
7249 snapshot,
7250 active_child,
7251 REGISTRY_RELEASE_TIMEOUT,
7252 )
7253 .await?
7254 };
7255
7256 match wait_outcome {
7257 RegistrationWaitOutcome::Registered => {
7258 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7259 Ok(())
7260 }
7261 RegistrationWaitOutcome::Exited(exit_report) => {
7262 if let Some(active_child) = child.as_mut() {
7263 active_child.drain_stderr(&spec.module_id).await;
7264 }
7265 let mut exited_child = child.take().expect("exited reload child is still stored");
7268 #[cfg(test)]
7269 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7270 gate.reached.notify_one();
7271 gate.resume.notified().await;
7272 }
7273 let result = handle_reload_child_registration_failure(
7274 spec,
7275 runtime,
7276 registry,
7277 process_liveness,
7278 snapshot,
7279 child,
7280 ReloadRegistrationFailure {
7281 exit_report: registration_failure_exit_report(exit_report),
7282 reason: exited_child
7283 .spawn_failure
7284 .clone()
7285 .unwrap_or_else(|| "new child exited before registering".to_string()),
7286 },
7287 )
7288 .await;
7289 exited_child.release_roster();
7290 result
7291 }
7292 RegistrationWaitOutcome::TimedOut => {
7293 let mut timed_out_child = child
7294 .take()
7295 .expect("timed-out reload child is still running");
7296 timed_out_child
7297 .start_kill()
7298 .map_err(|source| SuperviseError::Kill {
7299 module_id: spec.module_id.clone(),
7300 source,
7301 })?;
7302 let status = timed_out_child
7303 .wait()
7304 .await
7305 .map_err(|source| SuperviseError::Wait {
7306 module_id: spec.module_id.clone(),
7307 source,
7308 })?;
7309 timed_out_child.drain_stderr(&spec.module_id).await;
7310 handle_reload_child_registration_failure(
7311 spec,
7312 runtime,
7313 registry,
7314 process_liveness,
7315 snapshot,
7316 child,
7317 ReloadRegistrationFailure {
7318 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7319 snapshot,
7320 &timed_out_child,
7321 &status,
7322 )),
7323 reason: format!(
7324 "new child did not register within {:?}",
7325 REGISTRY_RELEASE_TIMEOUT
7326 ),
7327 },
7328 )
7329 .await
7330 }
7331 }
7332}
7333
7334async fn set_child_enabled(
7335 spec: &ModuleSpec,
7336 runtime: &SupervisorRuntimeConfig,
7337 registry: &Registry,
7338 process_liveness: &SupervisorProcessLiveness,
7339 snapshot: &SharedSnapshot,
7340 child: &mut Option<SupervisedChild>,
7341 enabled: bool,
7342) -> Result<bool, SuperviseError> {
7343 let (current_enabled, current_state, respawn_pending) = {
7344 let state = lock_snapshot(snapshot)?;
7345 (state.enabled, state.state, state.respawn_pending)
7346 };
7347 let revive_terminal = enabled
7355 && current_enabled
7356 && child.is_none()
7357 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7358 || (current_state == ModuleState::Restarting && !respawn_pending));
7359 if current_enabled == enabled && !revive_terminal {
7360 return Ok(false);
7361 }
7362
7363 if enabled {
7364 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7365 state.enabled = true;
7366 state.state = ModuleState::Starting;
7367 clear_current_process_facts(state);
7368 })?;
7369 #[cfg(test)]
7370 if runtime.test_seed_stale_facts_before_enable_spawn {
7371 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7372 state.process_alive = true;
7373 state.pid = Some(41);
7374 state.spawned_at_ms = Some(42);
7375 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7376 state.spawned_file_identity = Some(SpawnedFileIdentity {
7377 device: 43,
7378 inode: 44,
7379 #[cfg(windows)]
7380 windows_image: None,
7381 });
7382 })?;
7383 }
7384 release_dead_registration(
7385 registry,
7386 runtime.forwarding.as_deref(),
7387 snapshot,
7388 &spec.module_id,
7389 )
7390 .await?;
7391 reset_restart_count(snapshot, &spec.module_id)?;
7392 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7393 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7394 Ok(next_child) => next_child,
7395 Err(err) => {
7396 if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7397 state.state = ModuleState::Failed;
7398 clear_current_process_facts(state);
7399 }) {
7400 error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7401 }
7402 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7403 return Err(err);
7404 }
7405 };
7406 *child = Some(next_child);
7407 debug!(module_id = %spec.module_id, "supervised module enabled");
7408 Ok(true)
7409 } else {
7410 let stop_notice = begin_forwarding_drain_if_configured(
7411 spec,
7412 runtime,
7413 registry,
7414 snapshot,
7415 Some(false),
7416 RouteCloseReason::Disable,
7417 )
7418 .await?;
7419 drain_optional_child(
7420 &spec.module_id,
7421 spec.protocol,
7422 stop_notice,
7423 registry,
7424 runtime.forwarding.as_deref(),
7425 snapshot,
7426 &runtime.terminal_ring,
7427 &runtime.spawn_events,
7428 child,
7429 runtime.drain_timeout,
7430 ModuleState::Disabled,
7431 Some(false),
7432 )
7433 .await?;
7434 debug!(module_id = %spec.module_id, "supervised module disabled");
7435 Ok(true)
7436 }
7437}
7438
7439#[allow(clippy::too_many_arguments)]
7440async fn on_child_exit(
7441 spec: &ModuleSpec,
7442 policy: RestartPolicy,
7443 registry: &Registry,
7444 snapshot: &SharedSnapshot,
7445 terminal_ring: &Arc<Mutex<TerminalRing>>,
7446 spawn_events: &SpawnEventFeed,
7447 roster: &ChildRoster,
7448 exit_report: ExitReport,
7449) -> NextAction {
7450 if roster.is_closed() {
7456 return on_child_exit_during_daemon_shutdown(
7457 spec,
7458 registry,
7459 snapshot,
7460 terminal_ring,
7461 spawn_events,
7462 exit_report,
7463 )
7464 .await;
7465 }
7466 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7481 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7482 match exit_report.kind {
7483 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7484 info!(
7485 module_id = %spec.module_id,
7486 exit_code = ?exit_report.code,
7487 exit_signal = ?exit_report.signal,
7488 "supervised module exited cleanly"
7489 );
7490 if let Err(err) = publish_after_terminal_record(
7491 &spec.module_id,
7492 || {
7493 record_terminal(
7494 &spec.module_id,
7495 terminal_ring,
7496 spawn_events,
7497 &exit_report,
7498 TerminalDisposition::Stopped,
7499 )
7500 },
7501 || {
7502 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7503 state.state = ModuleState::Stopped;
7504 clear_current_process_facts(state);
7505 state.last_exit = Some(exit_report.clone());
7506 })
7507 },
7508 ) {
7509 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7510 }
7511 let registration_released = match wait_for_registration_release(
7512 registry,
7513 &spec.module_id,
7514 REGISTRY_RELEASE_TIMEOUT,
7515 )
7516 .await
7517 {
7518 Ok(()) => true,
7519 Err(err) => {
7520 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7521 false
7522 }
7523 };
7524 NextAction::Stop {
7525 registration_released,
7526 }
7527 }
7528 ExitKind::Clean | ExitKind::Crash => {
7529 if unrequested_clean_exit_of_protocol_none {
7530 warn!(
7531 module_id = %spec.module_id,
7532 exit_code = ?exit_report.code,
7533 exit_signal = ?exit_report.signal,
7534 "protocol-none module exited cleanly without a stop request; handling it as a crash"
7535 );
7536 } else {
7537 warn!(
7538 module_id = %spec.module_id,
7539 exit_code = ?exit_report.code,
7540 exit_signal = ?exit_report.signal,
7541 "supervised module exited abnormally (crash)"
7542 );
7543 }
7544 let mut restart_schedule = None;
7545 let mut disposition = TerminalDisposition::Disabled;
7546 let mut final_state = ModuleState::Disabled;
7547 let mut disposition_detail = lock_snapshot(snapshot)
7551 .ok()
7552 .and_then(|mut state| state.spawn_failure.take());
7553 let now = Instant::now();
7554 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7555 if state.enabled {
7556 if let Some(schedule) = state.next_crash_restart(&policy, now) {
7557 state.respawn_pending = true;
7563 final_state = ModuleState::Restarting;
7564 restart_schedule = Some(schedule);
7565 disposition = TerminalDisposition::Restarting;
7566 } else {
7567 final_state = ModuleState::Failed;
7568 disposition = TerminalDisposition::Failed;
7569 let budget = policy.budget_exhausted_detail();
7570 disposition_detail =
7571 Some(disposition_detail.take().map_or_else(
7572 || budget.clone(),
7573 |cause| format!("{cause}; {budget}"),
7574 ));
7575 }
7576 } else {
7577 disposition = TerminalDisposition::Disabled;
7578 }
7579 }) {
7580 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7581 return NextAction::Stop {
7582 registration_released: false,
7583 };
7584 }
7585 if disposition == TerminalDisposition::Failed {
7586 error!(
7591 module_id = %spec.module_id,
7592 max_restarts = policy.max_restarts,
7593 window_secs = policy.window.as_secs(),
7594 "module stopped: {}",
7595 policy.budget_exhausted_detail()
7596 );
7597 }
7598 if let Err(err) = publish_after_terminal_record(
7599 &spec.module_id,
7600 || {
7601 record_terminal_with_detail(
7602 &spec.module_id,
7603 terminal_ring,
7604 spawn_events,
7605 &exit_report,
7606 disposition,
7607 disposition_detail,
7608 )
7609 },
7610 || {
7611 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7612 state.state = final_state;
7613 clear_current_process_facts(state);
7614 state.last_exit = Some(exit_report.clone());
7615 })
7616 },
7617 ) {
7618 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7619 }
7620
7621 if let Some(schedule) = restart_schedule {
7622 NextAction::Restart {
7623 schedule: Some(schedule),
7624 }
7625 } else {
7626 let registration_released = match wait_for_registration_release(
7627 registry,
7628 &spec.module_id,
7629 REGISTRY_RELEASE_TIMEOUT,
7630 )
7631 .await
7632 {
7633 Ok(()) => true,
7634 Err(err) => {
7635 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7636 false
7637 }
7638 };
7639 NextAction::Stop {
7640 registration_released,
7641 }
7642 }
7643 }
7644 ExitKind::DeliberateSeverance => {
7645 warn!(
7646 module_id = %spec.module_id,
7647 exit_code = ?exit_report.code,
7648 exit_signal = ?exit_report.signal,
7649 "supervised module exited after deliberate connection severance"
7650 );
7651 let should_restart = match lock_snapshot(snapshot) {
7652 Ok(state) => state.enabled,
7653 Err(err) => {
7654 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7655 return NextAction::Stop {
7656 registration_released: false,
7657 };
7658 }
7659 };
7660 let final_state = if should_restart {
7661 ModuleState::Restarting
7662 } else {
7663 ModuleState::Disabled
7664 };
7665 if let Err(err) = publish_after_terminal_record(
7666 &spec.module_id,
7667 || {
7668 record_terminal(
7669 &spec.module_id,
7670 terminal_ring,
7671 spawn_events,
7672 &exit_report,
7673 terminal_disposition(final_state),
7674 )
7675 },
7676 || {
7677 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7678 state.state = final_state;
7679 clear_current_process_facts(state);
7680 state.last_exit = Some(exit_report.clone());
7681 state.lifetime_restarts += 1;
7682 })
7683 },
7684 ) {
7685 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7686 return NextAction::Stop {
7687 registration_released: false,
7688 };
7689 }
7690
7691 if should_restart {
7692 NextAction::Restart { schedule: None }
7693 } else {
7694 let registration_released = match wait_for_registration_release(
7695 registry,
7696 &spec.module_id,
7697 REGISTRY_RELEASE_TIMEOUT,
7698 )
7699 .await
7700 {
7701 Ok(()) => true,
7702 Err(err) => {
7703 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7704 false
7705 }
7706 };
7707 NextAction::Stop {
7708 registration_released,
7709 }
7710 }
7711 }
7712 }
7713}
7714
7715async fn on_child_exit_during_daemon_shutdown(
7716 spec: &ModuleSpec,
7717 registry: &Registry,
7718 snapshot: &SharedSnapshot,
7719 terminal_ring: &Arc<Mutex<TerminalRing>>,
7720 spawn_events: &SpawnEventFeed,
7721 exit_report: ExitReport,
7722) -> NextAction {
7723 info!(
7724 module_id = %spec.module_id,
7725 exit_code = ?exit_report.code,
7726 exit_signal = ?exit_report.signal,
7727 exit_kind = ?exit_report.kind,
7728 "supervised module exited during daemon shutdown; not restarting it"
7729 );
7730 if let Err(err) = publish_after_terminal_record(
7731 &spec.module_id,
7732 || {
7733 record_terminal(
7734 &spec.module_id,
7735 terminal_ring,
7736 spawn_events,
7737 &exit_report,
7738 TerminalDisposition::DaemonShutdown,
7739 )
7740 },
7741 || {
7742 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7743 state.state = ModuleState::Stopped;
7744 clear_current_process_facts(state);
7745 state.last_exit = Some(exit_report.clone());
7746 })
7747 },
7748 ) {
7749 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7750 }
7751 let registration_released =
7752 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7753 .await
7754 .is_ok();
7755 NextAction::Stop {
7756 registration_released,
7757 }
7758}
7759
7760fn publish_after_terminal_record<T>(
7765 module_id: &str,
7766 record: impl FnOnce(),
7767 publish: impl FnOnce() -> T,
7768) -> T {
7769 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7770 let published = publish();
7771 if let Err(panic) = recorded {
7772 error!(
7773 module_id,
7774 "terminal recording panicked; published final state before resuming unwind"
7775 );
7776 std::panic::resume_unwind(panic);
7777 }
7778 published
7779}
7780
7781fn fail_after_wait_error(
7782 module_id: &str,
7783 snapshot: &SharedSnapshot,
7784 terminal_ring: &Arc<Mutex<TerminalRing>>,
7785 spawn_events: &SpawnEventFeed,
7786 detail: String,
7787) {
7788 let report = wait_error_exit_report();
7789 publish_after_terminal_record(
7790 module_id,
7791 || {
7792 record_terminal_with_detail(
7793 module_id,
7794 terminal_ring,
7795 spawn_events,
7796 &report,
7797 TerminalDisposition::Failed,
7798 Some(detail),
7799 )
7800 },
7801 || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7802 );
7803}
7804
7805#[cfg(test)]
7806fn record_wait_error_terminal(
7807 module_id: &str,
7808 terminal_ring: &Arc<Mutex<TerminalRing>>,
7809 spawn_events: &SpawnEventFeed,
7810) {
7811 record_terminal(
7812 module_id,
7813 terminal_ring,
7814 spawn_events,
7815 &wait_error_exit_report(),
7816 TerminalDisposition::Failed,
7817 );
7818}
7819
7820fn record_terminal(
7821 module_id: &str,
7822 terminal_ring: &Arc<Mutex<TerminalRing>>,
7823 spawn_events: &SpawnEventFeed,
7824 exit_report: &ExitReport,
7825 disposition: TerminalDisposition,
7826) {
7827 record_terminal_with_detail(
7828 module_id,
7829 terminal_ring,
7830 spawn_events,
7831 exit_report,
7832 disposition,
7833 None,
7834 );
7835}
7836
7837fn durable_terminal_history_of(
7841 terminal_ring: &Mutex<TerminalRing>,
7842 module_id: &str,
7843) -> subc_control::TerminalHistory {
7844 let read = terminal_ring
7845 .lock()
7846 .unwrap_or_else(|p| p.into_inner())
7847 .capture_durable_history();
7848 read.read(module_id)
7849}
7850
7851fn record_terminal_with_detail(
7852 module_id: &str,
7853 terminal_ring: &Arc<Mutex<TerminalRing>>,
7854 spawn_events: &SpawnEventFeed,
7855 exit_report: &ExitReport,
7856 disposition: TerminalDisposition,
7857 disposition_detail: Option<String>,
7858) {
7859 #[cfg(test)]
7860 terminal_history_tests::before_terminal_record(spawn_events);
7861 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7862 let record = TerminalRecord {
7863 exit_code: exit_report.code,
7864 exit_signal: exit_report.signal,
7865 at_ms: exit_report.at_ms,
7866 disposition,
7867 exit_kind: exit_report.kind.into(),
7868 disposition_detail,
7869 };
7870 terminal_ring
7871 .lock()
7872 .unwrap_or_else(|poisoned| poisoned.into_inner())
7873 .record_exit(module_id, record);
7874}
7875
7876fn untrack_if_registration_released(
7877 process_liveness: &SupervisorProcessLiveness,
7878 registry: &Registry,
7879 module_id: &str,
7880 snapshot: &SharedSnapshot,
7881) {
7882 match registry.get_module(module_id) {
7883 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
7884 Ok(Some(_)) => {}
7885 Err(err) => {
7886 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
7887 }
7888 }
7889}
7890
7891#[cfg(test)]
7905fn apply_wire_spawn_args(
7906 command: &mut Command,
7907 spec: &ModuleSpec,
7908 connection_file_path: Option<&std::path::Path>,
7909 handle: Option<&SupervisorHandle>,
7910) -> Result<Option<NonceHandoff>, SuperviseError> {
7911 apply_wire_spawn_args_for_role(
7912 command,
7913 spec,
7914 connection_file_path,
7915 handle,
7916 SpawnRole::Plain,
7917 )
7918}
7919
7920#[cfg(unix)]
7925type NonceHandoff = subc_os::LaunchNonceHandoff;
7926#[cfg(windows)]
7927type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
7928#[cfg(not(any(unix, windows)))]
7929type NonceHandoff = std::convert::Infallible;
7930
7931fn apply_wire_spawn_args_for_role(
7952 command: &mut Command,
7953 spec: &ModuleSpec,
7954 connection_file_path: Option<&std::path::Path>,
7955 handle: Option<&SupervisorHandle>,
7956 role: SpawnRole,
7957) -> Result<Option<NonceHandoff>, SuperviseError> {
7958 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
7959 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
7964 #[cfg(windows)]
7965 {
7966 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
7967 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
7968 }
7969 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
7971 if spec.protocol == ModuleProtocol::None {
7972 return Ok(None);
7973 }
7974 if let Some(connection_file_path) = connection_file_path {
7975 command.arg(SUBC_ARG).arg(connection_file_path);
7976 }
7977
7978 let nonce = generate_launch_nonce()?;
7982 if let Some(handle) = handle {
7983 match role {
7984 SpawnRole::Plain => {
7985 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
7986 if spec.reserved {
7987 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
7988 }
7989 }
7990 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
7991 }
7992 }
7993 #[cfg(unix)]
7994 let handoff = {
7995 let handoff =
7996 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
7997 program: spec.program.clone(),
7998 source,
7999 cgroup_path: None,
8000 })?;
8001 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8002 Some(handoff)
8003 };
8004 #[cfg(windows)]
8005 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8006 #[cfg(not(any(unix, windows)))]
8007 let handoff = None;
8008 #[cfg(not(unix))]
8012 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8013 Ok(handoff)
8014}
8015
8016fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8017 command.env_remove(CK_LOG_ENV);
8018 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8025 for (key, value) in &spec.env {
8026 if matches!(
8030 key.as_str(),
8031 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8032 ) || key == SUBC_SPAWN_ROLE_ENV
8033 {
8034 continue;
8035 }
8036 command.env(key, value);
8037 }
8038}
8039
8040#[cfg(windows)]
8041fn prepare_windows_nonce_pipe(
8042 command: &mut Command,
8043 spec: &ModuleSpec,
8044 nonce: &str,
8045) -> Option<NonceHandoff> {
8046 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8047 Ok(handoff) => {
8048 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8049 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8054 Some(handoff)
8055 }
8056 Err(error) => {
8057 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8058 None
8059 }
8060 }
8061}
8062
8063#[cfg(windows)]
8064fn start_windows_nonce_pipe(
8065 handoff: Option<NonceHandoff>,
8066 pid: u32,
8067 spec: &ModuleSpec,
8068) -> Option<subc_os::LaunchNoncePipeDelivery> {
8069 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8073 Ok(delivery) => Some(delivery),
8074 Err(error) => {
8075 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8076 None
8077 }
8078 })
8079}
8080
8081fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8082 #[cfg(windows)]
8083 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8084 return Some(
8085 if snapshot
8086 .pipe_consumed
8087 .as_ref()
8088 .is_some_and(|consumed| consumed.get().is_some())
8089 {
8090 "pipe"
8091 } else {
8092 "env"
8093 }
8094 .to_owned(),
8095 );
8096 }
8097 let _ = snapshot;
8098 None
8099}
8100
8101#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8104enum SpawnRole {
8105 Plain,
8106 SwapCandidate,
8107}
8108
8109fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8112 if role == SpawnRole::SwapCandidate {
8113 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8114 }
8115}
8116
8117fn previous_instance_error(
8118 spec: &ModuleSpec,
8119 pid: u32,
8120 reason: crate::live_children::RetainedReason,
8121) -> SuperviseError {
8122 SuperviseError::Spawn {
8123 program: spec.program.clone(),
8124 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8125 cgroup_path: None,
8126 }
8127}
8128
8129fn spawn_child(
8130 spec: &ModuleSpec,
8131 connection_file_path: Option<&std::path::Path>,
8132 handle: Option<&SupervisorHandle>,
8133 ring: &Arc<Mutex<StderrRing>>,
8134 capture_logs_dir: Option<&std::path::Path>,
8135 roster: &ChildRoster,
8136 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8137) -> Result<SupervisedChild, SuperviseError> {
8138 spawn_child_in_slot(
8139 spec,
8140 connection_file_path,
8141 handle,
8142 ring,
8143 capture_logs_dir,
8144 roster,
8145 #[cfg(target_os = "linux")]
8146 cgroup_placement,
8147 SpawnRole::Plain,
8148 false,
8149 )
8150}
8151
8152#[allow(clippy::too_many_arguments)]
8165fn spawn_child_in_slot(
8166 spec: &ModuleSpec,
8167 connection_file_path: Option<&std::path::Path>,
8168 handle: Option<&SupervisorHandle>,
8169 ring: &Arc<Mutex<StderrRing>>,
8170 capture_logs_dir: Option<&std::path::Path>,
8171 roster: &ChildRoster,
8172 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8173 role: SpawnRole,
8174 alternate_slot: bool,
8175) -> Result<SupervisedChild, SuperviseError> {
8176 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8177 return Err(previous_instance_error(spec, pid, reason));
8178 }
8179 if roster.is_closed() {
8180 return Err(SuperviseError::Spawn {
8181 program: spec.program.clone(),
8182 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8183 cgroup_path: None,
8184 });
8185 }
8186 #[cfg(target_os = "linux")]
8187 let cgroup_name = {
8188 if cgroup_placement.is_none() {
8193 swap::cgroup_name(&spec.module_id, alternate_slot)
8194 } else {
8195 let nonce = generate_launch_nonce()?;
8196 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8197 let mut end = spec.module_id.len().min(64);
8200 while !spec.module_id.is_char_boundary(end) {
8201 end -= 1;
8202 }
8203 format!(
8204 "{}_{suffix}",
8205 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8206 )
8207 }
8208 };
8209 #[cfg(not(target_os = "linux"))]
8210 let _ = alternate_slot;
8211 #[cfg(target_os = "macos")]
8212 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8213 #[cfg(windows)]
8214 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8215 program: spec.program.clone(),
8216 source,
8217 cgroup_path: None,
8218 })?;
8219 #[cfg(windows)]
8220 let mut command = Command::new(image_capture.path());
8221 #[cfg(not(any(target_os = "macos", windows)))]
8222 let mut command = Command::new(&spec.program);
8223 command.args(&spec.args);
8224 apply_child_env(&mut command, spec);
8254 apply_spawn_role(&mut command, role);
8255 let nonce_handoff =
8256 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8257
8258 #[cfg(target_os = "linux")]
8259 let cgroup_path = cgroup_placement
8260 .map(|placement| placement.module_path(&cgroup_name))
8261 .transpose()
8262 .map_err(|source| SuperviseError::Cgroup {
8263 module_id: spec.module_id.clone(),
8264 source,
8265 })?;
8266 #[cfg(not(target_os = "linux"))]
8267 let cgroup_path: Option<PathBuf> = None;
8268 #[cfg(target_os = "linux")]
8269 if let Some(path) = &cgroup_path {
8270 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8271 if let Some(placement) = cgroup_placement {
8272 remove_module_cgroup(placement, &cgroup_name);
8273 }
8274 return Err(error);
8275 }
8276 }
8277
8278 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8279 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8280 match ChildOutputSink::open(&path, capture_retention(spec)) {
8281 Ok(sink) => sink,
8282 Err(error) => {
8283 warn!(
8284 module_id = %spec.module_id,
8285 path = %path.display(),
8286 error = %error,
8287 "could not open child output capture file; forwarding to stderr"
8288 );
8289 ChildOutputSink::Stderr
8290 }
8291 }
8292 } else {
8293 ChildOutputSink::Stderr
8294 };
8295
8296 command.stdout(Stdio::piped());
8297 command.stderr(Stdio::piped());
8298 command.kill_on_drop(true);
8299 #[cfg(unix)]
8316 command.process_group(0);
8317 command.stdin(Stdio::null());
8318 #[cfg(unix)]
8322 if let Some(handoff) = nonce_handoff {
8323 handoff.install_last(command.as_std_mut());
8324 }
8325 #[cfg(not(any(unix, windows)))]
8326 let _ = nonce_handoff;
8327
8328 #[cfg(windows)]
8333 {
8334 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8335 program: spec.program.clone(),
8336 source,
8337 cgroup_path: None,
8338 })?;
8339 subc_jobobject::suspend_on_create_async(&mut command);
8340 }
8341 let mut child = match command.spawn() {
8342 Ok(child) => child,
8343 Err(source) => {
8344 #[cfg(target_os = "linux")]
8345 if let Some(placement) = cgroup_placement {
8346 remove_module_cgroup(placement, &cgroup_name);
8347 }
8348 return Err(SuperviseError::Spawn {
8349 program: spec.program.clone(),
8350 source,
8351 cgroup_path,
8352 });
8353 }
8354 };
8355 #[cfg(test)]
8356 spawn_record_tests::after_create(&child, spec);
8357 #[cfg(target_os = "macos")]
8362 drop(exec_ack);
8363
8364 #[cfg(windows)]
8369 let windows_image = match image_capture.bind_tokio(&child) {
8370 Ok(image) => Arc::new(image),
8371 Err(source) => {
8372 kill_and_reap_rejected_child(&mut child);
8373 return Err(SuperviseError::Spawn {
8374 program: spec.program.clone(),
8375 source,
8376 cgroup_path: None,
8377 });
8378 }
8379 };
8380 #[cfg(windows)]
8383 let job = contain_spawned_child(&mut child, spec)?;
8384 let spawned_at_ms = unix_ms_now();
8385 #[cfg(not(windows))]
8386 let spawned_from = spec.program.clone();
8387 #[cfg(windows)]
8388 let spawned_from = windows_image.path().to_path_buf();
8389 #[cfg(not(windows))]
8390 let spawned_file_identity = spawned_file_identity(&spawned_from);
8391 #[cfg(windows)]
8392 let spawned_file_identity = Some(SpawnedFileIdentity {
8393 device: 0,
8394 inode: 0,
8395 windows_image: Some(Arc::clone(&windows_image)),
8396 });
8397 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8398 program: spec.program.clone(),
8399 source: io::Error::other("spawned child exposed no live pid"),
8400 cgroup_path: cgroup_path.clone(),
8401 })?;
8402 #[cfg(windows)]
8403 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8404 #[cfg(not(windows))]
8405 let process_start_time = crate::provenance::process_start_time(pid);
8406 #[cfg(windows)]
8407 let process_start_time = Some(windows_image.start_time());
8408 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8409 #[cfg(all(test, target_os = "macos"))]
8410 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8411 #[cfg(not(windows))]
8416 let recorded_image = observe_spawned_image(pid);
8417 #[cfg(target_os = "macos")]
8421 let recorded_image = if privacy_exec.is_some() {
8422 None
8423 } else {
8424 recorded_image
8425 };
8426 #[cfg(target_os = "linux")]
8427 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8428 #[cfg(not(target_os = "linux"))]
8429 let recorded_cgroup_name = None;
8430 let roster_guard = roster.admit(
8431 spec.module_id.clone(),
8432 pid,
8433 spec.protocol,
8434 process_start_time,
8435 crate::child_roster::RecordedIdentity {
8436 #[cfg(not(windows))]
8437 start_time: recorded_image.map(|image| image.start_time),
8438 #[cfg(windows)]
8439 start_time: process_start_time,
8440 #[cfg(not(windows))]
8441 executable: recorded_image
8442 .and_then(|image| image.executable)
8443 .map(crate::live_children::ExecutableIdentity::from),
8444 #[cfg(windows)]
8445 executable: Some(crate::live_children::ExecutableIdentity::from(
8446 windows_image.identity(),
8447 )),
8448 cgroup_name: recorded_cgroup_name,
8449 #[cfg(target_os = "linux")]
8450 cgroup_placement: cgroup_placement.cloned(),
8451 },
8452 );
8453 let roster_guard = match roster_guard {
8454 Ok(guard) => guard,
8455 Err(source) => {
8456 #[cfg(windows)]
8457 if let Some(job) = &job {
8458 let _ = job.terminate();
8459 }
8460 #[cfg(target_os = "linux")]
8461 kill_module_cgroup(cgroup_placement, &cgroup_name);
8462 kill_and_reap_rejected_child(&mut child);
8463 #[cfg(target_os = "linux")]
8464 if let Some(placement) = cgroup_placement {
8465 remove_module_cgroup(placement, &cgroup_name);
8466 }
8467 return Err(SuperviseError::Spawn {
8468 program: spec.program.clone(),
8469 source: io::Error::other(format!(
8470 "could not durably record module child: {source}"
8471 )),
8472 cgroup_path,
8473 });
8474 }
8475 };
8476 if roster.is_closed() {
8485 #[cfg(target_os = "linux")]
8487 kill_module_cgroup(cgroup_placement, &cgroup_name);
8488 if let Err(error) = child.start_kill() {
8489 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8490 }
8491 #[cfg(target_os = "linux")]
8492 if let Some(placement) = cgroup_placement {
8493 while matches!(child.try_wait(), Ok(None)) {
8497 std::thread::yield_now();
8498 }
8499 if matches!(
8500 subc_cgroup::kill_module(Some(placement), &cgroup_name),
8501 subc_cgroup::KillOutcome::Killed
8502 ) {
8503 if let Ok(path) = placement.module_path(&cgroup_name) {
8504 while std::fs::read_to_string(path.join("cgroup.events"))
8505 .ok()
8506 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8507 {
8508 std::thread::yield_now();
8509 }
8510 }
8511 }
8512 remove_module_cgroup(placement, &cgroup_name);
8513 }
8514 drop(roster_guard);
8515 return Err(SuperviseError::Spawn {
8516 program: spec.program.clone(),
8517 source: io::Error::other(
8518 "the daemon began shutting down while this process was starting; ended it",
8519 ),
8520 cgroup_path,
8521 });
8522 }
8523
8524 #[cfg(windows)]
8525 if let Err(error) = resume_suspended_child(pid, spec) {
8526 kill_and_reap_rejected_child(&mut child);
8527 return Err(error);
8528 }
8529
8530 let stdout_pump = match child.stdout.take() {
8531 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8532 None => {
8533 warn!(
8534 module_id = %spec.module_id,
8535 "spawned child exposed no stdout pipe; file capture will be incomplete"
8536 );
8537 None
8538 }
8539 };
8540 let stderr_pump = match child.stderr.take() {
8541 Some(stderr) => {
8542 let generation = ring
8543 .lock()
8544 .unwrap_or_else(|poisoned| poisoned.into_inner())
8545 .begin_process();
8546 Some(StderrPump {
8547 task: tokio::spawn(pump_stderr_to(
8548 stderr,
8549 Arc::clone(ring),
8550 generation,
8551 output_sink,
8552 )),
8553 generation,
8554 })
8555 }
8556 None => {
8557 ring.lock()
8561 .unwrap_or_else(|poisoned| poisoned.into_inner())
8562 .mark_not_captured("stderr pipe was not available on spawn");
8563 warn!(
8564 module_id = %spec.module_id,
8565 "spawned child exposed no stderr pipe; tail will be unavailable"
8566 );
8567 None
8568 }
8569 };
8570
8571 let exec = Arc::new(crate::activity::ExecOutcome::new());
8572 #[cfg(target_os = "macos")]
8573 let pending_exec = privacy_exec.is_some();
8574 #[cfg(not(target_os = "macos"))]
8575 let pending_exec = false;
8576 if !pending_exec {
8577 exec.finish(crate::activity::ExecState::CommittedExec);
8578 }
8579 Ok(SupervisedChild {
8580 child,
8581 exec: crate::activity::ExecOwner(exec),
8582 #[cfg(windows)]
8583 nonce_delivery,
8584 protocol: spec.protocol,
8585 #[cfg(target_os = "linux")]
8586 module_id: cgroup_name,
8587 #[cfg(target_os = "linux")]
8588 cgroup_placement: cgroup_placement.cloned(),
8589 #[cfg(windows)]
8590 job,
8591 stdout_pump,
8592 stderr_pump,
8593 stderr_ring: Arc::clone(ring),
8594 spawned_at_ms,
8595 spawned_from,
8596 spawned_file_identity,
8597 process_start_time,
8598 process_identity,
8599 pid,
8600 roster_guard: Some(roster_guard),
8601 #[cfg(target_os = "macos")]
8602 privacy_exec,
8603 #[cfg(target_os = "macos")]
8604 report_ready: Arc::new(OnceLock::new()),
8605 spawn_failure: None,
8606 })
8607}
8608
8609#[cfg(test)]
8610mod spawn_record_tests {
8611 use super::*;
8612
8613 thread_local! {
8614 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8615 }
8616
8617 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8618 LAST_PID.set(child.id());
8619 #[cfg(windows)]
8620 if let Some((_, address)) = _spec
8621 .env
8622 .iter()
8623 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8624 {
8625 use std::io::{Read, Write};
8626 let mut stream = std::net::TcpStream::connect(address).unwrap();
8627 stream
8628 .write_all(&child.id().unwrap().to_ne_bytes())
8629 .unwrap();
8630 let mut release = [0];
8631 stream.read_exact(&mut release).unwrap();
8632 }
8633 }
8634
8635 #[cfg(windows)]
8636 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8637 if spec
8638 .env
8639 .iter()
8640 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8641 {
8642 Err(io::Error::other("injected module job assignment failure"))
8643 } else {
8644 Ok(())
8645 }
8646 }
8647
8648 fn spec() -> ModuleSpec {
8649 ModuleSpec {
8650 module_id: "spawn-record-worker".into(),
8651 #[cfg(unix)]
8652 program: "/bin/sleep".into(),
8653 #[cfg(windows)]
8654 program: std::env::current_exe().unwrap(),
8655 #[cfg(unix)]
8656 args: vec!["60".into()],
8657 #[cfg(windows)]
8658 args: vec![
8659 "--exact".into(),
8660 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8661 "--nocapture".into(),
8662 ],
8663 env: vec![],
8664 reserved: false,
8665 reserved_prefixes: vec![],
8666 protocol: ModuleProtocol::None,
8667 overlap: Default::default(),
8668 }
8669 }
8670
8671 #[tokio::test]
8672 async fn record_sync_failure_fails_spawn_and_reaps_child() {
8673 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8674 let supervisor =
8675 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8676 .with_live_children_record(home.join("live-children.json"));
8677 crate::live_children::fail_next_file_sync();
8678 let result = supervisor.spawn(spec());
8679 let failed = result.is_err();
8680 if let Ok(module) = result {
8681 module.stop().await.unwrap();
8682 }
8683 assert!(failed, "record failure must fail spawn");
8684 let pid = LAST_PID.get().expect("the real OS child was created");
8685 assert!(
8686 subc_os::Process::open(pid).unwrap().is_none(),
8687 "child {pid} survived rejected admission or was not reaped"
8688 );
8689 }
8690
8691 #[cfg(target_os = "macos")]
8692 #[test]
8693 #[ignore = "manual on-device latency measurement"]
8694 fn durable_record_write_latency_50() {
8695 let home = cortexkit_test_support::ScratchDir::new("record-latency");
8696 let path = home.join("live-children.json");
8697 let children = vec![crate::live_children::LiveChild {
8698 module_id: "latency-worker".into(),
8699 pid: std::process::id(),
8700 protocol: ModuleProtocol::None,
8701 start_time: Some(1),
8702 executable: None,
8703 cgroup_name: None,
8704 }];
8705 let mut samples = Vec::new();
8706 for _ in 0..50 {
8707 let start = std::time::Instant::now();
8708 crate::live_children::write_record(&path, &children).unwrap();
8709 samples.push(start.elapsed());
8710 }
8711 samples.sort();
8712 eprintln!(
8713 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8714 (samples[24] + samples[25]) / 2,
8715 samples[47]
8716 );
8717 }
8718
8719 #[cfg(windows)]
8720 #[test]
8721 fn windows_store_touch_fixture() {
8722 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8723 std::fs::write(path, b"module ran").unwrap();
8724 }
8725 }
8726
8727 #[cfg(windows)]
8728 #[tokio::test]
8729 async fn windows_daemon_fixture() {
8730 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8731 return;
8732 };
8733 let mut spec = spec();
8734 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8735 let supervisor =
8736 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8737 let _module = supervisor.spawn(spec).unwrap();
8738 }
8739
8740 #[cfg(windows)]
8741 #[test]
8742 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8743 use std::io::Read;
8744 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8745 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8748 command
8749 .args([
8750 "--exact",
8751 "supervise::spawn_record_tests::windows_daemon_fixture",
8752 "--nocapture",
8753 ])
8754 .env(
8755 "SUBC_TEST_DAEMON_BARRIER",
8756 listener.local_addr().unwrap().to_string(),
8757 );
8758 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8759 listener.set_nonblocking(true).unwrap();
8760 let deadline = std::time::Instant::now() + Duration::from_secs(30);
8761 let mut stream = loop {
8762 if let Ok((stream, _)) = listener.accept() {
8763 break stream;
8764 }
8765 assert!(
8766 daemon.child.try_wait().unwrap().is_none(),
8767 "daemon failed before rendezvous"
8768 );
8769 assert!(
8770 std::time::Instant::now() < deadline,
8771 "daemon did not reach CreateProcess failpoint"
8772 );
8773 std::thread::yield_now();
8774 };
8775 stream
8776 .set_read_timeout(Some(Duration::from_secs(30)))
8777 .unwrap();
8778 let mut pid = [0; 4];
8779 stream.read_exact(&mut pid).unwrap();
8780 let pid = u32::from_ne_bytes(pid);
8781 assert!(subc_jobobject::process_exists(pid));
8782 daemon.child.kill().unwrap();
8785 daemon.child.wait().unwrap();
8786 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8787 if !exited {
8788 daemon.job.terminate().unwrap();
8789 }
8790 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8791 }
8792
8793 #[cfg(windows)]
8794 #[tokio::test]
8795 async fn windows_containment_failure_never_resumes() {
8796 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8797 let sentinel = home.join("store-touched");
8798 let mut spec = spec();
8799 spec.env
8800 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8801 spec.env.push((
8802 "SUBC_TEST_STORE_TOUCH".into(),
8803 sentinel.display().to_string(),
8804 ));
8805 let supervisor =
8806 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8807 let result = supervisor.spawn(spec);
8808 if let Ok(module) = result {
8811 let _ = tokio::time::timeout(Duration::from_secs(10), async {
8812 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8813 tokio::task::yield_now().await;
8814 }
8815 })
8816 .await;
8817 let _ = module.stop().await;
8818 panic!(
8819 "assignment failure resumed a module; store touched={}",
8820 sentinel.exists()
8821 );
8822 }
8823 assert!(
8824 !sentinel.exists(),
8825 "refused module executed store-touch sentinel"
8826 );
8827 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8828 }
8829
8830 #[cfg(windows)]
8831 #[tokio::test]
8832 async fn windows_nested_daemon_fixture() {
8833 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8834 return;
8835 };
8836 let mut spec = spec();
8837 spec.env.push((
8838 "SUBC_TEST_STORE_TOUCH".into(),
8839 sentinel.to_str().unwrap().into(),
8840 ));
8841 let supervisor =
8842 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8843 let module = supervisor
8844 .spawn(spec)
8845 .expect("per-module job must nest below daemon and host jobs");
8846 tokio::time::timeout(Duration::from_secs(10), async {
8847 while !PathBuf::from(&sentinel).exists() {
8848 tokio::task::yield_now().await;
8849 }
8850 })
8851 .await
8852 .unwrap();
8853 module.stop().await.unwrap();
8854 }
8855
8856 #[cfg(windows)]
8857 #[test]
8858 fn windows_module_job_nests_under_host_job() {
8859 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
8860 let sentinel = home.join("nested-store-touched");
8861 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8862 command
8863 .args([
8864 "--exact",
8865 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
8866 "--nocapture",
8867 ])
8868 .env("SUBC_TEST_NESTED_STORE", &sentinel);
8869 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8870 assert!(daemon.child.wait().unwrap().success());
8871 assert!(sentinel.exists(), "nested protocol:none module never ran");
8872 }
8873}
8874
8875#[cfg(target_os = "linux")]
8876pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
8877 use subc_cgroup::KillOutcome;
8878 match subc_cgroup::kill_module(placement, module_id) {
8879 KillOutcome::Killed => {}
8880 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
8881 debug!(
8882 module_id,
8883 "cgroup tree kill unavailable; using direct-child kill"
8884 );
8885 }
8886 KillOutcome::IoError { path, error } => {
8887 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
8888 }
8889 }
8890}
8891
8892#[cfg(windows)]
8900fn contain_spawned_child(
8901 child: &mut Child,
8902 spec: &ModuleSpec,
8903) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
8904 let assignment = || {
8905 let job = subc_jobobject::JobObject::new()?;
8906 #[cfg(test)]
8907 spawn_record_tests::assignment_failpoint(spec)?;
8908 job.assign(child)?;
8909 Ok::<_, io::Error>(job)
8910 };
8911 match assignment() {
8912 Ok(job) => Ok(Some(job)),
8913 Err(source) => {
8914 kill_and_reap_rejected_child(child);
8915 Err(SuperviseError::Spawn {
8916 program: spec.program.clone(),
8917 source: io::Error::other(format!(
8918 "Windows module job containment failed: {source}"
8919 )),
8920 cgroup_path: None,
8921 })
8922 }
8923 }
8924}
8925
8926fn kill_and_reap_rejected_child(child: &mut Child) {
8929 #[cfg(unix)]
8930 if let Some(pid) = child
8931 .id()
8932 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
8933 {
8934 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
8935 }
8936 let _ = child.start_kill();
8937 while matches!(child.try_wait(), Ok(None)) {
8938 std::thread::yield_now();
8939 }
8940}
8941
8942#[cfg(windows)]
8947fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
8948 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
8949 let _ = std::process::Command::new("taskkill.exe")
8953 .args(["/PID", &pid.to_string(), "/T", "/F"])
8954 .stdin(Stdio::null())
8955 .stdout(Stdio::null())
8956 .stderr(Stdio::null())
8957 .status();
8958 return Err(SuperviseError::Spawn {
8959 program: spec.program.clone(),
8960 source,
8961 cgroup_path: None,
8962 });
8963 }
8964 Ok(())
8965}
8966
8967#[cfg(target_os = "linux")]
8968fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
8969 match placement.remove_module(module_id) {
8970 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
8971 Err(error) => warn!(
8972 module_id,
8973 error = %error,
8974 "could not remove module cgroup after process exit; continuing teardown"
8975 ),
8976 }
8977}
8978
8979#[cfg(target_os = "linux")]
8980async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
8981 if matches!(
8985 subc_cgroup::kill_module(Some(placement), module_id),
8986 subc_cgroup::KillOutcome::Killed
8987 ) {
8988 if let Ok(path) = placement.module_path(module_id) {
8989 while std::fs::read_to_string(path.join("cgroup.events"))
8990 .ok()
8991 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8992 {
8993 sleep(Duration::from_millis(1)).await;
8994 }
8995 }
8996 }
8997 remove_module_cgroup(placement, module_id);
8998}
8999
9000#[cfg(target_os = "linux")]
9001fn apply_cgroup_placement(
9002 command: &mut Command,
9003 spec: &ModuleSpec,
9004 path: &std::path::Path,
9005) -> Result<(), SuperviseError> {
9006 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9007 module_id: spec.module_id.clone(),
9008 source,
9009 })
9010}
9011
9012fn capture_retention(spec: &ModuleSpec) -> Retention {
9013 let defaults = Retention::default();
9014 let value = |name: &str| {
9015 spec.env
9016 .iter()
9017 .rev()
9018 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9019 };
9020 Retention {
9021 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9022 .and_then(|value| value.parse().ok())
9023 .unwrap_or(defaults.max_file_mb),
9024 keep: value(CAPTURE_KEEP_ENV)
9025 .and_then(|value| value.parse().ok())
9026 .unwrap_or(defaults.keep),
9027 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9028 .and_then(|value| value.parse().ok())
9029 .unwrap_or(defaults.max_age_days),
9030 }
9031}
9032
9033fn generate_launch_nonce() -> Result<String, SuperviseError> {
9036 let mut bytes = [0u8; 32];
9037 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9038 reason: source.to_string(),
9039 })?;
9040 let mut hex = String::with_capacity(64);
9041 for b in bytes {
9042 use std::fmt::Write;
9043 let _ = write!(hex, "{b:02x}");
9044 }
9045 Ok(hex)
9046}
9047
9048fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9051 if a.len() != b.len() {
9052 return false;
9053 }
9054 let mut diff = 0u8;
9055 for (x, y) in a.iter().zip(b.iter()) {
9056 diff |= x ^ y;
9057 }
9058 diff == 0
9059}
9060
9061#[cfg(not(windows))]
9065fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9066 let process = subc_os::Process::open(pid).ok().flatten()?;
9067 #[cfg(target_os = "linux")]
9068 return sample_after_linux_exec(
9069 || {
9070 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9071 let committed = linux_exec_committed(&stat);
9072 #[cfg(test)]
9073 spawn_image_tests::report_exec_state(committed);
9074 committed
9075 },
9076 || process.observe(),
9077 );
9078 #[cfg(not(target_os = "linux"))]
9079 process.observe()
9080}
9081
9082#[cfg(target_os = "linux")]
9086fn linux_exec_committed(stat: &str) -> Option<bool> {
9087 const PF_FORKNOEXEC: u64 = 0x40;
9088 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9089 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9090}
9091
9092#[cfg(target_os = "linux")]
9093fn sample_after_linux_exec(
9094 mut exec_committed: impl FnMut() -> Option<bool>,
9095 observe: impl FnOnce() -> Option<subc_os::Observation>,
9096) -> Option<subc_os::Observation> {
9097 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9098 while !exec_committed()? {
9099 if std::time::Instant::now() >= deadline {
9103 return None;
9104 }
9105 std::thread::sleep(Duration::from_micros(50));
9106 }
9107 observe()
9108}
9109
9110#[cfg(all(test, target_os = "linux"))]
9111mod spawn_image_tests {
9112 use super::*;
9113 use std::cell::{Cell, RefCell};
9114
9115 enum ExecEvent {
9116 Uncommitted,
9117 Image(Option<subc_os::Observation>),
9118 }
9119
9120 thread_local! {
9121 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9122 }
9123
9124 pub(super) fn report_exec_state(committed: Option<bool>) {
9125 if committed == Some(false) {
9126 EXEC_EVENTS.with(|events| {
9127 if let Some(events) = events.borrow_mut().take() {
9128 let _ = events.send(ExecEvent::Uncommitted);
9129 }
9130 });
9131 }
9132 }
9133
9134 #[test]
9135 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9136 use std::io::{Read, Write};
9137 use std::os::fd::AsRawFd;
9138
9139 let descriptor = std::fs::File::open("/dev/null").unwrap();
9140 let mut command = std::process::Command::new("/bin/sleep");
9141 command.arg("30");
9142 let (mut ready, mut release) =
9143 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9144 .unwrap();
9145 let spawn = std::thread::spawn(move || command.spawn());
9146 let mut message = [0u8; 8];
9147 ready.read_exact(&mut message).unwrap();
9148 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9149 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9150 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9151 let (events, received) = std::sync::mpsc::channel();
9152 let observer = std::thread::spawn(move || {
9153 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9154 let image = observe_spawned_image(pid);
9155 let _ = events.send(ExecEvent::Image(image));
9156 });
9157 let first = received.recv_timeout(Duration::from_secs(5));
9161 release.write_all(&[1]).unwrap();
9162 let mut child = spawn.join().unwrap().unwrap();
9163 let after = match first {
9164 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9165 Ok(ExecEvent::Image(image)) => image,
9166 _ => None,
9167 },
9168 Ok(ExecEvent::Image(image)) => image,
9169 Err(_) => None,
9170 };
9171 observer.join().unwrap();
9172 child.kill().unwrap();
9173 child.wait().unwrap();
9174
9175 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9176 assert_eq!(
9177 inherited.unwrap().executable,
9178 subc_os::file_identity(&std::env::current_exe().unwrap())
9179 );
9180 assert_eq!(
9181 after.unwrap().executable,
9182 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9183 );
9184 }
9185
9186 #[test]
9187 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9188 let committed = Cell::new(false);
9189 let inherited = subc_os::Observation {
9190 start_time: 23,
9191 executable: Some(subc_os::FileIdentity {
9192 device: 29,
9193 inode: 31,
9194 }),
9195 };
9196 let module = subc_os::Observation {
9197 start_time: 23,
9198 executable: Some(subc_os::FileIdentity {
9199 device: 29,
9200 inode: 37,
9201 }),
9202 };
9203 let polls = Cell::new(0);
9204 let image = sample_after_linux_exec(
9205 || {
9206 polls.set(polls.get() + 1);
9207 committed.set(polls.get() == 2);
9210 Some(committed.get())
9211 },
9212 || Some(if committed.get() { module } else { inherited }),
9213 );
9214 assert_eq!(image, Some(module));
9215 assert_eq!(polls.get(), 2);
9216 }
9217
9218 #[test]
9219 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9220 assert_eq!(
9221 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9222 Some(false)
9223 );
9224 assert_eq!(
9225 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9226 Some(true)
9227 );
9228 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9229 assert_eq!(linux_exec_committed("17 (a) R"), None);
9230 }
9231
9232 #[test]
9233 fn unreadable_exec_state_never_samples_an_inherited_image() {
9234 assert_eq!(
9235 sample_after_linux_exec(
9236 || None,
9237 || panic!("must not sample before exec confirmation")
9238 ),
9239 None
9240 );
9241 }
9242
9243 #[test]
9244 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9245 assert_eq!(
9246 sample_after_linux_exec(
9247 || Some(false),
9248 || panic!("must not sample after an exec timeout")
9249 ),
9250 None
9251 );
9252 }
9253}
9254
9255#[cfg(windows)]
9258fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9259 let mut program = spec.program.clone();
9260 if program.is_absolute() || program.components().count() > 1 {
9261 match subc_os::ExecutableCapture::open(&program) {
9264 Ok(capture) => return Ok(capture),
9265 Err(error)
9266 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9267 Err(error) => return Err(error),
9268 }
9269 program.set_extension("exe");
9270 return subc_os::ExecutableCapture::open(&program);
9271 }
9272 if program.extension().is_none() {
9273 program.set_extension("exe");
9274 }
9275 let custom_path = spec
9276 .env
9277 .iter()
9278 .rev()
9279 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9280 let mut directories = Vec::new();
9281 if let Some(parent) = std::env::current_exe()?.parent() {
9284 directories.push(parent.to_path_buf());
9285 }
9286 directories.push(std::env::current_dir()?);
9287 if let Some(windows) = std::env::var_os("SystemRoot") {
9288 let windows = PathBuf::from(windows);
9289 directories.push(windows.join("System32"));
9290 directories.push(windows.clone());
9291 }
9292 let path = custom_path
9293 .map(|(_, value)| std::ffi::OsString::from(value))
9294 .or_else(|| std::env::var_os("PATH"));
9295 if let Some(path) = path {
9296 directories.extend(std::env::split_paths(&path));
9297 }
9298 for directory in directories {
9299 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9300 Ok(capture) => return Ok(capture),
9301 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9302 Err(error) => return Err(error),
9303 }
9304 }
9305 Err(io::Error::new(
9306 io::ErrorKind::NotFound,
9307 "resolved executable not found",
9308 ))
9309}
9310
9311fn spawn_and_mark_running(
9312 spec: &ModuleSpec,
9313 runtime: &SupervisorRuntimeConfig,
9314 snapshot: &SharedSnapshot,
9315) -> Result<SupervisedChild, SuperviseError> {
9316 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9317 let child = spawn_child(
9318 spec,
9319 runtime.connection_file_path.as_deref(),
9320 runtime.supervisor_handle.as_ref(),
9321 &runtime.stderr_ring,
9322 runtime.capture_logs_dir.as_deref(),
9323 &runtime.child_roster,
9324 #[cfg(target_os = "linux")]
9325 runtime.cgroup_placement.as_ref(),
9326 )?;
9327 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9328 if let Some(spawn) = activity_spawn {
9329 spawn.complete(snapshot);
9330 }
9331 Ok(child)
9332}
9333
9334enum RegistrationWaitOutcome {
9335 Registered,
9336 Exited(ExitReport),
9337 TimedOut,
9338}
9339
9340struct ReloadRegistrationFailure {
9341 exit_report: ExitReport,
9342 reason: String,
9343}
9344
9345#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9346enum BusyGaugeObservation {
9347 Quiescent,
9348 Busy,
9349 Omitted,
9350}
9351
9352fn busy_gauge_observation(metrics: Option<&Value>, gauges: &[String]) -> BusyGaugeObservation {
9353 let Some(metrics) = metrics.and_then(Value::as_object) else {
9354 return BusyGaugeObservation::Omitted;
9355 };
9356 let mut sum = 0u128;
9357 for gauge in gauges {
9358 let Some(value) = metrics.get(gauge) else {
9359 return BusyGaugeObservation::Omitted;
9360 };
9361 let Some(value) = value.as_u64() else {
9362 return BusyGaugeObservation::Busy;
9363 };
9364 sum = sum.saturating_add(u128::from(value));
9365 }
9366 if sum == 0 {
9367 BusyGaugeObservation::Quiescent
9368 } else {
9369 BusyGaugeObservation::Busy
9370 }
9371}
9372
9373fn declared_busy_gauges(
9374 registry: &Registry,
9375 module_id: &str,
9376) -> Result<Vec<String>, SuperviseError> {
9377 busy_gauges_of(
9378 registry
9379 .get_module(module_id)
9380 .map_err(SuperviseError::Registry)?,
9381 )
9382}
9383
9384fn declared_busy_gauges_for_connection(
9388 registry: &Registry,
9389 connection_id: ConnectionId,
9390) -> Result<Vec<String>, SuperviseError> {
9391 busy_gauges_of(
9392 registry
9393 .get_module_by_connection(connection_id)
9394 .map_err(SuperviseError::Registry)?,
9395 )
9396}
9397
9398fn busy_gauges_of(
9399 registration: Option<crate::registry::ModuleRegistration>,
9400) -> Result<Vec<String>, SuperviseError> {
9401 let Some(registration) = registration else {
9402 return Ok(Vec::new());
9403 };
9404 let Some(self_signals) = registration.manifest.self_signals else {
9405 return Ok(Vec::new());
9406 };
9407
9408 let mut gauges = Vec::new();
9409 for declaration in self_signals {
9410 if declaration.kind != SelfSignalKind::Busy {
9411 continue;
9412 }
9413 match declaration.anchored_to {
9414 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9415 gauges.extend(declared)
9416 }
9417 _ => {
9418 gauges.push(String::new());
9421 }
9422 }
9423 }
9424 Ok(gauges)
9425}
9426
9427async fn wait_for_forwarding_quiescence(
9434 forwarding: &ForwardingTable,
9435 module_id: &str,
9436 runtime: &SupervisorRuntimeConfig,
9437 endpoint: crate::ModuleEndpointId,
9438 deadline: Instant,
9439 busy_gauges: &[String],
9440 scope: DrainScope,
9441) -> Result<bool, SuperviseError> {
9442 let mut gauges_quiescent = busy_gauges.is_empty();
9443 let mut next_probe_at = Instant::now();
9444 let mut omission_counted = false;
9445
9446 loop {
9447 let now = Instant::now();
9448 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9449 let report = match scope {
9450 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9451 DrainScope::Endpoint(endpoint) => {
9452 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9453 }
9454 };
9455 gauges_quiescent = match report {
9456 Ok(report) => match busy_gauge_observation(report.metrics.as_ref(), busy_gauges) {
9457 BusyGaugeObservation::Quiescent => true,
9458 BusyGaugeObservation::Busy => false,
9459 BusyGaugeObservation::Omitted => {
9460 if !omission_counted {
9461 forwarding
9462 .counters()
9463 .increment_drains_with_undeclared_gauge();
9464 omission_counted = true;
9465 }
9466 false
9467 }
9468 },
9469 Err(err) => {
9470 warn!(
9471 module_id,
9472 error = %err,
9473 "drain health.check did not produce declared busy gauges; treating module as busy"
9474 );
9475 false
9476 }
9477 };
9478 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9481 }
9482
9483 let in_flight = forwarding
9484 .endpoint_in_flight_count(endpoint)
9485 .map_err(SuperviseError::Forwarding)?;
9486 if in_flight == 0 && gauges_quiescent {
9487 return Ok(true);
9488 }
9489
9490 let now = Instant::now();
9491 if now >= deadline {
9492 return Ok(false);
9493 }
9494 let mut wait = deadline
9495 .saturating_duration_since(now)
9496 .min(REGISTRY_RELEASE_POLL);
9497 if !busy_gauges.is_empty() {
9498 wait = wait.min(next_probe_at.saturating_duration_since(now));
9499 }
9500 sleep(wait).await;
9501 }
9502}
9503
9504fn drain_busy_reprobe_interval() -> Duration {
9508 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9509}
9510
9511fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9519 match wait_result {
9520 Ok(drained) => *drained,
9521 Err(_) => false,
9522 }
9523}
9524
9525fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9526 for released in released_routes {
9527 let frame = match Frame::build_with_version(
9528 released.negotiated_ver,
9529 FrameType::Goodbye,
9530 control_flags(),
9531 released.channel,
9532 released.epoch,
9533 0,
9534 Vec::new(),
9535 ) {
9536 Ok(frame) => frame,
9537 Err(err) => {
9538 warn!(
9539 route_channel = released.channel,
9540 error = %err,
9541 "failed to build supervisor drain route GOODBYE frame"
9542 );
9543 continue;
9544 }
9545 };
9546 if !released.close_on_delivery_failure() {
9547 crate::forwarding::send_module_route_goodbye(
9548 &forwarding.counters(),
9549 &released.sink,
9550 frame,
9551 released.module_id.as_deref(),
9552 "supervisor drain",
9553 );
9554 continue;
9555 }
9556 if let Err(err) = released.sink.try_send(frame) {
9557 warn!(
9558 target_connection_id = released.connection_id.get(),
9559 route_channel = released.channel,
9560 error = %err,
9561 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9562 );
9563 let _ = forwarding.escalate_client_delivery_failure(
9564 released.connection_id,
9565 released.channel,
9566 released.epoch,
9567 CloseReason::new(
9568 "route_goodbye_delivery_failed",
9569 format!(
9570 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9571 released.channel
9572 ),
9573 ),
9574 crate::forwarding::UndeliveredFrame {
9575 module_id: released.module_id.as_deref(),
9576 sink: &released.sink,
9577 },
9578 );
9579 }
9580 }
9581}
9582
9583fn send_module_draining(
9584 module_id: &str,
9585 reason: RouteCloseReason,
9586 deadline_ms: u64,
9587 target: &ModuleDrainTarget,
9588) {
9589 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9590 reason,
9591 deadline_ms,
9592 }) {
9593 Ok(body) => body,
9594 Err(err) => {
9595 warn!(
9596 module_id,
9597 error = %err,
9598 "failed to encode module draining command"
9599 );
9600 return;
9601 }
9602 };
9603 let frame = match Frame::build_with_version(
9604 target.negotiated_ver,
9605 FrameType::Push,
9606 control_flags(),
9607 0,
9608 0,
9609 0,
9610 body,
9611 ) {
9612 Ok(frame) => frame,
9613 Err(err) => {
9614 warn!(
9615 module_id,
9616 error = %err,
9617 "failed to build module draining command frame"
9618 );
9619 return;
9620 }
9621 };
9622 if let Err(err) = target.sink.try_send(frame) {
9623 warn!(
9624 module_id,
9625 target_connection_id = target.endpoint.connection_id.get(),
9626 error = %err,
9627 "module draining command was not delivered to peer"
9628 );
9629 }
9630}
9631
9632fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9634 match Frame::build_with_version(
9635 negotiated_ver,
9636 FrameType::Goodbye,
9637 control_flags(),
9638 0,
9639 0,
9640 0,
9641 Vec::new(),
9642 ) {
9643 Ok(frame) => Some(frame),
9644 Err(err) => {
9645 warn!(
9646 module_id,
9647 error = %err,
9648 "failed to build module GOODBYE frame"
9649 );
9650 None
9651 }
9652 }
9653}
9654
9655#[cfg(unix)]
9669async fn send_module_goodbyes_for_daemon_shutdown(
9670 forwarding: &Arc<ForwardingTable>,
9671 reason: &CloseReason,
9672 wait_for_flush: bool,
9673) {
9674 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
9675 let targets = match forwarding.module_connections() {
9676 Ok(targets) => targets,
9677 Err(err) => {
9678 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
9679 return;
9680 }
9681 };
9682 let deadline = Instant::now() + GOODBYE_BUDGET;
9683 let mut sends = tokio::task::JoinSet::new();
9684 for target in targets {
9685 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
9686 continue;
9687 };
9688 if !wait_for_flush {
9689 if let Err(err) = target.sink.try_send(frame) {
9690 debug!(
9691 module_id = %target.module_id,
9692 error = %err,
9693 "shutdown module GOODBYE was not queued"
9694 );
9695 }
9696 continue;
9697 }
9698 let forwarding = Arc::clone(forwarding);
9699 let reason = reason.clone();
9700 sends.spawn(async move {
9701 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
9702 Ok(Ok(())) => {}
9703 Ok(Err(err)) => debug!(
9704 module_id = %target.module_id,
9705 error = %err,
9706 "module connection closed before its shutdown GOODBYE was written"
9707 ),
9708 Err(_) => warn!(
9709 module_id = %target.module_id,
9710 budget = ?GOODBYE_BUDGET,
9711 "shutdown module GOODBYE was not written within its budget; closing anyway"
9712 ),
9713 }
9714 forwarding.request_connection_close(target.endpoint.connection_id, reason);
9715 });
9716 }
9717 while sends.join_next().await.is_some() {}
9719}
9720
9721fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
9722 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
9723 return;
9724 };
9725 if let Err(err) = target.sink.try_send(frame) {
9726 warn!(
9727 module_id,
9728 target_connection_id = target.endpoint.connection_id.get(),
9729 error = %err,
9730 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
9731 );
9732 forwarding.request_connection_close(
9733 target.endpoint.connection_id,
9734 CloseReason::new(
9735 "module_goodbye_delivery_failed",
9736 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
9737 ),
9738 );
9739 }
9740}
9741
9742#[derive(Clone, Copy)]
9743struct ForwardingDrainContext<'a> {
9744 spec: &'a ModuleSpec,
9745 runtime: &'a SupervisorRuntimeConfig,
9746 registry: &'a Registry,
9747 scope: DrainScope,
9748}
9749
9750#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9752enum DrainScope {
9753 Active,
9756 Endpoint(crate::ModuleEndpointId),
9761}
9762
9763#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9771enum StopNotice {
9772 SentOverConnection,
9775 NoConnection,
9779 NotSent,
9783}
9784
9785async fn begin_forwarding_drain(
9786 spec: &ModuleSpec,
9787 runtime: &SupervisorRuntimeConfig,
9788 registry: &Registry,
9789 snapshot: &SharedSnapshot,
9790 enabled: Option<bool>,
9791 reason: RouteCloseReason,
9792) -> Result<StopNotice, SuperviseError> {
9793 let Some(forwarding) = runtime.forwarding.as_ref() else {
9794 return Err(SuperviseError::ReloadUnavailable {
9795 module_id: spec.module_id.clone(),
9796 reason: "supervisor was not configured with a forwarding table".to_string(),
9797 });
9798 };
9799
9800 begin_forwarding_drain_with(
9801 forwarding,
9802 ForwardingDrainContext {
9803 spec,
9804 runtime,
9805 registry,
9806 scope: DrainScope::Active,
9807 },
9808 snapshot,
9809 enabled,
9810 reason,
9811 runtime.drain_timeout,
9812 )
9813 .await
9814}
9815
9816async fn begin_forwarding_drain_if_configured(
9817 spec: &ModuleSpec,
9818 runtime: &SupervisorRuntimeConfig,
9819 registry: &Registry,
9820 snapshot: &SharedSnapshot,
9821 enabled: Option<bool>,
9822 reason: RouteCloseReason,
9823) -> Result<StopNotice, SuperviseError> {
9824 begin_forwarding_drain_with_timeout(
9825 spec,
9826 runtime,
9827 registry,
9828 snapshot,
9829 enabled,
9830 reason,
9831 runtime.drain_timeout,
9832 )
9833 .await
9834}
9835
9836async fn begin_forwarding_drain_with_timeout(
9840 spec: &ModuleSpec,
9841 runtime: &SupervisorRuntimeConfig,
9842 registry: &Registry,
9843 snapshot: &SharedSnapshot,
9844 enabled: Option<bool>,
9845 reason: RouteCloseReason,
9846 drain_timeout: Duration,
9847) -> Result<StopNotice, SuperviseError> {
9848 let Some(forwarding) = runtime.forwarding.as_ref() else {
9849 return Ok(StopNotice::NotSent);
9850 };
9851
9852 begin_forwarding_drain_with(
9853 forwarding,
9854 ForwardingDrainContext {
9855 spec,
9856 runtime,
9857 registry,
9858 scope: DrainScope::Active,
9859 },
9860 snapshot,
9861 enabled,
9862 reason,
9863 drain_timeout,
9864 )
9865 .await
9866}
9867
9868async fn begin_forwarding_drain_with(
9869 forwarding: &ForwardingTable,
9870 context: ForwardingDrainContext<'_>,
9871 snapshot: &SharedSnapshot,
9872 enabled: Option<bool>,
9873 reason: RouteCloseReason,
9874 drain_timeout: Duration,
9875) -> Result<StopNotice, SuperviseError> {
9876 let ForwardingDrainContext {
9877 spec,
9878 runtime,
9879 registry,
9880 scope,
9881 } = context;
9882 debug_assert_ne!(reason, RouteCloseReason::Crash);
9883 let terminal = matches!(reason, RouteCloseReason::Disable);
9884 let drain_started_at = Instant::now();
9885 let drain_deadline = drain_started_at + drain_timeout;
9886 let deadline_ms =
9887 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
9888 let busy_gauges = match scope {
9889 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
9890 DrainScope::Endpoint(endpoint) => {
9891 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
9892 }
9893 };
9894
9895 let gate_started = Instant::now();
9898 let drain_target = match scope {
9899 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
9900 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
9901 }
9902 .map_err(SuperviseError::Forwarding)?;
9903 info!(
9908 module_id = %spec.module_id,
9909 ?reason,
9910 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
9911 connected = drain_target.is_some(),
9912 "module drain began; route admission closed"
9913 );
9914 if scope == DrainScope::Active {
9915 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9916 state.state = ModuleState::Draining;
9917 state.draining_to_replace =
9918 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
9919 if let Some(enabled) = enabled {
9920 state.enabled = enabled;
9921 }
9922 })?;
9923 }
9924
9925 let Some(target) = drain_target.as_ref() else {
9926 return Ok(StopNotice::NoConnection);
9930 };
9931 {
9932 send_module_draining(&spec.module_id, reason, deadline_ms, target);
9933 #[cfg(feature = "test-support")]
9934 forwarding.wait_before_route_closing_for_test().await;
9935 let routes = forwarding
9936 .endpoint_routes(target.endpoint)
9937 .map_err(SuperviseError::Forwarding)?;
9938 let routes_notified = routes.len();
9939 crate::control::send_route_control_pushes(
9940 forwarding,
9941 routes.clone(),
9942 ClientControlPush::RouteClosing {
9943 module_id: spec.module_id.clone(),
9944 channels: Vec::new(),
9945 reason,
9946 },
9947 );
9948 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
9949
9950 let wait_result = wait_for_forwarding_quiescence(
9956 forwarding,
9957 &spec.module_id,
9958 runtime,
9959 target.endpoint,
9960 drain_deadline,
9961 &busy_gauges,
9962 scope,
9963 )
9964 .await;
9965 let drained = drained_after_quiescence_wait(&wait_result);
9966 if let Err(err) = &wait_result {
9967 error!(
9968 module_id = %spec.module_id,
9969 ?reason,
9970 error = %err,
9971 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
9972 );
9973 } else if !drained {
9974 let holdouts = forwarding
9980 .endpoint_drain_holdouts(target.endpoint)
9981 .unwrap_or_default();
9982 warn!(
9983 module_id = %spec.module_id,
9984 waited = ?drain_timeout,
9985 ?reason,
9986 held_requests = holdouts.requests,
9987 held_routes = holdouts.routes,
9988 total_routes = holdouts.total_routes,
9989 top_connections = ?holdouts.top_connections,
9990 held = %holdouts
9993 .held
9994 .iter()
9995 .map(|(channel, corr)| format!("{channel}:{corr}"))
9996 .collect::<Vec<_>>()
9997 .join(","),
9998 "route drain timed out before request quiescence; forcing teardown"
9999 );
10000 }
10001 crate::control::send_route_control_pushes(
10002 forwarding,
10003 routes,
10004 ClientControlPush::RouteClosed {
10005 module_id: spec.module_id.clone(),
10006 channels: Vec::new(),
10007 reason,
10008 drained,
10009 abandoned: target.abandoned_bindings.len() as u32,
10010 excluded_subscriptions: target.excluded_subscriptions,
10011 terminal: Some(terminal),
10012 },
10013 );
10014 wait_result?;
10015
10016 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10022 Ok(routes) => routes,
10023 Err(err) => {
10024 warn!(
10025 module_id = %spec.module_id,
10026 ?reason,
10027 error = %err,
10028 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10029 );
10030 send_module_goodbye(&spec.module_id, forwarding, target);
10031 return Err(SuperviseError::Forwarding(err));
10032 }
10033 };
10034 let route_goodbye_count = released_routes.len();
10035 send_route_goodbyes(forwarding, released_routes);
10036 send_module_goodbye(&spec.module_id, forwarding, target);
10037
10038 info!(
10044 module_id = %spec.module_id,
10045 ?reason,
10046 routes_notified,
10047 route_goodbyes = route_goodbye_count,
10048 abandoned_reservations = target.abandoned_bindings.len(),
10049 excluded_subscriptions = target.excluded_subscriptions,
10050 drained,
10051 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10052 );
10053 }
10054
10055 Ok(StopNotice::SentOverConnection)
10056}
10057
10058async fn wait_for_registration_after_reload(
10061 registry: &Registry,
10062 module_id: &str,
10063 snapshot: &SharedSnapshot,
10064 child: &mut SupervisedChild,
10065 wait: Duration,
10066) -> Result<RegistrationWaitOutcome, SuperviseError> {
10067 wait_for_slot_registration(
10068 registry,
10069 crate::registry::RegistrationSlot::Active(module_id),
10070 module_id,
10071 snapshot,
10072 child,
10073 wait,
10074 )
10075 .await
10076}
10077
10078async fn wait_for_slot_registration(
10086 registry: &Registry,
10087 slot: crate::registry::RegistrationSlot<'_>,
10088 module_id: &str,
10089 snapshot: &SharedSnapshot,
10090 child: &mut SupervisedChild,
10091 wait: Duration,
10092) -> Result<RegistrationWaitOutcome, SuperviseError> {
10093 let deadline = Instant::now() + wait;
10094 loop {
10095 if registry
10096 .registration(slot)
10097 .map_err(SuperviseError::Registry)?
10098 .is_some()
10099 {
10100 return Ok(RegistrationWaitOutcome::Registered);
10101 }
10102
10103 let now = Instant::now();
10104 if now >= deadline {
10105 return Ok(RegistrationWaitOutcome::TimedOut);
10106 }
10107 let remaining = deadline.saturating_duration_since(now);
10108 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10109
10110 tokio::select! {
10111 wait_result = child.wait() => {
10112 let status = wait_result.map_err(|source| SuperviseError::Wait {
10113 module_id: module_id.to_string(),
10114 source,
10115 })?;
10116 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10117 snapshot,
10118 child,
10119 &status,
10120 )));
10121 }
10122 _ = sleep(poll) => {}
10123 }
10124 }
10125}
10126
10127fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10128 if exit_report.kind != ExitKind::DeliberateSeverance {
10131 exit_report.kind = ExitKind::Crash;
10132 }
10133 exit_report
10134}
10135
10136async fn handle_reload_child_registration_failure(
10137 spec: &ModuleSpec,
10138 runtime: &SupervisorRuntimeConfig,
10139 registry: &Registry,
10140 process_liveness: &SupervisorProcessLiveness,
10141 snapshot: &SharedSnapshot,
10142 _child: &mut Option<SupervisedChild>,
10143 failure: ReloadRegistrationFailure,
10144) -> Result<(), SuperviseError> {
10145 let ReloadRegistrationFailure {
10146 exit_report,
10147 reason,
10148 } = failure;
10149 match on_child_exit(
10150 spec,
10151 runtime.restart_policy,
10152 registry,
10153 snapshot,
10154 &runtime.terminal_ring,
10155 &runtime.spawn_events,
10156 &runtime.child_roster,
10157 exit_report,
10158 )
10159 .await
10160 {
10161 NextAction::Stop {
10162 registration_released,
10163 } => {
10164 if registration_released {
10165 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10166 }
10167 }
10168 NextAction::Restart { schedule } => {
10169 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10170 schedule.delay
10171 });
10172 if let Some(schedule) = schedule {
10173 log_crash_respawn(&spec.module_id, schedule);
10174 }
10175 schedule_respawn(
10176 runtime,
10177 snapshot,
10178 &spec.module_id,
10179 delay,
10180 RespawnKind::Spawn,
10181 )?;
10182 }
10183 }
10184 Err(SuperviseError::ReloadFailed {
10185 module_id: spec.module_id.clone(),
10186 reason,
10187 })
10188}
10189
10190async fn handle_reload_spawn_failure(
10191 spec: &ModuleSpec,
10192 runtime: &SupervisorRuntimeConfig,
10193 process_liveness: &SupervisorProcessLiveness,
10194 snapshot: &SharedSnapshot,
10195 _child: &mut Option<SupervisedChild>,
10196 reason: String,
10197) -> Result<(), SuperviseError> {
10198 let now = Instant::now();
10199 let mut schedule = None;
10200 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10201 clear_current_process_facts(state);
10202 if state.enabled {
10203 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10204 state.state = if schedule.is_some() {
10205 ModuleState::Restarting
10206 } else {
10207 ModuleState::Failed
10208 };
10209 } else {
10210 state.state = ModuleState::Disabled;
10211 }
10212 })?;
10213 if let Some(schedule) = schedule {
10214 schedule_respawn(
10215 runtime,
10216 snapshot,
10217 &spec.module_id,
10218 schedule.delay,
10219 RespawnKind::Spawn,
10220 )?;
10221 } else {
10222 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10223 }
10224 Err(SuperviseError::ReloadFailed {
10225 module_id: spec.module_id.clone(),
10226 reason,
10227 })
10228}
10229
10230fn control_flags() -> Flags {
10231 Flags::new(false, Priority::Passive, false)
10232}
10233
10234#[allow(clippy::too_many_arguments)]
10235async fn drain_optional_child(
10236 module_id: &str,
10237 protocol: ModuleProtocol,
10238 stop_notice: StopNotice,
10239 registry: &Registry,
10240 forwarding: Option<&ForwardingTable>,
10241 snapshot: &SharedSnapshot,
10242 terminal_ring: &Arc<Mutex<TerminalRing>>,
10243 spawn_events: &SpawnEventFeed,
10244 child: &mut Option<SupervisedChild>,
10245 drain_timeout: Duration,
10246 final_state: ModuleState,
10247 enabled: Option<bool>,
10248) -> Result<(), SuperviseError> {
10249 if let Some(child) = child.take() {
10250 drain_child_to_state(
10251 module_id,
10252 protocol,
10253 stop_notice,
10254 registry,
10255 forwarding,
10256 snapshot,
10257 terminal_ring,
10258 spawn_events,
10259 child,
10260 drain_timeout,
10261 final_state,
10262 enabled,
10263 )
10264 .await
10265 } else {
10266 update_snapshot(snapshot, Some(module_id), |state| {
10267 state.state = final_state;
10268 if let Some(enabled) = enabled {
10269 state.enabled = enabled;
10270 }
10271 clear_current_process_facts(state);
10272 })?;
10273 release_dead_registration(registry, forwarding, snapshot, module_id).await
10274 }
10275}
10276
10277#[allow(clippy::too_many_arguments)]
10278async fn drain_child_to_state(
10279 module_id: &str,
10280 _protocol: ModuleProtocol,
10281 stop_notice: StopNotice,
10282 registry: &Registry,
10283 forwarding: Option<&ForwardingTable>,
10284 snapshot: &SharedSnapshot,
10285 terminal_ring: &Arc<Mutex<TerminalRing>>,
10286 spawn_events: &SpawnEventFeed,
10287 mut child: SupervisedChild,
10288 drain_timeout: Duration,
10289 final_state: ModuleState,
10290 enabled: Option<bool>,
10291) -> Result<(), SuperviseError> {
10292 let protocol = child.protocol;
10293 update_snapshot(snapshot, Some(module_id), |state| {
10294 state.state = ModuleState::Draining;
10295 state.draining_to_replace = final_state == ModuleState::Restarting;
10296 if let Some(enabled) = enabled {
10297 state.enabled = enabled;
10298 }
10299 })?;
10300
10301 if stop_notice != StopNotice::SentOverConnection {
10312 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10313 info!(
10314 module_id,
10315 pid = child.pid,
10316 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10317 "module has no connection yet; requesting stop by signal"
10318 );
10319 }
10320 request_graceful_stop(module_id, &child);
10321 }
10322
10323 let exit_report = match timeout(drain_timeout, child.wait()).await {
10324 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10325 Ok(Err(source)) => {
10326 fail_after_wait_error(
10327 module_id,
10328 snapshot,
10329 terminal_ring,
10330 spawn_events,
10331 format!("failed to wait during drain: {source}"),
10332 );
10333 return Err(SuperviseError::Wait {
10334 module_id: module_id.to_string(),
10335 source,
10336 });
10337 }
10338 Err(_) => {
10339 warn!(
10352 module_id,
10353 pid = child.pid,
10354 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10355 reason = ?final_state,
10356 ?stop_notice,
10357 "drain budget expired before the module exited; killing it"
10358 );
10359 child.start_kill().map_err(|source| {
10360 fail_after_wait_error(
10361 module_id,
10362 snapshot,
10363 terminal_ring,
10364 spawn_events,
10365 format!("failed to kill during drain: {source}"),
10366 );
10367 SuperviseError::Kill {
10368 module_id: module_id.to_string(),
10369 source,
10370 }
10371 })?;
10372 let status = child.wait().await.map_err(|source| {
10373 fail_after_wait_error(
10374 module_id,
10375 snapshot,
10376 terminal_ring,
10377 spawn_events,
10378 format!("failed to wait after drain kill: {source}"),
10379 );
10380 SuperviseError::Wait {
10381 module_id: module_id.to_string(),
10382 source,
10383 }
10384 })?;
10385 classify_reaped_child_exit(snapshot, &child, &status)
10386 }
10387 };
10388
10389 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10390 publish_after_terminal_record(
10391 module_id,
10392 || {
10393 record_terminal_with_detail(
10394 module_id,
10395 terminal_ring,
10396 spawn_events,
10397 &exit_report,
10398 terminal_disposition(final_state),
10399 detail,
10400 )
10401 },
10402 || {
10403 update_snapshot(snapshot, Some(module_id), |state| {
10404 state.state = final_state;
10405 if let Some(enabled) = enabled {
10406 state.enabled = enabled;
10407 }
10408 clear_current_process_facts(state);
10409 state.last_exit = Some(exit_report.clone());
10410 if exit_report.kind == ExitKind::DeliberateSeverance {
10411 state.lifetime_restarts += 1;
10412 }
10413 })
10414 },
10415 )?;
10416 child.drain_stderr(module_id).await;
10417
10418 release_dead_registration(registry, forwarding, snapshot, module_id).await
10419}
10420
10421#[cfg(unix)]
10441fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10442 let Some(pid) = child
10443 .id()
10444 .and_then(|pid| i32::try_from(pid).ok())
10445 .and_then(rustix::process::Pid::from_raw)
10446 else {
10447 debug!(
10448 module_id,
10449 "no pid to signal for teardown; falling through to the drain wait"
10450 );
10451 return;
10452 };
10453 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10454 Ok(()) => debug!(
10455 module_id,
10456 "sent SIGTERM to a module nothing else asked to stop"
10457 ),
10458 Err(err) => debug!(
10459 module_id,
10460 error = %err,
10461 "SIGTERM to module failed; the drain wait and kill still apply"
10462 ),
10463 }
10464}
10465
10466#[cfg(not(unix))]
10474fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10475 debug!(
10476 module_id,
10477 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10478 );
10479}
10480
10481fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10482 match final_state {
10483 ModuleState::Stopped => TerminalDisposition::Stopped,
10484 ModuleState::Disabled => TerminalDisposition::Disabled,
10485 ModuleState::Restarting => TerminalDisposition::Restarting,
10486 ModuleState::Failed => TerminalDisposition::Failed,
10487 ModuleState::Starting
10488 | ModuleState::Running
10489 | ModuleState::Unresponsive
10490 | ModuleState::Draining => {
10491 unreachable!("terminal exits only finish in terminal or restarting states")
10492 }
10493 }
10494}
10495
10496async fn release_dead_registration(
10504 registry: &Registry,
10505 forwarding: Option<&ForwardingTable>,
10506 snapshot: &SharedSnapshot,
10507 module_id: &str,
10508) -> Result<(), SuperviseError> {
10509 let result = async {
10510 let registration = registry
10511 .get_module(module_id)
10512 .map_err(SuperviseError::Registry)?;
10513 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10514 Ok(()) => return Ok(()),
10515 Err(SuperviseError::RegistrationStillActive { .. }) => {}
10516 Err(err) => return Err(err),
10517 }
10518 let pid = lock_snapshot(snapshot)?.reaped_pid;
10519 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10520 warn!(
10521 module_id,
10522 pid,
10523 connection_id = registration.connection_id.get(),
10524 "reaped module registration outlived release grace; closing dead connection"
10525 );
10526 forwarding.request_connection_close(
10527 registration.connection_id,
10528 CloseReason::new(
10529 "supervised_process_reaped",
10530 format!("module '{module_id}' pid {pid} exited"),
10531 ),
10532 );
10533 wait_for_slot_registration_release(
10534 registry,
10535 crate::registry::RegistrationSlot::Connection(registration.connection_id),
10536 REGISTRY_RELEASE_TIMEOUT,
10537 )
10538 .await?;
10539 }
10540 wait_for_registration_release(registry, module_id, Duration::ZERO).await
10541 }
10542 .await;
10543 if let Err(err) = &result {
10544 fail_snapshot(snapshot, Some(module_id), None);
10545 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10546 }
10547 result
10548}
10549
10550async fn wait_for_registration_release(
10553 registry: &Registry,
10554 module_id: &str,
10555 wait: Duration,
10556) -> Result<(), SuperviseError> {
10557 wait_for_slot_registration_release(
10558 registry,
10559 crate::registry::RegistrationSlot::Active(module_id),
10560 wait,
10561 )
10562 .await
10563}
10564
10565async fn wait_for_slot_registration_release(
10573 registry: &Registry,
10574 slot: crate::registry::RegistrationSlot<'_>,
10575 wait: Duration,
10576) -> Result<(), SuperviseError> {
10577 let deadline = Instant::now() + wait;
10578 let mut release_events = registration_release_events().subscribe();
10579 let still_active = |registration: &crate::registry::ModuleRegistration| {
10580 SuperviseError::RegistrationStillActive {
10581 module_id: registration.manifest.module_id.clone(),
10582 waited: wait,
10583 }
10584 };
10585 loop {
10586 let _observed_generation = *release_events.borrow_and_update();
10587 let Some(registration) = registry
10588 .registration(slot)
10589 .map_err(SuperviseError::Registry)?
10590 else {
10591 return Ok(());
10592 };
10593
10594 let now = Instant::now();
10595 if now >= deadline {
10596 return Err(still_active(®istration));
10597 }
10598
10599 let remaining = deadline.saturating_duration_since(now);
10600 match timeout(remaining, release_events.changed()).await {
10601 Ok(Ok(())) | Ok(Err(_)) => {}
10602 Err(_) => return Err(still_active(®istration)),
10603 }
10604 }
10605}
10606
10607#[cfg(test)]
10608mod slot_registration_wait_tests {
10609 use super::*;
10610 use crate::registry::{ConnectionId, RegistrationSlot};
10611 use subc_protocol::manifest::ModuleManifest;
10612
10613 #[tokio::test]
10614 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10615 let registry = Arc::new(Registry::default());
10616 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
10617 let runtime = supervisor.runtime_config();
10618 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10619 let spec = ModuleSpec {
10620 module_id: "enable-stale-registration".to_string(),
10621 program: PathBuf::from("/missing/enable-retry-test"),
10622 args: Vec::new(),
10623 env: Vec::new(),
10624 reserved: false,
10625 reserved_prefixes: Vec::new(),
10626 protocol: ModuleProtocol::Subc,
10627 overlap: Default::default(),
10628 };
10629 let connection = ConnectionId::new(90);
10630 registry
10631 .register_with_control_ops(
10632 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10633 1,
10634 connection,
10635 Vec::new(),
10636 )
10637 .unwrap();
10638 let mut child = None;
10639 let err = set_child_enabled(
10640 &spec,
10641 &runtime,
10642 ®istry,
10643 &supervisor.process_liveness,
10644 &snapshot,
10645 &mut child,
10646 true,
10647 )
10648 .await
10649 .unwrap_err();
10650 assert!(matches!(
10651 err,
10652 SuperviseError::RegistrationStillActive { .. }
10653 ));
10654 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10655 assert!(child.is_none());
10656 registry.deregister_connection(connection).unwrap();
10657 let err = set_child_enabled(
10658 &spec,
10659 &runtime,
10660 ®istry,
10661 &supervisor.process_liveness,
10662 &snapshot,
10663 &mut child,
10664 true,
10665 )
10666 .await
10667 .unwrap_err();
10668 assert!(
10669 matches!(err, SuperviseError::Spawn { .. }),
10670 "second enable must attempt a spawn: {err}"
10671 );
10672 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10673 }
10674
10675 const INCUMBENT: u64 = 1;
10676 const CANDIDATE: u64 = 2;
10677
10678 fn swapped_registry() -> Arc<Registry> {
10679 let registry = Arc::new(Registry::default());
10680 let manifest = ModuleManifest::builder("m", "0.1.0").build();
10681 registry
10682 .register_with_control_ops(
10683 manifest.clone(),
10684 1,
10685 ConnectionId::new(INCUMBENT),
10686 Vec::new(),
10687 )
10688 .unwrap();
10689 registry
10690 .register_candidate_with_control_ops(
10691 manifest,
10692 1,
10693 ConnectionId::new(CANDIDATE),
10694 Vec::new(),
10695 )
10696 .unwrap();
10697 registry
10698 }
10699
10700 #[tokio::test]
10704 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
10705 let registry = swapped_registry();
10706 registry.promote_candidate("m").unwrap().unwrap();
10707
10708 assert!(matches!(
10709 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
10710 Err(SuperviseError::RegistrationStillActive { .. })
10711 ));
10712
10713 assert!(matches!(
10715 wait_for_slot_registration_release(
10716 ®istry,
10717 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10718 Duration::from_millis(50),
10719 )
10720 .await,
10721 Err(SuperviseError::RegistrationStillActive { .. })
10722 ));
10723
10724 let releaser = Arc::clone(®istry);
10725 let release = tokio::spawn(async move {
10726 sleep(Duration::from_millis(20)).await;
10727 releaser
10728 .deregister_connection(ConnectionId::new(INCUMBENT))
10729 .unwrap();
10730 notify_registration_release();
10731 });
10732 wait_for_slot_registration_release(
10733 ®istry,
10734 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10735 Duration::from_secs(5),
10736 )
10737 .await
10738 .expect("the incumbent's own registration is released");
10739 release.await.unwrap();
10740 assert!(registry.get_module("m").unwrap().is_some());
10741 }
10742
10743 #[tokio::test]
10746 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
10747 let registry = swapped_registry();
10748 assert!(matches!(
10749 wait_for_slot_registration_release(
10750 ®istry,
10751 RegistrationSlot::Candidate("m"),
10752 Duration::from_millis(50),
10753 )
10754 .await,
10755 Err(SuperviseError::RegistrationStillActive { .. })
10756 ));
10757 registry
10758 .deregister_connection(ConnectionId::new(CANDIDATE))
10759 .unwrap();
10760 wait_for_slot_registration_release(
10761 ®istry,
10762 RegistrationSlot::Candidate("m"),
10763 Duration::from_millis(50),
10764 )
10765 .await
10766 .expect("a candidate slot with no candidate is released");
10767 assert!(registry
10768 .registration(RegistrationSlot::Active("m"))
10769 .unwrap()
10770 .is_some());
10771 }
10772}
10773
10774fn classify_exit(status: &ExitStatus) -> ExitReport {
10775 ExitReport {
10776 kind: if status.success() {
10777 ExitKind::Clean
10778 } else {
10779 ExitKind::Crash
10780 },
10781 code: status.code(),
10782 signal: exit_signal(status),
10783 at_ms: unix_ms_now(),
10784 }
10785}
10786
10787fn wait_error_exit_report() -> ExitReport {
10793 ExitReport {
10794 kind: ExitKind::Crash,
10795 code: None,
10796 signal: None,
10797 at_ms: unix_ms_now(),
10798 }
10799}
10800
10801#[cfg(unix)]
10802fn exit_signal(status: &ExitStatus) -> Option<i32> {
10803 use std::os::unix::process::ExitStatusExt;
10804
10805 status.signal()
10806}
10807
10808#[cfg(not(unix))]
10809fn exit_signal(_status: &ExitStatus) -> Option<i32> {
10810 None
10811}
10812
10813fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
10819 update_snapshot(snapshot, Some(module_id), |state| {
10820 state.clear_crash_restarts();
10821 })
10822}
10823
10824fn set_running(
10825 snapshot: &SharedSnapshot,
10826 child: &SupervisedChild,
10827 module_id: &str,
10828 spawn_events: &SpawnEventFeed,
10829) -> Result<(), SuperviseError> {
10830 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10831 module_id: Some(module_id.to_string()),
10832 })?;
10833 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
10834 if std::mem::take(&mut state.coalesced_restart_pending) {
10835 let generation = state.spawn_generation;
10836 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
10837 }
10838 state.drain_disposition_detail = None;
10839 state.spawn_failure = None;
10840 state.waiting_for_previous_pid = None;
10841 state.waiting_for_previous_detail = None;
10842 state.in_alternate_slot = false;
10845 state.configuration_updated_since_spawn = false;
10846 state.spawned_protocol = Some(child.protocol);
10847 state.state = ModuleState::Running;
10848 state.enabled = true;
10849 state.process_alive = true;
10850 state.pid = child.id();
10851 #[cfg(windows)]
10852 {
10853 state.pipe_consumed = child
10854 .nonce_delivery
10855 .as_ref()
10856 .map(|delivery| delivery.consumed());
10857 }
10858 #[cfg(target_os = "macos")]
10859 {
10860 state.report_ready = Some(Arc::clone(&child.report_ready));
10861 }
10862 state.spawned_at_ms = Some(child.spawned_at_ms);
10863 state.exec_outcome = Some(Arc::clone(&child.exec.0));
10864 state.spawned_from = Some(child.spawned_from.clone());
10865 state.spawned_file_identity = child.spawned_file_identity.clone();
10866 state.process_start_time = child.process_start_time;
10867 Ok(())
10868}
10869
10870fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
10871 state.waiting_for_previous_pid = None;
10872 state.waiting_for_previous_detail = None;
10873 state.process_alive = false;
10874 state.spawned_protocol = None;
10875 state.pid = None;
10876 #[cfg(windows)]
10877 {
10878 state.pipe_consumed = None;
10879 }
10880 #[cfg(target_os = "macos")]
10881 {
10882 state.report_ready = None;
10883 }
10884 state.spawned_at_ms = None;
10885 state.exec_outcome = None;
10886 state.spawned_from = None;
10887 state.spawned_file_identity = None;
10888 state.process_start_time = None;
10889 state.deliberate_severance = None;
10890}
10891
10892#[cfg(test)]
10893fn record_deliberate_severance(
10894 snapshot: &SharedSnapshot,
10895 identity: ProcessIdentity,
10896) -> Result<(), SuperviseError> {
10897 update_snapshot(snapshot, None, |state| {
10898 state.deliberate_severance = Some(identity);
10899 })
10900}
10901
10902fn apply_deliberate_severance_marker(
10903 snapshot: &SharedSnapshot,
10904 exited_identity: Option<ProcessIdentity>,
10905 mut exit_report: ExitReport,
10906) -> ExitReport {
10907 let marker = lock_snapshot(snapshot)
10908 .ok()
10909 .and_then(|mut state| state.deliberate_severance.take());
10910 if marker.is_some() && marker == exited_identity {
10911 exit_report.kind = ExitKind::DeliberateSeverance;
10912 }
10913 exit_report
10914}
10915
10916fn classify_reaped_child_exit(
10917 snapshot: &SharedSnapshot,
10918 child: &SupervisedChild,
10919 status: &ExitStatus,
10920) -> ExitReport {
10921 let _ = update_snapshot(snapshot, None, |state| {
10922 state.reaped_pid = Some(child.pid);
10923 state.spawn_failure = child.spawn_failure.clone();
10924 });
10925 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
10926}
10927
10928fn fail_snapshot(
10929 snapshot: &SharedSnapshot,
10930 module_id: Option<&str>,
10931 last_exit: Option<ExitReport>,
10932) {
10933 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
10934 state.state = ModuleState::Failed;
10935 clear_current_process_facts(state);
10936 if let Some(last_exit) = last_exit {
10937 state.last_exit = Some(last_exit);
10938 }
10939 }) {
10940 error!(error = %err, "failed to mark supervisor state failed");
10941 }
10942}
10943
10944fn update_snapshot(
10945 snapshot: &SharedSnapshot,
10946 module_id: Option<&str>,
10947 update: impl FnOnce(&mut SupervisorSnapshot),
10948) -> Result<(), SuperviseError> {
10949 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10950 module_id: module_id.map(ToOwned::to_owned),
10951 })?;
10952 update(&mut state);
10953 Ok(())
10954}
10955
10956const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
10957
10958fn lock_snapshot_for_control<'a>(
10959 snapshot: &'a SharedSnapshot,
10960 module_id: &str,
10961 caller: &'static str,
10962) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
10963 let started_at = Instant::now();
10964 let guard = lock_snapshot(snapshot)?;
10965 let waited = started_at.elapsed();
10966 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
10967 warn!(
10968 module_id = %module_id,
10969 waited_ms = waited.as_millis() as u64,
10970 caller = %caller,
10971 "slow snapshot lock"
10972 );
10973 }
10974 Ok(guard)
10975}
10976
10977fn lock_snapshot(
10978 snapshot: &SharedSnapshot,
10979) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
10980 snapshot
10981 .lock()
10982 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
10983}
10984
10985#[cfg(test)]
10986mod terminal_history_tests {
10987 use std::{
10988 path::PathBuf,
10989 sync::Arc,
10990 time::{Duration, Instant},
10991 };
10992
10993 use tokio::{sync::mpsc, time::sleep};
10994
10995 use super::{
10996 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
10997 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
10998 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
10999 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11000 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11001 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11002 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11003 SupervisorHealthStatus, SupervisorSnapshot,
11004 };
11005 use super::Instant as ClockInstant;
11010 use crate::{
11011 registry::Registry,
11012 terminal_ring::{TerminalRing, TerminalRingConfig},
11013 };
11014 use std::sync::Mutex;
11015 use subc_control::TerminalDisposition;
11016
11017 pub(super) fn fake_aft_stub_path() -> PathBuf {
11022 let mut path = std::env::current_exe().expect("current_exe available in tests");
11023 path.pop();
11024 path.pop();
11025 path.push(if cfg!(windows) {
11026 "fake-aft-stub.exe"
11027 } else {
11028 "fake-aft-stub"
11029 });
11030 assert!(
11031 path.exists(),
11032 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11033 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11034 path.display()
11035 );
11036 path
11037 }
11038
11039 #[test]
11040 fn reserved_never_spawned_refuses_every_hello() {
11041 let supervisor = SupervisorHandle::default();
11046 supervisor.apply_identity_configuration(&ModuleSpec {
11047 module_id: "never-spawned".to_string(),
11048 program: PathBuf::from("/usr/bin/false"),
11049 args: Vec::new(),
11050 env: Vec::new(),
11051 reserved: true,
11052 reserved_prefixes: Vec::new(),
11053 protocol: ModuleProtocol::Subc,
11054 overlap: Default::default(),
11055 });
11056 assert!(
11057 supervisor
11058 .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11059 .is_some(),
11060 "forged nonce must refuse on a reserved never-spawned id"
11061 );
11062 assert!(
11063 supervisor
11064 .reserved_hello_rejection("never-spawned", None)
11065 .is_some(),
11066 "absent nonce must refuse on a reserved never-spawned id"
11067 );
11068 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11070 supervisor.apply_identity_configuration(&ModuleSpec {
11071 module_id: "never-spawned".to_string(),
11072 program: PathBuf::from("/usr/bin/false"),
11073 args: Vec::new(),
11074 env: Vec::new(),
11075 reserved: true,
11076 reserved_prefixes: Vec::new(),
11077 protocol: ModuleProtocol::Subc,
11078 overlap: Default::default(),
11079 });
11080 assert!(supervisor
11081 .reserved_hello_rejection("never-spawned", Some("minted"))
11082 .is_none());
11083 assert!(supervisor
11084 .reserved_hello_rejection("never-spawned", Some("forged"))
11085 .is_some());
11086 }
11087
11088 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11091 let now = ClockInstant::now();
11092 for _ in 0..count {
11093 state.crash_restarts.push_back(now);
11094 }
11095 }
11096
11097 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11101 let aged = state
11102 .crash_restarts
11103 .front()
11104 .expect("a crash restart must be recorded before it can be aged")
11105 .checked_sub(window + Duration::from_secs(1))
11106 .expect("the test clock is far enough from its origin to age an instant");
11107 state.crash_restarts[0] = aged;
11108 }
11109
11110 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11111 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11112 seed_crash_restarts(&mut state, count);
11113 state
11114 }
11115
11116 #[test]
11117 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11118 let policy = RestartPolicy::new(3, Duration::ZERO);
11119 let now = ClockInstant::now();
11120 assert!(daemon_will_restart(
11121 &mut snapshot_with_restarts(true, 2),
11122 &policy,
11123 now
11124 ));
11125 assert!(!daemon_will_restart(
11126 &mut snapshot_with_restarts(true, 3),
11127 &policy,
11128 now
11129 ));
11130 assert!(!daemon_will_restart(
11131 &mut snapshot_with_restarts(false, 0),
11132 &policy,
11133 now
11134 ));
11135 }
11136
11137 #[test]
11138 fn crash_restart_backoff_escalates_with_in_window_count() {
11139 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11140 .with_max_backoff(Duration::from_secs(30));
11141 let now = ClockInstant::now();
11142 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11143 let schedules = (0..4)
11144 .map(|_| {
11145 state
11146 .next_crash_restart(&policy, now)
11147 .expect("the test policy allows four crash restarts")
11148 })
11149 .collect::<Vec<_>>();
11150
11151 assert_eq!(
11152 schedules
11153 .iter()
11154 .map(|schedule| schedule.restart_in_window)
11155 .collect::<Vec<_>>(),
11156 vec![0, 1, 2, 3]
11157 );
11158 assert_eq!(
11159 schedules
11160 .iter()
11161 .map(|schedule| schedule.delay)
11162 .collect::<Vec<_>>(),
11163 vec![
11164 Duration::from_millis(100),
11165 Duration::from_secs(1),
11166 Duration::from_secs(10),
11167 Duration::from_secs(30),
11168 ]
11169 );
11170 }
11171
11172 #[test]
11173 fn crash_restart_backoff_resets_after_ring_clear() {
11174 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11175 let now = ClockInstant::now();
11176 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11177 assert_eq!(
11178 state.next_crash_restart(&policy, now).unwrap().delay,
11179 Duration::from_millis(100)
11180 );
11181 assert_eq!(
11182 state.next_crash_restart(&policy, now).unwrap().delay,
11183 Duration::from_secs(1)
11184 );
11185
11186 state.clear_crash_restarts();
11187 let schedule = state
11188 .next_crash_restart(&policy, now)
11189 .expect("a cleared ring must allow another restart");
11190 assert_eq!(schedule.restart_in_window, 0);
11191 assert_eq!(schedule.delay, Duration::from_millis(100));
11192 }
11193
11194 #[test]
11195 fn crash_restart_backoff_ignores_aged_restarts() {
11196 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11197 let now = ClockInstant::now();
11198 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11199 state
11200 .next_crash_restart(&policy, now)
11201 .expect("the first restart is allowed");
11202 state
11203 .next_crash_restart(&policy, now)
11204 .expect("the second restart is allowed");
11205 state.crash_restarts[0] = now
11206 .checked_sub(policy.window + Duration::from_secs(1))
11207 .expect("the fake clock can age a restart past the window");
11208
11209 let schedule = state
11210 .next_crash_restart(&policy, now)
11211 .expect("an aged restart must release its slot");
11212 assert_eq!(schedule.restart_in_window, 1);
11213 assert_eq!(schedule.delay, Duration::from_secs(1));
11214 assert_eq!(state.crash_restarts.len(), 2);
11215 }
11216
11217 #[test]
11221 fn a_budget_spent_before_the_window_no_longer_refuses() {
11222 let policy = RestartPolicy::new(3, Duration::ZERO);
11223 let mut state = snapshot_with_restarts(true, 3);
11224 let now = ClockInstant::now();
11225 assert!(!daemon_will_restart(&mut state, &policy, now));
11226
11227 assert!(daemon_will_restart(
11228 &mut state,
11229 &policy,
11230 now + policy.window + Duration::from_secs(1)
11231 ));
11232 assert!(
11233 state.crash_restarts.is_empty(),
11234 "reading the budget must drop the instants that left the window"
11235 );
11236 }
11237
11238 fn module_with_recovery_snapshot(
11239 state: ModuleState,
11240 enabled: bool,
11241 restart_count: u32,
11242 ) -> SupervisedModule {
11243 let registry = Arc::new(Registry::default());
11244 let supervisor =
11245 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
11246 let runtime = supervisor.runtime_config();
11247 let spec = ModuleSpec {
11248 module_id: "recovery-snapshot".to_string(),
11249 program: fake_aft_stub_path(),
11250 args: Vec::new(),
11251 env: Vec::new(),
11252 reserved: false,
11253 reserved_prefixes: Vec::new(),
11254 protocol: ModuleProtocol::Subc,
11255 overlap: Default::default(),
11256 };
11257 let mut snapshot = SupervisorSnapshot::new(state, enabled);
11258 seed_crash_restarts(&mut snapshot, restart_count);
11259 let (commands, _rx) = mpsc::channel(4);
11262 SupervisedModule {
11263 inner: Arc::new(SupervisedModuleInner {
11264 module_id: spec.module_id.clone(),
11265 registry,
11266 snapshot: Arc::new(Mutex::new(snapshot)),
11267 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11268 spec,
11269 health: runtime.health,
11270 })),
11271 stderr_ring: runtime.stderr_ring,
11272 terminal_ring: runtime.terminal_ring,
11273 commands,
11274 monitor: Mutex::new(None),
11275 restart_policy: runtime.restart_policy,
11276 effective_drain_timeout: runtime.effective_drain_timeout,
11277 provenance_probe: supervisor.provenance_probe.clone(),
11278 }),
11279 }
11280 }
11281
11282 #[cfg(target_os = "linux")]
11283 #[tokio::test]
11284 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11285 let supervisor =
11286 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11287 .with_cgroup_placement(None);
11288 let result = supervisor.spawn(ModuleSpec {
11289 module_id: "no-cgroup-placement".to_string(),
11290 program: fake_aft_stub_path(),
11291 args: Vec::new(),
11292 env: Vec::new(),
11293 reserved: false,
11294 reserved_prefixes: Vec::new(),
11295 protocol: ModuleProtocol::Subc,
11296 overlap: Default::default(),
11297 });
11298
11299 assert!(
11300 result.is_ok(),
11301 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11302 );
11303 }
11304
11305 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11306 async fn undecided_snapshot_uses_shared_restart_predicate() {
11307 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11308 .will_recover_after_connection_loss()
11309 .unwrap());
11310 assert!(
11311 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11312 .will_recover_after_connection_loss()
11313 .unwrap()
11314 );
11315 }
11316
11317 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11318 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11319 assert!(
11320 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11321 .will_recover_after_connection_loss()
11322 .unwrap()
11323 );
11324 }
11325
11326 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11327 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11328 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11329 .will_recover_after_connection_loss()
11330 .unwrap());
11331 assert!(
11332 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11333 .will_recover_after_connection_loss()
11334 .unwrap()
11335 );
11336 }
11337
11338 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11339 async fn warming_snapshot_is_limited_to_startup_phases() {
11340 for state in [
11341 ModuleState::Starting,
11342 ModuleState::Running,
11343 ModuleState::Restarting,
11344 ] {
11345 assert!(
11346 module_with_recovery_snapshot(state, true, 0)
11347 .is_warming()
11348 .unwrap(),
11349 "{state:?} should be warming"
11350 );
11351 }
11352 for state in [
11353 ModuleState::Unresponsive,
11354 ModuleState::Draining,
11355 ModuleState::Stopped,
11356 ModuleState::Failed,
11357 ModuleState::Disabled,
11358 ] {
11359 assert!(
11360 !module_with_recovery_snapshot(state, true, 0)
11361 .is_warming()
11362 .unwrap(),
11363 "{state:?} should not be warming"
11364 );
11365 }
11366 }
11367
11368 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11369 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11370 let registry = Arc::new(Registry::default());
11371 let supervisor =
11372 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
11373 let module = supervisor
11374 .spawn(ModuleSpec {
11375 module_id: "terminal-history".to_string(),
11376 program: fake_aft_stub_path(),
11377 args: Vec::new(),
11378 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11379 reserved: false,
11380 reserved_prefixes: Vec::new(),
11381 protocol: ModuleProtocol::Subc,
11382 overlap: Default::default(),
11383 })
11384 .unwrap();
11385
11386 let deadline = Instant::now() + Duration::from_secs(5);
11387 loop {
11388 let history = module.terminal_history();
11389 if history.entries.len() == 2 {
11390 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11391 assert_eq!(history.dropped, 0);
11392 assert_eq!(
11393 history
11394 .entries
11395 .iter()
11396 .map(|entry| entry.exit_code)
11397 .collect::<Vec<_>>(),
11398 vec![Some(23), Some(23)]
11399 );
11400 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11401 return;
11402 }
11403 assert!(
11404 Instant::now() < deadline,
11405 "module did not retain two terminal exits: {history:?}"
11406 );
11407 sleep(Duration::from_millis(10)).await;
11408 }
11409 }
11410
11411 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11415 async fn disable_during_crash_backoff_cancels_pending_respawn() {
11416 let backoff = Duration::from_secs(2);
11417 let supervisor = Supervisor::new_for_test(
11418 Arc::new(Registry::default()),
11419 RestartPolicy::new(10, backoff),
11420 );
11421 let module = supervisor
11422 .spawn(ModuleSpec {
11423 module_id: "disable-during-backoff".to_string(),
11424 program: fake_aft_stub_path(),
11425 args: Vec::new(),
11426 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11427 reserved: false,
11428 reserved_prefixes: Vec::new(),
11429 protocol: ModuleProtocol::Subc,
11430 overlap: Default::default(),
11431 })
11432 .unwrap();
11433
11434 let deadline = Instant::now() + Duration::from_secs(5);
11436 loop {
11437 if module.status().unwrap().state == ModuleState::Restarting {
11438 break;
11439 }
11440 assert!(
11441 Instant::now() < deadline,
11442 "module never entered the crash backoff"
11443 );
11444 sleep(Duration::from_millis(10)).await;
11445 }
11446
11447 let started = Instant::now();
11448 module.set_enabled(false).await.unwrap();
11449 let waited = started.elapsed();
11450
11451 assert!(
11452 waited < backoff / 2,
11453 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11454 );
11455 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11456
11457 sleep(backoff + Duration::from_millis(500)).await;
11459 let status = module.status().unwrap();
11460 assert_eq!(status.state, ModuleState::Disabled);
11461 assert_eq!(
11462 status.spawn_generation, 1,
11463 "module respawned after the operator disabled it"
11464 );
11465 }
11466
11467 #[cfg(unix)]
11470 fn protocol_none_sigterm_exits_clean_spec(
11471 module_id: &str,
11472 dir: &std::path::Path,
11473 ) -> (ModuleSpec, PathBuf, PathBuf) {
11474 let ready = dir.join("ready");
11475 let marker = dir.join("sigterm");
11476 let spec = ModuleSpec {
11477 module_id: module_id.to_string(),
11478 program: fake_aft_stub_path(),
11479 args: Vec::new(),
11480 env: vec![
11481 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11482 (
11483 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11484 marker.display().to_string(),
11485 ),
11486 (
11487 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11488 ready.display().to_string(),
11489 ),
11490 ],
11491 reserved: false,
11492 reserved_prefixes: Vec::new(),
11493 protocol: ModuleProtocol::None,
11494 overlap: Default::default(),
11495 };
11496 (spec, ready, marker)
11497 }
11498
11499 #[cfg(unix)]
11503 async fn wait_for_file(path: &std::path::Path) {
11504 let deadline = Instant::now() + Duration::from_secs(10);
11505 while !path.exists() {
11506 assert!(
11507 Instant::now() < deadline,
11508 "{} never appeared",
11509 path.display()
11510 );
11511 sleep(Duration::from_millis(10)).await;
11512 }
11513 }
11514
11515 #[cfg(unix)]
11519 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11520 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11521 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11522 let (spec, ready, marker) =
11523 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11524 let supervisor = Supervisor::new_for_test(
11525 Arc::new(Registry::default()),
11526 RestartPolicy::new(3, Duration::ZERO),
11527 );
11528 let module = supervisor.spawn(spec).unwrap();
11529 wait_for_file(&ready).await;
11530 let deadline = Instant::now() + Duration::from_secs(10);
11534 let first_pid = loop {
11535 let status = module.status().unwrap();
11536 if status.state == ModuleState::Running {
11537 if let Some(pid) = status.pid {
11538 break pid;
11539 }
11540 }
11541 assert!(
11542 Instant::now() < deadline,
11543 "a running module must report its pid after exec confirmation: {status:?}"
11544 );
11545 sleep(Duration::from_millis(10)).await;
11546 };
11547
11548 rustix::process::kill_process(
11549 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11550 rustix::process::Signal::TERM,
11551 )
11552 .unwrap();
11553
11554 let deadline = Instant::now() + Duration::from_secs(10);
11555 let respawned = loop {
11556 let status = module.status().unwrap();
11557 if status.state == ModuleState::Running
11558 && status.pid.is_some_and(|pid| pid != first_pid)
11559 {
11560 break status;
11561 }
11562 assert!(
11563 Instant::now() < deadline,
11564 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11565 );
11566 sleep(Duration::from_millis(10)).await;
11567 };
11568 assert_eq!(respawned.spawn_generation, 2);
11569 assert!(
11570 marker.exists(),
11571 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11572 );
11573
11574 let history = module.terminal_history();
11575 assert_eq!(history.entries.len(), 1, "{history:?}");
11576 let entry = &history.entries[0];
11577 assert_eq!(entry.exit_code, Some(0));
11578 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11579 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11580
11581 module.stop().await.unwrap();
11582 }
11583
11584 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11588 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11589 let supervisor = Supervisor::new_for_test(
11590 Arc::new(Registry::default()),
11591 RestartPolicy::new(1, Duration::ZERO),
11592 );
11593 let module = supervisor
11594 .spawn(ModuleSpec {
11595 module_id: "none-clean-exit-budget".to_string(),
11596 program: fake_aft_stub_path(),
11597 args: Vec::new(),
11598 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11599 reserved: false,
11600 reserved_prefixes: Vec::new(),
11601 protocol: ModuleProtocol::None,
11602 overlap: Default::default(),
11603 })
11604 .unwrap();
11605
11606 let deadline = Instant::now() + Duration::from_secs(10);
11609 loop {
11610 let status = module.status().unwrap();
11611 if status.state == ModuleState::Failed {
11612 break;
11613 }
11614 assert!(
11615 Instant::now() < deadline,
11616 "module never exhausted its budget: {status:?} {:?}",
11617 module.terminal_history()
11618 );
11619 sleep(Duration::from_millis(10)).await;
11620 }
11621 let history = module.terminal_history();
11622 assert_eq!(
11623 history
11624 .entries
11625 .iter()
11626 .map(|entry| (entry.exit_code, entry.disposition.clone()))
11627 .collect::<Vec<_>>(),
11628 vec![
11629 (Some(0), TerminalDisposition::Restarting),
11630 (Some(0), TerminalDisposition::Failed),
11631 ]
11632 );
11633 let detail = history.entries[1]
11634 .disposition_detail
11635 .as_deref()
11636 .expect("a budget failure names the budget");
11637 assert!(detail.contains("max_restarts=1"), "{detail}");
11638 assert_eq!(module.status().unwrap().spawn_generation, 2);
11639 }
11640
11641 #[test]
11642 fn restart_budget_failure_is_published_after_its_terminal_record() {
11643 use std::io::{Read, Write};
11644 let supervisor = Supervisor::new_for_test(
11645 Arc::new(Registry::default()),
11646 RestartPolicy::new(0, Duration::ZERO),
11647 );
11648 let runtime = supervisor.runtime_config();
11649 let spec = ModuleSpec {
11650 protocol: ModuleProtocol::None,
11651 ..windowed_crash_spec("budget-publication-order")
11652 };
11653 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11654 ModuleState::Running,
11655 true,
11656 )));
11657 lock_snapshot(&snapshot).unwrap().process_alive = true;
11658 let reaped_snapshot = snapshot.clone();
11659 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11660 let history = runtime.terminal_ring.clone();
11661 let reap = std::thread::spawn(move || {
11662 tokio::runtime::Builder::new_current_thread()
11663 .enable_all()
11664 .build()
11665 .unwrap()
11666 .block_on(on_child_exit(
11667 &spec,
11668 runtime.restart_policy,
11669 &supervisor.registry,
11670 &reaped_snapshot,
11671 &runtime.terminal_ring,
11672 &runtime.spawn_events,
11673 &runtime.child_roster,
11674 ExitReport {
11675 kind: ExitKind::Clean,
11676 code: Some(0),
11677 signal: None,
11678 at_ms: 1,
11679 },
11680 ))
11681 });
11682 let mut reached = [0];
11683 barrier.read_exact(&mut reached).unwrap();
11684 assert_eq!(&reached, b"R");
11685 let (before_state, before_alive) = {
11686 let state = lock_snapshot(&snapshot).unwrap();
11687 (state.state, state.process_alive)
11688 };
11689 let before_history = history.lock().unwrap().snapshot();
11690 barrier.write_all(b"X").unwrap();
11691 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
11692 assert!(before_history.entries.is_empty());
11693 assert_ne!(
11694 before_state,
11695 ModuleState::Failed,
11696 "Failed was visible before its terminal record could be written"
11697 );
11698 assert!(before_alive);
11699 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11700 let history = history.lock().unwrap().snapshot();
11701 assert_eq!(history.entries.len(), 1);
11702 assert_eq!(history.entries[0].exit_code, Some(0));
11703 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11704 }
11705
11706 #[test]
11707 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
11708 use std::io::{Read, Write};
11709
11710 let supervisor = Supervisor::new_for_test(
11711 Arc::new(Registry::default()),
11712 RestartPolicy::new(3, Duration::ZERO),
11713 );
11714 let mut runtime = supervisor.runtime_config();
11715 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
11716 let snapshot = module.inner.snapshot.clone();
11717 lock_snapshot(&snapshot).unwrap().process_alive = true;
11718 runtime.terminal_ring = module.inner.terminal_ring.clone();
11719 let spec = windowed_crash_spec(module.module_id());
11720 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11721 let reaped_snapshot = snapshot.clone();
11722 let reap = std::thread::spawn(move || {
11723 tokio::runtime::Builder::new_current_thread()
11724 .enable_all()
11725 .build()
11726 .unwrap()
11727 .block_on(on_child_exit(
11728 &spec,
11729 runtime.restart_policy,
11730 &supervisor.registry,
11731 &reaped_snapshot,
11732 &runtime.terminal_ring,
11733 &runtime.spawn_events,
11734 &runtime.child_roster,
11735 ExitReport {
11736 kind: ExitKind::Crash,
11737 code: Some(2),
11738 signal: None,
11739 at_ms: 1,
11740 },
11741 ))
11742 });
11743
11744 let mut reached = [0];
11748 barrier.read_exact(&mut reached).unwrap();
11749 assert_eq!(&reached, b"R");
11750 let before_status = module.status().unwrap();
11751 let before_history = module.terminal_history();
11752 let before_recovers = module.will_recover_after_connection_loss().unwrap();
11753 barrier.write_all(b"X").unwrap();
11754 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
11755
11756 assert_eq!(before_status.state, ModuleState::Running);
11757 assert!(before_status.process_alive);
11758 assert_eq!(before_status.restart_count, 3);
11759 assert!(before_history.entries.is_empty());
11760 assert!(
11761 before_recovers,
11762 "connection cleanup would report a terminal close for a scheduled crash restart"
11763 );
11764 let restarting = module.status().unwrap();
11765 assert_eq!(restarting.state, ModuleState::Restarting);
11766 assert_eq!(restarting.restart_count, 3);
11767 assert!(module.will_recover_after_connection_loss().unwrap());
11768 let history = module.terminal_history();
11769 assert_eq!(history.entries.len(), 1);
11770 assert_eq!(
11771 history.entries[0].disposition,
11772 TerminalDisposition::Restarting
11773 );
11774
11775 update_snapshot(&snapshot, Some(module.module_id()), |state| {
11778 state.state = ModuleState::Running;
11779 state.respawn_pending = false;
11780 })
11781 .unwrap();
11782 assert!(!module.will_recover_after_connection_loss().unwrap());
11783 update_snapshot(&snapshot, Some(module.module_id()), |state| {
11784 state.enabled = false;
11785 state.respawn_pending = true;
11786 })
11787 .unwrap();
11788 assert!(!module.will_recover_after_connection_loss().unwrap());
11789 }
11790
11791 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
11792 use std::net::{TcpListener, TcpStream};
11793 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
11794 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
11795 let (barrier, _) = listener.accept().unwrap();
11796 for stream in [&writer, &barrier] {
11797 stream
11798 .set_read_timeout(Some(Duration::from_secs(10)))
11799 .unwrap();
11800 stream
11801 .set_write_timeout(Some(Duration::from_secs(10)))
11802 .unwrap();
11803 }
11804 *events.1.lock().unwrap() = Some(writer);
11805 barrier
11806 }
11807
11808 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
11809 use std::io::{Read, Write};
11810 let barrier = events.1.lock().unwrap().take();
11811 if let Some(mut barrier) = barrier {
11812 barrier.write_all(b"R").unwrap();
11813 let mut release = [0];
11814 barrier.read_exact(&mut release).unwrap();
11815 assert_eq!(&release, b"X");
11816 }
11817 }
11818
11819 #[test]
11820 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
11821 use std::io::{Read, Write};
11822
11823 let supervisor = Supervisor::new_for_test(
11824 Arc::new(Registry::default()),
11825 RestartPolicy::new(0, Duration::ZERO),
11826 );
11827 let mut runtime = supervisor.runtime_config();
11828 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
11829 let snapshot = module.inner.snapshot.clone();
11830 runtime.terminal_ring = module.inner.terminal_ring.clone();
11831 let spec = ModuleSpec {
11832 module_id: module.module_id().into(),
11833 program: fake_aft_stub_path(),
11834 protocol: ModuleProtocol::None,
11835 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11836 ..windowed_crash_spec(module.module_id())
11837 };
11838 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11839 let reap = std::thread::spawn(move || {
11840 tokio::runtime::Builder::new_current_thread()
11841 .enable_all()
11842 .build()
11843 .unwrap()
11844 .block_on(async {
11845 let mut child =
11846 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11847 health_restart_child(
11848 &spec,
11849 &runtime,
11850 &supervisor.registry,
11851 &supervisor.process_liveness,
11852 &snapshot,
11853 &mut child,
11854 SupervisorHealthStatus::Failing,
11855 Some("health probe did not answer"),
11856 1,
11857 )
11858 .await
11859 .unwrap();
11860 assert!(child.is_none());
11861 });
11862 });
11863 let mut reached = [0];
11868 barrier.read_exact(&mut reached).unwrap();
11869 assert_eq!(&reached, b"R");
11870 let before_status = module.status().unwrap();
11871 let before_history = module.terminal_history();
11872 barrier.write_all(b"X").unwrap();
11873 reap.join().unwrap();
11874 assert!(before_history.entries.is_empty());
11875 assert_ne!(
11876 before_status.state,
11877 ModuleState::Failed,
11878 "Failed was visible before its terminal record could be written"
11879 );
11880 assert!(
11881 before_status.process_alive,
11882 "process death was visible before its terminal record could be written"
11883 );
11884 let failed = module.status().unwrap();
11885 assert_eq!(failed.state, ModuleState::Failed);
11886 assert!(!failed.process_alive);
11887 let history = module.terminal_history();
11888 assert_eq!(history.entries.len(), 1);
11889 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11890 assert_eq!(
11891 history.entries[0].disposition_detail.as_deref(),
11892 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
11893 );
11894 }
11895
11896 #[test]
11897 fn observed_exits_publish_process_death_after_terminal_history() {
11898 use std::io::{Read, Write};
11899 for (kind, enabled, shutdown, wait_error, expected_state) in [
11900 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
11901 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
11902 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
11903 (
11904 ExitKind::DeliberateSeverance,
11905 true,
11906 false,
11907 false,
11908 ModuleState::Restarting,
11909 ),
11910 #[cfg(unix)]
11911 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
11912 (ExitKind::Crash, true, false, true, ModuleState::Failed),
11913 ] {
11914 let supervisor = Supervisor::new_for_test(
11915 Arc::new(Registry::default()),
11916 RestartPolicy::new(1, Duration::ZERO),
11917 );
11918 let runtime = supervisor.runtime_config();
11919 #[cfg(unix)]
11920 if shutdown {
11921 runtime.child_roster.close();
11922 }
11923 #[cfg(not(unix))]
11924 let _ = shutdown;
11925 let spec = windowed_crash_spec("exit-publication-order");
11926 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11927 ModuleState::Running,
11928 enabled,
11929 )));
11930 lock_snapshot(&snapshot).unwrap().process_alive = true;
11931 let reaped_snapshot = snapshot.clone();
11932 let history = runtime.terminal_ring.clone();
11933 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11934 let reap = std::thread::spawn(move || {
11935 if wait_error {
11936 super::fail_after_wait_error(
11937 &spec.module_id,
11938 &reaped_snapshot,
11939 &runtime.terminal_ring,
11940 &runtime.spawn_events,
11941 "wait failed".into(),
11942 );
11943 } else {
11944 tokio::runtime::Builder::new_current_thread()
11945 .enable_all()
11946 .build()
11947 .unwrap()
11948 .block_on(on_child_exit(
11949 &spec,
11950 runtime.restart_policy,
11951 &supervisor.registry,
11952 &reaped_snapshot,
11953 &runtime.terminal_ring,
11954 &runtime.spawn_events,
11955 &runtime.child_roster,
11956 ExitReport {
11957 kind,
11958 code: Some(0),
11959 signal: None,
11960 at_ms: 1,
11961 },
11962 ));
11963 }
11964 });
11965 let mut reached = [0];
11966 barrier.read_exact(&mut reached).unwrap();
11967 assert_eq!(&reached, b"R");
11968 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
11969 let before_history = history.lock().unwrap().snapshot();
11970 barrier.write_all(b"X").unwrap();
11971 reap.join().unwrap();
11972 assert!(
11973 before_alive,
11974 "{kind:?}: death was published before recording"
11975 );
11976 assert!(before_history.entries.is_empty());
11977 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
11978 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
11979 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
11980 }
11981 }
11982
11983 #[tokio::test]
11984 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
11985 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
11986 let path = dir.join("terminals.jsonl");
11987 std::fs::create_dir(&path).unwrap();
11988 let supervisor = Supervisor::new_for_test(
11989 Arc::new(Registry::default()),
11990 RestartPolicy::new(0, Duration::ZERO),
11991 )
11992 .with_terminal_journal(path, "health-budget-journal-failure".into());
11993 let runtime = supervisor.runtime_config();
11994 let spec = ModuleSpec {
11995 program: fake_aft_stub_path(),
11996 protocol: ModuleProtocol::None,
11997 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11998 ..windowed_crash_spec("health-budget-journal-failure")
11999 };
12000 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12001 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12002 health_restart_child(
12003 &spec,
12004 &runtime,
12005 &supervisor.registry,
12006 &supervisor.process_liveness,
12007 &snapshot,
12008 &mut child,
12009 SupervisorHealthStatus::Failing,
12010 None,
12011 1,
12012 )
12013 .await
12014 .unwrap();
12015 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12016 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12017 let history = runtime
12018 .terminal_ring
12019 .lock()
12020 .unwrap()
12021 .durable_history(&spec.module_id);
12022 assert!(history.journal_write_failures > 0);
12023 assert_eq!(history.entries.len(), 1);
12024 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12025 }
12026
12027 #[test]
12028 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12029 let supervisor = Supervisor::new_for_test(
12030 Arc::new(Registry::default()),
12031 RestartPolicy::new(0, Duration::ZERO),
12032 );
12033 let runtime = supervisor.runtime_config();
12034 let spec = ModuleSpec {
12035 program: fake_aft_stub_path(),
12036 protocol: ModuleProtocol::None,
12037 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12038 ..windowed_crash_spec("health-budget-recording-panic")
12039 };
12040 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12041 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12042 tokio::runtime::Builder::new_current_thread()
12043 .enable_all()
12044 .build()
12045 .unwrap()
12046 .block_on(async {
12047 let mut child =
12048 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12049 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12050 health_restart_child(
12051 &spec,
12052 &runtime,
12053 &supervisor.registry,
12054 &supervisor.process_liveness,
12055 &snapshot,
12056 &mut child,
12057 SupervisorHealthStatus::Failing,
12058 None,
12059 1,
12060 )
12061 .await
12062 .unwrap();
12063 });
12064 }));
12065 let panic = result.expect_err("recording must still unwind");
12066 assert_eq!(
12067 panic.downcast_ref::<String>().map(String::as_str),
12068 Some("spawn event sequence exhausted")
12069 );
12070 let state = lock_snapshot(&snapshot).unwrap();
12071 assert_eq!(state.state, ModuleState::Failed);
12072 assert!(!state.process_alive);
12073 assert!(state.last_exit.is_some());
12074 }
12075
12076 #[tokio::test]
12077 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12078 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12079 let path = dir.join("terminals.jsonl");
12080 std::fs::create_dir(&path).unwrap();
12081 let supervisor = Supervisor::new_for_test(
12082 Arc::new(Registry::default()),
12083 RestartPolicy::new(0, Duration::ZERO),
12084 )
12085 .with_terminal_journal(path, "budget-journal-failure".into());
12086 let runtime = supervisor.runtime_config();
12087 let spec = windowed_crash_spec("budget-journal-failure");
12088 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12089 assert!(matches!(
12090 on_child_exit(
12091 &spec,
12092 runtime.restart_policy,
12093 &supervisor.registry,
12094 &snapshot,
12095 &runtime.terminal_ring,
12096 &runtime.spawn_events,
12097 &runtime.child_roster,
12098 crash_exit_report(1),
12099 )
12100 .await,
12101 NextAction::Stop { .. }
12102 ));
12103 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12104 let history = runtime
12105 .terminal_ring
12106 .lock()
12107 .unwrap()
12108 .durable_history(&spec.module_id);
12109 assert!(history.journal_write_failures > 0);
12110 assert_eq!(history.entries.len(), 1);
12111 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12112 }
12113
12114 #[test]
12115 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12116 let supervisor = Supervisor::new_for_test(
12117 Arc::new(Registry::default()),
12118 RestartPolicy::new(0, Duration::ZERO),
12119 );
12120 let runtime = supervisor.runtime_config();
12121 let spec = windowed_crash_spec("budget-recording-panic");
12122 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12123 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12124 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12127 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12128 tokio::runtime::Builder::new_current_thread()
12129 .enable_all()
12130 .build()
12131 .unwrap()
12132 .block_on(on_child_exit(
12133 &spec,
12134 runtime.restart_policy,
12135 &supervisor.registry,
12136 &snapshot,
12137 &runtime.terminal_ring,
12138 &runtime.spawn_events,
12139 &runtime.child_roster,
12140 crash_exit_report(1),
12141 ))
12142 }));
12143 let panic = result.err().expect("recording must still unwind");
12144 assert_eq!(
12145 panic.downcast_ref::<String>().map(String::as_str),
12146 Some("spawn event sequence exhausted")
12147 );
12148 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12149 }
12150
12151 #[cfg(unix)]
12154 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12155 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12156 for disable in [false, true] {
12157 let label = if disable {
12158 "none-requested-disable"
12159 } else {
12160 "none-requested-stop"
12161 };
12162 let dir = cortexkit_test_support::ScratchDir::new(label);
12163 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12164 let supervisor = Supervisor::new_for_test(
12165 Arc::new(Registry::default()),
12166 RestartPolicy::new(3, Duration::ZERO),
12167 );
12168 let module = supervisor.spawn(spec).unwrap();
12169 wait_for_file(&ready).await;
12170
12171 if disable {
12172 module.set_enabled(false).await.unwrap();
12173 } else {
12174 module.stop().await.unwrap();
12175 }
12176 assert!(
12177 marker.exists(),
12178 "{label}: the child must have left through its SIGTERM handler with exit 0"
12179 );
12180
12181 sleep(Duration::from_millis(500)).await;
12184 let status = module.status().unwrap();
12185 let expected = if disable {
12186 ModuleState::Disabled
12187 } else {
12188 ModuleState::Stopped
12189 };
12190 assert_eq!(status.state, expected, "{label}");
12191 assert_eq!(
12192 status.spawn_generation, 1,
12193 "{label}: respawned after a requested stop"
12194 );
12195 let history = module.terminal_history();
12196 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12197 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12198 assert_ne!(
12199 history.entries[0].disposition,
12200 TerminalDisposition::Restarting,
12201 "{label}"
12202 );
12203 }
12204 }
12205
12206 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12209 async fn subc_wire_clean_exit_is_still_a_stop() {
12210 let supervisor = Supervisor::new_for_test(
12211 Arc::new(Registry::default()),
12212 RestartPolicy::new(3, Duration::ZERO),
12213 );
12214 let module = supervisor
12215 .spawn(ModuleSpec {
12216 module_id: "wire-clean-exit".to_string(),
12217 program: fake_aft_stub_path(),
12218 args: Vec::new(),
12219 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12220 reserved: false,
12221 reserved_prefixes: Vec::new(),
12222 protocol: ModuleProtocol::Subc,
12223 overlap: Default::default(),
12224 })
12225 .unwrap();
12226
12227 let deadline = Instant::now() + Duration::from_secs(10);
12228 while module.terminal_history().entries.is_empty() {
12229 assert!(Instant::now() < deadline, "module never exited");
12230 sleep(Duration::from_millis(10)).await;
12231 }
12232 sleep(Duration::from_millis(500)).await;
12234 let status = module.status().unwrap();
12235 assert_eq!(status.state, ModuleState::Stopped);
12236 assert_eq!(status.spawn_generation, 1);
12237 let history = module.terminal_history();
12238 assert_eq!(history.entries.len(), 1, "{history:?}");
12239 assert_eq!(history.entries[0].exit_code, Some(0));
12240 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12241 }
12242
12243 #[cfg(unix)]
12244 #[tokio::test]
12245 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12246 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12247 let record = dir.join("live-children.json");
12248 let supervisor = Supervisor::new_for_test(
12249 Arc::new(Registry::default()),
12250 RestartPolicy::new(0, Duration::ZERO),
12251 );
12252 let mut runtime = supervisor.runtime_config();
12253 runtime.child_roster.record_to(record.clone());
12254 let gate = Arc::new(super::ReloadExitRecordGate::default());
12255 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12256 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12257 let spec = ModuleSpec {
12258 module_id: "reload-exit-roster".into(),
12259 program: fake_aft_stub_path(),
12260 args: Vec::new(),
12261 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12262 reserved: false,
12263 reserved_prefixes: Vec::new(),
12264 protocol: ModuleProtocol::Subc,
12265 overlap: Default::default(),
12266 };
12267 let mut child = None;
12268 let reload = super::finish_reload_child(
12269 &spec,
12270 &runtime,
12271 &supervisor.registry,
12272 &supervisor.process_liveness,
12273 &snapshot,
12274 &mut child,
12275 );
12276 tokio::pin!(reload);
12277 tokio::select! {
12278 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12279 _ = gate.reached.notified() => {}
12280 }
12281 assert!(runtime
12282 .terminal_ring
12283 .lock()
12284 .unwrap()
12285 .snapshot()
12286 .entries
12287 .is_empty());
12288 assert_eq!(
12289 crate::live_children::read_record(&record).unwrap().len(),
12290 1,
12291 "shutdown must still wait for the reaped child until its terminal record exists"
12292 );
12293 runtime.child_roster.close();
12294 gate.resume.notify_one();
12295 assert!(reload.await.is_err());
12296 assert!(crate::live_children::read_record(&record)
12297 .unwrap()
12298 .is_empty());
12299 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12300 assert_eq!(history.entries.len(), 1);
12301 assert_eq!(
12302 history.entries[0].disposition,
12303 TerminalDisposition::DaemonShutdown
12304 );
12305 }
12306
12307 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12311 async fn every_restart_increment_path_advances_lifetime_count() {
12312 let supervisor = Supervisor::new_for_test(
12313 Arc::new(Registry::default()),
12314 RestartPolicy::new(1, Duration::ZERO),
12315 );
12316 let runtime = supervisor.runtime_config();
12317 let spec = ModuleSpec {
12318 module_id: "lifetime-increment-path".to_string(),
12319 program: PathBuf::from("/unused/lifetime-increment-path"),
12320 args: Vec::new(),
12321 env: Vec::new(),
12322 reserved: false,
12323 reserved_prefixes: Vec::new(),
12324 protocol: ModuleProtocol::Subc,
12325 overlap: Default::default(),
12326 };
12327
12328 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12329 assert!(matches!(
12330 on_child_exit(
12331 &spec,
12332 runtime.restart_policy,
12333 &supervisor.registry,
12334 &crash_snapshot,
12335 &runtime.terminal_ring,
12336 &runtime.spawn_events,
12337 &runtime.child_roster,
12338 ExitReport {
12339 kind: ExitKind::Crash,
12340 code: Some(1),
12341 signal: None,
12342 at_ms: 1,
12343 },
12344 )
12345 .await,
12346 NextAction::Restart { schedule: _ }
12347 ));
12348 let (crash_restarts, crash_lifetime) = {
12349 let state = lock_snapshot(&crash_snapshot).unwrap();
12350 (state.crash_restarts.len(), state.lifetime_restarts)
12351 };
12352 assert_eq!(crash_restarts, 1);
12353 assert_eq!(crash_lifetime, 1);
12354
12355 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12356 let mut health_child = None;
12357 assert!(matches!(
12358 health_restart_child(
12359 &spec,
12360 &runtime,
12361 &supervisor.registry,
12362 &supervisor.process_liveness,
12363 &health_snapshot,
12364 &mut health_child,
12365 SupervisorHealthStatus::Failing,
12366 None,
12367 2,
12368 )
12369 .await,
12370 Ok(())
12371 ));
12372 assert!(health_child.is_none());
12373 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12374 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12375 let (health_restarts, health_lifetime) = {
12376 let state = lock_snapshot(&health_snapshot).unwrap();
12377 (state.crash_restarts.len(), state.lifetime_restarts)
12378 };
12379 assert_eq!(health_restarts, 1);
12380 assert_eq!(health_lifetime, 1);
12381
12382 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12383 let mut reload_child = None;
12384 assert!(matches!(
12385 handle_reload_spawn_failure(
12386 &spec,
12387 &runtime,
12388 &supervisor.process_liveness,
12389 &reload_snapshot,
12390 &mut reload_child,
12391 "forced reload spawn failure".to_string(),
12392 )
12393 .await,
12394 Err(SuperviseError::ReloadFailed { .. })
12395 ));
12396 let (reload_restarts, reload_lifetime) = {
12397 let state = lock_snapshot(&reload_snapshot).unwrap();
12398 (state.crash_restarts.len(), state.lifetime_restarts)
12399 };
12400 assert_eq!(reload_restarts, 1);
12401 assert_eq!(reload_lifetime, 1);
12402 }
12403
12404 #[tokio::test]
12405 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12406 let supervisor = Supervisor::new_for_test(
12407 Arc::new(Registry::default()),
12408 RestartPolicy::new(3, Duration::ZERO),
12409 );
12410 let runtime = supervisor.runtime_config();
12411 let spec = ModuleSpec {
12412 module_id: "deliberately-severed".to_string(),
12413 program: PathBuf::from("/unused/deliberately-severed"),
12414 args: Vec::new(),
12415 env: Vec::new(),
12416 reserved: false,
12417 reserved_prefixes: Vec::new(),
12418 protocol: ModuleProtocol::Subc,
12419 overlap: Default::default(),
12420 };
12421 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12422 let process = ProcessIdentity {
12423 pid: 41,
12424 start_time: 101,
12425 };
12426 record_deliberate_severance(&snapshot, process).unwrap();
12427 let exit_report = apply_deliberate_severance_marker(
12428 &snapshot,
12429 Some(process),
12430 ExitReport {
12431 kind: ExitKind::Crash,
12432 code: Some(1),
12433 signal: None,
12434 at_ms: 1,
12435 },
12436 );
12437 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12438
12439 assert!(matches!(
12440 on_child_exit(
12441 &spec,
12442 runtime.restart_policy,
12443 &supervisor.registry,
12444 &snapshot,
12445 &runtime.terminal_ring,
12446 &runtime.spawn_events,
12447 &runtime.child_roster,
12448 exit_report,
12449 )
12450 .await,
12451 NextAction::Restart { schedule: _ }
12452 ));
12453 let state = lock_snapshot(&snapshot).unwrap();
12454 assert_eq!(state.lifetime_restarts, 1);
12455 assert_eq!(state.crash_restarts.len(), 0);
12456 }
12457
12458 #[tokio::test]
12459 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12460 let supervisor = Supervisor::new_for_test(
12461 Arc::new(Registry::default()),
12462 RestartPolicy::new(3, Duration::ZERO),
12463 );
12464 let runtime = supervisor.runtime_config();
12465 let spec = ModuleSpec {
12466 module_id: "genuine-crash".to_string(),
12467 program: PathBuf::from("/unused/genuine-crash"),
12468 args: Vec::new(),
12469 env: Vec::new(),
12470 reserved: false,
12471 reserved_prefixes: Vec::new(),
12472 protocol: ModuleProtocol::Subc,
12473 overlap: Default::default(),
12474 };
12475 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12476
12477 assert!(matches!(
12478 on_child_exit(
12479 &spec,
12480 runtime.restart_policy,
12481 &supervisor.registry,
12482 &snapshot,
12483 &runtime.terminal_ring,
12484 &runtime.spawn_events,
12485 &runtime.child_roster,
12486 ExitReport {
12487 kind: ExitKind::Crash,
12488 code: Some(1),
12489 signal: None,
12490 at_ms: 1,
12491 },
12492 )
12493 .await,
12494 NextAction::Restart { schedule: _ }
12495 ));
12496 let state = lock_snapshot(&snapshot).unwrap();
12497 assert_eq!(state.lifetime_restarts, 1);
12498 assert_eq!(state.crash_restarts.len(), 1);
12499 }
12500
12501 fn crash_exit_report(at_ms: u64) -> ExitReport {
12502 ExitReport {
12503 kind: ExitKind::Crash,
12504 code: Some(1),
12505 signal: None,
12506 at_ms,
12507 }
12508 }
12509
12510 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12511 ModuleSpec {
12512 module_id: module_id.to_string(),
12513 program: PathBuf::from("/unused").join(module_id),
12514 args: Vec::new(),
12515 env: Vec::new(),
12516 reserved: false,
12517 reserved_prefixes: Vec::new(),
12518 protocol: ModuleProtocol::Subc,
12519 overlap: Default::default(),
12520 }
12521 }
12522
12523 #[tokio::test]
12529 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12530 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12531 let supervisor = Supervisor::new_for_test(
12532 Arc::new(Registry::default()),
12533 RestartPolicy::new(2, Duration::ZERO),
12534 );
12535 let runtime = supervisor.runtime_config();
12536 let spec = windowed_crash_spec("crash-loop-in-window");
12537 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12538
12539 for attempt in 1..=2 {
12540 assert!(
12541 matches!(
12542 on_child_exit(
12543 &spec,
12544 runtime.restart_policy,
12545 &supervisor.registry,
12546 &snapshot,
12547 &runtime.terminal_ring,
12548 &runtime.spawn_events,
12549 &runtime.child_roster,
12550 crash_exit_report(attempt),
12551 )
12552 .await,
12553 NextAction::Restart { schedule: _ }
12554 ),
12555 "crash {attempt} is inside the budget and must respawn"
12556 );
12557 }
12558
12559 assert!(matches!(
12560 on_child_exit(
12561 &spec,
12562 runtime.restart_policy,
12563 &supervisor.registry,
12564 &snapshot,
12565 &runtime.terminal_ring,
12566 &runtime.spawn_events,
12567 &runtime.child_roster,
12568 crash_exit_report(3),
12569 )
12570 .await,
12571 NextAction::Stop { .. }
12572 ));
12573
12574 {
12575 let state = lock_snapshot(&snapshot).unwrap();
12576 assert_eq!(state.state, ModuleState::Failed);
12577 assert_eq!(state.crash_restarts.len(), 2);
12578 assert_eq!(state.lifetime_restarts, 2);
12579 }
12580
12581 let history = runtime
12582 .terminal_ring
12583 .lock()
12584 .expect("terminal ring is not poisoned")
12585 .snapshot();
12586 let last = history
12587 .entries
12588 .last()
12589 .expect("the refused crash is retained");
12590 assert_eq!(last.disposition, TerminalDisposition::Failed);
12591 assert_eq!(
12592 last.disposition_detail.as_deref(),
12593 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12594 );
12595
12596 let captured = crate::router::test_log::captured_logs(&logs);
12597 assert!(
12598 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12599 "the stop must be logged with its window: {captured}"
12600 );
12601 }
12602
12603 #[tokio::test]
12611 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12612 let supervisor = Supervisor::new_for_test(
12613 Arc::new(Registry::default()),
12614 RestartPolicy::new(2, Duration::ZERO),
12615 );
12616 let runtime = supervisor.runtime_config();
12617 let spec = windowed_crash_spec("crash-across-windows");
12618 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12619
12620 for attempt in 1..=2 {
12621 assert!(matches!(
12622 on_child_exit(
12623 &spec,
12624 runtime.restart_policy,
12625 &supervisor.registry,
12626 &snapshot,
12627 &runtime.terminal_ring,
12628 &runtime.spawn_events,
12629 &runtime.child_roster,
12630 crash_exit_report(attempt),
12631 )
12632 .await,
12633 NextAction::Restart { schedule: _ }
12634 ));
12635 }
12636
12637 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12640 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12641 })
12642 .unwrap();
12643
12644 assert!(
12645 matches!(
12646 on_child_exit(
12647 &spec,
12648 runtime.restart_policy,
12649 &supervisor.registry,
12650 &snapshot,
12651 &runtime.terminal_ring,
12652 &runtime.spawn_events,
12653 &runtime.child_roster,
12654 crash_exit_report(3),
12655 )
12656 .await,
12657 NextAction::Restart { schedule: _ }
12658 ),
12659 "a crash older than the window must not hold a budget slot"
12660 );
12661
12662 let state = lock_snapshot(&snapshot).unwrap();
12663 assert_eq!(state.state, ModuleState::Restarting);
12664 assert_eq!(
12665 state.crash_restarts.len(),
12666 2,
12667 "the aged instant is dropped and the new one takes its place"
12668 );
12669 assert_eq!(
12670 state.lifetime_restarts, 3,
12671 "the ledger counts every restart, including the ones the window forgot"
12672 );
12673 }
12674
12675 #[tokio::test]
12680 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
12681 let supervisor = Supervisor::new_for_test(
12682 Arc::new(Registry::default()),
12683 RestartPolicy::new(2, Duration::ZERO),
12684 );
12685 let runtime = supervisor.runtime_config();
12686 let spec = windowed_crash_spec("operator-cleared-budget");
12687 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12688
12689 for attempt in 1..=2 {
12690 assert!(matches!(
12691 on_child_exit(
12692 &spec,
12693 runtime.restart_policy,
12694 &supervisor.registry,
12695 &snapshot,
12696 &runtime.terminal_ring,
12697 &runtime.spawn_events,
12698 &runtime.child_roster,
12699 crash_exit_report(attempt),
12700 )
12701 .await,
12702 NextAction::Restart { schedule: _ }
12703 ));
12704 }
12705
12706 reset_restart_count(&snapshot, &spec.module_id).unwrap();
12707 {
12708 let state = lock_snapshot(&snapshot).unwrap();
12709 assert!(
12710 state.crash_restarts.is_empty(),
12711 "an operator restart returns the full budget"
12712 );
12713 assert_eq!(
12714 state.lifetime_restarts, 2,
12715 "clearing the budget must not unmake the crashes"
12716 );
12717 }
12718
12719 assert!(
12720 matches!(
12721 on_child_exit(
12722 &spec,
12723 runtime.restart_policy,
12724 &supervisor.registry,
12725 &snapshot,
12726 &runtime.terminal_ring,
12727 &runtime.spawn_events,
12728 &runtime.child_roster,
12729 crash_exit_report(3),
12730 )
12731 .await,
12732 NextAction::Restart { schedule: _ }
12733 ),
12734 "the cleared budget must be spendable again"
12735 );
12736 let state = lock_snapshot(&snapshot).unwrap();
12737 assert_eq!(state.crash_restarts.len(), 1);
12738 assert_eq!(state.lifetime_restarts, 3);
12739 }
12740
12741 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12742 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
12743 let severed = ProcessIdentity {
12744 pid: 41,
12745 start_time: 101,
12746 };
12747 let successor = ProcessIdentity {
12748 pid: 41,
12749 start_time: 202,
12750 };
12751 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12752 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
12753 state.pid = Some(successor.pid);
12754 state.process_start_time = Some(successor.start_time);
12755 })
12756 .unwrap();
12757 assert!(!module.record_deliberate_severance(severed).unwrap());
12758
12759 let exit_report = apply_deliberate_severance_marker(
12760 &module.inner.snapshot,
12761 Some(successor),
12762 ExitReport {
12763 kind: ExitKind::Crash,
12764 code: Some(1),
12765 signal: None,
12766 at_ms: 1,
12767 },
12768 );
12769
12770 assert_eq!(exit_report.kind, ExitKind::Crash);
12771 }
12772
12773 #[tokio::test]
12774 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
12775 let registry = Registry::default();
12776 let supervisor = Supervisor::new_for_test(
12777 Arc::new(Registry::default()),
12778 RestartPolicy::new(3, Duration::ZERO),
12779 );
12780 let runtime = supervisor.runtime_config();
12781 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12782 let spec = ModuleSpec {
12783 module_id: "drain-deliberate-severance".to_string(),
12784 program: fake_aft_stub_path(),
12785 args: Vec::new(),
12786 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12787 reserved: false,
12788 reserved_prefixes: Vec::new(),
12789 protocol: ModuleProtocol::Subc,
12790 overlap: Default::default(),
12791 };
12792 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12793 let process = ProcessIdentity {
12794 pid: 41,
12795 start_time: 101,
12796 };
12797 child.process_identity = Some(process);
12798 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12799 state.pid = Some(process.pid);
12800 state.process_start_time = Some(process.start_time);
12801 })
12802 .unwrap();
12803 record_deliberate_severance(&snapshot, process).unwrap();
12804
12805 drain_child_to_state(
12806 &spec.module_id,
12807 spec.protocol,
12808 StopNotice::SentOverConnection,
12811 ®istry,
12812 None,
12813 &snapshot,
12814 &runtime.terminal_ring,
12815 &runtime.spawn_events,
12816 child,
12817 Duration::from_secs(1),
12818 ModuleState::Stopped,
12819 Some(false),
12820 )
12821 .await
12822 .unwrap();
12823
12824 let state = lock_snapshot(&snapshot).unwrap();
12825 assert_eq!(
12826 state.last_exit.as_ref().map(|exit| exit.kind),
12827 Some(ExitKind::DeliberateSeverance)
12828 );
12829 assert_eq!(state.lifetime_restarts, 1);
12830 assert_eq!(state.crash_restarts.len(), 0);
12831 drop(state);
12832 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12833 assert_eq!(
12834 history.entries[0].exit_kind,
12835 subc_control::TerminalExitKind::DeliberateSeverance
12836 );
12837 }
12838
12839 #[tokio::test]
12840 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
12841 let registry = Registry::default();
12842 let supervisor = Supervisor::new_for_test(
12843 Arc::new(Registry::default()),
12844 RestartPolicy::new(3, Duration::ZERO),
12845 );
12846 let runtime = supervisor.runtime_config();
12847 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12848 let spec = ModuleSpec {
12849 module_id: "ordinary-drain".to_string(),
12850 program: fake_aft_stub_path(),
12851 args: Vec::new(),
12852 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12853 reserved: false,
12854 reserved_prefixes: Vec::new(),
12855 protocol: ModuleProtocol::Subc,
12856 overlap: Default::default(),
12857 };
12858 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12859
12860 drain_child_to_state(
12861 &spec.module_id,
12862 spec.protocol,
12863 StopNotice::SentOverConnection,
12866 ®istry,
12867 None,
12868 &snapshot,
12869 &runtime.terminal_ring,
12870 &runtime.spawn_events,
12871 child,
12872 Duration::from_secs(1),
12873 ModuleState::Stopped,
12874 Some(false),
12875 )
12876 .await
12877 .unwrap();
12878
12879 let state = lock_snapshot(&snapshot).unwrap();
12880 assert_eq!(
12881 state.last_exit.as_ref().map(|exit| exit.kind),
12882 Some(ExitKind::Crash)
12883 );
12884 assert_eq!(state.lifetime_restarts, 0);
12885 assert_eq!(state.crash_restarts.len(), 0);
12886 }
12887
12888 #[test]
12889 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
12890 assert!(!include_str!("server.rs")
12896 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
12897 }
12898
12899 #[test]
12906 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
12907 assert!(drained_after_quiescence_wait(&Ok(true)));
12908 assert!(!drained_after_quiescence_wait(&Ok(false)));
12909 assert!(!drained_after_quiescence_wait(&Err(
12910 SuperviseError::StatePoisoned { module_id: None }
12911 )));
12912 }
12913
12914 #[test]
12923 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
12924 let ring = Arc::new(Mutex::new(TerminalRing::new(
12925 TerminalRingConfig::default(),
12926 0,
12927 )));
12928 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
12929
12930 let snapshot = ring.lock().unwrap().snapshot();
12931 assert_eq!(snapshot.entries.len(), 1);
12932 let entry = &snapshot.entries[0];
12933 assert_eq!(entry.exit_code, None);
12934 assert_eq!(entry.exit_signal, None);
12935 assert_eq!(entry.disposition, TerminalDisposition::Failed);
12936 }
12937
12938 #[test]
12939 fn wait_error_exit_path_preserves_spawn_event_density() {
12940 let feed = super::SpawnEventFeed::default();
12941 feed.configure_incarnation("wait-error-density".to_string());
12942 feed.emit_spawned("wait-error", 41, 1);
12943 let ring = Arc::new(Mutex::new(TerminalRing::new(
12944 TerminalRingConfig::default(),
12945 0,
12946 )));
12947
12948 record_wait_error_terminal("wait-error", &ring, &feed);
12949 feed.emit_spawned("after-wait-error", 42, 2);
12950
12951 let state = feed.0.lock().unwrap();
12952 let sequences = state
12953 .events
12954 .iter()
12955 .map(|event| event.cursor.seq)
12956 .collect::<Vec<_>>();
12957 assert_eq!(sequences, vec![1, 2, 3]);
12958 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
12959 assert_eq!(state.events[1].exit_code, None);
12960 assert_eq!(state.events[1].exit_signal, None);
12961 }
12962
12963 #[test]
12967 fn wait_error_exit_report_is_classified_as_a_crash() {
12968 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
12969 }
12970}
12971
12972#[cfg(all(test, unix))]
12973mod health_event_tests {
12974 use super::*;
12975 use cortexkit_test_support::ScratchDir;
12976 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
12977
12978 struct Actor {
12979 module: SupervisedModule,
12980 registry: Arc<Registry>,
12981 forwarding: Arc<ForwardingTable>,
12982 spec: ModuleSpec,
12983 health: HealthConfig,
12984 rx: mpsc::Receiver<crate::router::OutboundFrame>,
12985 _home: ScratchDir,
12986 }
12987
12988 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
12989 let deadline = std::time::Instant::now() + Duration::from_secs(10);
12993 let tick = Instant::now();
12994 loop {
12995 if let Some(value) = observe() {
12996 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
12997 return value;
12998 }
12999 assert!(
13000 std::time::Instant::now() < deadline,
13001 "timed out waiting for {reason}"
13002 );
13003 tokio::task::yield_now().await;
13004 }
13005 }
13006
13007 impl Actor {
13008 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13009 let home = ScratchDir::new("health-events");
13010 let registry = Arc::new(Registry::default());
13011 let forwarding = Arc::new(ForwardingTable::default());
13012 let health = HealthConfig {
13013 cadence,
13014 deadline: Duration::from_secs(3600),
13015 ..HealthConfig::default()
13016 };
13017 let supervisor =
13018 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13019 .with_forwarding(forwarding.clone())
13020 .with_health_config(health.clone());
13021 let spec = ModuleSpec {
13022 module_id: "event-health-child".into(),
13023 program: PathBuf::from("/bin/sh"),
13024 args: vec!["-c".into(), "exec sleep 600".into()],
13027 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13028 .into_iter()
13029 .map(|key| (key.into(), home.path().display().to_string()))
13030 .collect(),
13031 reserved: false,
13032 reserved_prefixes: vec![],
13033 protocol,
13034 overlap: Default::default(),
13035 };
13036 let module = supervisor.spawn(spec.clone()).unwrap();
13037 let (_, rx) = mpsc::channel(8);
13038 let actor = Self {
13039 module,
13040 registry,
13041 forwarding,
13042 spec,
13043 health,
13044 rx,
13045 _home: home,
13046 };
13047 actor
13048 .wait_for_select(|checkpoint| checkpoint.generation > 0)
13049 .await;
13050 actor
13051 }
13052
13053 fn turns(&self) -> u64 {
13054 lock_snapshot(&self.module.inner.snapshot)
13055 .unwrap()
13056 .actor_turns
13057 }
13058
13059 async fn wait_for_select(
13060 &self,
13061 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13062 ) -> ActorSelectCheckpoint {
13063 wait_for("actor select checkpoint", || {
13064 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13065 state.actor_select.filter(|checkpoint| {
13066 checkpoint.generation == state.spawn_generation
13067 && state.state == ModuleState::Running
13068 && state.process_alive
13069 && state.reported_pid().is_some()
13070 && expected(checkpoint)
13071 })
13072 })
13073 .await
13074 }
13075
13076 fn hello(&mut self, connection: u64, health: bool) {
13077 let manifest =
13078 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13079 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13080 .build();
13081 let connection = ConnectionId::new(connection);
13082 let (tx, rx) = mpsc::channel(8);
13083 self.rx = rx;
13084 self.forwarding
13085 .register_module_connection(
13086 connection,
13087 self.spec.module_id.clone(),
13088 subc_protocol::PROTOCOL_VERSION,
13089 Concurrency::ModuleManaged,
13090 FrameSink::new(tx),
13091 )
13092 .unwrap();
13093 self.registry
13094 .register_with_control_ops(
13095 manifest,
13096 subc_protocol::PROTOCOL_VERSION,
13097 connection,
13098 if health {
13099 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13100 } else {
13101 vec![]
13102 },
13103 )
13104 .unwrap();
13105 }
13106
13107 async fn probe(&mut self) -> crate::router::OutboundFrame {
13108 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13109 Ok(frame) => Some(frame),
13110 Err(mpsc::error::TryRecvError::Empty) => None,
13111 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13112 })
13113 .await;
13114 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13115 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13116 frame
13117 }
13118
13119 fn no_probe(&mut self) {
13120 assert!(matches!(
13121 self.rx.try_recv(),
13122 Err(mpsc::error::TryRecvError::Empty)
13123 ));
13124 }
13125
13126 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13127 self.forwarding
13128 .complete_module_control_rpc(
13129 ConnectionId::new(connection),
13130 frame.header.corr,
13131 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13132 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13133 status: HealthStatus::Ok,
13134 detail: None,
13135 metrics: None,
13136 }),
13137 )
13138 .unwrap();
13139 }
13140 }
13141
13142 #[tokio::test(start_paused = true)]
13143 async fn no_health_children_park_for_sixty_seconds() {
13144 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13145 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13146 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13147 unadvertised.hello(1, false);
13148 for _ in 0..6000 {
13153 tokio::time::advance(Duration::from_millis(10)).await;
13154 tokio::task::yield_now().await;
13155 }
13156 for actor in [&none, &unregistered, &unadvertised] {
13157 assert!(
13158 actor.turns() <= 5,
13159 "no-health actor woke {} times",
13160 actor.turns()
13161 );
13162 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13163 }
13164 }
13165
13166 #[tokio::test(start_paused = true)]
13167 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13168 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13172 let tick = Instant::now();
13173 actor.hello(1, true);
13174 actor.probe().await;
13175 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13176 }
13177
13178 #[tokio::test(start_paused = true)]
13179 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13180 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13181 let hello_tick = Instant::now();
13182 actor.hello(1, true);
13183 let armed = actor
13184 .wait_for_select(|checkpoint| {
13185 checkpoint.registered_connection == Some(ConnectionId::new(1))
13186 && checkpoint.next_probe_at.is_some()
13187 })
13188 .await;
13189 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13190 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13191 let first_deadline = hello_tick + delay;
13192 assert_eq!(armed.next_probe_at, Some(first_deadline));
13193 tokio::time::advance(Duration::from_secs(29)).await;
13194 let turns = actor.turns();
13195 actor
13196 .registry
13197 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13198 .unwrap();
13199 let updated = actor
13200 .wait_for_select(|checkpoint| checkpoint.turn > turns)
13201 .await;
13202 assert_eq!(
13203 updated.next_probe_at,
13204 Some(first_deadline),
13205 "catalog update must not reset cadence"
13206 );
13207 actor.no_probe();
13208 tokio::time::advance(first_deadline - Instant::now()).await;
13209 let frame = actor.probe().await;
13210 actor.answer(1, frame);
13211 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13212 let next_deadline = Instant::now() + next;
13213 let rearmed = actor
13214 .wait_for_select(|checkpoint| {
13215 checkpoint
13216 .next_probe_at
13217 .is_some_and(|deadline| deadline > first_deadline)
13218 })
13219 .await;
13220 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13221 tokio::time::advance(next - Duration::from_nanos(1)).await;
13222 actor.no_probe();
13223 tokio::time::advance(Duration::from_nanos(1)).await;
13224 actor.probe().await;
13225 }
13226
13227 #[tokio::test(start_paused = true)]
13228 async fn disconnect_disarms_and_reconnect_rearms_health() {
13229 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13230 actor.hello(1, true);
13231 actor
13232 .wait_for_select(|checkpoint| {
13233 checkpoint.registered_connection == Some(ConnectionId::new(1))
13234 && checkpoint.next_probe_at.is_some()
13235 })
13236 .await;
13237 let turns = actor.turns();
13238 let tick = Instant::now();
13239 actor
13240 .registry
13241 .deregister_connection(ConnectionId::new(1))
13242 .unwrap();
13243 let disarmed = actor
13244 .wait_for_select(|checkpoint| {
13245 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13246 })
13247 .await;
13248 assert_eq!(disarmed.next_probe_at, None);
13249 assert_eq!(disarmed.wake_after, None);
13250 assert_eq!(
13251 Instant::now(),
13252 tick,
13253 "disconnect must disarm in the notification tick"
13254 );
13255 tokio::time::advance(Duration::from_secs(60)).await;
13256 actor.no_probe();
13257 let reconnect_tick = Instant::now();
13258 actor.hello(2, true);
13259 let rearmed = actor
13260 .wait_for_select(|checkpoint| {
13261 checkpoint.registered_connection == Some(ConnectionId::new(2))
13262 && checkpoint.next_probe_at.is_some()
13263 })
13264 .await;
13265 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13266 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13267 tokio::time::advance(delay).await;
13268 actor.probe().await;
13269 }
13270
13271 #[tokio::test(start_paused = true)]
13272 async fn reregistration_without_health_disarms_the_old_schedule() {
13273 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13274 actor.hello(1, true);
13275 actor
13276 .wait_for_select(|checkpoint| {
13277 checkpoint.registered_connection == Some(ConnectionId::new(1))
13278 && checkpoint.next_probe_at.is_some()
13279 })
13280 .await;
13281 actor
13282 .registry
13283 .deregister_connection(ConnectionId::new(1))
13284 .unwrap();
13285 actor.hello(2, false);
13286 let disarmed = actor
13287 .wait_for_select(|checkpoint| {
13288 checkpoint.registered_connection == Some(ConnectionId::new(2))
13289 })
13290 .await;
13291 assert_eq!(disarmed.next_probe_at, None);
13292 assert_eq!(disarmed.wake_after, None);
13293 tokio::time::advance(Duration::from_secs(60)).await;
13294 actor.no_probe();
13295 }
13296
13297 #[tokio::test(start_paused = true)]
13298 async fn restart_waits_for_its_own_hello_before_rearming_health() {
13299 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13300 actor.hello(1, true);
13301 let armed = actor
13302 .wait_for_select(|checkpoint| {
13303 checkpoint.registered_connection == Some(ConnectionId::new(1))
13304 && checkpoint.next_probe_at.is_some()
13305 })
13306 .await;
13307 actor
13311 .registry
13312 .deregister_connection(ConnectionId::new(1))
13313 .unwrap();
13314 actor.module.restart(Some(0)).await.unwrap();
13315 let parked = actor
13316 .wait_for_select(|checkpoint| {
13317 checkpoint.generation > armed.generation
13318 && checkpoint.registered_connection.is_none()
13319 && checkpoint.next_probe_at.is_none()
13320 })
13321 .await;
13322 assert_eq!(parked.wake_after, None);
13323 actor.rx.close();
13324 while actor.rx.try_recv().is_ok() {}
13327 let turns = actor.turns();
13328 tokio::time::advance(Duration::from_secs(60)).await;
13329 let after = actor.turns();
13330 eprintln!(
13331 "restart park turns: before={turns} after={after} delta={}",
13332 after - turns
13333 );
13334 assert_eq!(after, turns, "replacement must park before HELLO");
13335 let hello_tick = Instant::now();
13336 actor.hello(2, true);
13337 let rearmed = actor
13338 .wait_for_select(|checkpoint| {
13339 checkpoint.registered_connection == Some(ConnectionId::new(2))
13340 && checkpoint.next_probe_at.is_some()
13341 })
13342 .await;
13343 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13344 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13345 tokio::time::advance(delay).await;
13346 actor.probe().await;
13347 }
13348
13349 #[tokio::test(start_paused = true)]
13350 async fn rescan_adds_and_removes_http_health_without_polling() {
13351 use tokio::{
13352 io::{AsyncReadExt, AsyncWriteExt},
13353 net::TcpListener,
13354 };
13355 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13356 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13357 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13358 actor.health.cadence = Duration::from_millis(1);
13360 let (started_tx, mut started_rx) = mpsc::channel(8);
13361 let peer = tokio::spawn(async move {
13362 loop {
13363 let (mut socket, _) = listener.accept().await.unwrap();
13364 let mut request = [0; 1024];
13365 assert!(socket.read(&mut request).await.unwrap() > 0);
13366 started_tx.send(()).await.unwrap();
13367 socket
13368 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13369 .await
13370 .unwrap();
13371 }
13372 });
13373 let tick = Instant::now();
13374 let turns = actor.turns();
13375 actor
13376 .module
13377 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13378 .await
13379 .unwrap();
13380 let armed = actor
13381 .wait_for_select(|checkpoint| {
13382 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13383 })
13384 .await;
13385 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13386 assert_eq!(Instant::now(), tick);
13387 tokio::time::advance(Duration::from_millis(1)).await;
13388 let mut started = false;
13389 wait_for("successful HTTP probe", || {
13390 started |= started_rx.try_recv().is_ok();
13391 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13392 .then_some(())
13393 })
13394 .await;
13395 assert!(started, "added HTTP check must start at its first deadline");
13396 assert_eq!(
13397 actor.module.status().unwrap().health.status,
13398 SupervisorHealthStatus::Ok
13399 );
13400 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13401 actor.health.http = None;
13402 let turns = actor.turns();
13403 actor
13404 .module
13405 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13406 .await
13407 .unwrap();
13408 let parked = actor
13409 .wait_for_select(|checkpoint| {
13410 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13411 })
13412 .await;
13413 assert_eq!(parked.wake_after, None);
13414 let turns = actor.turns();
13415 tokio::time::advance(Duration::from_secs(60)).await;
13416 assert_eq!(
13417 actor.turns(),
13418 turns,
13419 "removed HTTP check must park the actor"
13420 );
13421 assert!(
13422 started_rx.try_recv().is_err(),
13423 "removed HTTP check must stay disarmed"
13424 );
13425 assert_eq!(
13426 actor.module.status().unwrap().health,
13427 ModuleHealthStatus::default()
13428 );
13429 peer.abort();
13430 }
13431}
13432
13433#[cfg(test)]
13434mod drain_busy_reprobe_tests {
13435 use super::*;
13439 use std::sync::Mutex as StdMutex;
13440 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13441
13442 const MODULE_ID: &str = "drain-reprobe-module";
13443 const CONNECTION: u64 = 7;
13444 const GAUGE: &str = "runs_in_flight";
13445 const CADENCE: Duration = Duration::from_secs(30);
13448 const CEILING: Duration = Duration::from_secs(30);
13449
13450 #[derive(Clone, Copy)]
13452 enum Answer {
13453 BusyUntil { until: Instant, latency: Duration },
13456 Silent,
13458 }
13459
13460 struct Drain {
13461 forwarding: Arc<ForwardingTable>,
13462 runtime: SupervisorRuntimeConfig,
13463 endpoint: crate::ModuleEndpointId,
13464 probes: Arc<StdMutex<Vec<Instant>>>,
13466 responder: tokio::task::JoinHandle<()>,
13467 }
13468
13469 impl Drain {
13470 fn start(answer: Answer) -> Self {
13471 let registry = Arc::new(Registry::default());
13472 let forwarding = Arc::new(ForwardingTable::default());
13473 let health = HealthConfig {
13474 cadence: CADENCE,
13475 ..HealthConfig::default()
13476 };
13477 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
13478 .with_forwarding(forwarding.clone())
13479 .with_health_config(health)
13480 .runtime_config();
13481 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
13482 let endpoint = forwarding
13483 .register_module_connection(
13484 ConnectionId::new(CONNECTION),
13485 MODULE_ID.to_string(),
13486 subc_protocol::PROTOCOL_VERSION,
13487 Concurrency::ModuleManaged,
13488 FrameSink::new(tx),
13489 )
13490 .unwrap();
13491 let probes = Arc::new(StdMutex::new(Vec::new()));
13492 let responder = tokio::spawn({
13493 let forwarding = forwarding.clone();
13494 let probes = probes.clone();
13495 async move {
13496 while let Some(frame) = rx.recv().await {
13497 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13498 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13499 probes.lock().unwrap().push(Instant::now());
13500 let Answer::BusyUntil { until, latency } = answer else {
13501 continue;
13502 };
13503 sleep(latency).await;
13504 let busy = u64::from(Instant::now() < until);
13505 let _ = forwarding.complete_module_control_rpc(
13508 ConnectionId::new(CONNECTION),
13509 frame.header.corr,
13510 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13511 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13512 status: HealthStatus::Ok,
13513 detail: None,
13514 metrics: Some(serde_json::json!({ GAUGE: busy })),
13515 }),
13516 );
13517 }
13518 }
13519 });
13520 Self {
13521 forwarding,
13522 runtime,
13523 endpoint,
13524 probes,
13525 responder,
13526 }
13527 }
13528
13529 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
13532 let started = Instant::now();
13533 let drained = wait_for_forwarding_quiescence(
13534 &self.forwarding,
13535 MODULE_ID,
13536 &self.runtime,
13537 self.endpoint,
13538 started + ceiling,
13539 &[GAUGE.to_string()],
13540 DrainScope::Active,
13541 )
13542 .await
13543 .unwrap();
13544 (drained, started.elapsed())
13545 }
13546
13547 fn probes(&self) -> Vec<Instant> {
13548 self.probes.lock().unwrap().clone()
13549 }
13550 }
13551
13552 impl Drop for Drain {
13553 fn drop(&mut self) {
13554 self.responder.abort();
13555 }
13556 }
13557
13558 #[tokio::test(start_paused = true)]
13559 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
13560 let drain = Drain::start(Answer::BusyUntil {
13561 until: Instant::now() + Duration::from_secs(1),
13562 latency: Duration::ZERO,
13563 });
13564 let (drained, elapsed) = drain.run(CEILING).await;
13565 assert!(
13566 drained,
13567 "gauges that read zero during the drain must let it finish drained, \
13568 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
13569 );
13570 assert!(
13571 elapsed < Duration::from_secs(3),
13572 "drain took {elapsed:?} after the gauges went quiet at 1s"
13573 );
13574 let probes = drain.probes();
13575 assert!(
13576 probes.len() >= 2,
13577 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
13578 );
13579 }
13580
13581 #[tokio::test(start_paused = true)]
13582 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
13583 let started = Instant::now();
13584 let drain = Drain::start(Answer::BusyUntil {
13585 until: started + Duration::from_secs(3600),
13586 latency: Duration::ZERO,
13587 });
13588 let (drained, elapsed) = drain.run(CEILING).await;
13589 assert!(!drained, "a module busy throughout must not drain");
13590 assert!(
13591 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
13592 "a busy module is held until the ceiling and no longer: {elapsed:?}"
13593 );
13594 let probes = drain.probes();
13597 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
13598 assert!(
13599 probes.len() as u128 <= max_probes,
13600 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
13601 probes.len()
13602 );
13603 for pair in probes.windows(2) {
13604 assert!(
13605 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
13606 "consecutive probes {:?} apart",
13607 pair[1] - pair[0]
13608 );
13609 }
13610 }
13611
13612 #[tokio::test(start_paused = true)]
13613 async fn a_slow_probe_is_never_overlapped_by_the_next() {
13614 let latency = Duration::from_secs(1);
13617 let drain = Drain::start(Answer::BusyUntil {
13618 until: Instant::now() + Duration::from_secs(3600),
13619 latency,
13620 });
13621 let (drained, _) = drain.run(Duration::from_secs(5)).await;
13622 assert!(!drained);
13623 let probes = drain.probes();
13624 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
13625 for pair in probes.windows(2) {
13626 assert!(
13627 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
13628 "a probe started {:?} after the previous one, before it could finish",
13629 pair[1] - pair[0]
13630 );
13631 }
13632 }
13633
13634 #[tokio::test(start_paused = true)]
13635 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
13636 let drain = Drain::start(Answer::Silent);
13637 let ceiling = Duration::from_secs(2);
13638 let (drained, elapsed) = drain.run(ceiling).await;
13639 assert!(!drained, "an unanswered probe counts as busy");
13640 assert!(
13641 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
13642 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
13643 );
13644 assert_eq!(
13645 drain.probes().len(),
13646 1,
13647 "the unanswered probe holds the only slot until the ceiling"
13648 );
13649 }
13650}
13651
13652#[cfg(test)]
13653mod health_evidence_tests {
13654 use super::{HealthProbeError, HealthProbeEvidence};
13655 use std::collections::HashSet;
13656
13657 #[test]
13665 fn only_a_dead_lane_is_proof_of_death() {
13666 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
13667 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
13671 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
13672 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
13673 }
13674
13675 #[test]
13681 fn every_evidence_class_has_a_distinct_label() {
13682 let labels = [
13683 HealthProbeError::lane_dead("").label(),
13684 HealthProbeError::no_answer("").label(),
13685 HealthProbeError::bad_answer("").label(),
13686 HealthProbeError::misconfigured("").label(),
13687 ];
13688 let unique: HashSet<_> = labels.iter().collect();
13689 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
13690 }
13691
13692 #[test]
13698 fn classification_preserves_the_original_message() {
13699 let err = HealthProbeError::no_answer("module did not answer within 5s");
13700 assert_eq!(err.to_string(), "module did not answer within 5s");
13701 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13702 }
13703}
13704
13705#[cfg(test)]
13706mod health_tombstone_tests {
13707 use std::{path::PathBuf, sync::Arc, time::Duration};
13708
13709 use subc_protocol::{
13710 manifest::Concurrency,
13711 session::{HealthStatus, ModuleControlResponse},
13712 };
13713 use tokio::sync::mpsc;
13714
13715 use super::{
13716 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
13717 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
13718 };
13719 use crate::{
13720 control::ControlHandler,
13721 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
13722 registry::{ConnectionId, Registry},
13723 router::FrameSink,
13724 };
13725
13726 struct ProbeHarness {
13727 spec: ModuleSpec,
13728 runtime: SupervisorRuntimeConfig,
13729 forwarding: Arc<ForwardingTable>,
13730 module_connection: ConnectionId,
13731 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13732 handler: ControlHandler,
13733 module: super::SupervisedModule,
13734 }
13735
13736 fn probe_harness() -> ProbeHarness {
13737 let registry = Arc::new(Registry::default());
13738 let forwarding = Arc::new(ForwardingTable::default());
13739 let supervisor_handle = super::SupervisorHandle::new();
13740 let health = HealthConfig {
13741 http: None,
13742 cadence: Duration::from_secs(30),
13743 deadline: Duration::from_secs(5),
13744 failure_threshold: 3,
13745 on_degraded: HealthAction::Report,
13746 on_failing: HealthAction::Report,
13747 critical: false,
13748 };
13749 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13750 .with_forwarding(Arc::clone(&forwarding))
13751 .with_handle(supervisor_handle.clone())
13752 .with_health_config(health);
13753 let spec = ModuleSpec {
13754 module_id: "late-health-module".to_string(),
13755 program: PathBuf::from("disabled-module"),
13756 args: Vec::new(),
13757 env: Vec::new(),
13758 reserved: false,
13759 reserved_prefixes: Vec::new(),
13760 protocol: ModuleProtocol::Subc,
13761 overlap: Default::default(),
13762 };
13763 let module = supervisor
13764 .supervise_configured(spec.clone(), false)
13765 .unwrap();
13766 let runtime = supervisor.runtime_config();
13767 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
13768 .with_supervisor(supervisor_handle);
13769 let module_connection = ConnectionId::new(700);
13770 let (module_tx, module_rx) = mpsc::channel(8);
13771 forwarding
13772 .register_module_connection(
13773 module_connection,
13774 spec.module_id.clone(),
13775 subc_protocol::PROTOCOL_VERSION,
13776 Concurrency::ModuleManaged,
13777 FrameSink::new(module_tx),
13778 )
13779 .unwrap();
13780
13781 ProbeHarness {
13782 spec,
13783 runtime,
13784 forwarding,
13785 module_connection,
13786 module_rx,
13787 handler,
13788 module,
13789 }
13790 }
13791
13792 async fn finish_after(
13793 harness: &mut ProbeHarness,
13794 stall: Duration,
13795 ) -> ModuleControlRpcCompletion {
13796 assert!(stall > harness.runtime.health.deadline);
13797 let deadline = harness.runtime.health.deadline;
13798 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13799 let answer = async {
13800 let frame = harness.module_rx.recv().await.expect("health.check frame");
13801 tokio::time::advance(deadline).await;
13802 tokio::task::yield_now().await;
13803 tokio::time::advance(stall - deadline).await;
13804 harness
13805 .forwarding
13806 .complete_module_control_rpc(
13807 harness.module_connection,
13808 frame.header.corr,
13809 Some("health.check"),
13810 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13811 status: HealthStatus::Ok,
13812 detail: None,
13813 metrics: None,
13814 }),
13815 )
13816 .unwrap()
13817 };
13818 let (probe_result, completion) = tokio::join!(probe, answer);
13819 let err = probe_result.expect_err("probe must miss its deadline");
13820 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13821 completion
13822 }
13823
13824 async fn time_out_without_answer(harness: &mut ProbeHarness) {
13825 let deadline = harness.runtime.health.deadline;
13826 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13827 let exhaust_deadline = async {
13828 let _frame = harness.module_rx.recv().await.expect("health.check frame");
13829 tokio::time::advance(deadline).await;
13830 tokio::task::yield_now().await;
13831 };
13832 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
13833 let err = probe_result.expect_err("probe must miss its deadline");
13834 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13835 }
13836
13837 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
13838 let registry = Arc::clone(&harness.module.inner.registry);
13839 let snapshot = Arc::clone(&harness.module.inner.snapshot);
13840 let process_liveness = super::SupervisorProcessLiveness::default();
13841 let mut child = None;
13842 let cycle = super::run_health_probe_cycle(
13843 &harness.spec,
13844 &harness.runtime,
13845 ®istry,
13846 &process_liveness,
13847 &snapshot,
13848 &mut child,
13849 );
13850 let peer = async {
13851 let frame = harness.module_rx.recv().await.expect("health.check frame");
13852 if answer {
13853 harness
13854 .forwarding
13855 .complete_module_control_rpc(
13856 harness.module_connection,
13857 frame.header.corr,
13858 Some("health.check"),
13859 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13860 status: HealthStatus::Ok,
13861 detail: None,
13862 metrics: Some(serde_json::json!({"ready": true})),
13863 }),
13864 )
13865 .unwrap();
13866 } else {
13867 tokio::time::advance(harness.runtime.health.deadline).await;
13868 tokio::task::yield_now().await;
13869 }
13870 };
13871 tokio::join!(cycle, peer);
13872 }
13873
13874 #[tokio::test(start_paused = true)]
13875 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
13876 let mut harness = probe_harness();
13877 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
13881 monitor.abort();
13882 let _ = monitor.await;
13883 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
13884 state.enabled = true;
13885 state.state = super::ModuleState::Running;
13886 state.process_alive = true;
13887 })
13888 .unwrap();
13889
13890 run_probe_cycle(&mut harness, true).await;
13891 assert_eq!(
13892 harness.module.status().unwrap().health.status,
13893 super::SupervisorHealthStatus::Ok
13894 );
13895
13896 for failures in 1..harness.runtime.health.failure_threshold {
13897 run_probe_cycle(&mut harness, false).await;
13898 let status = harness.module.status().unwrap();
13899 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
13900 assert_eq!(status.health.consecutive_failures, failures);
13901 assert!(status.health.last_probe_ms.is_some());
13902 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
13903 assert!(status.health.metrics.is_none());
13904 assert_eq!(status.state, super::ModuleState::Running);
13905 assert!(status.process_alive);
13906 assert_eq!(status.restart_count, 0);
13907 assert_eq!(status.lifetime_restarts, 0);
13908 assert!(status.health.last_action.is_none());
13909 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
13910 }
13911
13912 run_probe_cycle(&mut harness, true).await;
13913 let recovered = harness.module.status().unwrap();
13914 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
13915 assert_eq!(recovered.health.consecutive_failures, 0);
13916 assert!(recovered.health.detail.is_none());
13917 assert_eq!(
13918 recovered.health.metrics,
13919 Some(serde_json::json!({"ready": true}))
13920 );
13921 assert_eq!(recovered.lifetime_restarts, 0);
13922
13923 for failures in 1..=harness.runtime.health.failure_threshold {
13924 run_probe_cycle(&mut harness, false).await;
13925 let status = harness.module.status().unwrap();
13926 assert_eq!(status.health.consecutive_failures, failures);
13927 if failures < harness.runtime.health.failure_threshold {
13928 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
13929 assert_eq!(status.state, super::ModuleState::Running);
13930 assert_eq!(status.lifetime_restarts, 0);
13931 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
13932 } else {
13933 assert_eq!(
13934 status.health.status,
13935 super::SupervisorHealthStatus::Unresponsive
13936 );
13937 assert_eq!(status.state, super::ModuleState::Restarting);
13938 assert_eq!(status.restart_count, 1);
13939 assert_eq!(status.lifetime_restarts, 1);
13940 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
13941 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
13942 }
13943 }
13944 }
13945
13946 #[tokio::test(start_paused = true)]
13947 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
13948 let mut harness = probe_harness();
13949
13950 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
13951 let first_latency = match &first {
13952 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
13953 other => panic!("late answer was not retained: {other:?}"),
13954 };
13955 assert!(harness.handler.observe_module_control_completion(first));
13956
13957 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
13958 let second_latency = match &second {
13959 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
13960 other => panic!("late answer was not retained: {other:?}"),
13961 };
13962 assert!(harness.handler.observe_module_control_completion(second));
13963
13964 assert_eq!(first_latency, Duration::from_secs(8));
13965 assert_eq!(
13966 second_latency - first_latency,
13967 Duration::from_secs(3),
13968 "latency must grow linearly with the additional stall"
13969 );
13970 let health = harness.module.status().unwrap().health;
13971 assert_eq!(health.late_answer_count, 2);
13972 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
13973 }
13974
13975 #[tokio::test(start_paused = true)]
13983 async fn late_answer_clears_the_consecutive_failure_streak() {
13984 let mut harness = probe_harness();
13985
13986 time_out_without_answer(&mut harness).await;
13988 harness
13989 .module
13990 .record_health_probe_failure_for_test("[no-answer] test miss")
13991 .unwrap();
13992 assert_eq!(
13993 harness.module.status().unwrap().health.consecutive_failures,
13994 1,
13995 "precondition: the miss must be on the streak before the late answer"
13996 );
13997
13998 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14000 assert!(matches!(
14001 late,
14002 ModuleControlRpcCompletion::LateHealthAnswer { .. }
14003 ));
14004 assert!(harness.handler.observe_module_control_completion(late));
14005
14006 let health = harness.module.status().unwrap().health;
14007 assert_eq!(
14008 health.consecutive_failures, 0,
14009 "a late answer is an answer: the streak must reset"
14010 );
14011 assert_eq!(health.late_answer_count, 1);
14012 }
14013
14014 #[tokio::test(start_paused = true)]
14015 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14016 let mut harness = probe_harness();
14017
14018 for _ in 0..20 {
14019 time_out_without_answer(&mut harness).await;
14020 assert_eq!(
14021 harness.forwarding.health_probe_tombstone_count().unwrap(),
14022 1
14023 );
14024 }
14025 }
14026}
14027
14028#[cfg(test)]
14029mod child_env_tests {
14030 #[cfg(windows)]
14031 use super::{
14032 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14033 };
14034 #[cfg(windows)]
14035 use std::sync::{Arc, OnceLock};
14036 #[cfg(windows)]
14037 #[test]
14038 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14039 let spec = spec(Vec::new());
14040 let mut command = Command::new("unused");
14041 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14042 .unwrap()
14043 .unwrap();
14044 let name = command
14045 .as_std()
14046 .get_envs()
14047 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14048 .and_then(|(_, value)| value)
14049 .unwrap();
14050 assert_eq!(name, OsStr::new(handoff.name()));
14051 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14052 assert!(command.as_std().get_envs().any(|(key, value)| key
14053 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14054 && value == Some(OsStr::new("env"))));
14055 assert!(command
14056 .as_std()
14057 .get_envs()
14058 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14059 }
14060
14061 #[cfg(windows)]
14062 #[test]
14063 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14064 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14065 snapshot.process_alive = true;
14066 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14067 assert_eq!(
14068 observed_launch_nonce_source(&snapshot).as_deref(),
14069 Some("env")
14070 );
14071 let consumed = Arc::new(OnceLock::new());
14072 snapshot.pipe_consumed = Some(consumed.clone());
14073 assert_eq!(
14074 observed_launch_nonce_source(&snapshot).as_deref(),
14075 Some("env")
14076 );
14077 consumed.set(()).unwrap();
14078 assert_eq!(
14079 observed_launch_nonce_source(&snapshot).as_deref(),
14080 Some("pipe")
14081 );
14082 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14083 assert_eq!(
14084 observed_launch_nonce_source(&snapshot).as_deref(),
14085 Some("env")
14086 );
14087 clear_current_process_facts(&mut snapshot);
14088 assert_eq!(observed_launch_nonce_source(&snapshot), None);
14089 }
14090 use super::{
14091 apply_child_env, apply_spawn_role, apply_wire_spawn_args, ModuleProtocol, ModuleSpec,
14092 SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG, SUBC_LAUNCH_NONCE_ENV,
14093 SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14094 };
14095 use std::{ffi::OsStr, path::PathBuf};
14096 use tokio::process::Command;
14097
14098 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14099 ModuleSpec {
14100 module_id: "env-plan".to_string(),
14101 program: PathBuf::from("/nonexistent"),
14102 args: Vec::new(),
14103 env,
14104 reserved: false,
14105 reserved_prefixes: Vec::new(),
14106 protocol: ModuleProtocol::Subc,
14107 overlap: Default::default(),
14108 }
14109 }
14110
14111 #[test]
14125 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14126 let mut command = Command::new("/nonexistent");
14127 apply_child_env(&mut command, &spec(Vec::new()));
14128 let removed = command
14129 .as_std()
14130 .get_envs()
14131 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14132 assert!(
14133 removed,
14134 "ambient CK_LOG must be explicitly removed for an unconfigured module"
14135 );
14136
14137 let mut configured = Command::new("/nonexistent");
14138 apply_child_env(
14139 &mut configured,
14140 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14141 );
14142 let effective = configured
14143 .as_std()
14144 .get_envs()
14145 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14146 .last()
14147 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14148 assert_eq!(
14149 effective,
14150 Some(Some("debug".to_string())),
14151 "a module's configured CK_LOG must survive the ambient removal"
14152 );
14153 }
14154
14155 #[test]
14164 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14165 let connection_file = std::path::Path::new("/run/subc-connection.json");
14166 let handle = SupervisorHandle::new();
14167
14168 let mut none_spec = spec(Vec::new());
14169 none_spec.protocol = ModuleProtocol::None;
14170 let mut none = Command::new("/nonexistent");
14171 let none_handoff =
14172 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14173 .expect("protocol-none spawn args apply");
14174 assert!(
14175 none_handoff.is_none(),
14176 "protocol:none spawn must not receive a nonce descriptor"
14177 );
14178 #[cfg(windows)]
14179 assert!(!none.as_std().get_envs().any(|(key, value)| key
14180 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14181 && value.is_some()));
14182 assert!(
14183 !none.as_std().get_envs().any(|(key, value)| key
14184 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14185 && value.is_some()),
14186 "protocol:none spawn must not name a nonce descriptor"
14187 );
14188 let none_args: Vec<String> = none
14189 .as_std()
14190 .get_args()
14191 .map(|a| a.to_string_lossy().into_owned())
14192 .collect();
14193 assert!(
14194 !none_args.iter().any(|a| a == SUBC_ARG),
14195 "protocol:none argv must not carry --subc; got {none_args:?}"
14196 );
14197 let none_has_nonce = none
14198 .as_std()
14199 .get_envs()
14200 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
14201 assert!(
14202 !none_has_nonce,
14203 "protocol:none spawn must not receive a launch nonce"
14204 );
14205 let none_has_module_id = none
14206 .as_std()
14207 .get_envs()
14208 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
14209 assert!(
14210 none_has_module_id,
14211 "SUBC_MODULE_ID is inert and stays on every path"
14212 );
14213 assert!(
14214 handle.spawn_nonce(&none_spec.module_id).is_none(),
14215 "no nonce record for a process that will never present one"
14216 );
14217
14218 let wire_spec = spec(Vec::new());
14220 let mut wire = Command::new("/nonexistent");
14221 let wire_handoff =
14222 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
14223 .expect("subc-wire spawn args apply");
14224 let wire_fd_env = wire
14225 .as_std()
14226 .get_envs()
14227 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
14228 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14229 #[cfg(unix)]
14230 assert_eq!(
14231 wire_fd_env,
14232 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
14233 "a subc-wire spawn names the pipe it will receive at descriptor 3"
14234 );
14235 #[cfg(windows)]
14236 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
14237 #[cfg(not(any(unix, windows)))]
14238 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
14239 let wire_args: Vec<String> = wire
14240 .as_std()
14241 .get_args()
14242 .map(|a| a.to_string_lossy().into_owned())
14243 .collect();
14244 assert_eq!(
14245 wire_args,
14246 vec![
14247 SUBC_ARG.to_string(),
14248 connection_file.to_string_lossy().into_owned()
14249 ],
14250 "a subc-wire spawn still carries --subc <path>"
14251 );
14252 assert_eq!(
14253 wire.as_std()
14254 .get_envs()
14255 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
14256 !cfg!(unix),
14257 "only Windows supplies the environment nonce"
14258 );
14259 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
14260 }
14261
14262 #[test]
14272 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
14273 let role = |command: &Command| {
14274 command
14275 .as_std()
14276 .get_envs()
14277 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
14278 .last()
14279 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
14280 };
14281 let forged = spec(vec![(
14282 SUBC_SPAWN_ROLE_ENV.to_string(),
14283 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
14284 )]);
14285
14286 let mut plain = Command::new("/nonexistent");
14287 apply_child_env(&mut plain, &forged);
14288 apply_spawn_role(&mut plain, SpawnRole::Plain);
14289 assert_eq!(
14290 role(&plain),
14291 Some(None),
14292 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
14293 );
14294
14295 let mut candidate = Command::new("/nonexistent");
14296 apply_child_env(&mut candidate, &spec(Vec::new()));
14297 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
14298 assert_eq!(
14299 role(&candidate),
14300 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
14301 );
14302 }
14303
14304 #[test]
14310 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
14311 let mut command = Command::new("/nonexistent");
14312 apply_child_env(
14313 &mut command,
14314 &spec(vec![
14315 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
14316 ("KEPT".to_string(), "yes".to_string()),
14317 ]),
14318 );
14319 let keys: Vec<String> = command
14320 .as_std()
14321 .get_envs()
14322 .filter(|(_, value)| value.is_some())
14323 .map(|(key, _)| key.to_string_lossy().into_owned())
14324 .collect();
14325 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
14326 assert!(
14327 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
14328 "daemon-private capture key leaked to the child: {keys:?}"
14329 );
14330 }
14331}
14332
14333#[cfg(test)]
14334mod jitter_tests {
14335 use super::jittered_health_delay;
14336 use std::{collections::HashSet, time::Duration};
14337
14338 const FLEET: [&str; 14] = [
14347 "aft",
14348 "alfonso-core",
14349 "magic-context",
14350 "broca",
14351 "thalamus",
14352 "quota",
14353 "engram",
14354 "plexus",
14355 "cerebellum",
14356 "astrocyte",
14357 "synapse",
14358 "subc-mcp",
14359 "cortexkit-credentials",
14360 "subc-federation",
14361 ];
14362
14363 #[test]
14371 fn probe_delays_disperse_across_the_fleet() {
14372 let cadence = Duration::from_secs(30);
14373 let delays: HashSet<Duration> = FLEET
14374 .iter()
14375 .map(|id| jittered_health_delay(id, 0, cadence))
14376 .collect();
14377 assert_eq!(
14378 delays.len(),
14379 FLEET.len(),
14380 "every supervised module must land on its own probe offset"
14381 );
14382 }
14383
14384 #[test]
14390 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
14391 let cadence = Duration::from_secs(30);
14392 let span = cadence / 10;
14393 for id in FLEET {
14394 for probe_index in 0..8 {
14395 let delay = jittered_health_delay(id, probe_index, cadence);
14396 assert!(
14397 delay >= cadence,
14398 "{id}#{probe_index}: jitter must not shorten the cadence"
14399 );
14400 assert!(
14401 delay < cadence + span,
14402 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
14403 );
14404 }
14405 }
14406 }
14407
14408 #[test]
14414 fn a_module_offset_is_stable_across_restarts() {
14415 let cadence = Duration::from_secs(30);
14416 for id in FLEET {
14417 assert_eq!(
14418 jittered_health_delay(id, 0, cadence),
14419 jittered_health_delay(id, 0, cadence),
14420 "{id}: the same module and probe index must produce the same offset"
14421 );
14422 }
14423 }
14424
14425 #[test]
14427 fn zero_cadence_yields_zero_delay() {
14428 assert_eq!(
14429 jittered_health_delay("aft", 0, Duration::ZERO),
14430 Duration::ZERO
14431 );
14432 }
14433}
14434
14435#[cfg(all(test, target_os = "linux"))]
14436mod cgroup_placement_tests {
14437 use super::{
14438 apply_cgroup_placement, remove_module_cgroup, ModuleProtocol, ModuleSpec, SuperviseError,
14439 SupervisedChild,
14440 };
14441 use crate::stderr_tail::{StderrRing, StderrTailConfig};
14442 use cortexkit_test_support::ScratchDir;
14443 use std::{
14444 fs, io,
14445 path::{Path, PathBuf},
14446 sync::{Arc, Mutex},
14447 };
14448 use tokio::process::Command;
14449
14450 #[tokio::test]
14451 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
14452 use super::*;
14453 let dir = ScratchDir::new("unique-spawn-cgroups");
14454 let root = PathBuf::from(format!(
14455 "/sys/fs/cgroup/subc-unique-test-{}-{}",
14456 std::process::id(),
14457 unix_ms_now()
14458 ));
14459 if let Err(error) = fs::create_dir(&root) {
14460 assert!(
14461 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
14462 "required cgroup test cannot execute: {error}"
14463 );
14464 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
14465 return;
14466 }
14467 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
14468 let group_count = || {
14469 fs::read_dir(root.join("subc-modules"))
14470 .unwrap()
14471 .map(|entry| entry.unwrap().file_type().unwrap())
14472 .filter(|kind| kind.is_dir())
14473 .count()
14474 };
14475 let supervisor =
14476 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
14477 .with_cgroup_placement(Some(placement.clone()));
14478 let runtime = supervisor.runtime_config();
14479 let mut spec = ModuleSpec {
14480 module_id: "unique-spawn".into(),
14481 program: PathBuf::from("/bin/sleep"),
14482 args: vec!["60".into()],
14483 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14484 .into_iter()
14485 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
14486 .collect(),
14487 reserved: false,
14488 reserved_prefixes: vec![],
14489 protocol: ModuleProtocol::None,
14490 overlap: Default::default(),
14491 };
14492 let spawn = |spec: &ModuleSpec| {
14493 spawn_child(
14494 spec,
14495 None,
14496 None,
14497 &runtime.stderr_ring,
14498 None,
14499 &runtime.child_roster,
14500 Some(&placement),
14501 )
14502 .unwrap()
14503 };
14504 let mut live = spawn(&spec);
14505 for _ in 0..3 {
14506 let next = spawn(&spec);
14508 assert_ne!(live.module_id, next.module_id);
14509 live.start_kill().unwrap();
14510 live.wait().await.unwrap();
14511 live = next;
14512 assert!(
14513 live.child.try_wait().unwrap().is_none(),
14514 "retiring the old slot must not kill the replacement"
14515 );
14516 assert_eq!(
14517 group_count(),
14518 1,
14519 "only the live spawn's cgroup should remain"
14520 );
14521 }
14522 supervisor.begin_daemon_shutdown();
14523 let reap = tokio::spawn(async move {
14524 live.wait().await.unwrap();
14525 });
14526 supervisor
14527 .end_children_for_daemon_shutdown(false, std::future::pending())
14528 .await;
14529 reap.await.unwrap();
14530 assert_eq!(group_count(), 0);
14531 spec.program = PathBuf::from("/bin/true");
14533 spec.args.clear();
14534 let fresh_roster = ChildRoster::default();
14535 let mut short = spawn_child(
14536 &spec,
14537 None,
14538 None,
14539 &runtime.stderr_ring,
14540 None,
14541 &fresh_roster,
14542 Some(&placement),
14543 )
14544 .unwrap();
14545 short.wait().await.unwrap();
14546 assert_eq!(group_count(), 0);
14547 spec.module_id = "_".repeat(255);
14548 let mut long_id = spawn_child(
14549 &spec,
14550 None,
14551 None,
14552 &runtime.stderr_ring,
14553 None,
14554 &fresh_roster,
14555 Some(&placement),
14556 )
14557 .unwrap();
14558 long_id.wait().await.unwrap();
14559 assert_eq!(
14560 group_count(),
14561 0,
14562 "valid long module IDs must not exceed cgroup NAME_MAX"
14563 );
14564 fs::remove_dir(root.join("subc-modules")).unwrap();
14565 fs::remove_dir(root).unwrap();
14566 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
14567 }
14568
14569 #[test]
14570 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
14571 let path = Path::new("/definitely-missing-subc-cgroup");
14572 let mut command = Command::new("true");
14573 let error = apply_cgroup_placement(
14574 &mut command,
14575 &ModuleSpec {
14576 module_id: "broken-cgroup".to_string(),
14577 program: PathBuf::from("true"),
14578 args: Vec::new(),
14579 env: Vec::new(),
14580 reserved: false,
14581 reserved_prefixes: Vec::new(),
14582 protocol: ModuleProtocol::Subc,
14583 overlap: Default::default(),
14584 },
14585 path,
14586 )
14587 .expect_err("a parent cgroup open failure must reject the supervised spawn");
14588 let reason = error.to_string();
14589
14590 assert!(
14591 matches!(error, SuperviseError::Cgroup { .. }),
14592 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
14593 );
14594 assert!(
14595 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
14596 "parent cgroup open failure must name cgroup.procs: {reason}"
14597 );
14598 }
14599
14600 #[tokio::test]
14601 async fn reaping_a_child_removes_its_empty_module_cgroup() {
14602 let root = ScratchDir::new("supervisor-reap-cgroup");
14603 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14604 let placement = subc_cgroup::prepare_at(&root)
14605 .expect("prepare scratch cgroup root")
14606 .expect("scratch root has a cgroup.procs marker");
14607 let module_id = "reaped-module";
14608 let module = placement
14609 .module_path(module_id)
14610 .expect("create scratch module cgroup");
14611 let child = Command::new("true")
14612 .env("XDG_DATA_HOME", root.path())
14613 .env("XDG_RUNTIME_DIR", root.path())
14614 .env("XDG_CONFIG_HOME", root.path())
14615 .spawn()
14616 .expect("spawn short-lived child");
14617 let pid = child.id().expect("spawned child has pid");
14618 let mut child = SupervisedChild {
14619 child,
14620 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
14621 protocol: ModuleProtocol::Subc,
14622 module_id: module_id.to_string(),
14623 cgroup_placement: Some(placement),
14624 stdout_pump: None,
14625 stderr_pump: None,
14626 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
14627 spawned_at_ms: 0,
14628 spawned_from: PathBuf::from("true"),
14629 spawned_file_identity: None,
14630 process_start_time: None,
14631 process_identity: None,
14632 pid,
14633 roster_guard: None,
14634 #[cfg(target_os = "macos")]
14635 privacy_exec: None,
14636 spawn_failure: None,
14637 };
14638
14639 child.wait().await.expect("reap short-lived child");
14640
14641 assert!(
14642 !module.exists(),
14643 "reaping the supervised child must remove its empty cgroup"
14644 );
14645 }
14646
14647 #[test]
14648 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
14649 let root = ScratchDir::new("supervisor-non-empty-cgroup");
14650 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14651 let placement = subc_cgroup::prepare_at(&root)
14652 .expect("prepare scratch cgroup root")
14653 .expect("scratch root has a cgroup.procs marker");
14654 let module = placement
14655 .module_path("surviving-module")
14656 .expect("create scratch module cgroup");
14657 fs::write(module.join("surviving-process"), b"still present")
14658 .expect("make scratch cgroup non-empty");
14659 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
14660
14661 remove_module_cgroup(&placement, "surviving-module");
14662
14663 let logs = crate::router::test_log::captured_logs(&logs);
14664 assert!(
14665 module.exists(),
14666 "failed removal must leave the cgroup intact"
14667 );
14668 assert!(
14669 logs.contains("could not remove module cgroup after process exit; continuing teardown")
14670 && logs.contains("surviving-module"),
14671 "best-effort removal must report the failure without returning it: {logs}"
14672 );
14673 }
14674
14675 #[test]
14676 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
14677 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
14678 let reason = SuperviseError::Spawn {
14679 program: PathBuf::from("/bin/true"),
14680 source: io::Error::from_raw_os_error(13),
14681 cgroup_path: Some(cgroup_path.clone()),
14682 }
14683 .to_string();
14684
14685 assert!(
14686 reason.contains(&cgroup_path.display().to_string()),
14687 "a pre_exec spawn failure must name the cgroup path: {reason}"
14688 );
14689 }
14690}
14691
14692#[cfg(test)]
14693mod spawn_subscriber_lag_tests {
14694 use super::*;
14695
14696 #[tokio::test]
14701 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
14702 let feed = SpawnEventFeed::default();
14703 feed.configure_incarnation("lag-incarnation".to_string());
14704 let (tx, mut rx) = mpsc::channel(1);
14707 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
14708 .expect("subscribe");
14709 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
14710 for index in 0..emitted {
14711 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
14712 tokio::task::yield_now().await;
14715 }
14716 assert_eq!(
14717 feed.subscriber_count(),
14718 0,
14719 "the lagged subscriber must be removed"
14720 );
14721
14722 let mut data = Vec::new();
14723 let mut last = None;
14724 loop {
14725 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14726 .await
14727 .expect("the forwarder must finish once the subscriber is dropped");
14728 let Some(outbound) = next else { break };
14729 let frame = outbound.frame;
14730 if frame.header.ty == FrameType::StreamData {
14731 assert!(last.is_none(), "no data may follow the terminal frame");
14732 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
14733 data.push(event.cursor.seq);
14734 } else {
14735 assert!(last.is_none(), "exactly one terminal frame");
14736 last = Some(frame);
14737 }
14738 }
14739 assert!(!data.is_empty(), "queued frames drain before the terminal");
14740 for pair in data.windows(2) {
14741 assert_eq!(
14742 pair[1],
14743 pair[0] + 1,
14744 "queued frames arrive dense and in order"
14745 );
14746 }
14747 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
14748 assert_eq!(terminal.header.ty, FrameType::Error);
14749 assert_eq!(terminal.header.corr, 7);
14750 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
14751 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
14752 let detail = body.detail.expect("lagged error carries detail");
14753 assert_eq!(
14754 detail["first_undelivered_cursor"]["seq"],
14755 data.last().unwrap() + 1,
14756 "the named cursor is the first event the subscriber did not receive"
14757 );
14758 assert_eq!(
14759 detail["first_undelivered_cursor"]["daemon_incarnation"],
14760 "lag-incarnation"
14761 );
14762 }
14763}
14764
14765#[cfg(test)]
14766mod terminal_history_read_concurrency_tests {
14767 use super::*;
14768 use crate::terminal_journal::read_pause;
14769 use cortexkit_test_support::ScratchDir;
14770 use std::sync::mpsc as std_mpsc;
14771
14772 fn journaled_ring(
14773 journal: &Arc<crate::terminal_journal::TerminalJournal>,
14774 ) -> Arc<Mutex<TerminalRing>> {
14775 Arc::new(Mutex::new(
14776 TerminalRing::new(TerminalRingConfig::default(), 1)
14777 .with_journal(Some(Arc::clone(journal))),
14778 ))
14779 }
14780
14781 fn crash(at_ms: u64) -> ExitReport {
14782 ExitReport {
14783 kind: ExitKind::Crash,
14784 code: Some(1),
14785 signal: None,
14786 at_ms,
14787 }
14788 }
14789
14790 fn record_within(
14793 module_id: &'static str,
14794 ring: &Arc<Mutex<TerminalRing>>,
14795 at_ms: u64,
14796 bound: Duration,
14797 ) -> bool {
14798 let ring = Arc::clone(ring);
14799 let (done, done_rx) = std_mpsc::channel();
14800 std::thread::spawn(move || {
14801 record_terminal(
14802 module_id,
14803 &ring,
14804 &SpawnEventFeed::default(),
14805 &crash(at_ms),
14806 TerminalDisposition::Restarting,
14807 );
14808 let _ = done.send(());
14809 });
14810 done_rx.recv_timeout(bound).is_ok()
14811 }
14812
14813 #[test]
14818 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
14819 let dir = ScratchDir::new("terminal-history-concurrent-read");
14820 let path = dir.join("terminals.jsonl");
14821 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
14822 path.clone(),
14823 "daemon".into(),
14824 ));
14825 let reader_ring = journaled_ring(&journal);
14826 let other_ring = journaled_ring(&journal);
14827 assert!(record_within(
14828 "reader-module",
14829 &reader_ring,
14830 10,
14831 Duration::from_secs(5)
14832 ));
14833
14834 let (started, release) = read_pause::install(&path);
14835 let reading = {
14836 let ring = Arc::clone(&reader_ring);
14837 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
14838 };
14839 started
14840 .recv_timeout(Duration::from_secs(5))
14841 .expect("the history read reached its pause");
14842
14843 let bound = Duration::from_secs(1);
14844 assert!(
14845 record_within("other-module", &other_ring, 20, bound),
14846 "another module's exit waited on a history read (journal writer held)"
14847 );
14848 assert!(
14849 record_within("reader-module", &reader_ring, 30, bound),
14850 "the read module's own exit waited on its history read (ring held)"
14851 );
14852
14853 drop(release);
14854 let paused = reading.join().unwrap();
14855 assert_eq!(
14856 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14857 vec![10],
14858 "an exit recorded after the read began lands in neither half of it"
14859 );
14860 assert_eq!(paused.journal_skipped_lines, 0);
14861 assert_eq!(paused.journal_read_errors, 0);
14862
14863 let after = durable_terminal_history_of(&reader_ring, "reader-module");
14864 assert_eq!(
14865 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14866 vec![10, 30],
14867 "the next read merges ring and journal with no duplicate"
14868 );
14869 assert_eq!(after.journal_skipped_lines, 0);
14870 }
14871}
14872
14873#[cfg(test)]
14878mod stderr_settle_tests {
14879 use std::{
14880 future::Future,
14881 io,
14882 pin::Pin,
14883 sync::{Arc, Mutex},
14884 task::{Context, Poll},
14885 time::Duration,
14886 };
14887
14888 use tokio::{
14889 io::{AsyncRead, ReadBuf},
14890 sync::oneshot,
14891 time::Instant,
14892 };
14893
14894 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
14895 use crate::stderr_tail::{
14896 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
14897 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
14898 };
14899
14900 const BOUND: Duration = Duration::from_millis(250);
14901
14902 struct HeldReader {
14906 before: Option<Vec<u8>>,
14907 gate: Option<oneshot::Receiver<()>>,
14908 after: io::Cursor<Vec<u8>>,
14909 }
14910
14911 impl AsyncRead for HeldReader {
14912 fn poll_read(
14913 mut self: Pin<&mut Self>,
14914 cx: &mut Context<'_>,
14915 buf: &mut ReadBuf<'_>,
14916 ) -> Poll<io::Result<()>> {
14917 if let Some(bytes) = self.before.take() {
14918 buf.put_slice(&bytes);
14919 return Poll::Ready(Ok(()));
14920 }
14921 if let Some(gate) = self.gate.as_mut() {
14922 match Pin::new(gate).poll(cx) {
14923 Poll::Pending => return Poll::Pending,
14924 Poll::Ready(_) => self.gate = None,
14925 }
14926 }
14927 Pin::new(&mut self.after).poll_read(cx, buf)
14928 }
14929 }
14930
14931 struct DiscardSink;
14932
14933 impl OutputSink for DiscardSink {
14934 fn write_line(&mut self, _line: &[u8]) {}
14935 }
14936
14937 fn line(text: &str) -> TailEntry {
14938 TailEntry::Line {
14939 text: text.to_string(),
14940 truncated: false,
14941 at_ms: None,
14942 }
14943 }
14944
14945 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
14946 ring.lock().unwrap()
14947 }
14948
14949 fn held_pump(
14953 ring: &Arc<Mutex<StderrRing>>,
14954 before: &str,
14955 after: &str,
14956 ) -> (StderrPump, oneshot::Sender<()>) {
14957 let generation = lock(ring).begin_process();
14958 let (release, gate) = oneshot::channel();
14959 let reader = HeldReader {
14960 before: Some(before.as_bytes().to_vec()),
14961 gate: Some(gate),
14962 after: io::Cursor::new(after.as_bytes().to_vec()),
14963 };
14964 let task = tokio::spawn(pump_stderr_to(
14965 reader,
14966 Arc::clone(ring),
14967 generation,
14968 DiscardSink,
14969 ));
14970 (StderrPump { task, generation }, release)
14971 }
14972
14973 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
14974 for _ in 0..1000 {
14975 if done(&lock(ring)) {
14976 return;
14977 }
14978 tokio::time::sleep(Duration::from_millis(1)).await;
14979 }
14980 panic!(
14981 "ring never reached the expected state: {:?}",
14982 lock(ring).snapshot(None, None)
14983 );
14984 }
14985
14986 #[tokio::test(start_paused = true)]
14987 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
14988 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14989 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
14990
14991 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
14992 let before_release = lock(&ring).snapshot(None, None);
14993 assert!(
14994 matches!(before_release.capture, CaptureState::Incomplete { .. }),
14995 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
14996 );
14997
14998 let next = lock(&ring).begin_process();
15001 lock(&ring).push_line_from(next, "next process booting");
15002 release.send(()).unwrap();
15003 wait_until(&ring, |ring| {
15004 ring.snapshot(None, None).capture == CaptureState::Captured
15005 })
15006 .await;
15007
15008 assert_eq!(
15009 untimed(lock(&ring).snapshot(None, None).entries),
15010 vec![
15011 line("booting"),
15012 line("config error: missing storage"),
15013 TailEntry::ProcessStart,
15014 line("next process booting"),
15015 ],
15016 "the crash's last line must survive a slow reader and stay in the crashed process's section"
15017 );
15018 }
15019
15020 #[tokio::test(start_paused = true)]
15021 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15022 ) {
15023 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15024 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15027
15028 let started = Instant::now();
15029 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15030 assert_eq!(
15031 started.elapsed(),
15032 BOUND,
15033 "the restart must wait exactly the bound for a pipe that stays open, no longer"
15034 );
15035
15036 let next = lock(&ring).begin_process();
15037 lock(&ring).push_line_from(next, "next process booting");
15038 tokio::time::sleep(Duration::from_secs(60)).await;
15039
15040 let snapshot = lock(&ring).snapshot(None, None);
15041 match &snapshot.capture {
15042 CaptureState::Incomplete { reason } => assert!(
15043 reason.contains("had not reached EOF") && reason.contains("250ms"),
15044 "the reason must say what is missing and after how long: {reason}"
15045 ),
15046 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15047 }
15048 assert_eq!(
15049 untimed(snapshot.entries),
15050 vec![
15051 line("parent exiting"),
15052 TailEntry::ProcessStart,
15053 line("next process booting"),
15054 ]
15055 );
15056 }
15057
15058 #[tokio::test(start_paused = true)]
15059 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15060 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15061 let (pump, release) = held_pump(&ring, "one\n", "two\n");
15062 release.send(()).unwrap();
15063
15064 settle_stderr_pump("clean", &ring, pump, BOUND).await;
15065
15066 let snapshot = lock(&ring).snapshot(None, None);
15067 assert_eq!(snapshot.capture, CaptureState::Captured);
15068 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15069 }
15070
15071 #[tokio::test(start_paused = true)]
15072 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15073 const BURST: usize = 150;
15074 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15075 let path = capture.join("two-pipe.stderr.log");
15076 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15077 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15078 let generation = lock(&ring).begin_process();
15079 let stdout = (0..BURST)
15080 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15081 .collect::<String>();
15082 let stderr = (0..BURST)
15083 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15084 .collect::<String>();
15085 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15090 let (release, gate) = oneshot::channel();
15091 let reader = HeldReader {
15092 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15093 gate: Some(gate),
15094 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15095 };
15096 let (finished, completion) = oneshot::channel();
15097 let stdout_sink = sink.clone();
15098 let pump = tokio::spawn(async move {
15099 pump_stdout_to(reader, stdout_sink).await;
15100 finished.send(()).unwrap();
15101 });
15102 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15103
15104 let started = Instant::now();
15105 settle_stdout_pump("two-pipe", pump, BOUND).await;
15106 assert_eq!(
15107 started.elapsed(),
15108 BOUND,
15109 "a late reader must not hold up restart"
15110 );
15111 assert_eq!(
15112 std::fs::read_to_string(&path).unwrap().lines().count(),
15113 BURST + 1
15114 );
15115
15116 release
15117 .send(())
15118 .expect("stdout reader must survive the exit drain bound");
15119 tokio::time::timeout(Duration::from_secs(1), completion)
15120 .await
15121 .expect("released stdout reader must reach EOF")
15122 .expect("stdout capture must finish at EOF, not be cancelled");
15123 let contents = std::fs::read_to_string(&path).unwrap();
15124 let mut lines = contents
15125 .lines()
15126 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15127 .collect::<Vec<_>>();
15128 let mut expected = (0..BURST)
15129 .flat_map(|i| {
15130 [
15131 format!("out-{i:04}-{}", "o".repeat(64)),
15132 format!("err-{i:04}-{}", "e".repeat(64)),
15133 ]
15134 })
15135 .collect::<Vec<_>>();
15136 lines.sort_unstable();
15137 expected.sort_unstable();
15138 assert_eq!(
15139 lines, expected,
15140 "both pipes must retain every complete line exactly once"
15141 );
15142 }
15143}
15144
15145#[cfg(all(test, windows))]
15159mod job_containment_tests {
15160 use super::*;
15161 use cortexkit_test_support::ScratchDir;
15162 use std::{
15163 path::{Path, PathBuf},
15164 sync::{Arc, Mutex},
15165 time::{Duration, Instant},
15166 };
15167
15168 fn stub_path() -> PathBuf {
15174 let mut path = std::env::current_exe().expect("current_exe available in tests");
15175 path.pop();
15176 path.pop();
15177 path.push("fake-aft-stub.exe");
15178 assert!(
15179 path.exists(),
15180 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15181 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15182 path.display()
15183 );
15184 path
15185 }
15186
15187 fn read_grandchild_pid(path: &Path) -> u32 {
15189 let deadline = Instant::now() + Duration::from_secs(10);
15190 loop {
15191 if let Ok(contents) = std::fs::read_to_string(path) {
15192 if let Ok(pid) = contents.trim().parse() {
15193 return pid;
15194 }
15195 }
15196 assert!(
15197 Instant::now() < deadline,
15198 "the stub never recorded a grandchild pid at {}",
15199 path.display()
15200 );
15201 std::thread::sleep(Duration::from_millis(10));
15202 }
15203 }
15204
15205 struct Fixture {
15208 _dir: ScratchDir,
15209 module_id: String,
15210 grandchild: u32,
15211 child: Option<SupervisedChild>,
15212 registry: Arc<Registry>,
15213 snapshot: Arc<Mutex<SupervisorSnapshot>>,
15214 terminal_ring: Arc<Mutex<TerminalRing>>,
15215 spawn_events: SpawnEventFeed,
15216 }
15217
15218 fn fixture(label: &str, module_id: &str) -> Fixture {
15219 let dir = ScratchDir::new(label);
15220 let pid_file = dir.join("grandchild.pid");
15221 let supervisor = Supervisor::new_for_test(
15222 Arc::new(Registry::default()),
15223 RestartPolicy::new(3, Duration::ZERO),
15224 );
15225 let runtime = supervisor.runtime_config();
15226 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15227 let spec = ModuleSpec {
15228 module_id: module_id.to_string(),
15229 program: stub_path(),
15230 args: Vec::new(),
15234 env: vec![
15235 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
15236 (
15237 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
15238 pid_file.display().to_string(),
15239 ),
15240 ],
15241 reserved: false,
15242 reserved_prefixes: Vec::new(),
15243 protocol: ModuleProtocol::Subc,
15244 overlap: Default::default(),
15245 };
15246 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
15247 .expect("spawn the supervised fixture");
15248 let grandchild = read_grandchild_pid(&pid_file);
15249 Fixture {
15250 _dir: dir,
15251 module_id: module_id.to_string(),
15252 grandchild,
15253 child: Some(child),
15254 registry: Arc::new(Registry::default()),
15255 snapshot,
15256 terminal_ring: Arc::clone(&runtime.terminal_ring),
15257 spawn_events: SpawnEventFeed::default(),
15258 }
15259 }
15260
15261 impl Fixture {
15262 async fn drain(&mut self) {
15264 let child = self
15265 .child
15266 .take()
15267 .expect("the fixture child is still present");
15268 drain_child_to_state(
15269 &self.module_id,
15270 ModuleProtocol::Subc,
15271 StopNotice::NotSent,
15274 &self.registry,
15275 None,
15276 &self.snapshot,
15277 &self.terminal_ring,
15278 &self.spawn_events,
15279 child,
15280 Duration::from_millis(500),
15281 ModuleState::Stopped,
15282 Some(false),
15283 )
15284 .await
15285 .expect("drain the supervised fixture");
15286 }
15287 }
15288
15289 #[tokio::test]
15295 async fn teardown_reaps_the_grandchild() {
15296 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
15297 let grandchild = fixture.grandchild;
15298
15299 assert!(
15300 subc_jobobject::process_exists(grandchild),
15301 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
15302 );
15303
15304 fixture.drain().await;
15305
15306 assert!(
15307 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15308 "grandchild {grandchild} outlived module teardown: the tree was not contained"
15309 );
15310 }
15311
15312 #[test]
15325 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
15326 let dir = ScratchDir::new("teardown-uncontained");
15327 let pid_file = dir.join("grandchild.pid");
15328 let mut child = std::process::Command::new(stub_path())
15329 .env("FAKE_AFT_NEVER_CONNECT", "1")
15330 .env(
15331 "FAKE_AFT_GRANDCHILD_PID_FILE",
15332 pid_file.display().to_string(),
15333 )
15334 .stdin(std::process::Stdio::null())
15335 .stdout(std::process::Stdio::null())
15336 .stderr(std::process::Stdio::null())
15337 .spawn()
15338 .expect("spawn the uncontained fixture");
15339 let grandchild = read_grandchild_pid(&pid_file);
15340
15341 child.kill().expect("kill the direct child");
15343 let _ = child.wait();
15344
15345 assert!(
15346 subc_jobobject::process_exists(grandchild),
15347 "grandchild {grandchild} died with the direct child, so this control no longer \
15348 distinguishes contained from uncontained teardown and the regression test is \
15349 passing vacuously"
15350 );
15351
15352 kill_tree(grandchild);
15355 }
15356
15357 #[tokio::test]
15366 async fn dropping_containment_reaps_the_grandchild() {
15367 let mut fixture = fixture("drop-containment", "tree-drop");
15368 let grandchild = fixture.grandchild;
15369
15370 assert!(subc_jobobject::process_exists(grandchild));
15371
15372 fixture.child.as_mut().expect("child present").job = None;
15374
15375 assert!(
15376 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15377 "grandchild {grandchild} survived the containment handle closing, so a daemon \
15378 crash would leave the tree behind"
15379 );
15380 }
15381
15382 fn kill_tree(pid: u32) {
15384 let _ = std::process::Command::new("taskkill.exe")
15385 .args(["/PID", &pid.to_string(), "/T", "/F"])
15386 .stdin(std::process::Stdio::null())
15387 .stdout(std::process::Stdio::null())
15388 .stderr(std::process::Stdio::null())
15389 .status();
15390 assert!(
15391 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
15392 "could not clean up grandchild {pid}"
15393 );
15394 }
15395}
15396
15397#[cfg(test)]
15398mod privacy_trampoline_configuration_tests {
15399 #[tokio::test]
15400 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15401 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
15402 #[cfg(target_os = "macos")]
15403 {
15404 let supervisor = super::Supervisor::new(
15405 std::sync::Arc::new(crate::Registry::default()),
15406 super::RestartPolicy::default(),
15407 );
15408 let error = supervisor.spawn(spec()).unwrap_err();
15409 assert!(
15410 error
15411 .to_string()
15412 .contains("no privacy trampoline configured"),
15413 "{error}"
15414 );
15415 }
15416 }
15417
15418 #[tokio::test]
15419 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15420 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
15421 #[cfg(target_os = "macos")]
15422 {
15423 let supervisor = super::Supervisor::new(
15424 std::sync::Arc::new(crate::Registry::default()),
15425 super::RestartPolicy::default(),
15426 )
15427 .with_privacy_trampoline(std::env::current_exe().unwrap());
15428 let error = supervisor.spawn(spec()).unwrap_err();
15429 assert!(
15430 error
15431 .to_string()
15432 .contains("binary does not implement the privacy trampoline protocol"),
15433 "{error}"
15434 );
15435 }
15436 }
15437
15438 #[cfg(target_os = "macos")]
15439 fn spec() -> super::ModuleSpec {
15440 super::ModuleSpec {
15441 module_id: "privacy-configuration".into(),
15442 program: "/bin/sleep".into(),
15443 args: vec!["30".into()],
15444 env: vec![],
15445 reserved: false,
15446 reserved_prefixes: vec![],
15447 protocol: subc_control::ModuleProtocol::None,
15448 overlap: super::ModuleOverlap::Exclusive,
15449 }
15450 }
15451}
15452
15453#[cfg(test)]
15454mod privacy_exec_boundary_tests {
15455 #[cfg(target_os = "macos")]
15456 use super::*;
15457 #[cfg(target_os = "macos")]
15458 use std::{
15459 io::{Read, Write},
15460 net::{TcpListener, TcpStream},
15461 };
15462
15463 #[cfg(target_os = "macos")]
15466 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
15467 if let Some((_, path)) = spec
15468 .env
15469 .iter()
15470 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
15471 {
15472 let mut barrier = TcpStream::connect(path).unwrap();
15473 barrier
15474 .set_read_timeout(Some(Duration::from_secs(30)))
15475 .unwrap();
15476 barrier.write_all(&pid.to_ne_bytes()).unwrap();
15477 let mut release = [0];
15478 barrier.read_exact(&mut release).unwrap();
15479 assert_eq!(&release, b"X");
15480 }
15481 }
15482
15483 #[cfg(target_os = "macos")]
15484 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
15485 ModuleSpec {
15486 module_id: "privacy-boundary".into(),
15487 program: program.into(),
15488 args: args.iter().map(|arg| (*arg).into()).collect(),
15489 env: vec![],
15490 reserved: false,
15491 reserved_prefixes: vec![],
15492 protocol: ModuleProtocol::None,
15493 overlap: ModuleOverlap::Exclusive,
15494 }
15495 }
15496
15497 #[cfg(target_os = "macos")]
15498 async fn accept(listener: TcpListener) -> TcpStream {
15499 let listener = tokio::net::TcpListener::from_std({
15501 listener.set_nonblocking(true).unwrap();
15502 listener
15503 })
15504 .unwrap();
15505 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
15506 .await
15507 .unwrap()
15508 .unwrap();
15509 let stream = stream.into_std().unwrap();
15510 stream.set_nonblocking(false).unwrap();
15511 stream
15512 .set_read_timeout(Some(Duration::from_secs(30)))
15513 .unwrap();
15514 stream
15515 }
15516
15517 #[tokio::test]
15518 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15519 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
15520 #[cfg(target_os = "macos")]
15521 {
15522 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
15523 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15526 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15527 let record = root.join("live-children.json");
15528 let supervisor =
15529 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15530 .with_live_children_record(&record);
15531 let runtime = supervisor.runtime_config();
15532 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15533 let mut spec = spec("/bin/sleep", &["30"]);
15534 spec.env = vec![
15535 (
15536 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
15537 exec_listener.local_addr().unwrap().to_string(),
15538 ),
15539 (
15540 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
15541 sample_listener.local_addr().unwrap().to_string(),
15542 ),
15543 ];
15544 let spawn = tokio::task::spawn_blocking(move || {
15545 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
15546 });
15547 let mut sample = accept(sample_listener).await;
15548 let mut pid = [0; 4];
15549 sample.read_exact(&mut pid).unwrap();
15550 let pid = u32::from_ne_bytes(pid);
15551 let mut exec = accept(exec_listener).await;
15552 let mut ready = [0];
15553 exec.read_exact(&mut ready).unwrap();
15554 assert_eq!(&ready, b"R");
15555 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
15558 assert_eq!(
15559 observe_spawned_image(pid).unwrap().executable,
15560 Some(trampoline)
15561 );
15562 sample.write_all(b"X").unwrap();
15563 let mut child = spawn.await.unwrap();
15564 let early = crate::live_children::read_record(&record).unwrap();
15565 assert_eq!(early.len(), 1);
15566 assert_eq!(early[0].pid, pid);
15567 assert_eq!(
15568 early[0].executable, None,
15569 "unconfirmed trampoline image entered the roster"
15570 );
15571 assert!(child.report_ready.get().is_none());
15572 child.privacy_exec.as_mut().unwrap().deadline =
15575 tokio::time::Instant::now() + Duration::from_secs(30);
15576 exec.write_all(b"X").unwrap();
15577 child.confirm_privacy_exec().await;
15578 assert_eq!(child.spawn_failure, None);
15579 assert!(child.report_ready.get().is_some());
15580 let confirmed = crate::live_children::read_record(&record).unwrap();
15581 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
15582 assert_ne!(module, trampoline);
15583 assert_eq!(confirmed[0].executable, Some(module.into()));
15584 child.start_kill().unwrap();
15585 child.wait().await.unwrap();
15586 child.release_roster();
15587 }
15588 }
15589
15590 #[tokio::test]
15591 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15592 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
15593 #[cfg(target_os = "macos")]
15594 {
15595 let registry = Arc::new(Registry::default());
15596 let policy = RestartPolicy::new(0, Duration::ZERO);
15597 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
15598 let runtime = supervisor.runtime_config();
15599 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15600 let spec = spec("/bin/sh", &["-c", "exit 121"]);
15601 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
15605 let pid = child.pid;
15606 tokio::task::spawn_blocking(move || {
15607 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
15608 })
15609 .await
15610 .unwrap()
15611 .unwrap();
15612 child.privacy_exec.as_mut().unwrap().deadline =
15613 tokio::time::Instant::now() + Duration::from_secs(30);
15614 let status = child.wait().await.unwrap();
15615 assert_eq!(status.code(), Some(121));
15616 assert!(child.privacy_exec.is_none());
15617 assert!(
15618 child.report_ready.get().is_none(),
15619 "an exited module must not publish a live pid"
15620 );
15621 let report = classify_reaped_child_exit(&snapshot, &child, &status);
15622 on_child_exit(
15623 &spec,
15624 policy,
15625 ®istry,
15626 &snapshot,
15627 &runtime.terminal_ring,
15628 &runtime.spawn_events,
15629 &runtime.child_roster,
15630 report,
15631 )
15632 .await;
15633 let state = lock_snapshot(&snapshot).unwrap();
15634 assert_eq!(state.state, ModuleState::Failed);
15635 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
15636 assert_eq!(state.reported_pid(), None);
15637 drop(state);
15638 let history = runtime.terminal_ring.lock().unwrap().snapshot();
15639 assert_eq!(history.entries.len(), 1);
15640 let terminal = &history.entries[0];
15641 assert_eq!(terminal.exit_code, Some(121));
15642 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
15643 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
15644 assert_eq!(
15645 terminal.disposition_detail.as_deref(),
15646 Some(policy.budget_exhausted_detail().as_str()),
15647 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
15648 );
15649 assert_eq!(child.spawn_failure, None);
15650 child.release_roster();
15651 }
15652 }
15653}
15654
15655#[cfg(all(test, unix))]
15659mod launch_nonce_descriptor_tests {
15660 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
15661 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15662 use cortexkit_test_support::ScratchDir;
15663 use std::{
15664 path::PathBuf,
15665 sync::{Arc, Mutex},
15666 time::{Duration, Instant},
15667 };
15668
15669 async fn probe(role: super::SpawnRole) {
15670 let scratch = ScratchDir::new("launch-nonce-descriptor");
15671 let fd_copy = scratch.join("from-descriptor");
15672 let env_copy = scratch.join("environment");
15673 let script = format!(
15674 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
15675 fd = fd_copy.display(), env = env_copy.display(),
15676 );
15677 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
15678 let spec = ModuleSpec {
15679 module_id: "nonce-descriptor-probe".to_string(),
15680 program: PathBuf::from("/bin/sh"),
15681 args: vec!["-c".to_string(), script],
15682 env: vec![
15683 xdg("XDG_DATA_HOME"),
15684 xdg("XDG_RUNTIME_DIR"),
15685 xdg("XDG_CONFIG_HOME"),
15686 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
15687 ],
15688 reserved: true,
15689 reserved_prefixes: Vec::new(),
15690 protocol: ModuleProtocol::Subc,
15691 overlap: Default::default(),
15692 };
15693 let handle = SupervisorHandle::new();
15694 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15695 let roster = ChildRoster::default();
15696 #[cfg(target_os = "macos")]
15697 {
15698 let path = super::test_privacy_trampoline();
15699 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
15700 }
15701 let child = super::spawn_child_in_slot(
15702 &spec,
15703 None,
15704 Some(&handle),
15705 &ring,
15706 None,
15707 &roster,
15708 #[cfg(target_os = "linux")]
15709 None,
15710 role,
15711 matches!(role, super::SpawnRole::SwapCandidate),
15712 )
15713 .expect("spawn probe");
15714 let deadline = Instant::now() + Duration::from_secs(10);
15715 while !(fd_copy.exists() && env_copy.exists()) {
15716 assert!(Instant::now() < deadline, "probe never wrote its copies");
15717 tokio::time::sleep(Duration::from_millis(20)).await;
15718 }
15719 let nonce = std::fs::read_to_string(fd_copy).unwrap();
15720 assert!(!nonce.is_empty());
15721 let environment = std::fs::read_to_string(env_copy).unwrap();
15722 assert!(environment
15723 .lines()
15724 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
15725 let copy = environment
15726 .lines()
15727 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
15728 assert_eq!(
15729 copy, None,
15730 "Unix children must never receive the environment nonce"
15731 );
15732 if matches!(role, super::SpawnRole::Plain) {
15733 assert_eq!(
15734 handle.spawn_nonce(&spec.module_id).as_deref(),
15735 Some(nonce.as_str())
15736 );
15737 }
15738 drop(child);
15739 }
15740
15741 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15742 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
15743 probe(super::SpawnRole::Plain).await;
15744 }
15745
15746 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15747 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
15748 probe(super::SpawnRole::SwapCandidate).await;
15749 }
15750}
15751
15752#[cfg(all(test, target_os = "linux"))]
15753mod cgroup_containment_tests {
15754 use super::*;
15755 use cortexkit_test_support::ScratchDir;
15756
15757 fn running(pid: u32) -> bool {
15758 std::fs::read_to_string(format!("/proc/{pid}/stat"))
15760 .ok()
15761 .and_then(|stat| {
15762 stat.rsplit_once(") ")
15763 .map(|(_, rest)| rest.starts_with('Z'))
15764 })
15765 .is_some_and(|zombie| !zombie)
15766 }
15767
15768 #[tokio::test]
15769 async fn linux_teardown_reaps_the_grandchild() {
15770 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
15771 }
15772
15773 #[tokio::test]
15774 async fn linux_shutdown_straggler_reaps_the_grandchild() {
15775 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
15776 }
15777
15778 async fn teardown_tree(test_name: &str, shutdown: bool) {
15779 let dir = ScratchDir::new(test_name);
15780 let root = PathBuf::from(format!(
15781 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
15782 std::process::id(),
15783 unix_ms_now()
15784 ));
15785 if let Err(error) = std::fs::create_dir(&root) {
15786 assert!(
15787 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15788 "required cgroup test cannot execute: {error}"
15789 );
15790 eprintln!(
15791 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
15792 root.display()
15793 );
15794 return;
15795 }
15796 let placement = subc_cgroup::prepare_at(&root)
15797 .expect("prepare isolated kernel cgroup")
15798 .expect("isolated cgroup is delegated");
15799 let module_id = "tree-teardown";
15800 let module = placement
15801 .module_path(module_id)
15802 .expect("create isolated module cgroup");
15803 if !module.join("cgroup.kill").exists() {
15804 std::fs::remove_dir(&module).unwrap();
15805 std::fs::remove_dir(root.join("subc-modules")).unwrap();
15806 std::fs::remove_dir(&root).unwrap();
15807 assert!(
15808 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15809 "required cgroup.kill interface unavailable"
15810 );
15811 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
15812 return;
15813 }
15814 let supervisor = Supervisor::new_for_test(
15815 Arc::new(Registry::default()),
15816 RestartPolicy::new(3, Duration::ZERO),
15817 )
15818 .with_cgroup_placement(Some(placement));
15819 let mut runtime = supervisor.runtime_config();
15820 runtime.child_roster = runtime
15821 .child_roster
15822 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
15823 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15824 let pid_file = dir.join("grandchild.pid");
15825 let spec = ModuleSpec {
15826 module_id: module_id.to_string(),
15827 program: PathBuf::from("/bin/sh"),
15828 args: vec![
15829 "-c".into(),
15830 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
15831 "fixture".into(),
15832 pid_file.display().to_string(),
15833 ],
15834 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15835 .into_iter()
15836 .map(|key| (key.to_string(), dir.display().to_string()))
15837 .collect(),
15838 reserved: false,
15839 reserved_prefixes: Vec::new(),
15840 protocol: ModuleProtocol::None,
15841 overlap: Default::default(),
15842 };
15843 let child =
15844 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
15845 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
15846 let grandchild: u32 = loop {
15847 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
15848 if let Ok(pid) = contents.trim().parse() {
15849 break pid;
15850 }
15851 }
15852 assert!(
15853 tokio::time::Instant::now() < deadline,
15854 "grandchild pid was not recorded"
15855 );
15856 tokio::time::sleep(Duration::from_millis(10)).await;
15857 };
15858 assert!(
15859 running(grandchild),
15860 "grandchild must be alive before teardown"
15861 );
15862 if shutdown {
15863 let mut child = child;
15864 crate::child_roster::end_children_for_daemon_shutdown(
15865 &runtime.child_roster,
15866 false,
15867 std::future::pending(),
15868 )
15869 .await;
15870 child.wait().await.expect("reap shutdown straggler");
15871 } else {
15872 drain_child_to_state(
15873 module_id,
15874 ModuleProtocol::None,
15875 StopNotice::NotSent,
15876 &Registry::default(),
15877 None,
15878 &snapshot,
15879 &runtime.terminal_ring,
15880 &SpawnEventFeed::default(),
15881 child,
15882 Duration::from_millis(100),
15883 ModuleState::Stopped,
15884 Some(false),
15885 )
15886 .await
15887 .expect("real supervisor teardown");
15888 }
15889 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
15890 while running(grandchild) && tokio::time::Instant::now() < deadline {
15891 tokio::time::sleep(Duration::from_millis(10)).await;
15892 }
15893 let survived = running(grandchild);
15894 if survived {
15896 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
15897 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
15898 tokio::time::sleep(Duration::from_millis(100)).await;
15899 }
15900 if module.exists() {
15901 std::fs::remove_dir(&module).expect("remove empty module cgroup");
15902 }
15903 std::fs::remove_dir(root.join("subc-modules")).unwrap();
15904 std::fs::remove_dir(&root).unwrap();
15905 assert!(
15906 !survived,
15907 "grandchild {grandchild} outlived module teardown"
15908 );
15909 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
15910 }
15911}