1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136 deadline: tokio::time::Instant,
137 expected: Option<subc_os::FileIdentity>,
138 trampoline: Option<subc_os::FileIdentity>,
139 script: bool,
140 module_id: String,
141 #[cfg(test)]
142 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147 let path = std::env::current_exe()
150 .unwrap()
151 .parent()
152 .unwrap()
153 .parent()
154 .unwrap()
155 .join("privacy-trampoline-fixture");
156 assert!(
160 path.exists(),
161 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162 --bins --features test-support` or `cargo test -p subc-daemon` first",
163 path.display()
164 );
165 path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170 use std::io::Read;
171 let mut probe = std::process::Command::new(path)
172 .args(["__disclaim-exec", "--probe"])
173 .stdin(Stdio::null())
174 .stdout(Stdio::piped())
175 .stderr(Stdio::piped())
176 .spawn()
177 .map_err(|error| {
178 format!(
179 "privacy trampoline probe failed for {}: {error}",
180 path.display()
181 )
182 })?;
183 let deadline = std::time::Instant::now() + Duration::from_secs(5);
184 let status = loop {
185 match probe.try_wait() {
186 Ok(Some(status)) => break status,
187 Ok(None) if std::time::Instant::now() < deadline => {
188 std::thread::sleep(Duration::from_millis(5))
189 }
190 result => {
191 let _ = probe.kill();
192 let _ = probe.wait();
193 return Err(format!(
194 "privacy trampoline probe failed or timed out for {}: {result:?}",
195 path.display()
196 ));
197 }
198 }
199 };
200 let mut answer = String::new();
201 if let Some(stdout) = probe.stdout.take() {
202 let _ = stdout.take(256).read_to_string(&mut answer);
203 }
204 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205 return Ok(());
206 }
207 let mut diagnostic = String::new();
208 if let Some(stderr) = probe.stderr.take() {
209 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210 }
211 let cause = diagnostic
212 .trim()
213 .strip_prefix("ck-subc: own privacy identity refused: ")
214 .unwrap_or("binary does not implement the privacy trampoline protocol");
215 Err(format!(
216 "{cause}: probe of {} exited {status}",
217 path.display()
218 ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223 spec: &ModuleSpec,
224 roster: &ChildRoster,
225) -> Result<
226 (
227 Command,
228 Option<PrivacyExec>,
229 subc_os::privacy_identity::ExecAcknowledgement,
230 ),
231 SuperviseError,
232> {
233 let failure = |cause: String| {
234 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235 SuperviseError::Spawn {
236 program: spec.program.clone(),
237 source: io::Error::other(cause),
238 cgroup_path: None,
239 }
240 };
241 let trampoline = roster.privacy_trampoline().map_err(failure)?;
242 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247 let path = spec
248 .env
249 .iter()
250 .find(|(key, _)| key == "PATH")
251 .map(|(_, value)| std::ffi::OsString::from(value))
252 .or_else(|| std::env::var_os("PATH"))
253 .unwrap_or_else(|| "/usr/bin:/bin".into());
254 std::env::split_paths(&path)
255 .map(|dir| dir.join(&spec.program))
256 .find(|path| path.is_file())
257 .unwrap_or_else(|| spec.program.clone())
258 } else {
259 spec.program.clone()
260 };
261 let expected = subc_os::file_identity(&program);
262 let trampoline_image = subc_os::file_identity(&trampoline);
263 let script = {
264 use std::io::Read;
265 let mut prefix = [0u8; 2];
266 std::fs::File::open(&program)
267 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268 };
269 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270 return Err(failure(
271 "privacy identity module executable is missing or is the trampoline itself".to_string(),
272 ));
273 }
274 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275 .map_err(|error| failure(error.to_string()))?;
276 let reader =
277 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278 let mut command = Command::new(&trampoline);
279 command
280 .arg("__disclaim-exec")
281 .arg(ack.fd().to_string())
282 .arg(&program);
283 ack.install(command.as_std_mut());
284 Ok((
285 command,
286 Some(PrivacyExec {
287 reader,
288 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289 expected,
290 trampoline: trampoline_image,
291 script,
292 module_id: spec.module_id.clone(),
293 #[cfg(test)]
294 confirmation_pause: spec
295 .env
296 .iter()
297 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298 .map(|(_, address)| (address.clone(), None)),
299 }),
300 ack,
301 ))
302}
303
304struct SupervisedChild {
305 child: Child,
306 exec: crate::activity::ExecOwner,
307 #[cfg(windows)]
308 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309 #[cfg(target_os = "macos")]
310 privacy_exec: Option<PrivacyExec>,
311 #[cfg(target_os = "macos")]
318 report_ready: Arc<OnceLock<()>>,
319 spawn_failure: Option<String>,
321 protocol: ModuleProtocol,
326 #[cfg(target_os = "linux")]
332 module_id: String,
333 #[cfg(target_os = "linux")]
334 cgroup_placement: Option<subc_cgroup::Placement>,
335 #[cfg(windows)]
357 job: Option<subc_jobobject::JobObject>,
358 stdout_pump: Option<JoinHandle<()>>,
359 stderr_pump: Option<StderrPump>,
360 stderr_ring: Arc<Mutex<StderrRing>>,
361 spawned_at: Instant,
364 spawned_at_ms: u64,
365 spawned_from: PathBuf,
366 launch_spec_digest: LaunchSpecDigest,
367 spawned_file_identity: Option<SpawnedFileIdentity>,
368 process_start_time: Option<u64>,
369 process_identity: Option<ProcessIdentity>,
370 pid: u32,
371 roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377 fn id(&self) -> Option<u32> {
378 Some(self.pid)
379 }
380
381 fn process_identity(&self) -> Option<ProcessIdentity> {
382 self.process_identity
383 }
384
385 async fn wait(&mut self) -> io::Result<ExitStatus> {
386 #[cfg(target_os = "macos")]
387 self.confirm_privacy_exec().await;
388 let result = self.child.wait().await;
396 #[cfg(target_os = "linux")]
397 if result.is_ok() {
398 if let Some(placement) = self.cgroup_placement.as_ref() {
399 cleanup_reaped_cgroup(placement, &self.module_id).await;
400 self.cgroup_placement = None;
403 }
404 }
405 result
406 }
407
408 #[cfg(target_os = "macos")]
409 async fn confirm_privacy_exec(&mut self) {
410 let Some(pending) = &mut self.privacy_exec else {
411 return;
412 };
413 let result = tokio::time::timeout_at(pending.deadline, async {
414 let mut record = Vec::new();
415 loop {
416 let mut ready = pending.reader.readable().await?;
417 let read = ready.try_io(|reader| {
418 use std::io::Read;
419 let mut reader = reader.get_ref();
420 let mut buffer = [0u8; 256];
421 reader.read(&mut buffer).map(|count| (count, buffer))
422 });
423 match read {
424 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425 Ok(Ok((count, buffer))) => {
426 if record.len() + count > 1024 {
427 return Err(io::Error::other(
428 "privacy exec refusal record is too long",
429 ));
430 }
431 record.extend_from_slice(&buffer[..count]);
432 }
433 Ok(Err(error)) => return Err(error),
434 Err(_) => continue,
435 }
436 }
437 })
438 .await;
439 #[cfg(test)]
440 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441 use tokio::io::{AsyncReadExt, AsyncWriteExt};
442 if stream.is_none() {
443 *stream = Some(
444 tokio::net::TcpStream::connect(address.as_str())
445 .await
446 .unwrap(),
447 );
448 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449 stream
450 .as_mut()
451 .unwrap()
452 .write_all(&self.pid.to_ne_bytes())
453 .await
454 .unwrap();
455 }
456 let mut reply = [0];
457 stream
458 .as_mut()
459 .unwrap()
460 .read_exact(&mut reply)
461 .await
462 .unwrap();
463 pending.confirmation_pause = None;
464 if &reply == b"F" {
465 Ok(Err(io::Error::other("injected late exec refusal")))
466 } else {
467 assert_eq!(&reply, b"X");
468 result
469 }
470 } else {
471 result
472 };
473 let pending = self.privacy_exec.as_ref().expect("pending exec");
476 let cause = match result {
477 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478 Ok(Err(error)) => Some(format!(
479 "privacy identity exec acknowledgement failed: {error}"
480 )),
481 Ok(Ok(record)) if !record.is_empty() => Some(
482 std::str::from_utf8(&record)
483 .ok()
484 .and_then(|record| {
485 record
486 .trim()
487 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488 })
489 .filter(|cause| !cause.is_empty())
490 .unwrap_or("invalid privacy exec refusal record")
491 .to_string(),
492 ),
493 Ok(Ok(_)) => match self.child.try_wait() {
494 Ok(Some(_status)) => None,
498 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499 Ok(None) => {
500 let image = observe_spawned_image(self.pid);
501 if let Some(image) = image.filter(|image| {
502 image.executable.is_some()
503 && image.executable != pending.trampoline
504 && (image.executable == pending.expected || pending.script)
505 }) {
506 if let Err(error) = self
507 .roster_guard
508 .as_ref()
509 .map(|guard| guard.confirm_executable(image))
510 .transpose()
511 {
512 Some(format!(
513 "could not durably record confirmed module image: {error}"
514 ))
515 } else {
516 let _ = self.report_ready.set(());
517 info!(module_id = %pending.module_id, pid = self.pid,
518 "module spawned with own privacy identity (responsibility disclaimed)");
519 None
520 }
521 } else if image.is_none()
522 || image.is_some_and(|image| image.executable.is_none())
523 {
524 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531 Ok(Ok(_status)) => None,
532 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534 }
535 } else {
536 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537 }
538 }
539 },
540 };
541 let pending = self.privacy_exec.take().expect("pending exec");
542 self.exec.0.finish(if cause.is_some() {
545 crate::activity::ExecState::Refused
546 } else {
547 crate::activity::ExecState::CommittedExec
548 });
549 if let Some(cause) = cause {
550 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551 self.spawn_failure = Some(cause);
552 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556 }
557 let _ = self.child.start_kill();
558 }
559 }
560
561 fn release_roster(&mut self) {
565 self.roster_guard = None;
566 }
567
568 fn start_kill(&mut self) -> io::Result<()> {
581 #[cfg(windows)]
582 if let Some(job) = &self.job {
583 if let Err(error) = job.terminate() {
584 debug!(
585 error = %error,
586 "job termination failed; the direct-child kill still owns the outcome"
587 );
588 }
589 }
590 #[cfg(target_os = "linux")]
591 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592 self.child.start_kill()
593 }
594
595 async fn drain_stderr(&mut self, module_id: &str) {
596 if let Some(pump) = self.stdout_pump.take() {
597 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598 }
599
600 let Some(pump) = self.stderr_pump.take() else {
601 return;
602 };
603 settle_stderr_pump(
604 module_id,
605 &self.stderr_ring,
606 pump,
607 STDERR_PUMP_DRAIN_TIMEOUT,
608 )
609 .await;
610 }
611}
612
613async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617 match timeout(bound, &mut pump).await {
618 Ok(Ok(())) => {}
619 Ok(Err(error)) => {
620 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621 }
622 Err(_) => {
623 drop(pump);
624 warn!(
625 module_id,
626 waited = ?bound,
627 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628 );
629 }
630 }
631}
632
633struct StderrPump {
636 task: JoinHandle<()>,
637 generation: u64,
638}
639
640async fn settle_stderr_pump(
646 module_id: &str,
647 ring: &Arc<Mutex<StderrRing>>,
648 pump: StderrPump,
649 bound: Duration,
650) {
651 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652 let StderrPump {
653 mut task,
654 generation,
655 } = pump;
656 lock().retire_pump(generation);
657 match timeout(bound, &mut task).await {
658 Ok(Ok(())) => {}
659 Ok(Err(err)) => {
660 let mut ring = lock();
661 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662 ring.finish_pump(generation);
663 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664 }
665 Err(_) => {
666 drop(task);
668 lock().mark_pump_late(
669 generation,
670 format!(
671 "stderr of the exited process had not reached EOF {bound:?} after it was \
672 retired (a descendant may still hold the pipe open); lines it still \
673 writes are kept in that process's section"
674 ),
675 );
676 warn!(
677 module_id,
678 waited = ?bound,
679 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680 );
681 }
682 }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687 EVENTS.get_or_init(|| {
688 let (sender, _receiver) = watch::channel(0);
689 sender
690 })
691}
692
693pub(crate) fn notify_registration_release() {
694 let events = registration_release_events();
695 let next_generation = (*events.borrow()).wrapping_add(1);
696 events.send_replace(next_generation);
697}
698
699#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702 pub module_id: String,
703 pub program: PathBuf,
704 pub args: Vec<String>,
705 pub env: Vec<(String, String)>,
706 pub reserved: bool,
711 pub reserved_prefixes: Vec<String>,
716 pub protocol: ModuleProtocol,
735 pub overlap: ModuleOverlap,
740}
741
742#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749 f.write_str("LaunchSpecDigest([redacted])")
750 }
751}
752
753impl LaunchSpecDigest {
754 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755 fn field(hash: &mut Sha256, bytes: &[u8]) {
756 hash.update((bytes.len() as u64).to_le_bytes());
757 hash.update(bytes);
758 }
759 let mut hash = Sha256::new();
760 if let Some(program) = spec.program.to_str() {
761 field(&mut hash, b"utf8-program");
762 field(&mut hash, program.as_bytes());
763 } else {
764 #[cfg(unix)]
767 {
768 use std::os::unix::ffi::OsStrExt;
769 field(&mut hash, b"unix-program");
770 field(&mut hash, spec.program.as_os_str().as_bytes());
771 }
772 #[cfg(windows)]
773 {
774 use std::os::windows::ffi::OsStrExt;
775 field(&mut hash, b"windows-program");
776 let bytes = spec
777 .program
778 .as_os_str()
779 .encode_wide()
780 .flat_map(u16::to_le_bytes)
781 .collect::<Vec<_>>();
782 field(&mut hash, &bytes);
783 }
784 #[cfg(not(any(unix, windows)))]
785 field(&mut hash, spec.program.to_string_lossy().as_bytes());
786 }
787 hash.update((spec.args.len() as u64).to_le_bytes());
788 for arg in &spec.args {
789 field(&mut hash, arg.as_bytes());
790 }
791 let mut env = spec.env.iter().collect::<Vec<_>>();
792 env.sort_unstable();
793 hash.update((env.len() as u64).to_le_bytes());
794 for (key, value) in env {
795 field(&mut hash, key.as_bytes());
796 field(&mut hash, value.as_bytes());
797 }
798 field(
799 &mut hash,
800 match spec.protocol {
801 ModuleProtocol::Subc => b"subc",
802 ModuleProtocol::None => b"none",
803 },
804 );
805 Self(hash.finalize().into())
806 }
807}
808
809#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817 #[default]
819 Exclusive,
820 Safe,
832}
833
834impl ModuleOverlap {
835 pub fn as_str(self) -> &'static str {
836 match self {
837 Self::Exclusive => "exclusive",
838 Self::Safe => "safe",
839 }
840 }
841}
842
843pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880 pub max_restarts: u32,
881 pub backoff: Duration,
884 pub max_backoff: Duration,
886 pub window: Duration,
890}
891
892impl RestartPolicy {
893 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897 Self {
898 max_restarts,
899 backoff,
900 max_backoff: DEFAULT_MAX_BACKOFF,
901 window: DEFAULT_RESTART_WINDOW,
902 }
903 }
904
905 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906 self.max_backoff = max_backoff;
907 self
908 }
909
910 pub fn with_window(mut self, window: Duration) -> Self {
911 self.window = window;
912 self
913 }
914
915 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920 if self.backoff.is_zero() || self.max_backoff.is_zero() {
921 return Duration::ZERO;
922 }
923
924 let mut delay = self.backoff;
925 for _ in 0..restart_in_window {
926 if delay >= self.max_backoff {
927 return self.max_backoff;
928 }
929 delay = delay
930 .checked_mul(10)
931 .unwrap_or(self.max_backoff)
932 .min(self.max_backoff);
933 }
934 delay.min(self.max_backoff)
935 }
936
937 fn budget_exhausted_detail(&self) -> String {
942 format!(
943 "crash budget exhausted: max_restarts={} within window_secs={}",
944 self.max_restarts,
945 self.window.as_secs()
946 )
947 }
948}
949
950impl Default for RestartPolicy {
951 fn default() -> Self {
952 Self {
953 max_restarts: DEFAULT_MAX_RESTARTS,
954 backoff: DEFAULT_BACKOFF,
955 max_backoff: DEFAULT_MAX_BACKOFF,
956 window: DEFAULT_RESTART_WINDOW,
957 }
958 }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963 restart_in_window: u32,
964 delay: Duration,
965}
966
967fn daemon_will_restart(
974 state: &mut SupervisorSnapshot,
975 policy: &RestartPolicy,
976 now: Instant,
977) -> bool {
978 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988 Report,
989 Restart,
990 Alert,
991}
992
993impl fmt::Display for HealthAction {
994 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995 f.write_str(match self {
996 Self::Report => "report",
997 Self::Restart => "restart",
998 Self::Alert => "alert",
999 })
1000 }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005 pub http: Option<String>,
1009 pub cadence: Duration,
1010 pub deadline: Duration,
1011 pub failure_threshold: u32,
1012 pub on_degraded: HealthAction,
1013 pub on_failing: HealthAction,
1014 pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018 fn default() -> Self {
1019 Self {
1020 http: None,
1021 cadence: DEFAULT_HEALTH_CADENCE,
1022 deadline: DEFAULT_HEALTH_DEADLINE,
1023 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024 on_degraded: HealthAction::Report,
1025 on_failing: HealthAction::Report,
1026 critical: false,
1027 }
1028 }
1029}
1030
1031#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050 pub status: SupervisorHealthStatus,
1051 pub last_probe_ms: Option<u64>,
1052 pub detail: Option<String>,
1053 pub metrics: Option<Value>,
1054 pub consecutive_failures: u32,
1055 pub late_answer_count: u64,
1058 pub last_late_answer_latency_ms: Option<u64>,
1060 pub last_action: Option<String>,
1061 pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068 fn default() -> Self {
1069 Self {
1070 status: SupervisorHealthStatus::Unknown,
1071 last_probe_ms: None,
1072 detail: None,
1073 metrics: None,
1074 consecutive_failures: 0,
1075 late_answer_count: 0,
1076 last_late_answer_latency_ms: None,
1077 last_action: None,
1078 last_action_ms: None,
1079 }
1080 }
1081}
1082
1083#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086 Starting,
1087 Running,
1088 Unresponsive,
1089 Restarting,
1090 Draining,
1091 Stopped,
1092 Failed,
1093 Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098 f.write_str(match self {
1099 Self::Starting => "starting",
1100 Self::Running => "running",
1101 Self::Unresponsive => "unresponsive",
1102 Self::Restarting => "restarting",
1103 Self::Draining => "draining",
1104 Self::Stopped => "stopped",
1105 Self::Failed => "failed",
1106 Self::Disabled => "disabled",
1107 })
1108 }
1109}
1110
1111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114 Clean,
1115 Crash,
1116 DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120 fn from(kind: ExitKind) -> Self {
1121 match kind {
1122 ExitKind::Clean => Self::Clean,
1123 ExitKind::Crash => Self::Crash,
1124 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125 }
1126 }
1127}
1128
1129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133 pub(crate) pid: u32,
1134 pub(crate) start_time: u64,
1135}
1136
1137#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140 pub kind: ExitKind,
1141 pub code: Option<i32>,
1142 pub signal: Option<i32>,
1143 pub at_ms: u64,
1144}
1145
1146#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150 pub module_id: String,
1151 pub waiting_for_previous_pid: Option<u32>,
1153 pub waiting_for_previous_detail: Option<String>,
1155 pub launch_nonce_source: Option<String>,
1160 pub state: ModuleState,
1161 pub enabled: bool,
1162 pub process_alive: bool,
1163 pub registration_active: bool,
1164 pub protocol: ModuleProtocol,
1169 pub live: bool,
1180 pub restart_count: u32,
1184 pub lifetime_restarts: u32,
1188 pub spawn_generation: u64,
1189 pub max_restarts: u32,
1194 pub restart_window: Duration,
1198 pub drain_timeout: Duration,
1202 pub restart_backoff: Duration,
1203 pub restart_max_backoff: Duration,
1204 pub pid: Option<u32>,
1209 pub spawned_at_ms: Option<u64>,
1210 pub spawned_from: Option<PathBuf>,
1211 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212 pub process_start_time: Option<u64>,
1213 pub last_exit: Option<ExitReport>,
1214 pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220 generation: u64,
1221 turn: u64,
1222 registered_connection: Option<ConnectionId>,
1223 next_probe_at: Option<Instant>,
1224 wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230 #[cfg(windows)]
1231 pipe_consumed: Option<Arc<OnceLock<()>>>,
1232 #[cfg(test)]
1234 actor_turns: u64,
1235 #[cfg(test)]
1238 actor_select: Option<ActorSelectCheckpoint>,
1239 state: ModuleState,
1240 enabled: bool,
1241 process_alive: bool,
1242 spawned_protocol: Option<ModuleProtocol>,
1243 spawn_failure: Option<String>,
1244 waiting_for_previous_pid: Option<u32>,
1245 waiting_for_previous_detail: Option<String>,
1246 crash_restarts: VecDeque<Instant>,
1252 lifetime_restarts: u32,
1253 spawn_generation: u64,
1262 pid: Option<u32>,
1263 #[cfg(target_os = "macos")]
1264 report_ready: Option<Arc<OnceLock<()>>>,
1265 reaped_pid: Option<u32>,
1267 respawn_pending: bool,
1269 coalesced_restart_pending: bool,
1271 spawned_at_ms: Option<u64>,
1272 spawned_from: Option<PathBuf>,
1273 launch_spec_digest: Option<LaunchSpecDigest>,
1274 spawned_file_identity: Option<SpawnedFileIdentity>,
1275 process_start_time: Option<u64>,
1276 deliberate_severance: Option<ProcessIdentity>,
1277 last_exit: Option<ExitReport>,
1278 drain_disposition_detail: Option<String>,
1280 health: ModuleHealthStatus,
1281 in_alternate_slot: bool,
1286 draining_to_replace: bool,
1293 configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302 fn reported_pid(&self) -> Option<u32> {
1307 #[cfg(target_os = "macos")]
1308 if self
1309 .report_ready
1310 .as_ref()
1311 .is_some_and(|ready| ready.get().is_none())
1312 {
1313 return None;
1314 }
1315 self.pid
1316 }
1317
1318 fn starting() -> Self {
1319 Self::new(ModuleState::Starting, true)
1320 }
1321
1322 fn disabled() -> Self {
1323 Self::new(ModuleState::Disabled, false)
1324 }
1325
1326 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330 while let Some(oldest) = self.crash_restarts.front() {
1331 if now.duration_since(*oldest) > window {
1332 self.crash_restarts.pop_front();
1333 } else {
1334 break;
1335 }
1336 }
1337 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338 }
1339
1340 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346 self.crash_restarts.push_back(now);
1347 while self.crash_restarts.len() > policy.max_restarts as usize {
1348 self.crash_restarts.pop_front();
1349 }
1350 self.lifetime_restarts += 1;
1351 }
1352
1353 fn next_crash_restart(
1357 &mut self,
1358 policy: &RestartPolicy,
1359 now: Instant,
1360 ) -> Option<CrashRestartSchedule> {
1361 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362 if restart_in_window >= policy.max_restarts {
1363 return None;
1364 }
1365 self.record_crash_restart(policy, now);
1366 Some(CrashRestartSchedule {
1367 restart_in_window,
1368 delay: policy.delay_for_restart(restart_in_window),
1369 })
1370 }
1371
1372 fn clear_crash_restarts(&mut self) {
1377 self.crash_restarts.clear();
1378 }
1379
1380 fn new(state: ModuleState, enabled: bool) -> Self {
1381 Self {
1382 exec_outcome: None,
1383 #[cfg(test)]
1384 actor_turns: 0,
1385 #[cfg(test)]
1386 actor_select: None,
1387 state,
1388 enabled,
1389 process_alive: false,
1390 #[cfg(windows)]
1391 pipe_consumed: None,
1392 spawned_protocol: None,
1393 spawn_failure: None,
1394 waiting_for_previous_pid: None,
1395 waiting_for_previous_detail: None,
1396 crash_restarts: VecDeque::new(),
1397 lifetime_restarts: 0,
1398 spawn_generation: 0,
1399 pid: None,
1400 #[cfg(target_os = "macos")]
1401 report_ready: None,
1402 reaped_pid: None,
1403 respawn_pending: false,
1404 coalesced_restart_pending: false,
1405 spawned_at_ms: None,
1406 spawned_from: None,
1407 launch_spec_digest: None,
1408 spawned_file_identity: None,
1409 process_start_time: None,
1410 deliberate_severance: None,
1411 last_exit: None,
1412 drain_disposition_detail: None,
1413 health: ModuleHealthStatus::default(),
1414 in_alternate_slot: false,
1415 draining_to_replace: false,
1416 configuration_updated_since_spawn: false,
1417 }
1418 }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427 version: u8,
1428 frames: mpsc::Sender<Frame>,
1429 lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437 daemon_incarnation: String,
1438 seq: u64,
1439 capacity: usize,
1440 live: HashMap<String, LiveSpawn>,
1441 generations: HashMap<String, u64>,
1442 events: VecDeque<SpawnEvent>,
1443 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447 fn default() -> Self {
1448 Self {
1449 daemon_incarnation: "unconfigured".to_string(),
1450 seq: 0,
1451 capacity: SPAWN_EVENT_RING_CAPACITY,
1452 live: HashMap::new(),
1453 generations: HashMap::new(),
1454 events: VecDeque::new(),
1455 subscribers: HashMap::new(),
1456 }
1457 }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462 Arc<Mutex<SpawnEventState>>,
1463 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468 ForeignIncarnation { current: String },
1469 TooOld { oldest: SpawnCursor },
1470 Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474 fn configure_incarnation(&self, daemon_incarnation: String) {
1475 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476 state.daemon_incarnation = daemon_incarnation;
1477 state.seq = 0;
1478 state.live.clear();
1479 state.generations.clear();
1480 state.events.clear();
1481 state.subscribers.clear();
1482 }
1483
1484 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485 SpawnCursor {
1486 daemon_incarnation: state.daemon_incarnation.clone(),
1487 seq: state.seq,
1488 }
1489 }
1490
1491 fn snapshot(&self) -> SpawnSnapshot {
1492 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495 SpawnSnapshot {
1496 cursor: Self::cursor(&state),
1497 ring_bound: state.capacity as u64,
1498 live,
1499 }
1500 }
1501
1502 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504 let generation = state
1505 .generations
1506 .get(module_id)
1507 .copied()
1508 .unwrap_or(0)
1509 .checked_add(1)
1510 .expect("spawn generation exhausted");
1511 state.generations.insert(module_id.to_string(), generation);
1512 let live = LiveSpawn {
1513 module_id: module_id.to_string(),
1514 spawn_generation: generation,
1515 pid,
1516 spawned_at_ms,
1517 };
1518 state.live.insert(module_id.to_string(), live);
1519 Self::emit_locked(
1520 &mut state,
1521 SpawnEventKind::Spawned,
1522 module_id.to_string(),
1523 generation,
1524 pid,
1525 None,
1526 None,
1527 );
1528 generation
1529 }
1530
1531 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533 let Some(live) = state.live.remove(module_id) else {
1534 warn!(
1535 module_id,
1536 "terminal record had no live spawn event identity"
1537 );
1538 return;
1539 };
1540 Self::emit_locked(
1541 &mut state,
1542 SpawnEventKind::Exited,
1543 module_id.to_string(),
1544 live.spawn_generation,
1545 live.pid,
1546 exit_code,
1547 exit_signal,
1548 );
1549 }
1550
1551 fn emit_superseded_exited(
1558 &self,
1559 module_id: &str,
1560 spawn_generation: u64,
1561 pid: u32,
1562 exit_code: Option<i32>,
1563 exit_signal: Option<i32>,
1564 ) {
1565 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566 if state
1567 .live
1568 .get(module_id)
1569 .is_some_and(|live| live.spawn_generation == spawn_generation)
1570 {
1571 state.live.remove(module_id);
1572 }
1573 Self::emit_locked(
1574 &mut state,
1575 SpawnEventKind::Exited,
1576 module_id.to_string(),
1577 spawn_generation,
1578 pid,
1579 exit_code,
1580 exit_signal,
1581 );
1582 }
1583
1584 #[allow(clippy::too_many_arguments)]
1585 fn emit_locked(
1586 state: &mut SpawnEventState,
1587 kind: SpawnEventKind,
1588 module_id: String,
1589 spawn_generation: u64,
1590 pid: u32,
1591 exit_code: Option<i32>,
1592 exit_signal: Option<i32>,
1593 ) {
1594 state.seq = state
1595 .seq
1596 .checked_add(1)
1597 .expect("spawn event sequence exhausted");
1598 let event = SpawnEvent {
1599 cursor: Self::cursor(state),
1600 kind,
1601 module_id,
1602 spawn_generation,
1603 pid,
1604 exit_code,
1605 exit_signal,
1606 };
1607 state.events.push_back(event.clone());
1608 while state.events.len() > state.capacity {
1609 state.events.pop_front();
1610 }
1611 let body = match serde_json::to_vec(&event) {
1612 Ok(body) => body,
1613 Err(error) => {
1614 error!(%error, "failed to serialize supervisor spawn event");
1615 return;
1616 }
1617 };
1618 state.subscribers.retain(|(connection_id, corr), subscriber| {
1619 let frame = Frame::build_with_version(
1620 subscriber.version,
1621 FrameType::StreamData,
1622 control_flags(),
1623 0,
1624 0,
1625 *corr,
1626 body.clone(),
1627 );
1628 match frame {
1629 Ok(frame) => {
1630 if subscriber.frames.try_send(frame).is_ok() {
1631 true
1632 } else {
1633 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634 if let Some(lagged) = subscriber.lagged.take() {
1635 let _ = lagged.send(event.cursor.clone());
1636 }
1637 false
1638 }
1639 }
1640 Err(error) => {
1641 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642 false
1643 }
1644 }
1645 });
1646 }
1647
1648 fn subscribe(
1649 &self,
1650 connection_id: ConnectionId,
1651 corr: u64,
1652 version: u8,
1653 since: Option<SpawnCursor>,
1654 sink: FrameSink,
1655 ) -> Result<(), SpawnSubscribeRefusal> {
1656 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658 {
1659 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660 let replay = if let Some(since) = since {
1661 if since.daemon_incarnation != state.daemon_incarnation {
1662 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663 current: state.daemon_incarnation.clone(),
1664 });
1665 }
1666 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667 if since.seq < oldest.seq.saturating_sub(1) {
1668 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669 }
1670 }
1671 state
1672 .events
1673 .iter()
1674 .filter(|event| event.cursor.seq > since.seq)
1675 .cloned()
1676 .collect::<Vec<_>>()
1677 } else {
1678 Vec::new()
1679 };
1680 for event in replay {
1681 let body = serde_json::to_vec(&event)
1682 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683 let frame = Frame::build_with_version(
1684 version,
1685 FrameType::StreamData,
1686 control_flags(),
1687 0,
1688 0,
1689 corr,
1690 body,
1691 )
1692 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693 frames
1694 .try_send(frame)
1695 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696 }
1697 state.subscribers.insert(
1698 (connection_id, corr),
1699 SpawnSubscriber {
1700 version,
1701 frames,
1702 lagged: Some(lagged),
1703 },
1704 );
1705 }
1706 tokio::spawn(async move {
1717 while let Some(frame) = receiver.recv().await {
1718 if sink.send(frame).await.is_err() {
1719 return;
1720 }
1721 }
1722 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723 return;
1724 };
1725 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726 Ok(frame) => {
1727 let _ = sink.send(frame).await;
1728 }
1729 Err(error) => {
1730 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731 }
1732 }
1733 });
1734 Ok(())
1735 }
1736
1737 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738 let Some(subscriber) = self
1739 .0
1740 .lock()
1741 .unwrap_or_else(|p| p.into_inner())
1742 .subscribers
1743 .remove(&(connection_id, corr))
1744 else {
1745 return false;
1746 };
1747 if let Ok(frame) = Frame::build_with_version(
1748 subscriber.version,
1749 FrameType::StreamEnd,
1750 control_flags(),
1751 0,
1752 0,
1753 corr,
1754 Vec::new(),
1755 ) {
1756 tokio::spawn(async move {
1757 let _ = subscriber.frames.send(frame).await;
1758 });
1759 }
1760 true
1761 }
1762
1763 fn remove_connection(&self, connection_id: ConnectionId) {
1764 self.0
1765 .lock()
1766 .unwrap_or_else(|p| p.into_inner())
1767 .subscribers
1768 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769 }
1770
1771 #[cfg(any(test, feature = "test-support"))]
1772 fn set_capacity(&self, capacity: usize) {
1773 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774 }
1775
1776 #[cfg(any(test, feature = "test-support"))]
1777 fn subscriber_count(&self) -> usize {
1778 self.0
1779 .lock()
1780 .unwrap_or_else(|p| p.into_inner())
1781 .subscribers
1782 .len()
1783 }
1784}
1785
1786fn spawn_subscriber_lagged_frame(
1789 version: u8,
1790 corr: u64,
1791 first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796 .to_string(),
1797 detail: Some(serde_json::json!({
1798 "first_undelivered_cursor": first_undelivered
1799 })),
1800 })
1801 .map_err(|error| error.to_string())?;
1802 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803 .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807 fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809 fn process_replacing(&self, _module_id: &str) -> bool {
1815 false
1816 }
1817}
1818
1819#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826 pub fn new() -> Self {
1827 Self::default()
1828 }
1829
1830 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831 let mut snapshots = self
1832 .snapshots
1833 .lock()
1834 .unwrap_or_else(|poisoned| poisoned.into_inner());
1835 snapshots.insert(module_id, snapshot);
1836 }
1837
1838 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839 let mut snapshots = self
1840 .snapshots
1841 .lock()
1842 .unwrap_or_else(|poisoned| poisoned.into_inner());
1843 let is_current = snapshots
1844 .get(module_id)
1845 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846 .unwrap_or(false);
1847 if is_current {
1848 snapshots.remove(module_id);
1849 }
1850 }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854 fn process_live(&self, module_id: &str) -> Option<bool> {
1855 let snapshot = {
1856 let snapshots = self
1857 .snapshots
1858 .lock()
1859 .unwrap_or_else(|poisoned| poisoned.into_inner());
1860 snapshots.get(module_id).cloned()
1861 }?;
1862 let snapshot = snapshot
1863 .lock()
1864 .unwrap_or_else(|poisoned| poisoned.into_inner());
1865 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866 }
1867
1868 fn process_replacing(&self, module_id: &str) -> bool {
1869 let Some(snapshot) = self
1870 .snapshots
1871 .lock()
1872 .unwrap_or_else(|poisoned| poisoned.into_inner())
1873 .get(module_id)
1874 .cloned()
1875 else {
1876 return false;
1877 };
1878 let snapshot = snapshot
1879 .lock()
1880 .unwrap_or_else(|poisoned| poisoned.into_inner());
1881 snapshot.enabled
1882 && match snapshot.state {
1883 ModuleState::Restarting => true,
1884 ModuleState::Draining => snapshot.draining_to_replace,
1885 ModuleState::Starting
1886 | ModuleState::Running
1887 | ModuleState::Unresponsive
1888 | ModuleState::Stopped
1889 | ModuleState::Failed
1890 | ModuleState::Disabled => false,
1891 }
1892 }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898 reached: tokio::sync::Notify,
1899 resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904 Spawn,
1905 Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910 deadline: Instant,
1911 kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918 start_requester: Option<Principal>,
1919 pending_activity_restart: bool,
1920 provenance_probe: ExecutableIdentityProbe,
1921 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925 restart_policy: RestartPolicy,
1926 drain_timeout: Duration,
1929 effective_drain_timeout: Arc<Mutex<Duration>>,
1932 default_drain_timeout: Duration,
1935 health: HealthConfig,
1936 connection_file_path: Option<PathBuf>,
1937 capture_logs_dir: Option<PathBuf>,
1938 forwarding: Option<Arc<ForwardingTable>>,
1939 supervisor_handle: Option<SupervisorHandle>,
1942 stderr_ring: Arc<Mutex<StderrRing>>,
1949 terminal_ring: Arc<Mutex<TerminalRing>>,
1950 spawn_events: SpawnEventFeed,
1951 child_roster: ChildRoster,
1952 #[cfg(target_os = "linux")]
1953 cgroup_placement: Option<subc_cgroup::Placement>,
1954 #[cfg(test)]
1955 test_seed_stale_facts_before_enable_spawn: bool,
1956 #[cfg(test)]
1957 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962 spec: ModuleSpec,
1963 health: HealthConfig,
1964}
1965
1966#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976 configured_ids: Arc<Mutex<HashSet<String>>>,
1988 spawn_events: SpawnEventFeed,
1989 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024 promotion_observer: PromotionObserverSlot,
2026 operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032pub(crate) trait SwapPromotionObserver: Send + Sync {
2041 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052 f.write_str("PromotionObserverSlot")
2053 }
2054}
2055
2056#[derive(Debug, Clone)]
2058struct OpenSwap {
2059 candidate_nonce: String,
2062 incumbent_nonce: Option<String>,
2067 candidate_admitted: bool,
2071}
2072
2073#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077 NotSwapping,
2080 Candidate,
2082 Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089 Exact {
2090 module_id: String,
2091 },
2092 Prefix {
2093 prefix: String,
2094 owner_module_id: String,
2095 },
2096}
2097
2098impl SupervisorHandle {
2099 pub fn new() -> Self {
2100 Self::default()
2101 }
2102
2103 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104 self.spawn_events.snapshot()
2105 }
2106
2107 pub(crate) fn subscribe_spawns(
2108 &self,
2109 connection_id: ConnectionId,
2110 corr: u64,
2111 version: u8,
2112 since: Option<SpawnCursor>,
2113 sink: FrameSink,
2114 ) -> Result<(), SpawnSubscribeRefusal> {
2115 self.spawn_events
2116 .subscribe(connection_id, corr, version, since, sink)
2117 }
2118
2119 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120 self.spawn_events.cancel(connection_id, corr)
2121 }
2122
2123 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124 self.spawn_events.remove_connection(connection_id);
2125 }
2126
2127 #[cfg(any(test, feature = "test-support"))]
2128 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129 assert!(capacity > 0, "spawn event capacity must be non-zero");
2130 self.spawn_events.set_capacity(capacity);
2131 }
2132
2133 #[cfg(any(test, feature = "test-support"))]
2134 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135 self.spawn_events.subscriber_count()
2136 }
2137
2138 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141 self.spawn_nonces
2142 .lock()
2143 .unwrap_or_else(|poisoned| poisoned.into_inner())
2144 .insert(module_id.to_string(), nonce);
2145 }
2146
2147 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150 self.reserved_nonces
2151 .lock()
2152 .unwrap_or_else(|poisoned| poisoned.into_inner())
2153 .insert(module_id.to_string(), Some(nonce));
2154 }
2155
2156 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158 let mut owners = self
2159 .reserved_prefix_owners
2160 .lock()
2161 .unwrap_or_else(|poisoned| poisoned.into_inner());
2162 owners.retain(|_, owner| owner != owner_module_id);
2163 for prefix in prefixes {
2164 owners.insert(prefix.clone(), owner_module_id.to_string());
2165 }
2166 }
2167
2168 #[cfg(test)]
2170 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171 self.spawn_nonces
2172 .lock()
2173 .unwrap_or_else(|poisoned| poisoned.into_inner())
2174 .get(module_id)
2175 .cloned()
2176 }
2177
2178 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180 let spawn_nonce = self
2181 .spawn_nonces
2182 .lock()
2183 .unwrap_or_else(|poisoned| poisoned.into_inner())
2184 .get(&spec.module_id)
2185 .cloned();
2186 let mut reserved_nonces = self
2187 .reserved_nonces
2188 .lock()
2189 .unwrap_or_else(|poisoned| poisoned.into_inner());
2190 if spec.reserved {
2191 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196 }
2197 drop(reserved_nonces);
2198 self.removal_tombstones
2202 .lock()
2203 .unwrap_or_else(|poisoned| poisoned.into_inner())
2204 .remove(&spec.module_id);
2205 }
2206
2207 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212 self.reserved_hello_rejection(module_id, presented)
2213 .is_none()
2214 }
2215
2216 pub(crate) fn reserved_hello_rejection(
2217 &self,
2218 module_id: &str,
2219 presented: Option<&str>,
2220 ) -> Option<ReservedHelloRejection> {
2221 let nonces = self
2222 .reserved_nonces
2223 .lock()
2224 .unwrap_or_else(|poisoned| poisoned.into_inner());
2225 if let Some(expected) = nonces.get(module_id) {
2226 let authorized = match expected {
2230 Some(expected) => {
2231 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2232 }
2233 None => false,
2234 };
2235 if authorized {
2236 return None;
2237 }
2238 return Some(ReservedHelloRejection::Exact {
2239 module_id: module_id.to_string(),
2240 });
2241 }
2242 drop(nonces);
2243
2244 let matched_prefix = self
2245 .reserved_prefix_owners
2246 .lock()
2247 .unwrap_or_else(|poisoned| poisoned.into_inner())
2248 .iter()
2249 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2250 .max_by_key(|(prefix, _)| prefix.len())
2251 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2252 let (prefix, owner_module_id) = matched_prefix?;
2253
2254 let authorized = presented.is_some_and(|presented| {
2255 self.spawn_nonces
2256 .lock()
2257 .unwrap_or_else(|poisoned| poisoned.into_inner())
2258 .get(&owner_module_id)
2259 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2260 || self.swap_nonce_matches(&owner_module_id, presented)
2263 });
2264 if authorized {
2265 None
2266 } else {
2267 Some(ReservedHelloRejection::Prefix {
2268 prefix,
2269 owner_module_id,
2270 })
2271 }
2272 }
2273
2274 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2279 if presented.is_empty() {
2280 return false;
2281 }
2282 let nonces = self
2283 .spawn_nonces
2284 .lock()
2285 .unwrap_or_else(|poisoned| poisoned.into_inner());
2286 let current = nonces
2287 .get(module_id)
2288 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2289 drop(nonces);
2290 current || self.swap_nonce_matches(module_id, presented)
2295 }
2296
2297 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2299 let swaps = self
2300 .swaps
2301 .lock()
2302 .unwrap_or_else(|poisoned| poisoned.into_inner());
2303 swaps.get(module_id).is_some_and(|swap| {
2304 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2305 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2306 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2307 })
2308 })
2309 }
2310
2311 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2314 let incumbent_nonce = self
2315 .spawn_nonces
2316 .lock()
2317 .unwrap_or_else(|poisoned| poisoned.into_inner())
2318 .get(module_id)
2319 .cloned();
2320 self.swaps
2321 .lock()
2322 .unwrap_or_else(|poisoned| poisoned.into_inner())
2323 .insert(
2324 module_id.to_string(),
2325 OpenSwap {
2326 candidate_nonce,
2327 incumbent_nonce,
2328 candidate_admitted: false,
2329 },
2330 );
2331 }
2332
2333 pub(crate) fn close_swap(&self, module_id: &str) {
2336 self.swaps
2337 .lock()
2338 .unwrap_or_else(|poisoned| poisoned.into_inner())
2339 .remove(module_id);
2340 }
2341
2342 pub(crate) fn set_swap_promotion_observer(
2345 &self,
2346 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2347 ) {
2348 *self
2349 .promotion_observer
2350 .0
2351 .lock()
2352 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2353 }
2354
2355 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2358 let observer = self
2359 .promotion_observer
2360 .0
2361 .lock()
2362 .unwrap_or_else(|poisoned| poisoned.into_inner())
2363 .as_ref()
2364 .and_then(std::sync::Weak::upgrade);
2365 if let Some(observer) = observer {
2366 observer.swap_promoted(registration);
2367 }
2368 }
2369
2370 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2372 self.swaps
2373 .lock()
2374 .unwrap_or_else(|poisoned| poisoned.into_inner())
2375 .contains_key(module_id)
2376 }
2377
2378 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2383 let candidate_nonce = self
2384 .swaps
2385 .lock()
2386 .unwrap_or_else(|poisoned| poisoned.into_inner())
2387 .get(module_id)
2388 .map(|swap| swap.candidate_nonce.clone());
2389 let Some(nonce) = candidate_nonce else {
2390 return;
2391 };
2392 self.set_spawn_nonce(module_id, nonce.clone());
2393 if reserved {
2394 self.set_reserved_nonce(module_id, nonce);
2395 }
2396 }
2397
2398 pub(crate) fn swap_hello_admission(
2413 &self,
2414 module_id: &str,
2415 presented: Option<&str>,
2416 ) -> SwapHelloAdmission {
2417 let swaps = self
2418 .swaps
2419 .lock()
2420 .unwrap_or_else(|poisoned| poisoned.into_inner());
2421 let Some(swap) = swaps.get(module_id) else {
2422 return SwapHelloAdmission::NotSwapping;
2423 };
2424 let Some(presented) = presented else {
2425 return SwapHelloAdmission::Refused;
2426 };
2427 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2428 return if swap.candidate_admitted {
2429 SwapHelloAdmission::Refused
2430 } else {
2431 SwapHelloAdmission::Candidate
2432 };
2433 }
2434 if swap
2435 .incumbent_nonce
2436 .as_deref()
2437 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2438 {
2439 return SwapHelloAdmission::NotSwapping;
2440 }
2441 SwapHelloAdmission::Refused
2442 }
2443
2444 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2447 if let Some(swap) = self
2448 .swaps
2449 .lock()
2450 .unwrap_or_else(|poisoned| poisoned.into_inner())
2451 .get_mut(module_id)
2452 {
2453 swap.candidate_admitted = true;
2454 }
2455 }
2456
2457 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2459 self.spawn_nonces
2460 .lock()
2461 .unwrap_or_else(|poisoned| poisoned.into_inner())
2462 .get(module_id)
2463 .cloned()
2464 }
2465
2466 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2468 self.reserved_nonces
2469 .lock()
2470 .unwrap_or_else(|poisoned| poisoned.into_inner())
2471 .get(module_id)
2472 .cloned()
2473 .flatten()
2474 }
2475
2476 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2477 self.mark_configured(module.module_id());
2481 let mut modules = self
2482 .modules
2483 .lock()
2484 .unwrap_or_else(|poisoned| poisoned.into_inner());
2485 modules.insert(module.module_id().to_string(), module)
2486 }
2487
2488 fn mark_configured(&self, module_id: &str) {
2492 self.configured_ids
2493 .lock()
2494 .unwrap_or_else(|poisoned| poisoned.into_inner())
2495 .insert(module_id.to_string());
2496 }
2497
2498 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2502 let modules = self
2503 .modules
2504 .lock()
2505 .unwrap_or_else(|poisoned| poisoned.into_inner());
2506 if !modules.contains_key(module_id) {
2507 self.configured_ids
2508 .lock()
2509 .unwrap_or_else(|poisoned| poisoned.into_inner())
2510 .remove(module_id);
2511 }
2512 }
2513
2514 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2522 self.configured_ids
2523 .lock()
2524 .unwrap_or_else(|poisoned| poisoned.into_inner())
2525 .contains(module_id)
2526 }
2527
2528 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2529 let modules = self
2530 .modules
2531 .lock()
2532 .unwrap_or_else(|poisoned| poisoned.into_inner());
2533 modules.get(module_id).cloned()
2534 }
2535
2536 pub(crate) fn record_late_health_answer(
2537 &self,
2538 module_id: &str,
2539 latency_ms: u64,
2540 ) -> Result<bool, SuperviseError> {
2541 let Some(module) = self.get(module_id) else {
2542 return Ok(false);
2543 };
2544 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2545 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2546 state.health.last_late_answer_latency_ms = Some(latency_ms);
2547 state.health.consecutive_failures = 0;
2555 })?;
2556 Ok(true)
2557 }
2558
2559 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2565 let Some(module) = self.get(module_id) else {
2566 return Ok(false);
2567 };
2568 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2569 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2570 return Ok(false);
2571 };
2572 drop(snapshot);
2573 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2574 }
2575
2576 pub fn list(&self) -> Vec<SupervisedModule> {
2577 let modules = self
2578 .modules
2579 .lock()
2580 .unwrap_or_else(|poisoned| poisoned.into_inner());
2581 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2582 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2583 modules
2584 }
2585
2586 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2587 self.activity_images
2588 .lock()
2589 .unwrap_or_else(|p| p.into_inner())
2590 .remove(module_id);
2591 self.spawn_nonces
2592 .lock()
2593 .unwrap_or_else(|poisoned| poisoned.into_inner())
2594 .remove(module_id);
2595 self.close_swap(module_id);
2596 let mut reserved_nonces = self
2597 .reserved_nonces
2598 .lock()
2599 .unwrap_or_else(|poisoned| poisoned.into_inner());
2600 if reserved_nonces.contains_key(module_id) {
2601 reserved_nonces.insert(module_id.to_string(), None);
2604 }
2605 drop(reserved_nonces);
2606 self.reserved_prefix_owners
2607 .lock()
2608 .unwrap_or_else(|poisoned| poisoned.into_inner())
2609 .retain(|_, owner| owner != module_id);
2610 let removed = self
2611 .modules
2612 .lock()
2613 .unwrap_or_else(|poisoned| poisoned.into_inner())
2614 .remove(module_id);
2615 self.configured_ids
2616 .lock()
2617 .unwrap_or_else(|poisoned| poisoned.into_inner())
2618 .remove(module_id);
2619 removed
2620 }
2621
2622 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2625 self.removal_tombstones
2626 .lock()
2627 .unwrap_or_else(|poisoned| poisoned.into_inner())
2628 .insert(module_id.to_string(), unix_ms_now());
2629 }
2630
2631 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2633 self.removal_tombstones
2634 .lock()
2635 .unwrap_or_else(|poisoned| poisoned.into_inner())
2636 .get(module_id)
2637 .copied()
2638 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2639 }
2640
2641 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2646 if self.get(module_id).is_some() {
2647 return false;
2648 }
2649 let mut reserved_nonces = self
2650 .reserved_nonces
2651 .lock()
2652 .unwrap_or_else(|poisoned| poisoned.into_inner());
2653 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2654 return false;
2655 }
2656 reserved_nonces.remove(module_id);
2657 true
2658 }
2659
2660 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2661 Arc::clone(&self.operation_lock)
2662 }
2663}
2664
2665#[derive(Debug, Clone)]
2667pub struct Supervisor {
2668 registry: Arc<Registry>,
2669 restart_policy: RestartPolicy,
2670 drain_timeout: Duration,
2671 connection_file_path: Option<PathBuf>,
2672 capture_logs_dir: Option<PathBuf>,
2673 forwarding: Option<Arc<ForwardingTable>>,
2674 process_liveness: Arc<SupervisorProcessLiveness>,
2675 supervisor_handle: Option<SupervisorHandle>,
2676 health: HealthConfig,
2677 daemon_start_clock: crate::clock::StartClock,
2678 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2679 spawn_events: SpawnEventFeed,
2680 provenance_probe: ExecutableIdentityProbe,
2681 child_roster: ChildRoster,
2684 #[cfg(target_os = "linux")]
2685 cgroup_placement: Option<subc_cgroup::Placement>,
2686 #[cfg(test)]
2687 test_after_first_spawn: AfterFirstSpawnHook,
2688}
2689
2690pub(crate) struct RegisteredConfiguredModule {
2694 spec: ModuleSpec,
2695 runtime: SupervisorRuntimeConfig,
2696 module: SupervisedModule,
2697 commands: mpsc::Receiver<SupervisorCommand>,
2698}
2699
2700#[cfg(test)]
2706#[derive(Clone, Default)]
2707struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2708
2709#[cfg(test)]
2710type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2711
2712#[cfg(test)]
2713impl fmt::Debug for AfterFirstSpawnHook {
2714 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2715 f.write_str("AfterFirstSpawnHook")
2716 }
2717}
2718
2719#[cfg(test)]
2720impl AfterFirstSpawnHook {
2721 fn run(&self, module_id: &str) {
2722 if let Some(hook) = &self.0 {
2723 hook(module_id);
2724 }
2725 }
2726}
2727
2728impl Supervisor {
2729 #[cfg(test)]
2730 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2731 let supervisor = Self::new(registry, policy);
2732 #[cfg(target_os = "macos")]
2733 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2734 supervisor
2735 }
2736 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2741 let path = path.into();
2742 #[cfg(target_os = "macos")]
2743 {
2744 let result = probe_privacy_trampoline(&path).map(|()| path);
2745 if let Err(cause) = &result {
2746 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2747 }
2748 self.child_roster.set_privacy_trampoline(result);
2749 }
2750 #[cfg(not(target_os = "macos"))]
2751 let _ = path;
2752 self
2753 }
2754 #[cfg(unix)]
2765 pub(crate) fn begin_daemon_shutdown(&self) {
2766 self.child_roster.close();
2767 if let Some(journal) = &self.terminal_journal {
2768 journal.stamp_shutdown();
2769 }
2770 }
2771
2772 #[cfg(unix)]
2776 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2777 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2778 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2779 let Some(forwarding) = &self.forwarding else {
2780 return Ok(());
2781 };
2782 let modules = forwarding
2783 .begin_daemon_drain()
2784 .map_err(SuperviseError::Forwarding)?;
2785 let deadline_ms =
2786 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2787 let mut notices = tokio::task::JoinSet::new();
2788 let mut drains = Vec::new();
2789 for (module_id, endpoint) in modules {
2790 let Some(target) = forwarding
2791 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2792 .map_err(SuperviseError::Forwarding)?
2793 else {
2794 continue;
2795 };
2796 let routes = forwarding
2797 .endpoint_routes(target.endpoint)
2798 .map_err(SuperviseError::Forwarding)?;
2799 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2805 reason: RouteCloseReason::Restart,
2806 deadline_ms,
2807 })
2808 .expect("module draining serializes");
2809 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2810 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2811 for route in routes {
2812 let client = route.goodbye_target;
2813 if let Some((_, channels)) = clients
2814 .iter_mut()
2815 .find(|(existing, _)| existing.connection_id == client.connection_id)
2816 {
2817 channels.push(client.channel);
2818 } else {
2819 let channel = client.channel;
2820 clients.push((client, vec![channel]));
2821 }
2822 }
2823 for (client, mut channels) in clients {
2824 channels.sort_unstable();
2825 channels.dedup();
2826 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2827 module_id: module_id.clone(),
2828 channels,
2829 reason: RouteCloseReason::Restart,
2830 })
2831 .expect("route closing serializes");
2832 recipients.push((client.sink, client.negotiated_ver, closing));
2833 }
2834 for (sink, version, body) in recipients {
2835 notices.spawn(async move {
2836 let frame = Frame::build_with_version(
2837 version,
2838 FrameType::Push,
2839 control_flags(),
2840 0,
2841 0,
2842 0,
2843 body,
2844 )
2845 .expect("bounded lifecycle notice frame builds");
2846 sink.send_flushed(frame).await
2847 });
2848 }
2849 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2850 drains.push((module_id, target.endpoint, gauges));
2851 }
2852 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2855 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2856 if !matches!(result, Ok(Ok(()))) {
2857 warn!(?result, "daemon shutdown notice delivery failed");
2858 }
2859 }
2860 notices.abort_all();
2861 let deadline = Instant::now() + DRAIN_BUDGET;
2862 let mut waits = tokio::task::JoinSet::new();
2863 for (module_id, endpoint, gauges) in drains {
2864 let forwarding = Arc::clone(forwarding);
2865 let mut runtime = self.runtime_config();
2866 runtime.health.cadence = Duration::from_millis(100);
2867 waits.spawn(async move {
2868 let wait_result = wait_for_forwarding_quiescence(
2869 &forwarding,
2870 &module_id,
2871 &runtime,
2872 endpoint,
2873 deadline,
2874 &gauges,
2875 DrainScope::Endpoint(endpoint),
2876 &mut DrainGaugeTrace::new(&gauges),
2877 )
2878 .await;
2879 forwarding
2883 .record_daemon_drain_result(
2884 endpoint,
2885 drained_after_quiescence_wait(&wait_result),
2886 )
2887 .map_err(SuperviseError::Forwarding)?;
2888 wait_result
2889 });
2890 }
2891 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2892 if !matches!(result, Ok(Ok(true))) {
2893 warn!(?result, "daemon shutdown drain did not reach quiescence");
2894 }
2895 }
2896 Ok(())
2897 }
2898
2899 #[cfg(unix)]
2911 pub(crate) async fn end_children_for_daemon_shutdown(
2912 &self,
2913 already_escalated: bool,
2914 escalate: impl std::future::Future<Output = ()>,
2915 ) {
2916 tokio::pin!(escalate);
2917 let mut escalated = already_escalated;
2918 if let Some(forwarding) = &self.forwarding {
2919 let reason = CloseReason::new(
2920 "daemon_shutdown",
2921 "the daemon is exiting after its shutdown notice and drain",
2922 );
2923 if escalated {
2924 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2927 } else {
2928 tokio::select! {
2929 biased;
2930 _ = escalate.as_mut() => {
2931 info!("second SIGTERM: abandoning module GOODBYE delivery");
2932 escalated = true;
2933 }
2934 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2935 }
2936 }
2937 let closed = forwarding.close_all_connections(&reason);
2938 debug!(closed, "closed established connections for daemon shutdown");
2939 }
2940 let escalated_here = escalated && !already_escalated;
2944 let remaining_escalate = async move {
2945 if escalated_here {
2946 std::future::pending::<()>().await;
2947 } else {
2948 escalate.await;
2949 }
2950 };
2951 crate::child_roster::end_children_for_daemon_shutdown(
2952 &self.child_roster,
2953 escalated,
2954 remaining_escalate,
2955 )
2956 .await;
2957 }
2958
2959 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2960 Self {
2961 registry,
2962 restart_policy,
2963 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2964 connection_file_path: None,
2965 capture_logs_dir: None,
2966 forwarding: None,
2967 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2968 supervisor_handle: None,
2969 health: HealthConfig::default(),
2970 daemon_start_clock: crate::clock::StartClock::capture(),
2971 terminal_journal: None,
2972 spawn_events: SpawnEventFeed::default(),
2973 provenance_probe: ExecutableIdentityProbe::default(),
2974 child_roster: ChildRoster::default(),
2975 #[cfg(target_os = "linux")]
2976 cgroup_placement: None,
2977 #[cfg(test)]
2978 test_after_first_spawn: AfterFirstSpawnHook::default(),
2979 }
2980 }
2981
2982 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2983 self.drain_timeout = drain_timeout;
2984 self
2985 }
2986
2987 pub fn with_process_liveness(
2988 mut self,
2989 process_liveness: Arc<SupervisorProcessLiveness>,
2990 ) -> Self {
2991 self.process_liveness = process_liveness;
2992 self
2993 }
2994
2995 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2996 self.connection_file_path = Some(connection_file_path.into());
2997 self
2998 }
2999
3000 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3002 self.capture_logs_dir = Some(logs_dir.into());
3003 self
3004 }
3005
3006 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3009 self.spawn_events.configure_incarnation(daemon_incarnation);
3013 self
3014 }
3015
3016 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3019 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3020 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3021 path,
3022 daemon_incarnation,
3023 )));
3024 this
3025 }
3026
3027 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3028 self.forwarding = Some(forwarding);
3029 self
3030 }
3031
3032 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3033 self.spawn_events = supervisor_handle.spawn_events.clone();
3034 self.supervisor_handle = Some(supervisor_handle);
3035 self
3036 }
3037
3038 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3039 self.health = health;
3040 self
3041 }
3042
3043 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3047 self.child_roster.record_to(path.into());
3048 self
3049 }
3050
3051 #[cfg(target_os = "linux")]
3052 pub fn with_cgroup_placement(
3053 mut self,
3054 cgroup_placement: Option<subc_cgroup::Placement>,
3055 ) -> Self {
3056 self.cgroup_placement = cgroup_placement;
3057 self
3058 }
3059
3060 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3066 validate_spec(&spec)?;
3067 self.establish_identity(&spec);
3068
3069 let runtime = self.runtime_config();
3070 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3071 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3072 let spawned = spawn_child(
3073 &spec,
3074 runtime.connection_file_path.as_deref(),
3075 self.supervisor_handle.as_ref(),
3076 &runtime.stderr_ring,
3077 runtime.capture_logs_dir.as_deref(),
3078 &runtime.child_roster,
3079 #[cfg(target_os = "linux")]
3080 runtime.cgroup_placement.as_ref(),
3081 );
3082 #[cfg(test)]
3083 self.test_after_first_spawn.run(&spec.module_id);
3084 let child = match spawned {
3085 Ok(child) => child,
3086 Err(err) => {
3087 self.abandon_unrostered(&spec.module_id);
3090 return Err(err);
3091 }
3092 };
3093 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3094 if let Some(spawn) = activity_spawn {
3095 spawn.complete(&snapshot);
3096 }
3097
3098 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3099 }
3100
3101 fn establish_identity(&self, spec: &ModuleSpec) {
3114 if let Some(supervisor_handle) = &self.supervisor_handle {
3115 supervisor_handle.apply_identity_configuration(spec);
3116 supervisor_handle.mark_configured(&spec.module_id);
3117 }
3118 }
3119
3120 fn abandon_unrostered(&self, module_id: &str) {
3123 if let Some(supervisor_handle) = &self.supervisor_handle {
3124 supervisor_handle.unmark_configured_unless_rostered(module_id);
3125 }
3126 }
3127
3128 fn mark_first_process_running(
3131 &self,
3132 spec: &ModuleSpec,
3133 runtime: &SupervisorRuntimeConfig,
3134 snapshot: &SharedSnapshot,
3135 child: &SupervisedChild,
3136 ) -> Result<(), SuperviseError> {
3137 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3138 self.abandon_unrostered(&spec.module_id);
3139 return Err(err);
3140 }
3141 self.process_liveness
3142 .track(spec.module_id.clone(), Arc::clone(snapshot));
3143 Ok(())
3144 }
3145
3146 pub fn supervise_configured(
3152 &self,
3153 spec: ModuleSpec,
3154 enabled: bool,
3155 ) -> Result<SupervisedModule, SuperviseError> {
3156 self.supervise_configured_requested(
3157 spec,
3158 enabled,
3159 self.health.clone(),
3160 None,
3161 self.restart_policy,
3162 None,
3163 )
3164 }
3165
3166 pub fn supervise_configured_with_health(
3172 &self,
3173 spec: ModuleSpec,
3174 enabled: bool,
3175 health: HealthConfig,
3176 drain_timeout_ms: Option<u64>,
3177 restart_policy: RestartPolicy,
3178 ) -> Result<SupervisedModule, SuperviseError> {
3179 self.supervise_configured_requested(
3180 spec,
3181 enabled,
3182 health,
3183 drain_timeout_ms,
3184 restart_policy,
3185 None,
3186 )
3187 }
3188
3189 pub(crate) fn supervise_configured_requested(
3190 &self,
3191 spec: ModuleSpec,
3192 enabled: bool,
3193 health: HealthConfig,
3194 drain_timeout_ms: Option<u64>,
3195 restart_policy: RestartPolicy,
3196 requester: Option<Principal>,
3197 ) -> Result<SupervisedModule, SuperviseError> {
3198 let registered = self.register_configured_with_health(
3199 spec,
3200 enabled,
3201 health,
3202 drain_timeout_ms,
3203 restart_policy,
3204 )?;
3205 self.start_registered_configured(registered, requester)
3206 }
3207
3208 pub(crate) fn register_configured_with_health(
3213 &self,
3214 spec: ModuleSpec,
3215 enabled: bool,
3216 health: HealthConfig,
3217 drain_timeout_ms: Option<u64>,
3218 restart_policy: RestartPolicy,
3219 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3220 validate_spec(&spec)?;
3221 self.establish_identity(&spec);
3222
3223 let mut runtime = self.runtime_config();
3224 runtime.health = health;
3225 runtime.restart_policy = restart_policy;
3226 if let Some(ms) = drain_timeout_ms {
3227 runtime.drain_timeout = Duration::from_millis(ms);
3228 *runtime
3229 .effective_drain_timeout
3230 .lock()
3231 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3232 }
3233 let snapshot = if enabled {
3234 SupervisorSnapshot::starting()
3235 } else {
3236 SupervisorSnapshot::disabled()
3237 };
3238 let snapshot = {
3239 let mut snapshot = snapshot;
3240 if enabled {
3241 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3242 {
3243 snapshot.waiting_for_previous_pid = Some(pid);
3244 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3245 }
3246 }
3247 snapshot
3248 };
3249 let (module, commands) =
3250 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3251 Ok(RegisteredConfiguredModule {
3252 spec,
3253 runtime,
3254 module,
3255 commands,
3256 })
3257 }
3258
3259 pub(crate) fn start_registered_configured(
3262 &self,
3263 registered: RegisteredConfiguredModule,
3264 requester: Option<Principal>,
3265 ) -> Result<SupervisedModule, SuperviseError> {
3266 let RegisteredConfiguredModule {
3267 spec,
3268 mut runtime,
3269 module,
3270 commands,
3271 } = registered;
3272 runtime.start_requester = requester;
3273 let snapshot = &module.inner.snapshot;
3274 let should_spawn = {
3275 let state = lock_snapshot(snapshot)?;
3276 state.enabled && state.waiting_for_previous_pid.is_none()
3277 };
3278 let child = if should_spawn {
3279 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3280 let spawned = spawn_child(
3281 &spec,
3282 runtime.connection_file_path.as_deref(),
3283 self.supervisor_handle.as_ref(),
3284 &runtime.stderr_ring,
3285 runtime.capture_logs_dir.as_deref(),
3286 &runtime.child_roster,
3287 #[cfg(target_os = "linux")]
3288 runtime.cgroup_placement.as_ref(),
3289 );
3290 #[cfg(test)]
3291 self.test_after_first_spawn.run(&spec.module_id);
3292 match spawned {
3293 Ok(child) => {
3294 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3295 if let Some(spawn) = activity_spawn {
3296 spawn.complete(snapshot);
3297 }
3298 Some(child)
3299 }
3300 Err(err) => {
3301 if runtime.health.critical {
3302 error!(
3303 module_id = %spec.module_id,
3304 program = %spec.program.display(),
3305 error = %err,
3306 "critical configured module failed to spawn; marking failed and alerting"
3307 );
3308 } else {
3309 error!(
3310 module_id = %spec.module_id,
3311 program = %spec.program.display(),
3312 error = %err,
3313 "configured module failed to spawn; marking failed and continuing"
3314 );
3315 }
3316 update_snapshot(snapshot, Some(&spec.module_id), |state| {
3317 state.state = ModuleState::Failed;
3318 clear_current_process_facts(state);
3319 })?;
3320 None
3321 }
3322 }
3323 } else {
3324 None
3325 };
3326 self.start_monitor(spec, runtime, &module, child, commands);
3327 Ok(module)
3328 }
3329
3330 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3331 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3332 SupervisorRuntimeConfig {
3333 start_requester: None,
3334 pending_activity_restart: false,
3335 provenance_probe: self.provenance_probe.clone(),
3336 scheduled_respawn: Arc::default(),
3337 deferred_reload_reply: Arc::default(),
3338 restart_policy: self.restart_policy,
3339 drain_timeout: self.drain_timeout,
3340 child_roster: self
3343 .child_roster
3344 .for_module(Arc::clone(&effective_drain_timeout)),
3345 effective_drain_timeout,
3346 default_drain_timeout: self.drain_timeout,
3347 health: self.health.clone(),
3348 connection_file_path: self.connection_file_path.clone(),
3349 capture_logs_dir: self.capture_logs_dir.clone(),
3350 forwarding: self.forwarding.clone(),
3351 supervisor_handle: self.supervisor_handle.clone(),
3352 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3353 terminal_ring: Arc::new(Mutex::new(
3354 TerminalRing::new(
3355 TerminalRingConfig::default(),
3356 self.daemon_start_clock.started_at_ms(),
3357 )
3358 .with_start_clock(self.daemon_start_clock)
3359 .with_journal(self.terminal_journal.clone())
3360 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3361 )),
3362 spawn_events: self.spawn_events.clone(),
3363 #[cfg(target_os = "linux")]
3364 cgroup_placement: self.cgroup_placement.clone(),
3365 #[cfg(test)]
3366 test_seed_stale_facts_before_enable_spawn: false,
3367 #[cfg(test)]
3368 test_reload_exit_record_gate: None,
3369 }
3370 }
3371
3372 fn supervised_module(
3373 &self,
3374 spec: ModuleSpec,
3375 runtime: SupervisorRuntimeConfig,
3376 snapshot: SharedSnapshot,
3377 child: Option<SupervisedChild>,
3378 ) -> SupervisedModule {
3379 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3380 self.start_monitor(spec, runtime, &module, child, commands);
3381 module
3382 }
3383
3384 fn register_module(
3385 &self,
3386 spec: &ModuleSpec,
3387 runtime: &SupervisorRuntimeConfig,
3388 snapshot: SharedSnapshot,
3389 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3390 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3391 spec: spec.clone(),
3392 health: runtime.health.clone(),
3393 }));
3394 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3395 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3396 let restart_policy = runtime.restart_policy;
3400 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3401 let (tx, rx) = mpsc::channel(4);
3402 let module_id = spec.module_id.clone();
3403 let module = SupervisedModule {
3404 inner: Arc::new(SupervisedModuleInner {
3405 module_id: module_id.clone(),
3406 registry: Arc::clone(&self.registry),
3407 snapshot,
3408 configuration,
3409 stderr_ring,
3410 terminal_ring,
3411 commands: tx,
3412 monitor: Mutex::new(None),
3413 restart_policy,
3414 effective_drain_timeout,
3415 provenance_probe: self.provenance_probe.clone(),
3416 }),
3417 };
3418 if let Some(supervisor_handle) = &self.supervisor_handle {
3421 supervisor_handle.insert(module.clone());
3422 }
3423 (module, rx)
3424 }
3425
3426 fn start_monitor(
3427 &self,
3428 spec: ModuleSpec,
3429 runtime: SupervisorRuntimeConfig,
3430 module: &SupervisedModule,
3431 child: Option<SupervisedChild>,
3432 commands: mpsc::Receiver<SupervisorCommand>,
3433 ) {
3434 let monitor = tokio::spawn(supervise_loop(
3435 spec,
3436 runtime,
3437 Arc::clone(&self.registry),
3438 Arc::clone(&self.process_liveness),
3439 Arc::clone(&module.inner.snapshot),
3440 child,
3441 commands,
3442 ));
3443 *module
3444 .inner
3445 .monitor
3446 .lock()
3447 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3448 }
3449}
3450
3451impl Default for Supervisor {
3452 fn default() -> Self {
3453 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3454 }
3455}
3456
3457#[derive(Clone)]
3459pub struct SupervisedModule {
3460 inner: Arc<SupervisedModuleInner>,
3461}
3462
3463struct SupervisedModuleInner {
3464 module_id: String,
3465 registry: Arc<Registry>,
3466 snapshot: SharedSnapshot,
3467 configuration: Arc<Mutex<SupervisedConfiguration>>,
3468 stderr_ring: Arc<Mutex<StderrRing>>,
3469 terminal_ring: Arc<Mutex<TerminalRing>>,
3470 commands: mpsc::Sender<SupervisorCommand>,
3471 monitor: Mutex<Option<JoinHandle<()>>>,
3472 restart_policy: RestartPolicy,
3476 effective_drain_timeout: Arc<Mutex<Duration>>,
3477 provenance_probe: ExecutableIdentityProbe,
3478}
3479
3480impl fmt::Debug for SupervisedModule {
3481 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3482 f.debug_struct("SupervisedModule")
3483 .field("module_id", &self.inner.module_id)
3484 .field("status", &self.status())
3485 .finish_non_exhaustive()
3486 }
3487}
3488
3489impl SupervisedModule {
3490 pub fn module_id(&self) -> &str {
3491 &self.inner.module_id
3492 }
3493
3494 #[cfg(test)]
3498 pub(crate) fn record_health_probe_failure_for_test(
3499 &self,
3500 detail: &str,
3501 ) -> Result<(), SuperviseError> {
3502 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3503 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3504 state.health.detail = Some(detail.to_string());
3505 })
3506 }
3507
3508 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3509 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3510 }
3511
3512 pub(crate) fn registration_end_reason(
3513 &self,
3514 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3515 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3516 Ok(match snapshot.state {
3517 ModuleState::Draining if snapshot.draining_to_replace => {
3518 Some(RegistrationEndReason::SupervisorRestart)
3519 }
3520 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3521 ModuleState::Restarting if snapshot.draining_to_replace => {
3522 Some(RegistrationEndReason::SupervisorRestart)
3523 }
3524 _ => None,
3525 })
3526 }
3527
3528 pub fn stderr_tail(
3535 &self,
3536 max_lines: Option<usize>,
3537 max_bytes: Option<usize>,
3538 ) -> StderrTailSnapshot {
3539 self.inner
3540 .stderr_ring
3541 .lock()
3542 .unwrap_or_else(|poisoned| poisoned.into_inner())
3543 .snapshot(max_lines, max_bytes)
3544 }
3545
3546 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3551 self.inner
3552 .terminal_ring
3553 .lock()
3554 .unwrap_or_else(|poisoned| poisoned.into_inner())
3555 .snapshot()
3556 }
3557
3558 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3563 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3564 }
3565
3566 pub(crate) async fn read_durable_terminal_history(
3571 &self,
3572 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3573 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3574 let module_id = self.inner.module_id.clone();
3575 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3576 .await
3577 }
3578
3579 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3580 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3581 .map(|(status, _)| status)
3582 }
3583
3584 pub(crate) fn record_deliberate_severance(
3585 &self,
3586 identity: ProcessIdentity,
3587 ) -> Result<bool, SuperviseError> {
3588 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3589 if snapshot.pid != Some(identity.pid)
3590 || snapshot.process_start_time != Some(identity.start_time)
3591 {
3592 return Ok(false);
3593 }
3594 snapshot.deliberate_severance = Some(identity);
3595 Ok(true)
3596 }
3597
3598 pub(crate) fn status_for_control(
3603 &self,
3604 caller: &'static str,
3605 ) -> Result<ModuleStatus, SuperviseError> {
3606 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3607 .map(|(status, _)| status)
3608 }
3609
3610 fn status_with_snapshot_lock(
3611 &self,
3612 snapshot: &SharedSnapshot,
3613 caller: Option<&'static str>,
3614 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3615 let mut guard = match caller {
3616 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3617 None => lock_snapshot(snapshot)?,
3618 };
3619 let restart_count =
3622 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3623 let snapshot = guard.clone();
3624 drop(guard);
3625 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3626 SuperviseError::StatePoisoned {
3627 module_id: Some(self.inner.module_id.clone()),
3628 }
3629 })?;
3630 let registration_active = self
3631 .inner
3632 .registry
3633 .get_module(&self.inner.module_id)
3634 .map_err(SuperviseError::Registry)?
3635 .is_some();
3636 let protocol = snapshot
3637 .spawned_protocol
3638 .unwrap_or(self.declared_protocol()?);
3639 let running_process =
3640 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3641 let live = match protocol {
3647 ModuleProtocol::Subc => running_process && registration_active,
3648 ModuleProtocol::None => running_process,
3649 };
3650
3651 Ok((
3652 ModuleStatus {
3653 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3654 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3655 module_id: self.inner.module_id.clone(),
3656 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3657 state: snapshot.state,
3658 enabled: snapshot.enabled,
3659 process_alive: snapshot.process_alive,
3660 registration_active,
3661 protocol,
3662 live,
3663 restart_count,
3664 lifetime_restarts: snapshot.lifetime_restarts,
3665 spawn_generation: snapshot.spawn_generation,
3666 max_restarts: self.inner.restart_policy.max_restarts,
3667 restart_window: self.inner.restart_policy.window,
3668 drain_timeout,
3669 restart_backoff: self.inner.restart_policy.backoff,
3670 restart_max_backoff: self.inner.restart_policy.max_backoff,
3671 pid: snapshot.reported_pid(),
3672 spawned_at_ms: snapshot.spawned_at_ms,
3673 spawned_from: snapshot.spawned_from,
3674 launch_spec_digest: snapshot.launch_spec_digest,
3675 process_start_time: snapshot.process_start_time,
3676 last_exit: snapshot.last_exit,
3677 health: snapshot.health,
3678 },
3679 snapshot.spawned_file_identity.clone(),
3680 ))
3681 }
3682
3683 #[cfg(test)]
3684 pub(crate) fn hold_snapshot_for_test(
3685 &self,
3686 acquired: std::sync::mpsc::Sender<()>,
3687 hold: Duration,
3688 ) -> std::thread::JoinHandle<()> {
3689 let snapshot = Arc::clone(&self.inner.snapshot);
3690 std::thread::spawn(move || {
3691 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3692 acquired
3693 .send(())
3694 .expect("test receiver waits for snapshot lock");
3695 std::thread::sleep(hold);
3696 })
3697 }
3698
3699 pub(crate) async fn status_and_running_image_agreement(
3705 &self,
3706 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3707 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3708 let image = self
3709 .inner
3710 .provenance_probe
3711 .observe(
3712 status.pid,
3713 status.spawned_from.as_deref(),
3714 identity,
3715 status.process_start_time,
3716 )
3717 .await;
3718 Ok((status, image))
3719 }
3720
3721 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3722 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3723 Ok(snapshot) => snapshot.clone(),
3724 Err(_) => {
3725 return subc_control::RunningImageAgreement::Unavailable {
3726 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3727 };
3728 }
3729 };
3730 self.inner
3731 .provenance_probe
3732 .observe(
3733 snapshot.reported_pid(),
3734 snapshot.spawned_from.as_deref(),
3735 snapshot.spawned_file_identity.clone(),
3736 snapshot.process_start_time,
3737 )
3738 .await
3739 }
3740
3741 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3744 #[cfg(windows)]
3745 {
3746 let image = lock_snapshot(&self.inner.snapshot)
3747 .ok()
3748 .and_then(|snapshot| {
3749 snapshot
3750 .spawned_file_identity
3751 .as_ref()
3752 .and_then(|identity| identity.windows_image.clone())
3753 });
3754 match image {
3755 Some(image) => {
3756 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3757 }
3758 None => subc_control::ChildResourceUsage::Unavailable {
3759 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3760 },
3761 }
3762 }
3763 #[cfg(not(windows))]
3764 {
3765 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3766 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3767 Err(_) => {
3768 return subc_control::ChildResourceUsage::Unavailable {
3769 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3770 }
3771 }
3772 };
3773 crate::child_resources::read(pid, start_time)
3774 }
3775 }
3776
3777 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3778 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3779 Ok(match snapshot.state {
3780 ModuleState::Restarting => true,
3781 ModuleState::Failed | ModuleState::Disabled => false,
3782 _ if snapshot.enabled && snapshot.respawn_pending => true,
3783 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3784 })
3785 }
3786
3787 #[cfg(test)]
3788 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3789 self.is_warming_with_snapshot_lock(None)
3790 }
3791
3792 pub(crate) fn is_warming_for_control(
3793 &self,
3794 caller: &'static str,
3795 ) -> Result<bool, SuperviseError> {
3796 self.is_warming_with_snapshot_lock(Some(caller))
3797 }
3798
3799 fn is_warming_with_snapshot_lock(
3800 &self,
3801 caller: Option<&'static str>,
3802 ) -> Result<bool, SuperviseError> {
3803 let snapshot = match caller {
3804 Some(caller) => {
3805 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3806 }
3807 None => lock_snapshot(&self.inner.snapshot)?,
3808 }
3809 .clone();
3810 Ok(matches!(
3811 snapshot.state,
3812 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3813 ))
3814 }
3815
3816 pub async fn drain(&self) -> Result<(), SuperviseError> {
3818 self.stop().await
3819 }
3820
3821 pub(crate) async fn retire_requested(
3822 &self,
3823 requester: Option<Principal>,
3824 ) -> Result<(), SuperviseError> {
3825 match self.state()? {
3826 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3827 ModuleState::Starting
3828 | ModuleState::Running
3829 | ModuleState::Unresponsive
3830 | ModuleState::Restarting
3831 | ModuleState::Draining
3832 | ModuleState::Disabled => {}
3833 }
3834
3835 let (reply_tx, reply_rx) = oneshot::channel();
3836 self.inner
3837 .commands
3838 .send(SupervisorCommand::Retire {
3839 requester,
3840 reply: reply_tx,
3841 })
3842 .await
3843 .map_err(|_| SuperviseError::CommandClosed {
3844 module_id: self.inner.module_id.clone(),
3845 })?;
3846 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3847 module_id: self.inner.module_id.clone(),
3848 })?
3849 }
3850
3851 pub async fn stop(&self) -> Result<(), SuperviseError> {
3852 self.stop_requested(None).await
3853 }
3854
3855 pub(crate) async fn stop_requested(
3856 &self,
3857 requester: Option<Principal>,
3858 ) -> Result<(), SuperviseError> {
3859 match self.state()? {
3860 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3861 ModuleState::Starting
3862 | ModuleState::Running
3863 | ModuleState::Unresponsive
3864 | ModuleState::Restarting
3865 | ModuleState::Draining
3866 | ModuleState::Disabled => {}
3867 }
3868
3869 let (reply_tx, reply_rx) = oneshot::channel();
3870 self.inner
3871 .commands
3872 .send(SupervisorCommand::Drain {
3873 requester,
3874 reply: reply_tx,
3875 })
3876 .await
3877 .map_err(|_| SuperviseError::CommandClosed {
3878 module_id: self.inner.module_id.clone(),
3879 })?;
3880 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3881 module_id: self.inner.module_id.clone(),
3882 })?
3883 }
3884
3885 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3886 self.restart_requested(drain_timeout_ms, None).await
3887 }
3888
3889 pub(crate) async fn restart_requested(
3890 &self,
3891 drain_timeout_ms: Option<u64>,
3892 requester: Option<Principal>,
3893 ) -> Result<(), SuperviseError> {
3894 let queued_at = Instant::now();
3895 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3896 let (reply_tx, reply_rx) = oneshot::channel();
3897 self.inner
3898 .commands
3899 .send(SupervisorCommand::Restart {
3900 requester,
3901 drain_timeout_ms,
3902 received_at_generation,
3903 queued_at,
3904 reply: reply_tx,
3905 })
3906 .await
3907 .map_err(|_| SuperviseError::CommandClosed {
3908 module_id: self.inner.module_id.clone(),
3909 })?;
3910 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3911 module_id: self.inner.module_id.clone(),
3912 })?
3913 }
3914
3915 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3920 self.swap_requested(ready_timeout, None).await
3921 }
3922
3923 pub(crate) async fn swap_requested(
3924 &self,
3925 ready_timeout: Option<Duration>,
3926 requester: Option<Principal>,
3927 ) -> Result<(), SuperviseError> {
3928 let (reply_tx, reply_rx) = oneshot::channel();
3929 self.inner
3930 .commands
3931 .send(SupervisorCommand::Swap {
3932 requester,
3933 ready_timeout,
3934 reply: reply_tx,
3935 })
3936 .await
3937 .map_err(|_| SuperviseError::CommandClosed {
3938 module_id: self.inner.module_id.clone(),
3939 })?;
3940 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3941 module_id: self.inner.module_id.clone(),
3942 })?
3943 }
3944
3945 pub async fn reload(&self) -> Result<(), SuperviseError> {
3946 self.reload_requested(None).await
3947 }
3948
3949 pub(crate) async fn reload_requested(
3950 &self,
3951 requester: Option<Principal>,
3952 ) -> Result<(), SuperviseError> {
3953 let (reply_tx, reply_rx) = oneshot::channel();
3954 self.inner
3955 .commands
3956 .send(SupervisorCommand::Reload {
3957 requester,
3958 reply: reply_tx,
3959 })
3960 .await
3961 .map_err(|_| SuperviseError::CommandClosed {
3962 module_id: self.inner.module_id.clone(),
3963 })?;
3964 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3965 module_id: self.inner.module_id.clone(),
3966 })?
3967 }
3968
3969 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3970 self.set_enabled_requested(enabled, None).await
3971 }
3972
3973 pub(crate) async fn set_enabled_requested(
3974 &self,
3975 enabled: bool,
3976 requester: Option<Principal>,
3977 ) -> Result<bool, SuperviseError> {
3978 let (reply_tx, reply_rx) = oneshot::channel();
3979 self.inner
3980 .commands
3981 .send(SupervisorCommand::SetEnabled {
3982 requester,
3983 enabled,
3984 reply: reply_tx,
3985 })
3986 .await
3987 .map_err(|_| SuperviseError::CommandClosed {
3988 module_id: self.inner.module_id.clone(),
3989 })?;
3990 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3991 module_id: self.inner.module_id.clone(),
3992 })?
3993 }
3994
3995 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3999 let configured = self
4000 .inner
4001 .configuration
4002 .lock()
4003 .map_err(|_| SuperviseError::StatePoisoned {
4004 module_id: Some(self.inner.module_id.clone()),
4005 })?
4006 .spec
4007 .protocol;
4008 let state = lock_snapshot(&self.inner.snapshot)?;
4009 Ok(state.spawned_protocol.unwrap_or(configured))
4010 }
4011
4012 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4013 let configuration =
4014 self.inner
4015 .configuration
4016 .lock()
4017 .map_err(|_| SuperviseError::StatePoisoned {
4018 module_id: Some(self.inner.module_id.clone()),
4019 })?;
4020 Ok((configuration.spec.clone(), configuration.health.clone()))
4021 }
4022
4023 #[cfg(any(test, feature = "test-support"))]
4027 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4028 let (_, health) = self.configuration()?;
4029 let drain_timeout_ms = u64::try_from(
4030 self.inner
4031 .effective_drain_timeout
4032 .lock()
4033 .unwrap_or_else(|poisoned| poisoned.into_inner())
4034 .as_millis(),
4035 )
4036 .ok();
4037 self.update_configuration(spec, health, drain_timeout_ms)
4038 .await
4039 }
4040
4041 pub(crate) async fn update_configuration(
4042 &self,
4043 spec: ModuleSpec,
4044 health: HealthConfig,
4045 drain_timeout_ms: Option<u64>,
4046 ) -> Result<(), SuperviseError> {
4047 if spec.module_id != self.inner.module_id {
4048 return Err(SuperviseError::InvalidSpec {
4049 reason: "a supervised module's module_id cannot be changed".to_string(),
4050 });
4051 }
4052 validate_spec(&spec)?;
4053 let (reply_tx, reply_rx) = oneshot::channel();
4054 self.inner
4055 .commands
4056 .send(SupervisorCommand::UpdateConfiguration {
4057 spec: spec.clone(),
4058 health: health.clone(),
4059 drain_timeout_ms,
4060 reply: reply_tx,
4061 })
4062 .await
4063 .map_err(|_| SuperviseError::CommandClosed {
4064 module_id: self.inner.module_id.clone(),
4065 })?;
4066 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4067 module_id: self.inner.module_id.clone(),
4068 })?;
4069 let mut configuration =
4070 self.inner
4071 .configuration
4072 .lock()
4073 .map_err(|_| SuperviseError::StatePoisoned {
4074 module_id: Some(self.inner.module_id.clone()),
4075 })?;
4076 configuration.spec = spec;
4077 configuration.health = health;
4078 Ok(())
4079 }
4080}
4081
4082impl Drop for SupervisedModuleInner {
4083 fn drop(&mut self) {
4084 let Ok(mut monitor) = self.monitor.lock() else {
4085 return;
4086 };
4087 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4088 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4089 state.state = ModuleState::Stopped;
4090 clear_current_process_facts(state);
4091 });
4092 monitor.abort();
4093 }
4094 let _ = monitor.take();
4095 }
4096}
4097
4098#[derive(Debug)]
4099enum SupervisorCommand {
4100 Drain {
4101 requester: Option<Principal>,
4102 reply: oneshot::Sender<Result<(), SuperviseError>>,
4103 },
4104 Retire {
4105 requester: Option<Principal>,
4106 reply: oneshot::Sender<Result<(), SuperviseError>>,
4107 },
4108 Restart {
4109 requester: Option<Principal>,
4110 drain_timeout_ms: Option<u64>,
4115 received_at_generation: u64,
4119 queued_at: Instant,
4122 reply: oneshot::Sender<Result<(), SuperviseError>>,
4123 },
4124 Reload {
4125 requester: Option<Principal>,
4126 reply: oneshot::Sender<Result<(), SuperviseError>>,
4127 },
4128 SetEnabled {
4129 requester: Option<Principal>,
4130 enabled: bool,
4131 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4132 },
4133 UpdateConfiguration {
4134 spec: ModuleSpec,
4135 health: HealthConfig,
4136 drain_timeout_ms: Option<u64>,
4139 reply: oneshot::Sender<()>,
4140 },
4141 Swap {
4142 requester: Option<Principal>,
4143 ready_timeout: Option<Duration>,
4146 reply: oneshot::Sender<Result<(), SuperviseError>>,
4148 },
4149}
4150
4151#[derive(Debug)]
4152pub enum SuperviseError {
4153 InvalidSpec {
4154 reason: String,
4155 },
4156 Spawn {
4157 program: PathBuf,
4158 source: io::Error,
4159 cgroup_path: Option<PathBuf>,
4160 },
4161 Cgroup {
4162 module_id: String,
4163 source: io::Error,
4164 },
4165 LaunchNonce {
4168 reason: String,
4169 },
4170 Wait {
4171 module_id: String,
4172 source: io::Error,
4173 },
4174 Kill {
4175 module_id: String,
4176 source: io::Error,
4177 },
4178 Forwarding(ForwardingError),
4179 Registry(RegistryError),
4180 ReloadUnavailable {
4181 module_id: String,
4182 reason: String,
4183 },
4184 Disabled {
4189 module_id: String,
4190 },
4191 ReloadFailed {
4192 module_id: String,
4193 reason: String,
4194 },
4195 RegistrationStillActive {
4196 module_id: String,
4197 waited: Duration,
4198 },
4199 StatePoisoned {
4200 module_id: Option<String>,
4201 },
4202 CommandClosed {
4203 module_id: String,
4204 },
4205 SwapInProgress {
4209 module_id: String,
4210 },
4211 SwapRefused {
4213 module_id: String,
4214 reason: SwapRefusal,
4215 },
4216 SwapFailed {
4220 module_id: String,
4221 arm: SwapFailureArm,
4222 detail: String,
4223 candidate_exit: Option<ExitReport>,
4226 },
4227}
4228
4229#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4231pub enum SwapRefusal {
4232 OverlapExclusive,
4234 NotRegistered,
4237 ProtocolNone,
4240 NotConfigured,
4243 AlreadySwapping,
4245}
4246
4247impl SwapRefusal {
4248 pub fn as_str(self) -> &'static str {
4249 match self {
4250 Self::OverlapExclusive => "overlap_exclusive",
4251 Self::NotRegistered => "not_registered",
4252 Self::ProtocolNone => "protocol_none",
4253 Self::NotConfigured => "not_configured",
4254 Self::AlreadySwapping => "already_swapping",
4255 }
4256 }
4257}
4258
4259#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4262pub enum SwapFailureArm {
4263 SpawnFailed,
4265 NeverRegistered,
4267 NeverReady,
4269 CandidateExited,
4271 CandidateUnhealthy,
4273 Interrupted,
4277 CutoverLost,
4282}
4283
4284impl SwapFailureArm {
4285 pub fn as_str(self) -> &'static str {
4286 match self {
4287 Self::SpawnFailed => "spawn_failed",
4288 Self::NeverRegistered => "never_registered",
4289 Self::NeverReady => "never_ready",
4290 Self::CandidateExited => "candidate_exited",
4291 Self::CandidateUnhealthy => "candidate_unhealthy",
4292 Self::Interrupted => "interrupted",
4293 Self::CutoverLost => "cutover_lost",
4294 }
4295 }
4296}
4297
4298impl fmt::Display for SuperviseError {
4299 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4300 match self {
4301 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4302 Self::Spawn {
4303 program,
4304 source,
4305 cgroup_path: Some(cgroup_path),
4306 } => write!(
4307 f,
4308 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4309 cgroup_path.display(),
4310 program.display()
4311 ),
4312 Self::Spawn {
4313 program,
4314 source,
4315 cgroup_path: None,
4316 } => write!(
4317 f,
4318 "failed to spawn module '{}': {source}",
4319 program.display()
4320 ),
4321 Self::Cgroup { module_id, source } => {
4322 write!(
4323 f,
4324 "failed to prepare cgroup for module '{module_id}': {source}"
4325 )
4326 }
4327 Self::LaunchNonce { reason } => {
4328 write!(
4329 f,
4330 "failed to generate reserved-module launch nonce: {reason}"
4331 )
4332 }
4333 Self::Wait { module_id, source } => {
4334 write!(f, "failed to wait for module '{module_id}': {source}")
4335 }
4336 Self::Kill { module_id, source } => {
4337 write!(f, "failed to kill module '{module_id}': {source}")
4338 }
4339 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4340 Self::Registry(err) => write!(f, "registry error: {err}"),
4341 Self::ReloadUnavailable { module_id, reason } => {
4342 write!(f, "reload unavailable for module '{module_id}': {reason}")
4343 }
4344 Self::Disabled { module_id } => {
4345 write!(
4346 f,
4347 "module '{module_id}' is disabled; enable it before restart or reload"
4348 )
4349 }
4350 Self::ReloadFailed { module_id, reason } => {
4351 write!(f, "reload failed for module '{module_id}': {reason}")
4352 }
4353 Self::RegistrationStillActive { module_id, waited } => write!(
4354 f,
4355 "module '{module_id}' registration remained active after waiting {waited:?}"
4356 ),
4357 Self::StatePoisoned { module_id } => match module_id {
4358 Some(module_id) => {
4359 write!(f, "supervisor state for module '{module_id}' was poisoned")
4360 }
4361 None => write!(f, "supervisor state was poisoned"),
4362 },
4363 Self::CommandClosed { module_id } => {
4364 write!(
4365 f,
4366 "supervisor command channel for module '{module_id}' is closed"
4367 )
4368 }
4369 Self::SwapInProgress { module_id } => write!(
4370 f,
4371 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4372 ),
4373 Self::SwapRefused { module_id, reason } => match reason {
4374 SwapRefusal::OverlapExclusive => write!(
4375 f,
4376 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4377 ),
4378 SwapRefusal::NotRegistered => write!(
4379 f,
4380 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4381 ),
4382 SwapRefusal::ProtocolNone => write!(
4383 f,
4384 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4385 ),
4386 SwapRefusal::NotConfigured => write!(
4387 f,
4388 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4389 ),
4390 SwapRefusal::AlreadySwapping => {
4391 write!(f, "module '{module_id}' is already being swapped")
4392 }
4393 },
4394 Self::SwapFailed {
4395 module_id,
4396 arm,
4397 detail,
4398 ..
4399 } => write!(
4400 f,
4401 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4402 arm.as_str()
4403 ),
4404 }
4405 }
4406}
4407
4408impl Error for SuperviseError {
4409 fn source(&self) -> Option<&(dyn Error + 'static)> {
4410 match self {
4411 Self::Spawn { source, .. }
4412 | Self::Cgroup { source, .. }
4413 | Self::Wait { source, .. }
4414 | Self::Kill { source, .. } => Some(source),
4415 Self::Forwarding(err) => Some(err),
4416 Self::Registry(err) => Some(err),
4417 Self::LaunchNonce { .. }
4418 | Self::InvalidSpec { .. }
4419 | Self::ReloadUnavailable { .. }
4420 | Self::Disabled { .. }
4421 | Self::ReloadFailed { .. }
4422 | Self::RegistrationStillActive { .. }
4423 | Self::StatePoisoned { .. }
4424 | Self::CommandClosed { .. }
4425 | Self::SwapInProgress { .. }
4426 | Self::SwapRefused { .. }
4427 | Self::SwapFailed { .. } => None,
4428 }
4429 }
4430}
4431
4432pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4433 if spec.module_id.trim().is_empty() {
4434 return Err(SuperviseError::InvalidSpec {
4435 reason: "module_id must not be empty".to_string(),
4436 });
4437 }
4438
4439 Ok(())
4440}
4441
4442#[derive(Debug, Default)]
4443struct HealthProbeRuntime {
4444 configured_health: Option<HealthConfig>,
4445 registered_connection: Option<crate::ConnectionId>,
4446 advertised: bool,
4447 next_probe_at: Option<Instant>,
4448 probe_index: u64,
4449}
4450
4451fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4452 lock_snapshot(snapshot)
4453 .ok()
4454 .and_then(|state| state.spawned_protocol)
4455 .unwrap_or(spec.protocol)
4456}
4457
4458impl HealthProbeRuntime {
4459 fn refresh_registration(
4460 &mut self,
4461 spec: &ModuleSpec,
4462 runtime: &SupervisorRuntimeConfig,
4463 registry: &Registry,
4464 snapshot: &SharedSnapshot,
4465 ) {
4466 if self.configured_health.as_ref() != Some(&runtime.health) {
4467 self.configured_health = Some(runtime.health.clone());
4468 self.next_probe_at = None;
4469 self.registered_connection = None;
4470 self.probe_index = 0;
4471 }
4472 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4476 self.registered_connection = None;
4477 self.advertised = runtime.health.http.is_some();
4478 if !self.advertised {
4479 self.next_probe_at = None;
4480 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4481 let unknown = ModuleHealthStatus::default();
4482 if state.health != unknown {
4483 state.health = unknown;
4484 }
4485 });
4486 } else if self.next_probe_at.is_none() {
4487 self.next_probe_at = Some(
4488 Instant::now()
4489 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4490 );
4491 }
4492 return;
4493 }
4494
4495 let registration = match registry.get_module(&spec.module_id) {
4496 Ok(registration) => registration,
4497 Err(err) => {
4498 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4499 self.advertised = false;
4500 self.next_probe_at = None;
4501 return;
4502 }
4503 };
4504
4505 let Some(registration) = registration else {
4506 self.registered_connection = None;
4507 self.advertised = false;
4508 self.next_probe_at = None;
4509 return;
4510 };
4511
4512 let advertised = registration
4513 .control_ops
4514 .iter()
4515 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4516 if !advertised {
4517 self.registered_connection = Some(registration.connection_id);
4518 self.advertised = false;
4519 self.next_probe_at = None;
4520 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4521 if state.health.status != SupervisorHealthStatus::Unknown
4522 || state.health.consecutive_failures != 0
4523 || state.health.last_probe_ms.is_some()
4524 || state.health.detail.is_some()
4525 || state.health.metrics.is_some()
4526 {
4527 state.health.status = SupervisorHealthStatus::Unknown;
4528 state.health.consecutive_failures = 0;
4529 state.health.last_probe_ms = None;
4530 state.health.detail = None;
4531 state.health.metrics = None;
4532 }
4533 });
4534 return;
4535 }
4536
4537 let reregistered = self.registered_connection != Some(registration.connection_id);
4538 self.registered_connection = Some(registration.connection_id);
4539 self.advertised = true;
4540 if reregistered || self.next_probe_at.is_none() {
4541 self.probe_index = 0;
4542 self.next_probe_at = Some(
4543 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4544 );
4545 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4546 state.health.status = SupervisorHealthStatus::Unknown;
4547 state.health.consecutive_failures = 0;
4548 state.health.detail = None;
4549 state.health.metrics = None;
4550 });
4551 }
4552 }
4553
4554 fn wake_after(&self) -> Option<Duration> {
4555 if !self.advertised {
4556 return None;
4557 }
4558 self.next_probe_at
4559 .map(|next| next.saturating_duration_since(Instant::now()))
4560 }
4561
4562 fn due(&self) -> bool {
4563 self.advertised
4564 && self
4565 .next_probe_at
4566 .is_some_and(|next| Instant::now() >= next)
4567 }
4568
4569 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4570 self.probe_index = self.probe_index.wrapping_add(1);
4571 self.next_probe_at = Some(
4572 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4573 );
4574 }
4575}
4576
4577#[derive(Debug)]
4612enum HealthProbeEvidence {
4613 LaneDead,
4615 NoAnswer,
4617 BadAnswer,
4619 Misconfigured,
4621}
4622
4623#[derive(Debug)]
4624struct HealthProbeError {
4625 evidence: HealthProbeEvidence,
4626 message: String,
4627}
4628
4629impl HealthProbeError {
4630 fn lane_dead(message: impl Into<String>) -> Self {
4631 Self::with(HealthProbeEvidence::LaneDead, message)
4632 }
4633
4634 fn no_answer(message: impl Into<String>) -> Self {
4635 Self::with(HealthProbeEvidence::NoAnswer, message)
4636 }
4637
4638 fn bad_answer(message: impl Into<String>) -> Self {
4639 Self::with(HealthProbeEvidence::BadAnswer, message)
4640 }
4641
4642 fn misconfigured(message: impl Into<String>) -> Self {
4643 Self::with(HealthProbeEvidence::Misconfigured, message)
4644 }
4645
4646 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4647 Self {
4648 evidence,
4649 message: message.into(),
4650 }
4651 }
4652
4653 #[allow(dead_code)]
4667 fn is_proof_of_death(&self) -> bool {
4668 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4669 }
4670
4671 fn label(&self) -> &'static str {
4679 match self.evidence {
4680 HealthProbeEvidence::LaneDead => "lane-dead",
4681 HealthProbeEvidence::NoAnswer => "no-answer",
4682 HealthProbeEvidence::BadAnswer => "bad-answer",
4683 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4684 }
4685 }
4686}
4687
4688impl fmt::Display for HealthProbeError {
4689 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4690 f.write_str(&self.message)
4691 }
4692}
4693
4694async fn run_health_probe_cycle(
4695 spec: &ModuleSpec,
4696 runtime: &SupervisorRuntimeConfig,
4697 registry: &Registry,
4698 process_liveness: &SupervisorProcessLiveness,
4699 snapshot: &SharedSnapshot,
4700 child: &mut Option<SupervisedChild>,
4701) {
4702 let now_ms = unix_ms_now();
4703 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4704 .then_some(runtime.health.http.as_deref())
4705 .flatten();
4706 let result = match http {
4707 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4708 None => probe_module_health(&spec.module_id, runtime, None).await,
4709 };
4710 match result {
4711 Ok(report) => {
4712 handle_health_report(
4713 spec,
4714 runtime,
4715 registry,
4716 process_liveness,
4717 snapshot,
4718 child,
4719 report,
4720 now_ms,
4721 )
4722 .await;
4723 }
4724 Err(err) => {
4725 if http.is_some() {
4726 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4727 state.health.status = SupervisorHealthStatus::Failing;
4728 });
4729 }
4730 handle_health_probe_failure(
4731 spec,
4732 runtime,
4733 registry,
4734 process_liveness,
4735 snapshot,
4736 child,
4737 err,
4738 now_ms,
4739 )
4740 .await;
4741 }
4742 }
4743}
4744
4745pub(crate) struct HttpProbeTarget<'a> {
4746 address: std::net::SocketAddr,
4747 localhost: bool,
4748 authority: &'a str,
4749 path: String,
4750}
4751
4752pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4755 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4756 return Err("must not contain whitespace, controls, or a fragment".into());
4757 }
4758 let rest = url
4759 .strip_prefix("http://")
4760 .ok_or("must use plain http://")?;
4761 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4762 let (authority, suffix) = rest.split_at(split);
4763 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4764 ("::1", rest)
4765 } else {
4766 let split = authority.find(':').unwrap_or(authority.len());
4767 authority.split_at(split)
4768 };
4769 let ip: std::net::IpAddr = match host {
4770 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4771 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4772 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4773 };
4774 let port = if port.is_empty() {
4775 80
4776 } else {
4777 port.strip_prefix(':')
4778 .and_then(|p| p.parse::<u16>().ok())
4779 .filter(|p| *p > 0)
4780 .ok_or("must have a valid nonzero TCP port")?
4781 };
4782 let path = if suffix.is_empty() {
4783 "/".into()
4784 } else if suffix.starts_with('?') {
4785 format!("/{suffix}")
4786 } else {
4787 suffix.into()
4788 };
4789 Ok(HttpProbeTarget {
4790 address: std::net::SocketAddr::new(ip, port),
4791 localhost: host == "localhost",
4792 authority,
4793 path,
4794 })
4795}
4796
4797async fn probe_http_health(
4798 url: &str,
4799 deadline: Duration,
4800) -> Result<HealthReport, HealthProbeError> {
4801 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4802 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4803 let mut response_status = String::new();
4806 let mut body = Vec::new();
4807 let probe = async {
4808 let connection = match tokio::net::TcpStream::connect(target.address).await {
4811 Err(_) if target.localhost => {
4812 tokio::net::TcpStream::connect((
4813 std::net::Ipv6Addr::LOCALHOST,
4814 target.address.port(),
4815 ))
4816 .await
4817 }
4818 result => result,
4819 };
4820 let mut stream = connection.map_err(|error| {
4821 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4822 })?;
4823 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4824 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4825 let mut reader = BufReader::new(stream);
4826 let mut budget = 16 * 1024;
4827 let status = http_line(&mut reader, &mut budget).await?;
4828 let mut words = status.split_ascii_whitespace();
4829 let version = words.next();
4830 let code = words
4831 .next()
4832 .filter(|word| word.len() == 3)
4833 .and_then(|word| word.parse::<u16>().ok());
4834 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4835 || !code.is_some_and(|code| (100..600).contains(&code))
4836 {
4837 return Err(HealthProbeError::bad_answer(format!(
4838 "invalid HTTP status: {status}"
4839 )));
4840 }
4841 let code = code.expect("validated status code");
4842 response_status = status.clone();
4843 let mut length = None;
4844 let mut chunked = false;
4845 loop {
4846 let line = http_line(&mut reader, &mut budget).await?;
4847 if line.is_empty() {
4848 break;
4849 }
4850 if let Some((name, value)) = line.split_once(':') {
4851 if name.eq_ignore_ascii_case("content-length") {
4852 length = Some(value.trim().parse::<u64>().map_err(|_| {
4853 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4854 })?);
4855 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4856 chunked = value.trim().eq_ignore_ascii_case("chunked");
4857 }
4858 }
4859 }
4860 if chunked {
4861 while body.len() < 200 {
4862 let line = http_line(&mut reader, &mut budget).await?;
4863 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4864 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4865 if size == 0 {
4866 break;
4867 }
4868 let count = size.min((200 - body.len()) as u64) as usize;
4869 let start = body.len();
4870 (&mut reader)
4871 .take(count as u64)
4872 .read_to_end(&mut body)
4873 .await
4874 .map_err(|error| {
4875 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4876 })?;
4877 if body.len() - start != count {
4878 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4879 }
4880 if size > count as u64 || body.len() == 200 {
4881 break;
4882 }
4883 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4884 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4885 }
4886 }
4887 } else {
4888 reader
4889 .take(length.unwrap_or(200).min(200))
4890 .read_to_end(&mut body)
4891 .await
4892 .map_err(|error| {
4893 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4894 })?;
4895 }
4896 if (200..300).contains(&code) {
4897 Ok(HealthReport::ok())
4898 } else {
4899 Err(HealthProbeError::bad_answer(
4900 "HTTP health endpoint returned non-2xx",
4901 ))
4902 }
4903 };
4904 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4905 Err(HealthProbeError::no_answer(format!(
4906 "HTTP probe timed out after {deadline:?}"
4907 )))
4908 });
4909 if let Err(error) = &mut result {
4910 if !response_status.is_empty() {
4911 error.message = format!(
4912 "{}; {response_status}: {}",
4913 error.message,
4914 String::from_utf8_lossy(&body)
4915 );
4916 }
4917 }
4918 result
4919}
4920
4921async fn http_line(
4922 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4923 remaining: &mut usize,
4924) -> Result<String, HealthProbeError> {
4925 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4926 let mut line = Vec::new();
4927 (&mut *reader)
4928 .take(*remaining as u64)
4929 .read_until(b'\n', &mut line)
4930 .await
4931 .map_err(|error| {
4932 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4933 })?;
4934 *remaining -= line.len();
4935 if !line.ends_with(b"\r\n") {
4936 return Err(HealthProbeError::bad_answer(
4937 "HTTP headers are incomplete or exceed 16 KiB",
4938 ));
4939 }
4940 line.truncate(line.len() - 2);
4941 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4942}
4943
4944async fn probe_module_health(
4945 module_id: &str,
4946 runtime: &SupervisorRuntimeConfig,
4947 drain_deadline: Option<Instant>,
4948) -> Result<HealthReport, HealthProbeError> {
4949 let Some(forwarding) = runtime.forwarding.as_ref() else {
4950 return Err(HealthProbeError::misconfigured(
4951 "supervisor was not configured with a forwarding table",
4952 ));
4953 };
4954 let probe_started_at = Instant::now();
4955 let mut deadline = probe_started_at + runtime.health.deadline;
4956 if let Some(drain_deadline) = drain_deadline {
4957 deadline = deadline.min(drain_deadline);
4958 }
4959 let pending = if drain_deadline.is_some() {
4960 forwarding.begin_drain_health_probe_rpc_for(
4961 module_id,
4962 MODULE_CONTROL_OP_HEALTH_CHECK,
4963 probe_started_at,
4964 deadline,
4965 )
4966 } else {
4967 forwarding.begin_health_probe_rpc_for(
4968 module_id,
4969 MODULE_CONTROL_OP_HEALTH_CHECK,
4970 probe_started_at,
4971 deadline,
4972 )
4973 }
4974 .map_err(|err| {
4975 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4978 })?;
4979 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4980}
4981
4982async fn probe_endpoint_health(
4989 endpoint: crate::ModuleEndpointId,
4990 runtime: &SupervisorRuntimeConfig,
4991 deadline_cap: Option<Instant>,
4992) -> Result<HealthReport, HealthProbeError> {
4993 let Some(forwarding) = runtime.forwarding.as_ref() else {
4994 return Err(HealthProbeError::misconfigured(
4995 "supervisor was not configured with a forwarding table",
4996 ));
4997 };
4998 let probe_started_at = Instant::now();
4999 let mut deadline = probe_started_at + runtime.health.deadline;
5000 if let Some(cap) = deadline_cap {
5001 deadline = deadline.min(cap);
5002 }
5003 let pending = forwarding
5004 .begin_endpoint_health_probe_rpc_for(
5005 endpoint,
5006 MODULE_CONTROL_OP_HEALTH_CHECK,
5007 probe_started_at,
5008 deadline,
5009 )
5010 .map_err(|err| {
5011 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5012 })?;
5013 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5014}
5015
5016async fn await_health_probe(
5018 forwarding: &ForwardingTable,
5019 pending: PendingModuleControlRpc,
5020 deadline: Instant,
5021 probe_budget: Duration,
5022) -> Result<HealthReport, HealthProbeError> {
5023 let PendingModuleControlRpc {
5024 endpoint,
5025 module_sink,
5026 negotiated_ver,
5027 corr,
5028 receiver,
5029 } = pending;
5030 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5031 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5032 })?;
5033 let frame = Frame::build_with_version(
5034 negotiated_ver,
5035 FrameType::Request,
5036 control_flags(),
5037 0,
5038 0,
5039 corr,
5040 body,
5041 )
5042 .map_err(|err| {
5043 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5044 })?;
5045
5046 match timeout_at(deadline, module_sink.send(frame)).await {
5052 Ok(Ok(())) => {}
5053 Ok(Err(err)) => {
5054 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5055 return Err(HealthProbeError::lane_dead(format!(
5058 "failed to send health.check: {err}"
5059 )));
5060 }
5061 Err(_elapsed) => {
5062 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5063 return Err(HealthProbeError::no_answer(
5067 "health.check send timed out before enqueue (module egress full)",
5068 ));
5069 }
5070 }
5071
5072 match timeout_at(deadline, receiver).await {
5073 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5077 response.health_report().ok_or_else(|| {
5078 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5079 })
5080 }
5081 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5082 format!("health.check rejected: {}", body.message),
5083 )),
5084 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5085 Err(HealthProbeError::lane_dead(message))
5086 }
5087 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5088 Err(HealthProbeError::bad_answer(message))
5089 }
5090 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5091 Err(HealthProbeError::bad_answer(format!(
5092 "expected module-control op '{expected}', got '{actual}'"
5093 )))
5094 }
5095 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5099 "module answered health.check after its daemon deadline",
5100 )),
5101 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5102 "health.check waiter was canceled before the module responded",
5103 )),
5104 Err(_) => {
5105 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5106 Err(HealthProbeError::no_answer(format!(
5107 "module did not answer health.check within {probe_budget:?}"
5108 )))
5109 }
5110 }
5111}
5112
5113#[allow(clippy::too_many_arguments)]
5114async fn handle_health_report(
5115 spec: &ModuleSpec,
5116 runtime: &SupervisorRuntimeConfig,
5117 registry: &Registry,
5118 process_liveness: &SupervisorProcessLiveness,
5119 snapshot: &SharedSnapshot,
5120 child: &mut Option<SupervisedChild>,
5121 report: HealthReport,
5122 now_ms: u64,
5123) {
5124 let status = supervisor_health_status(report.status);
5125 let detail = report.detail.clone();
5126 let metrics = truncate_health_metrics(report.metrics);
5127 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5128 state.health.status = status;
5129 state.health.last_probe_ms = Some(now_ms);
5130 state.health.detail = detail.clone();
5131 state.health.metrics = metrics.clone();
5132 state.health.consecutive_failures = 0;
5133 });
5134
5135 let action = match report.status {
5136 HealthStatus::Ok => return,
5137 HealthStatus::Degraded => runtime.health.on_degraded,
5138 HealthStatus::Failing => runtime.health.on_failing,
5139 };
5140 apply_l3_health_action(
5141 spec,
5142 runtime,
5143 registry,
5144 process_liveness,
5145 snapshot,
5146 child,
5147 status,
5148 detail.as_deref(),
5149 action,
5150 now_ms,
5151 )
5152 .await;
5153}
5154
5155#[allow(clippy::too_many_arguments)]
5156async fn handle_health_probe_failure(
5157 spec: &ModuleSpec,
5158 runtime: &SupervisorRuntimeConfig,
5159 registry: &Registry,
5160 process_liveness: &SupervisorProcessLiveness,
5161 snapshot: &SharedSnapshot,
5162 child: &mut Option<SupervisedChild>,
5163 err: HealthProbeError,
5164 now_ms: u64,
5165) {
5166 let threshold = runtime.health.failure_threshold.max(1);
5167 let mut failures = 0;
5168 let detail = format!("[{}] {err}", err.label());
5173 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5174 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5177 state.health.status = SupervisorHealthStatus::Unknown;
5178 }
5179 state.health.last_probe_ms = Some(now_ms);
5180 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5181 state.health.detail = Some(detail.clone());
5182 state.health.metrics = None;
5183 failures = state.health.consecutive_failures;
5184 });
5185
5186 if failures < threshold {
5187 warn!(
5188 module_id = %spec.module_id,
5189 consecutive_failures = failures,
5190 threshold,
5191 evidence = err.label(),
5192 detail = %detail,
5193 "health.check probe failed"
5194 );
5195 return;
5196 }
5197
5198 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5199 state.state = ModuleState::Unresponsive;
5200 state.health.status = SupervisorHealthStatus::Unresponsive;
5201 });
5202 if runtime.health.critical {
5206 error!(
5207 module_id = %spec.module_id,
5208 status = "unresponsive",
5209 evidence = err.label(),
5210 detail = %detail,
5211 "critical module health alert"
5212 );
5213 } else {
5214 warn!(
5215 module_id = %spec.module_id,
5216 status = "unresponsive",
5217 evidence = err.label(),
5218 detail = %detail,
5219 "module health threshold breached"
5220 );
5221 }
5222 if let Err(err) = health_restart_child(
5223 spec,
5224 runtime,
5225 registry,
5226 process_liveness,
5227 snapshot,
5228 child,
5229 SupervisorHealthStatus::Unresponsive,
5230 Some(&detail),
5231 now_ms,
5232 )
5233 .await
5234 {
5235 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5236 }
5237}
5238
5239#[allow(clippy::too_many_arguments)]
5240async fn apply_l3_health_action(
5241 spec: &ModuleSpec,
5242 runtime: &SupervisorRuntimeConfig,
5243 registry: &Registry,
5244 process_liveness: &SupervisorProcessLiveness,
5245 snapshot: &SharedSnapshot,
5246 child: &mut Option<SupervisedChild>,
5247 status: SupervisorHealthStatus,
5248 detail: Option<&str>,
5249 action: HealthAction,
5250 now_ms: u64,
5251) {
5252 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5253 match action {
5254 HealthAction::Report => {
5255 info!(
5256 module_id = %spec.module_id,
5257 status = ?status,
5258 detail,
5259 "module reported non-ok health"
5260 );
5261 }
5262 HealthAction::Alert => {
5263 error!(
5264 module_id = %spec.module_id,
5265 status = ?status,
5266 detail,
5267 "module health alert"
5268 );
5269 }
5270 HealthAction::Restart => {
5271 if let Err(err) = health_restart_child(
5272 spec,
5273 runtime,
5274 registry,
5275 process_liveness,
5276 snapshot,
5277 child,
5278 status,
5279 detail,
5280 now_ms,
5281 )
5282 .await
5283 {
5284 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5285 }
5286 }
5287 }
5288}
5289
5290#[allow(clippy::too_many_arguments)]
5291async fn health_restart_child(
5292 spec: &ModuleSpec,
5293 runtime: &SupervisorRuntimeConfig,
5294 registry: &Registry,
5295 process_liveness: &SupervisorProcessLiveness,
5296 snapshot: &SharedSnapshot,
5297 child: &mut Option<SupervisedChild>,
5298 status: SupervisorHealthStatus,
5299 detail: Option<&str>,
5300 now_ms: u64,
5301) -> Result<(), SuperviseError> {
5302 let (enabled, schedule) = {
5303 let mut state = lock_snapshot(snapshot)?;
5304 let enabled = state.enabled;
5305 let schedule = if enabled {
5306 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5307 } else {
5308 None
5309 };
5310 (enabled, schedule)
5311 };
5312
5313 if !enabled {
5314 return Err(SuperviseError::Disabled {
5315 module_id: spec.module_id.clone(),
5316 });
5317 }
5318
5319 if schedule.is_none() {
5320 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5321 error!(
5322 module_id = %spec.module_id,
5323 status = ?status,
5324 detail,
5325 max_restarts = runtime.restart_policy.max_restarts,
5326 window_secs = runtime.restart_policy.window.as_secs(),
5327 reason = %runtime.restart_policy.budget_exhausted_detail(),
5328 "health restart budget exhausted; marking module failed"
5329 );
5330 let stop_notice = begin_forwarding_drain_if_configured(
5331 spec,
5332 runtime,
5333 registry,
5334 snapshot,
5335 Some(true),
5336 RouteCloseReason::Disable,
5337 )
5338 .await?;
5339 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5340 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5341 })?;
5342 drain_optional_child(
5343 &spec.module_id,
5344 spec.protocol,
5345 stop_notice,
5346 registry,
5347 runtime.forwarding.as_deref(),
5348 snapshot,
5349 &runtime.terminal_ring,
5350 &runtime.spawn_events,
5351 child,
5352 runtime.drain_timeout,
5353 ModuleState::Failed,
5354 Some(true),
5355 )
5356 .await?;
5357 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5358 return Ok(());
5359 }
5360
5361 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5362 let mut restart_count = 0;
5363 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5364 restart_count = state.crash_restarts.len();
5365 state.state = ModuleState::Unresponsive;
5366 state.health.status = status;
5367 state.health.last_action = Some(HealthAction::Restart.to_string());
5368 state.health.last_action_ms = Some(now_ms);
5369 })?;
5370 warn!(
5371 module_id = %spec.module_id,
5372 status = ?status,
5373 detail,
5374 restart_count,
5375 restart_in_window = schedule.restart_in_window,
5376 delay_ms = schedule.delay.as_millis() as u64,
5377 "health-triggered module restart"
5378 );
5379
5380 let stop_notice = begin_forwarding_drain_if_configured(
5381 spec,
5382 runtime,
5383 registry,
5384 snapshot,
5385 Some(true),
5386 RouteCloseReason::Restart,
5387 )
5388 .await?;
5389 drain_optional_child(
5390 &spec.module_id,
5391 spec.protocol,
5392 stop_notice,
5393 registry,
5394 runtime.forwarding.as_deref(),
5395 snapshot,
5396 &runtime.terminal_ring,
5397 &runtime.spawn_events,
5398 child,
5399 runtime.drain_timeout,
5400 ModuleState::Restarting,
5401 Some(true),
5402 )
5403 .await?;
5404 schedule_respawn(
5405 runtime,
5406 snapshot,
5407 &spec.module_id,
5408 schedule.delay,
5409 RespawnKind::Spawn,
5410 )
5411}
5412
5413fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5414 if let Some(reply) = runtime
5415 .deferred_reload_reply
5416 .lock()
5417 .unwrap_or_else(|p| p.into_inner())
5418 .take()
5419 {
5420 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5421 module_id: module_id.to_string(),
5422 reason: reason.to_string(),
5423 }));
5424 }
5425}
5426
5427fn schedule_respawn(
5428 runtime: &SupervisorRuntimeConfig,
5429 snapshot: &SharedSnapshot,
5430 module_id: &str,
5431 delay: Duration,
5432 kind: RespawnKind,
5433) -> Result<(), SuperviseError> {
5434 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5435 update_snapshot(snapshot, Some(module_id), |state| {
5436 state.respawn_pending = true
5437 })?;
5438 *runtime
5439 .scheduled_respawn
5440 .lock()
5441 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5442 deadline: Instant::now() + delay,
5443 kind,
5444 });
5445 Ok(())
5446}
5447
5448fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5449 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5450 state.health.last_action = Some(action);
5451 state.health.last_action_ms = Some(now_ms);
5452 });
5453}
5454
5455fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5456 match status {
5457 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5458 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5459 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5460 }
5461}
5462
5463fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5475 let metrics = metrics?;
5476 match serde_json::to_vec(&metrics) {
5477 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5478 "truncated": true,
5479 "original_bytes": encoded.len(),
5480 })),
5481 Ok(_) | Err(_) => Some(metrics),
5482 }
5483}
5484
5485fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5491 if cadence.is_zero() {
5492 return Duration::ZERO;
5493 }
5494 let cadence_ms = cadence.as_millis() as u64;
5495 if cadence_ms == 0 {
5511 return cadence;
5512 }
5513 let jitter_span = (cadence_ms / 10).max(1);
5528 let hash = module_id.as_bytes().iter().fold(
5529 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5530 |acc, byte| {
5531 acc.wrapping_mul(1099511628211)
5532 .wrapping_add(u64::from(*byte))
5533 },
5534 );
5535 cadence + Duration::from_millis(hash % jitter_span)
5536}
5537
5538#[cfg(test)]
5539mod tests {
5540 use super::*;
5541
5542 #[test]
5543 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5544 let handle = SupervisorHandle::new();
5545 let module_id = "readded-tombstone";
5546 handle.record_rescan_removal(module_id);
5547 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5548
5549 handle.apply_identity_configuration(&ModuleSpec {
5550 module_id: module_id.to_string(),
5551 program: PathBuf::from("/test/module"),
5552 args: Vec::new(),
5553 env: Vec::new(),
5554 reserved: false,
5555 reserved_prefixes: Vec::new(),
5556 protocol: ModuleProtocol::Subc,
5557 overlap: Default::default(),
5558 });
5559
5560 assert!(
5561 handle.removal_tombstone_age_ms(module_id).is_none(),
5562 "a re-added module must not retain a stale removal tombstone"
5563 );
5564 }
5565
5566 #[derive(Debug, PartialEq, Eq)]
5569 struct OwnerInSpawnWindow {
5570 module_id: String,
5571 configured: bool,
5572 on_roster: bool,
5573 admission_refusal: Option<&'static str>,
5574 }
5575
5576 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5587 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5588 use crate::scopes::ScopeTable;
5589 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5590
5591 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5592 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5593 module_id: module_id.to_string(),
5594 program,
5595 args: Vec::new(),
5596 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5597 .into_iter()
5598 .map(|key| (key.to_string(), dir.path().display().to_string()))
5599 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5600 .collect(),
5601 reserved: false,
5602 reserved_prefixes: Vec::new(),
5603 protocol: ModuleProtocol::Subc,
5604 overlap: Default::default(),
5605 };
5606 let live = super::terminal_history_tests::fake_aft_stub_path();
5607 let missing = dir.path().join("definitely-missing-module");
5608
5609 let handle = SupervisorHandle::new();
5610 let mut supervisor =
5611 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5612 .with_handle(handle.clone());
5613 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5614 let hook_handle = handle.clone();
5615 let hook_observed = Arc::clone(&observed);
5616 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5617 let configured = hook_handle.is_configured(module_id);
5620 let selector = ScopeSelector::new(
5621 Principal::Reserved {
5622 module_id: module_id.to_string(),
5623 },
5624 "s".to_string(),
5625 Some(1),
5626 );
5627 let carrier = Principal::Reserved {
5628 module_id: "carrier".to_string(),
5629 };
5630 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5631 .admit(&carrier, module_id, &selector, configured)
5632 {
5633 Ok(_) => None,
5634 Err(refusal) => Some(refusal.code),
5635 };
5636 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5637 module_id: module_id.to_string(),
5638 configured,
5639 on_roster: hook_handle.get(module_id).is_some(),
5640 admission_refusal,
5641 });
5642 })));
5643
5644 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5645 let configured = supervisor
5646 .supervise_configured(stub("configured", live.clone()), true)
5647 .unwrap();
5648 let with_health = supervisor
5649 .supervise_configured_with_health(
5650 stub("with-health", live.clone()),
5651 true,
5652 HealthConfig::default(),
5653 None,
5654 RestartPolicy::default(),
5655 )
5656 .unwrap();
5657 let failed = supervisor
5660 .supervise_configured_with_health(
5661 stub("failed-spawn", missing.clone()),
5662 true,
5663 HealthConfig::default(),
5664 None,
5665 RestartPolicy::default(),
5666 )
5667 .unwrap();
5668 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5671
5672 let expected = [
5673 "plain",
5674 "configured",
5675 "with-health",
5676 "failed-spawn",
5677 "spawn-error",
5678 ]
5679 .into_iter()
5680 .map(|module_id| OwnerInSpawnWindow {
5681 module_id: module_id.to_string(),
5682 configured: true,
5683 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5684 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5685 })
5686 .collect::<Vec<_>>();
5687 assert_eq!(*observed.lock().unwrap(), expected);
5688
5689 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5690 assert!(
5691 handle.get(module_id).is_some(),
5692 "{module_id} is on the roster"
5693 );
5694 assert!(
5695 handle.is_configured(module_id),
5696 "{module_id} stays configured"
5697 );
5698 }
5699 assert!(handle.get("spawn-error").is_none());
5700 assert!(
5701 !handle.is_configured("spawn-error"),
5702 "a plain spawn that failed must not leave its module marked configured"
5703 );
5704
5705 handle.retire("failed-spawn");
5707 assert!(!handle.is_configured("failed-spawn"));
5708
5709 for module in [plain, configured, with_health] {
5710 module.stop().await.unwrap();
5711 }
5712 drop(failed);
5713 }
5714
5715 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5722 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5723 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5724 let spec = ModuleSpec {
5725 module_id: "spawned-not-admitted".to_string(),
5726 program: super::terminal_history_tests::fake_aft_stub_path(),
5727 args: Vec::new(),
5728 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5729 .into_iter()
5730 .map(|key| (key.to_string(), dir.path().display().to_string()))
5731 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5732 .collect(),
5733 reserved: false,
5734 reserved_prefixes: Vec::new(),
5735 protocol: ModuleProtocol::Subc,
5736 overlap: Default::default(),
5737 };
5738 let handle = SupervisorHandle::new();
5739 let mut supervisor =
5740 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5741 .with_handle(handle.clone());
5742 let observed = Arc::new(Mutex::new(None));
5743 let hook_handle = handle.clone();
5744 let hook_observed = Arc::clone(&observed);
5745 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5746 let state = hook_handle
5747 .get(module_id)
5748 .expect("a configured module is on the roster before it is spawned")
5749 .state()
5750 .unwrap();
5751 *hook_observed.lock().unwrap() = Some(state);
5752 })));
5753
5754 let module = supervisor.supervise_configured(spec, true).unwrap();
5755 assert_eq!(
5756 *observed.lock().unwrap(),
5757 Some(ModuleState::Starting),
5758 "between the spawn and its admission the supervisor still reports starting"
5759 );
5760 assert_eq!(module.state().unwrap(), ModuleState::Running);
5761 module.stop().await.unwrap();
5762 }
5763
5764 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5765 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5766 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5767 snapshot.process_alive = true;
5768 snapshot.pid = Some(41);
5769 snapshot.spawned_at_ms = Some(42);
5770 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5771 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5772 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5773 device: 43,
5774 inode: 44,
5775 #[cfg(windows)]
5776 windows_image: None,
5777 });
5778 })
5779 .unwrap();
5780 snapshot
5781 }
5782
5783 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5784 let snapshot = lock_snapshot(snapshot).unwrap();
5785 assert!(!snapshot.process_alive);
5786 assert_eq!(snapshot.pid, None);
5787 assert_eq!(snapshot.spawned_at_ms, None);
5788 assert_eq!(snapshot.spawned_from, None);
5789 assert_eq!(snapshot.launch_spec_digest, None);
5790 assert_eq!(snapshot.spawned_file_identity, None);
5791 }
5792
5793 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5794 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5795 let supervisor =
5796 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5797 let mut runtime = supervisor.runtime_config();
5798 runtime.test_seed_stale_facts_before_enable_spawn = true;
5799 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5800 let mut child = None;
5801 let spec = ModuleSpec {
5802 module_id: "failed-enable-clears-facts".to_string(),
5803 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5804 args: Vec::new(),
5805 env: Vec::new(),
5806 reserved: false,
5807 reserved_prefixes: Vec::new(),
5808 protocol: ModuleProtocol::Subc,
5809 overlap: Default::default(),
5810 };
5811
5812 let result = set_child_enabled(
5813 &spec,
5814 &runtime,
5815 &supervisor.registry,
5816 &supervisor.process_liveness,
5817 &snapshot,
5818 &mut child,
5819 true,
5820 )
5821 .await;
5822
5823 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5824 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5825 assert_snapshot_process_facts_cleared(&snapshot);
5826 }
5827
5828 #[tokio::test]
5829 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5830 let supervisor =
5831 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5832 let runtime = supervisor.runtime_config();
5833 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5834 ModuleState::Restarting,
5835 true,
5836 )));
5837 let spec = ModuleSpec {
5838 module_id: "start-stranded-restarting".to_string(),
5839 program: super::terminal_history_tests::fake_aft_stub_path(),
5840 args: Vec::new(),
5841 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5842 reserved: false,
5843 reserved_prefixes: Vec::new(),
5844 protocol: ModuleProtocol::None,
5845 overlap: Default::default(),
5846 };
5847 let mut child = None;
5848 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5849 assert!(!super::set_child_enabled(
5850 &spec,
5851 &runtime,
5852 &Registry::default(),
5853 &supervisor.process_liveness,
5854 &snapshot,
5855 &mut child,
5856 true
5857 )
5858 .await
5859 .unwrap());
5860 assert!(child.is_none());
5861 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5862 assert!(super::set_child_enabled(
5863 &spec,
5864 &runtime,
5865 &Registry::default(),
5866 &supervisor.process_liveness,
5867 &snapshot,
5868 &mut child,
5869 true
5870 )
5871 .await
5872 .unwrap());
5873 assert_eq!(
5874 lock_snapshot(&snapshot).unwrap().state,
5875 ModuleState::Running
5876 );
5877 let mut child = child.unwrap();
5878 child.start_kill().unwrap();
5879 child.wait().await.unwrap();
5880 }
5881
5882 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5883 async fn failed_reload_spawn_clears_current_process_facts() {
5884 let supervisor =
5885 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5886 let mut runtime = supervisor.runtime_config();
5887 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5888 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5889 let mut child = None;
5890 let spec = ModuleSpec {
5891 module_id: "failed-reload-clears-facts".to_string(),
5892 program: PathBuf::from("/unused/failed-reload-module"),
5893 args: Vec::new(),
5894 env: Vec::new(),
5895 reserved: false,
5896 reserved_prefixes: Vec::new(),
5897 protocol: ModuleProtocol::Subc,
5898 overlap: Default::default(),
5899 };
5900
5901 let result = handle_reload_spawn_failure(
5902 &spec,
5903 &runtime,
5904 &supervisor.process_liveness,
5905 &snapshot,
5906 &mut child,
5907 "forced reload spawn failure".to_string(),
5908 )
5909 .await;
5910
5911 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5912 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5913 assert_snapshot_process_facts_cleared(&snapshot);
5914 }
5915
5916 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5917 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5918 let supervisor =
5919 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5920 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5921 let module = supervisor.supervised_module(
5922 ModuleSpec {
5923 module_id: "drop-clears-facts".to_string(),
5924 program: PathBuf::from("/unused/drop-module"),
5925 args: Vec::new(),
5926 env: Vec::new(),
5927 reserved: false,
5928 reserved_prefixes: Vec::new(),
5929 protocol: ModuleProtocol::Subc,
5930 overlap: Default::default(),
5931 },
5932 supervisor.runtime_config(),
5933 Arc::clone(&snapshot),
5934 None,
5935 );
5936 assert!(!module
5937 .inner
5938 .monitor
5939 .lock()
5940 .unwrap()
5941 .as_ref()
5942 .unwrap()
5943 .is_finished());
5944
5945 drop(module);
5946
5947 assert_eq!(
5948 lock_snapshot(&snapshot).unwrap().state,
5949 ModuleState::Stopped
5950 );
5951 assert_snapshot_process_facts_cleared(&snapshot);
5952 }
5953
5954 #[cfg(unix)]
5955 #[tokio::test]
5956 async fn rescan_preserves_running_protocol_until_respawn() {
5957 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5958 let initial = ModuleSpec {
5959 module_id: "rescan-protocol".into(),
5960 program: PathBuf::from("/bin/sleep"),
5961 args: vec!["60".into()],
5962 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5963 .into_iter()
5964 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5965 .collect(),
5966 reserved: false,
5967 reserved_prefixes: vec![],
5968 protocol: ModuleProtocol::None,
5969 overlap: Default::default(),
5970 };
5971 let supervisor =
5972 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5973 let module = supervisor.spawn(initial.clone()).unwrap();
5974 assert!(module.status().unwrap().live);
5975 let mut next = initial;
5976 next.protocol = ModuleProtocol::Subc;
5977 module
5978 .update_configuration(next.clone(), HealthConfig::default(), None)
5979 .await
5980 .unwrap();
5981 assert!(
5982 module.status().unwrap().live,
5983 "rescan must not require HELLO from the old non-wire process"
5984 );
5985 let runtime = supervisor.runtime_config();
5986 let action = on_child_exit(
5987 &next,
5988 RestartPolicy::default(),
5989 &supervisor.registry,
5990 &module.inner.snapshot,
5991 &runtime.terminal_ring,
5992 &runtime.spawn_events,
5993 &runtime.child_roster,
5994 ExitReport {
5995 kind: ExitKind::Clean,
5996 code: Some(0),
5997 signal: None,
5998 at_ms: unix_ms_now(),
5999 },
6000 )
6001 .await;
6002 assert!(
6003 matches!(action, NextAction::Restart { .. }),
6004 "the old non-wire process's clean exit must restart"
6005 );
6006 module.drain().await.unwrap();
6007 }
6008
6009 #[cfg(unix)]
6010 #[tokio::test]
6011 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6012 use std::os::unix::fs::PermissionsExt;
6013 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6014 let script = dir.join("module.sh");
6015 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6016 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6017 let record_path = dir.join("live-children.json");
6018 let supervisor =
6019 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6020 .with_live_children_record(&record_path);
6021 for (program, args) in [
6022 (PathBuf::from("sleep"), vec!["60".into()]),
6023 (script, vec![]),
6024 ] {
6025 let spec = ModuleSpec {
6026 module_id: "image-identity".into(),
6027 program: program.clone(),
6028 args,
6029 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6030 .into_iter()
6031 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6032 .collect(),
6033 reserved: false,
6034 reserved_prefixes: vec![],
6035 protocol: ModuleProtocol::None,
6036 overlap: Default::default(),
6037 };
6038 let module = supervisor.spawn(spec).unwrap();
6039 #[cfg(target_os = "macos")]
6040 {
6041 let deadline = Instant::now() + Duration::from_secs(5);
6044 while crate::live_children::read_record(&record_path)
6045 .unwrap()
6046 .iter()
6047 .all(|entry| entry.executable.is_none())
6048 {
6049 assert!(Instant::now() < deadline, "module image was not confirmed");
6050 tokio::time::sleep(Duration::from_millis(5)).await;
6051 }
6052 }
6053 let entry = crate::live_children::read_record(&record_path)
6054 .unwrap()
6055 .pop()
6056 .unwrap();
6057 let observed = subc_os::Process::open(entry.pid)
6058 .unwrap()
6059 .unwrap()
6060 .observe()
6061 .unwrap();
6062 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6063 module.drain().await.unwrap();
6064 assert_eq!(
6065 verdict,
6066 crate::live_children::IdentityVerdict::Matches,
6067 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6068 );
6069 }
6070 }
6071
6072 #[cfg(unix)]
6073 fn http_fixture(
6074 dir: &std::path::Path,
6075 url: &str,
6076 threshold: u32,
6077 ) -> crate::daemon_config::ConfiguredModule {
6078 let path = dir.join("subc.jsonc");
6079 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6080 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6081 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6082 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6083 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6084 }}}).to_string()).unwrap();
6085 crate::daemon_config::load(&path)
6086 .unwrap()
6087 .unwrap()
6088 .modules
6089 .pop()
6090 .unwrap()
6091 }
6092
6093 #[cfg(unix)]
6094 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6095 timeout(Duration::from_secs(5), async {
6096 loop {
6097 if module.status().unwrap().health.status == status {
6098 break;
6099 }
6100 sleep(Duration::from_millis(5)).await;
6101 }
6102 })
6103 .await
6104 .unwrap_or_else(|_| {
6105 panic!(
6106 "expected {status:?}, got {:?}",
6107 module.status().unwrap().health
6108 )
6109 });
6110 }
6111
6112 #[cfg(unix)]
6113 #[tokio::test]
6114 async fn http_health_status_flips_ok_failing_ok() {
6115 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6116 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6117 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6118 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6119 let serving_status = status.clone();
6120 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6121 let server = tokio::spawn(async move {
6122 loop {
6123 let (mut stream, _) = listener.accept().await.unwrap();
6124 let mut request = [0u8; 2048];
6125 let count = stream.read(&mut request).await.unwrap();
6126 assert!(count > 0, "a probe must send an HTTP request");
6127 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6128 let body = if code == 200 {
6129 "ready"
6130 } else {
6131 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6132 };
6133 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6134 let _ = stream.write_all(response.as_bytes()).await;
6135 }
6136 });
6137 let configured = http_fixture(&dir, &url, 1000);
6138 let module =
6139 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6140 .supervise_configured_with_health(
6141 configured.module_spec(),
6142 true,
6143 configured.health,
6144 configured.drain_timeout_ms,
6145 configured.restart,
6146 )
6147 .unwrap();
6148 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6149 status.store(503, std::sync::atomic::Ordering::SeqCst);
6150 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6151 assert!(module
6152 .status()
6153 .unwrap()
6154 .health
6155 .detail
6156 .unwrap()
6157 .contains("scratch failure"));
6158 status.store(200, std::sync::atomic::Ordering::SeqCst);
6159 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6160 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6161 let before = module.status().unwrap();
6162 let (spec, mut health) = module.configuration().unwrap();
6163 health.http = None;
6164 module
6165 .update_configuration(spec.clone(), health.clone(), Some(10))
6166 .await
6167 .unwrap();
6168 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6169 health.http = Some(url);
6170 module
6171 .update_configuration(spec, health, Some(10))
6172 .await
6173 .unwrap();
6174 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6175 assert_eq!(
6176 module.status().unwrap().pid,
6177 before.pid,
6178 "changing a probe must apply live, not restart its process"
6179 );
6180 let (spec, mut health) = module.configuration().unwrap();
6181 health.failure_threshold = 2;
6182 module
6183 .update_configuration(spec, health, Some(10))
6184 .await
6185 .unwrap();
6186 status.store(503, std::sync::atomic::Ordering::SeqCst);
6187 timeout(Duration::from_secs(5), async {
6188 while module.status().unwrap().spawn_generation == before.spawn_generation {
6189 sleep(Duration::from_millis(5)).await;
6190 }
6191 })
6192 .await
6193 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6194 module.drain().await.unwrap();
6195 server.abort();
6196 }
6197
6198 #[cfg(unix)]
6199 #[tokio::test]
6200 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6201 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6202 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6203 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6204 drop(unused);
6205 let configured = http_fixture(&dir, &url, 2);
6206 let module =
6207 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6208 .supervise_configured_with_health(
6209 configured.module_spec(),
6210 true,
6211 configured.health,
6212 configured.drain_timeout_ms,
6213 configured.restart,
6214 )
6215 .unwrap();
6216 let before = module.status().unwrap().spawn_generation;
6217 timeout(Duration::from_secs(5), async {
6218 loop {
6219 let status = module.status().unwrap();
6220 if status.spawn_generation > before {
6221 assert!(status.lifetime_restarts > 0);
6222 break;
6223 }
6224 sleep(Duration::from_millis(5)).await;
6225 }
6226 })
6227 .await
6228 .expect("sustained HTTP refusal must trigger the health restart policy");
6229 module.drain().await.unwrap();
6230 }
6231
6232 #[cfg(unix)]
6233 #[tokio::test]
6234 async fn http_health_timeout_honours_deadline() {
6235 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6236 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6237 let server = tokio::spawn(async move {
6238 let _held = listener.accept().await.unwrap();
6239 std::future::pending::<()>().await;
6240 });
6241 let error = timeout(
6242 Duration::from_secs(1),
6243 probe_http_health(&url, Duration::from_millis(10)),
6244 )
6245 .await
6246 .expect("the probe must enforce its own deadline")
6247 .unwrap_err();
6248 server.abort();
6249 assert!(error.to_string().contains("timed out"));
6250 }
6251
6252 #[cfg(unix)]
6253 #[tokio::test]
6254 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6255 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6256 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6257 let url = format!(
6258 "http://localhost:{}/healthz",
6259 listener.local_addr().unwrap().port()
6260 );
6261 let server = tokio::spawn(async move {
6262 let (mut stream, _) = listener.accept().await.unwrap();
6263 let mut request = [0u8; 2048];
6264 assert!(stream.read(&mut request).await.unwrap() > 0);
6265 stream
6266 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6267 .await
6268 .unwrap();
6269 });
6270 assert_eq!(
6275 probe_http_health(&url, Duration::from_secs(10))
6276 .await
6277 .unwrap()
6278 .status,
6279 HealthStatus::Ok
6280 );
6281 server.await.unwrap();
6282 }
6283
6284 #[cfg(unix)]
6285 #[tokio::test]
6286 async fn http_health_timeout_keeps_partial_status_and_body() {
6287 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6288 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6289 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6290 let server = tokio::spawn(async move {
6291 let (mut stream, _) = listener.accept().await.unwrap();
6292 let mut request = [0u8; 2048];
6293 assert!(stream.read(&mut request).await.unwrap() > 0);
6294 stream
6295 .write_all(
6296 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6297 )
6298 .await
6299 .unwrap();
6300 std::future::pending::<()>().await;
6301 });
6302 let error = probe_http_health(&url, Duration::from_secs(1))
6303 .await
6304 .unwrap_err()
6305 .to_string();
6306 server.abort();
6307 assert!(
6308 error.contains("timed out")
6309 && error.contains("503 Unavailable")
6310 && error.contains("partial diagnostic"),
6311 "{error}"
6312 );
6313 }
6314
6315 #[cfg(unix)]
6316 #[tokio::test]
6317 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6318 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6319 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6320 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6321 let server = tokio::spawn(async move {
6322 let (mut stream, _) = listener.accept().await.unwrap();
6323 let mut request = [0u8; 2048];
6324 assert!(stream.read(&mut request).await.unwrap() > 0);
6325 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6326 let response = format!(
6327 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6328 body.len()
6329 );
6330 stream.write_all(response.as_bytes()).await.unwrap();
6331 });
6332 let error = probe_http_health(&url, Duration::from_secs(1))
6333 .await
6334 .unwrap_err()
6335 .to_string();
6336 server.await.unwrap();
6337 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6338 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6339 assert!(!error.contains("not-in-diagnostic"));
6340 }
6341
6342 #[cfg(unix)]
6343 #[tokio::test]
6344 async fn http_health_real_nats_server_monitoring() {
6345 if std::process::Command::new("nats-server")
6346 .arg("--version")
6347 .env("XDG_DATA_HOME", std::env::temp_dir())
6348 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6349 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6350 .output()
6351 .is_err()
6352 {
6353 eprintln!(
6354 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6355 );
6356 return;
6357 }
6358 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6359 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6360 let port = monitor.local_addr().unwrap().port();
6361 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6362 let client_port = client.local_addr().unwrap().port();
6363 let config = dir.join("server.conf");
6364 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6365 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6366 configured.program = PathBuf::from("nats-server");
6367 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6368 drop(monitor);
6369 drop(client);
6370 let module =
6371 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6372 .supervise_configured_with_health(
6373 configured.module_spec(),
6374 true,
6375 configured.health,
6376 configured.drain_timeout_ms,
6377 configured.restart,
6378 )
6379 .unwrap();
6380 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6381 module.drain().await.unwrap();
6382 }
6383
6384 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6385 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6386 let supervisor =
6387 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6388 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6389 let initial = ModuleSpec {
6390 module_id: "rescan-preserves-spawn-facts".to_string(),
6391 program: PathBuf::from("/spawned/module"),
6392 args: Vec::new(),
6393 env: Vec::new(),
6394 reserved: false,
6395 reserved_prefixes: Vec::new(),
6396 protocol: ModuleProtocol::Subc,
6397 overlap: Default::default(),
6398 };
6399 let module = supervisor.supervised_module(
6400 initial.clone(),
6401 supervisor.runtime_config(),
6402 snapshot,
6403 None,
6404 );
6405 let before = module.status().unwrap();
6406 let mut replacement = initial;
6407 replacement.program = PathBuf::from("/rescanned/replacement-module");
6408
6409 module
6410 .update_configuration(replacement, HealthConfig::default(), None)
6411 .await
6412 .unwrap();
6413
6414 let after = module.status().unwrap();
6415 assert_eq!(after.pid, before.pid);
6416 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6417 assert_eq!(after.spawned_from, before.spawned_from);
6418 drop(module);
6419 }
6420}
6421
6422fn unix_ms_now() -> u64 {
6423 SystemTime::now()
6424 .duration_since(UNIX_EPOCH)
6425 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6426 .unwrap_or(0)
6427}
6428
6429async fn supervise_loop(
6430 mut spec: ModuleSpec,
6431 mut runtime: SupervisorRuntimeConfig,
6432 registry: Arc<Registry>,
6433 process_liveness: Arc<SupervisorProcessLiveness>,
6434 snapshot: SharedSnapshot,
6435 mut child: Option<SupervisedChild>,
6436 mut commands: mpsc::Receiver<SupervisorCommand>,
6437) {
6438 let mut health_probe = HealthProbeRuntime::default();
6439 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6443 Ok(changes) => Some(changes),
6444 Err(err) => {
6445 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6448 None
6449 }
6450 };
6451 let mut pending_respawn: Option<PendingRespawn> = None;
6455 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6458 let mut previous_instance_recheck = tokio::time::interval_at(
6459 Instant::now() + Duration::from_secs(5),
6460 Duration::from_secs(5),
6461 );
6462 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6463 loop {
6464 #[cfg(test)]
6465 {
6466 lock_snapshot(&snapshot).unwrap().actor_select = None;
6467 }
6468 #[cfg(target_os = "macos")]
6469 if let Some(active) = child.as_mut() {
6470 active.confirm_privacy_exec().await;
6471 }
6472 if let Some(scheduled) = runtime
6473 .scheduled_respawn
6474 .lock()
6475 .unwrap_or_else(|p| p.into_inner())
6476 .take()
6477 {
6478 pending_respawn = Some(scheduled);
6479 }
6480 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6481 pending_respawn = None;
6482 cancel_deferred_reload(
6483 &runtime,
6484 &spec.module_id,
6485 "respawn cancelled by a supervisor command",
6486 );
6487 }
6488 if child.is_none()
6489 && pending_respawn.is_none()
6490 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6491 {
6492 cancel_deferred_reload(
6493 &runtime,
6494 &spec.module_id,
6495 "respawn cancelled before a replacement was spawned",
6496 );
6497 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6498 state.respawn_pending = false;
6499 state.coalesced_restart_pending = false;
6500 if matches!(
6501 state.state,
6502 ModuleState::Restarting
6503 | ModuleState::Starting
6504 | ModuleState::Draining
6505 | ModuleState::Unresponsive
6506 ) {
6507 error!(module_id = %spec.module_id, state = ?state.state,
6508 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6509 state.state = ModuleState::Failed;
6510 clear_current_process_facts(state);
6511 }
6512 });
6513 }
6514 if let Some(command) = requeued.pop_front() {
6515 if !handle_supervisor_command(
6516 command,
6517 &mut spec,
6518 &mut runtime,
6519 ®istry,
6520 &process_liveness,
6521 &snapshot,
6522 &mut child,
6523 &mut commands,
6524 &mut requeued,
6525 )
6526 .await
6527 {
6528 return;
6529 }
6530 if child.is_some() || !respawn_still_pending(&snapshot) {
6531 pending_respawn = None;
6532 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6533 state.respawn_pending = false
6534 });
6535 }
6536 continue;
6537 }
6538 if child.is_some() {
6539 #[cfg(test)]
6540 {
6541 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6542 }
6543 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6544 let wake_after = health_probe.wake_after();
6545 let probe_sleep = async move {
6546 match wake_after {
6547 Some(delay) => sleep(delay).await,
6548 None => std::future::pending().await,
6549 }
6550 };
6551 tokio::pin!(probe_sleep);
6552 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6557 #[cfg(test)]
6558 {
6559 let registration_pending = wire_child
6560 && registration_changes
6561 .as_ref()
6562 .is_some_and(|changes| changes.has_changed().unwrap());
6563 let mut state = lock_snapshot(&snapshot).unwrap();
6564 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6565 generation: state.spawn_generation,
6566 turn: state.actor_turns,
6567 registered_connection: health_probe.registered_connection,
6568 next_probe_at: health_probe.next_probe_at,
6569 wake_after,
6570 });
6571 }
6572 let active_child = child.as_mut().expect("child checked above");
6573 tokio::select! {
6574 wait_result = active_child.wait() => {
6575 #[cfg(test)]
6576 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6577 let exit_report = match wait_result {
6586 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6587 Err(err) => {
6588 active_child.drain_stderr(&spec.module_id).await;
6589 fail_after_wait_error(
6590 &spec.module_id,
6591 &snapshot,
6592 &runtime.terminal_ring,
6593 &runtime.spawn_events,
6594 format!("failed to wait for supervised module: {err}"),
6595 );
6596 untrack_if_registration_released(
6597 &process_liveness,
6598 ®istry,
6599 &spec.module_id,
6600 &snapshot,
6601 );
6602 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6603 child = None;
6604 continue;
6605 }
6606 };
6607 active_child.drain_stderr(&spec.module_id).await;
6608
6609 let next = on_child_exit(
6610 &spec,
6611 runtime.restart_policy,
6612 ®istry,
6613 &snapshot,
6614 &runtime.terminal_ring,
6615 &runtime.spawn_events,
6616 &runtime.child_roster,
6617 exit_report,
6618 ).await;
6619 runtime.start_requester = None;
6620 runtime.pending_activity_restart = false;
6621 active_child.release_roster();
6624 match next {
6625 NextAction::Stop { registration_released } => {
6626 if registration_released {
6627 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6628 }
6629 child = None;
6630 }
6631 NextAction::Restart { schedule } => {
6632 let delay = schedule.map_or(
6633 runtime.restart_policy.delay_for_restart(0),
6634 |schedule| schedule.delay,
6635 );
6636 if let Some(schedule) = schedule {
6637 log_crash_respawn(&spec.module_id, schedule);
6638 }
6639 child = None;
6647 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6648 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6649 }
6650 }
6651 }
6652 command = commands.recv() => {
6653 #[cfg(test)]
6654 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6655 let Some(command) = command else {
6656 return;
6657 };
6658 if !handle_supervisor_command(
6659 command,
6660 &mut spec,
6661 &mut runtime,
6662 ®istry,
6663 &process_liveness,
6664 &snapshot,
6665 &mut child,
6666 &mut commands,
6667 &mut requeued,
6668 ).await {
6669 return;
6670 }
6671 }
6672 _ = async {
6673 match registration_changes.as_mut() {
6674 Some(changes) => { let _ = changes.changed().await; }
6675 None => std::future::pending().await,
6676 }
6677 }, if wire_child => {
6678 #[cfg(test)]
6679 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6680 }
6681 _ = &mut probe_sleep => {
6682 #[cfg(test)]
6683 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6684 if health_probe.due() {
6685 run_health_probe_cycle(
6686 &spec,
6687 &runtime,
6688 ®istry,
6689 &process_liveness,
6690 &snapshot,
6691 &mut child,
6692 ).await;
6693 if child.is_none() {
6694 runtime.start_requester = None;
6695 runtime.pending_activity_restart = false;
6696 }
6697 if child.is_some() {
6698 health_probe.schedule_next(&spec, runtime.health.cadence);
6699 }
6700 }
6701 }
6702 }
6703 } else if let Some(pending) = pending_respawn {
6704 tokio::select! {
6705 _ = sleep_until(pending.deadline) => {
6706 pending_respawn = None;
6707 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6708 if !respawn_still_pending(&snapshot) {
6712 continue;
6713 }
6714 if runtime.child_roster.is_closed() {
6719 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6720 state.state = ModuleState::Stopped;
6721 });
6722 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6723 continue;
6724 }
6725 if let Err(err) = release_dead_registration(
6726 ®istry,
6727 runtime.forwarding.as_deref(),
6728 &snapshot,
6729 &spec.module_id,
6730 ).await {
6731 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6732 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6733 continue;
6734 }
6735
6736 if matches!(pending.kind, RespawnKind::Reload) {
6737 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6738 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
6739 if result.is_ok() && runtime.pending_activity_restart {
6740 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6741 }
6742 runtime.pending_activity_restart = false;
6743 if let Some(reply) = reply { let _ = reply.send(result); }
6744 continue;
6745 }
6746 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6747 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6748 Ok(next_child) => {
6749 child = Some(next_child);
6750 if runtime.pending_activity_restart {
6751 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6752 runtime.pending_activity_restart = false;
6753 }
6754 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6755 }
6756 Err(err) => {
6757 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6758 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6759 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6760 }
6761 }
6762 }
6763 command = commands.recv() => {
6764 let Some(command) = command else {
6765 return;
6766 };
6767 if !handle_supervisor_command(
6768 command,
6769 &mut spec,
6770 &mut runtime,
6771 ®istry,
6772 &process_liveness,
6773 &snapshot,
6774 &mut child,
6775 &mut commands,
6776 &mut requeued,
6777 ).await {
6778 return;
6779 }
6780 if child.is_some() || !respawn_still_pending(&snapshot) {
6785 pending_respawn = None;
6786 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6787 }
6788 }
6789 }
6790 } else {
6791 let command = if lock_snapshot(&snapshot)
6792 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6793 {
6794 tokio::select! {
6795 command = commands.recv() => command,
6796 _ = previous_instance_recheck.tick() => {
6797 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6798 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6799 if enabled && !runtime.child_roster.is_closed() {
6800 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6801 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6802 Ok(next_child) => child = Some(next_child),
6803 Err(error) => {
6804 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6805 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6806 }
6807 }
6808 } else {
6809 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6810 state.waiting_for_previous_pid = None;
6811 state.waiting_for_previous_detail = None;
6812 });
6813 }
6814 }
6815 continue;
6816 }
6817 }
6818 } else {
6819 commands.recv().await
6820 };
6821 let Some(command) = command else {
6822 return;
6823 };
6824 if !handle_supervisor_command(
6825 command,
6826 &mut spec,
6827 &mut runtime,
6828 ®istry,
6829 &process_liveness,
6830 &snapshot,
6831 &mut child,
6832 &mut commands,
6833 &mut requeued,
6834 )
6835 .await
6836 {
6837 return;
6838 }
6839 }
6840 }
6841}
6842
6843fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6844 info!(
6845 module_id,
6846 restart_in_window = schedule.restart_in_window,
6847 delay_ms = schedule.delay.as_millis() as u64,
6848 "respawning after crash"
6849 );
6850}
6851
6852fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6858 matches!(
6859 lock_snapshot(snapshot),
6860 Ok(state) if state.enabled && state.state == ModuleState::Restarting
6861 )
6862}
6863
6864enum NextAction {
6865 Stop {
6866 registration_released: bool,
6867 },
6868 Restart {
6869 schedule: Option<CrashRestartSchedule>,
6870 },
6871}
6872
6873#[allow(clippy::too_many_arguments)]
6874async fn handle_supervisor_command(
6875 command: SupervisorCommand,
6876 spec: &mut ModuleSpec,
6877 runtime: &mut SupervisorRuntimeConfig,
6878 registry: &Arc<Registry>,
6879 process_liveness: &SupervisorProcessLiveness,
6880 snapshot: &SharedSnapshot,
6881 child: &mut Option<SupervisedChild>,
6882 commands: &mut mpsc::Receiver<SupervisorCommand>,
6883 requeued: &mut VecDeque<SupervisorCommand>,
6884) -> bool {
6885 let command =
6886 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6887 match command {
6888 SupervisorCommand::Restart { reply, .. }
6889 | SupervisorCommand::Reload { reply, .. }
6890 | SupervisorCommand::Swap { reply, .. } => {
6891 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6892 return true;
6893 }
6894 SupervisorCommand::SetEnabled {
6895 enabled: true,
6896 reply,
6897 ..
6898 } => {
6899 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6900 return true;
6901 }
6902 other => other,
6903 }
6904 } else {
6905 command
6906 };
6907 match command {
6908 SupervisorCommand::Drain { requester, reply } => {
6909 let result = drain_optional_child(
6912 &spec.module_id,
6913 spec.protocol,
6914 StopNotice::NotSent,
6915 registry,
6916 runtime.forwarding.as_deref(),
6917 snapshot,
6918 &runtime.terminal_ring,
6919 &runtime.spawn_events,
6920 child,
6921 runtime.drain_timeout,
6922 ModuleState::Stopped,
6923 None,
6924 )
6925 .await;
6926 let registration_released = result.is_ok();
6927 if registration_released {
6928 runtime.record_activity_transition(
6929 "subc.module_stopped",
6930 &spec.module_id,
6931 requester,
6932 );
6933 }
6934 let _ = reply.send(result);
6935 if registration_released {
6936 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6937 }
6938 false
6939 }
6940 SupervisorCommand::Retire { requester, reply } => {
6941 let result = async {
6942 let stop_notice = begin_forwarding_drain_if_configured(
6943 spec,
6944 runtime,
6945 registry,
6946 snapshot,
6947 None,
6948 RouteCloseReason::Disable,
6949 )
6950 .await?;
6951 drain_optional_child(
6952 &spec.module_id,
6953 spec.protocol,
6954 stop_notice,
6955 registry,
6956 runtime.forwarding.as_deref(),
6957 snapshot,
6958 &runtime.terminal_ring,
6959 &runtime.spawn_events,
6960 child,
6961 runtime.drain_timeout,
6962 ModuleState::Stopped,
6963 None,
6964 )
6965 .await
6966 }
6967 .await;
6968 let registration_released = result.is_ok();
6969 if registration_released {
6970 runtime.record_activity_transition(
6971 "subc.module_stopped",
6972 &spec.module_id,
6973 requester,
6974 );
6975 }
6976 let _ = reply.send(result);
6977 if registration_released {
6978 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6979 }
6980 false
6981 }
6982 SupervisorCommand::Restart {
6983 requester,
6984 drain_timeout_ms,
6985 received_at_generation,
6986 queued_at,
6987 reply,
6988 } => {
6989 info!(
6993 module_id = %spec.module_id,
6994 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6995 "restart command dequeued"
6996 );
6997 let validation = match lock_snapshot(snapshot) {
7009 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7010 module_id: spec.module_id.clone(),
7011 }),
7012 Ok(_) => Ok(()),
7013 Err(err) => Err(err),
7014 };
7015 let initiated = validation.is_ok();
7016 let _ = reply.send(validation);
7017 let satisfied_by_generation = if initiated
7031 && child
7032 .as_ref()
7033 .is_some_and(|child| child.spawned_at > queued_at)
7034 {
7035 lock_snapshot(snapshot).ok().and_then(|state| {
7036 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7037 })
7038 } else {
7039 None
7040 };
7041 let satisfied_by_pending = initiated
7042 && child.is_none()
7043 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7044 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7048 if pending {
7049 state.coalesced_restart_pending = true;
7050 }
7051 pending
7052 });
7053 if satisfied_by_pending {
7054 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7055 } else if let Some(generation) = satisfied_by_generation {
7056 info!(
7057 module_id = %spec.module_id,
7058 received_at_generation,
7059 "restart already satisfied by generation {generation}; not restarting again"
7060 );
7061 } else if initiated {
7062 let drain_timeout = drain_timeout_ms
7065 .map(Duration::from_millis)
7066 .unwrap_or(runtime.drain_timeout);
7067 if let Err(err) = restart_child(
7068 spec,
7069 runtime,
7070 registry,
7071 process_liveness,
7072 snapshot,
7073 child,
7074 drain_timeout,
7075 )
7076 .await
7077 {
7078 warn!(
7079 module_id = %spec.module_id,
7080 error = %err,
7081 "operator restart failed after initiation ack; module state carries the outcome"
7082 );
7083 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7084 state.state = ModuleState::Failed;
7085 clear_current_process_facts(state);
7086 });
7087 } else {
7088 runtime.start_requester = requester;
7089 runtime.pending_activity_restart = true;
7090 }
7091 }
7092 true
7093 }
7094 SupervisorCommand::Reload { requester, reply } => {
7095 let result =
7096 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7097 if result.is_ok() {
7098 runtime.start_requester = requester;
7099 runtime.pending_activity_restart = true;
7100 }
7101 if result.is_ok()
7102 && runtime
7103 .scheduled_respawn
7104 .lock()
7105 .unwrap_or_else(|p| p.into_inner())
7106 .as_ref()
7107 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7108 {
7109 *runtime
7110 .deferred_reload_reply
7111 .lock()
7112 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7113 } else {
7114 let _ = reply.send(result);
7115 }
7116 true
7117 }
7118 SupervisorCommand::SetEnabled {
7119 enabled,
7120 requester,
7121 reply,
7122 } => {
7123 let previous_requester = runtime.start_requester.clone();
7124 if enabled {
7125 runtime.start_requester = requester.clone();
7126 }
7127 let result = set_child_enabled(
7128 spec,
7129 runtime,
7130 registry,
7131 process_liveness,
7132 snapshot,
7133 child,
7134 enabled,
7135 )
7136 .await;
7137 if matches!(result, Ok(true)) {
7138 runtime.pending_activity_restart = false;
7139 if enabled {
7140 runtime.start_requester = requester;
7141 } else {
7142 runtime.start_requester = None;
7143 runtime.record_activity_transition(
7144 "subc.module_disabled",
7145 &spec.module_id,
7146 requester,
7147 );
7148 }
7149 } else {
7150 runtime.start_requester = previous_requester;
7151 }
7152 let _ = reply.send(result);
7153 true
7154 }
7155 SupervisorCommand::UpdateConfiguration {
7156 spec: next_spec,
7157 health,
7158 drain_timeout_ms,
7159 reply,
7160 } => {
7161 if let Some(handle) = &runtime.supervisor_handle {
7162 handle.apply_identity_configuration(&next_spec);
7163 }
7164 *spec = next_spec;
7165 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7166 state.configuration_updated_since_spawn = true;
7167 });
7168 let health_changed = runtime.health != health;
7169 runtime.health = health;
7170 if health_changed {
7173 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7174 state.health = ModuleHealthStatus::default();
7175 });
7176 }
7177 runtime.drain_timeout = drain_timeout_ms
7178 .map(Duration::from_millis)
7179 .unwrap_or(runtime.default_drain_timeout);
7180 *runtime
7181 .effective_drain_timeout
7182 .lock()
7183 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7184 let _ = reply.send(());
7185 true
7186 }
7187 SupervisorCommand::Swap {
7188 requester,
7189 ready_timeout,
7190 reply,
7191 } => {
7192 let mut swap_runtime = runtime.clone();
7193 swap_runtime.start_requester = requester;
7194 let before = lock_snapshot(snapshot)
7195 .ok()
7196 .map(|state| state.spawn_generation);
7197 let end = swap::run_swap(
7198 spec,
7199 &swap_runtime,
7200 registry,
7201 process_liveness,
7202 snapshot,
7203 child,
7204 commands,
7205 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7206 reply,
7207 )
7208 .await;
7209 if before
7210 != lock_snapshot(snapshot)
7211 .ok()
7212 .map(|state| state.spawn_generation)
7213 {
7214 runtime.start_requester = swap_runtime.start_requester;
7215 }
7216 requeued.extend(end.requeue);
7217 true
7218 }
7219 }
7220}
7221
7222async fn restart_child(
7223 spec: &ModuleSpec,
7224 runtime: &SupervisorRuntimeConfig,
7225 registry: &Registry,
7226 process_liveness: &SupervisorProcessLiveness,
7227 snapshot: &SharedSnapshot,
7228 child: &mut Option<SupervisedChild>,
7229 drain_timeout: Duration,
7230) -> Result<(), SuperviseError> {
7231 if !lock_snapshot(snapshot)?.enabled {
7233 return Err(SuperviseError::Disabled {
7234 module_id: spec.module_id.clone(),
7235 });
7236 }
7237 let stop_notice = begin_forwarding_drain_with_timeout(
7238 spec,
7239 runtime,
7240 registry,
7241 snapshot,
7242 None,
7243 RouteCloseReason::Restart,
7244 drain_timeout,
7245 )
7246 .await?;
7247
7248 if child.is_some() {
7249 drain_optional_child(
7250 &spec.module_id,
7251 spec.protocol,
7252 stop_notice,
7253 registry,
7254 runtime.forwarding.as_deref(),
7255 snapshot,
7256 &runtime.terminal_ring,
7257 &runtime.spawn_events,
7258 child,
7259 drain_timeout,
7260 ModuleState::Restarting,
7261 Some(true),
7262 )
7263 .await?;
7264 } else {
7265 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7266 state.enabled = true;
7267 state.state = ModuleState::Restarting;
7268 clear_current_process_facts(state);
7269 })?;
7270 release_dead_registration(
7271 registry,
7272 runtime.forwarding.as_deref(),
7273 snapshot,
7274 &spec.module_id,
7275 )
7276 .await?;
7277 }
7278
7279 reset_restart_count(snapshot, &spec.module_id)?;
7280 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7281 schedule_respawn(
7282 runtime,
7283 snapshot,
7284 &spec.module_id,
7285 runtime.restart_policy.backoff,
7286 RespawnKind::Spawn,
7287 )
7288}
7289
7290async fn reload_child(
7291 spec: &ModuleSpec,
7292 runtime: &SupervisorRuntimeConfig,
7293 registry: &Registry,
7294 process_liveness: &SupervisorProcessLiveness,
7295 snapshot: &SharedSnapshot,
7296 child: &mut Option<SupervisedChild>,
7297) -> Result<(), SuperviseError> {
7298 if !lock_snapshot(snapshot)?.enabled {
7300 return Err(SuperviseError::Disabled {
7301 module_id: spec.module_id.clone(),
7302 });
7303 }
7304 let stop_notice = begin_forwarding_drain(
7305 spec,
7306 runtime,
7307 registry,
7308 snapshot,
7309 Some(true),
7310 RouteCloseReason::Reload,
7311 )
7312 .await?;
7313
7314 if child.is_some() {
7315 drain_optional_child(
7316 &spec.module_id,
7317 spec.protocol,
7318 stop_notice,
7319 registry,
7320 runtime.forwarding.as_deref(),
7321 snapshot,
7322 &runtime.terminal_ring,
7323 &runtime.spawn_events,
7324 child,
7325 runtime.drain_timeout,
7326 ModuleState::Restarting,
7327 Some(true),
7328 )
7329 .await?;
7330 } else {
7331 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7332 state.enabled = true;
7333 state.state = ModuleState::Restarting;
7334 clear_current_process_facts(state);
7335 })?;
7336 release_dead_registration(
7337 registry,
7338 runtime.forwarding.as_deref(),
7339 snapshot,
7340 &spec.module_id,
7341 )
7342 .await?;
7343 }
7344
7345 reset_restart_count(snapshot, &spec.module_id)?;
7346 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7347 schedule_respawn(
7348 runtime,
7349 snapshot,
7350 &spec.module_id,
7351 runtime.restart_policy.backoff,
7352 RespawnKind::Reload,
7353 )
7354}
7355
7356async fn finish_reload_child(
7357 spec: &ModuleSpec,
7358 runtime: &SupervisorRuntimeConfig,
7359 registry: &Registry,
7360 process_liveness: &SupervisorProcessLiveness,
7361 snapshot: &SharedSnapshot,
7362 child: &mut Option<SupervisedChild>,
7363) -> Result<(), SuperviseError> {
7364 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7365 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7366 Ok(next_child) => next_child,
7367 Err(err) => {
7368 return handle_reload_spawn_failure(
7369 spec,
7370 runtime,
7371 process_liveness,
7372 snapshot,
7373 child,
7374 format!("new child failed to spawn: {err}"),
7375 )
7376 .await;
7377 }
7378 };
7379 *child = Some(next_child);
7380
7381 let wait_outcome = {
7382 let active_child = child.as_mut().expect("new reload child was just stored");
7383 wait_for_registration_after_reload(
7384 registry,
7385 &spec.module_id,
7386 snapshot,
7387 active_child,
7388 REGISTRY_RELEASE_TIMEOUT,
7389 )
7390 .await?
7391 };
7392
7393 match wait_outcome {
7394 RegistrationWaitOutcome::Registered => {
7395 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7396 Ok(())
7397 }
7398 RegistrationWaitOutcome::Exited(exit_report) => {
7399 if let Some(active_child) = child.as_mut() {
7400 active_child.drain_stderr(&spec.module_id).await;
7401 }
7402 let mut exited_child = child.take().expect("exited reload child is still stored");
7405 #[cfg(test)]
7406 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7407 gate.reached.notify_one();
7408 gate.resume.notified().await;
7409 }
7410 let result = handle_reload_child_registration_failure(
7411 spec,
7412 runtime,
7413 registry,
7414 process_liveness,
7415 snapshot,
7416 child,
7417 ReloadRegistrationFailure {
7418 exit_report: registration_failure_exit_report(exit_report),
7419 reason: exited_child
7420 .spawn_failure
7421 .clone()
7422 .unwrap_or_else(|| "new child exited before registering".to_string()),
7423 },
7424 )
7425 .await;
7426 exited_child.release_roster();
7427 result
7428 }
7429 RegistrationWaitOutcome::TimedOut => {
7430 let mut timed_out_child = child
7431 .take()
7432 .expect("timed-out reload child is still running");
7433 timed_out_child
7434 .start_kill()
7435 .map_err(|source| SuperviseError::Kill {
7436 module_id: spec.module_id.clone(),
7437 source,
7438 })?;
7439 let status = timed_out_child
7440 .wait()
7441 .await
7442 .map_err(|source| SuperviseError::Wait {
7443 module_id: spec.module_id.clone(),
7444 source,
7445 })?;
7446 timed_out_child.drain_stderr(&spec.module_id).await;
7447 handle_reload_child_registration_failure(
7448 spec,
7449 runtime,
7450 registry,
7451 process_liveness,
7452 snapshot,
7453 child,
7454 ReloadRegistrationFailure {
7455 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7456 snapshot,
7457 &timed_out_child,
7458 &status,
7459 )),
7460 reason: format!(
7461 "new child did not register within {:?}",
7462 REGISTRY_RELEASE_TIMEOUT
7463 ),
7464 },
7465 )
7466 .await
7467 }
7468 }
7469}
7470
7471async fn set_child_enabled(
7472 spec: &ModuleSpec,
7473 runtime: &SupervisorRuntimeConfig,
7474 registry: &Registry,
7475 process_liveness: &SupervisorProcessLiveness,
7476 snapshot: &SharedSnapshot,
7477 child: &mut Option<SupervisedChild>,
7478 enabled: bool,
7479) -> Result<bool, SuperviseError> {
7480 let (current_enabled, current_state, respawn_pending) = {
7481 let state = lock_snapshot(snapshot)?;
7482 (state.enabled, state.state, state.respawn_pending)
7483 };
7484 let revive_terminal = enabled
7492 && current_enabled
7493 && child.is_none()
7494 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7495 || (current_state == ModuleState::Restarting && !respawn_pending));
7496 if current_enabled == enabled && !revive_terminal {
7497 return Ok(false);
7498 }
7499
7500 if enabled {
7501 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7502 state.enabled = true;
7503 state.state = ModuleState::Starting;
7504 clear_current_process_facts(state);
7505 })?;
7506 #[cfg(test)]
7507 if runtime.test_seed_stale_facts_before_enable_spawn {
7508 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7509 state.process_alive = true;
7510 state.pid = Some(41);
7511 state.spawned_at_ms = Some(42);
7512 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7513 state.spawned_file_identity = Some(SpawnedFileIdentity {
7514 device: 43,
7515 inode: 44,
7516 #[cfg(windows)]
7517 windows_image: None,
7518 });
7519 })?;
7520 }
7521 release_dead_registration(
7522 registry,
7523 runtime.forwarding.as_deref(),
7524 snapshot,
7525 &spec.module_id,
7526 )
7527 .await?;
7528 reset_restart_count(snapshot, &spec.module_id)?;
7529 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7530 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7531 Ok(next_child) => next_child,
7532 Err(err) => {
7533 if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7534 state.state = ModuleState::Failed;
7535 clear_current_process_facts(state);
7536 }) {
7537 error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7538 }
7539 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7540 return Err(err);
7541 }
7542 };
7543 *child = Some(next_child);
7544 debug!(module_id = %spec.module_id, "supervised module enabled");
7545 Ok(true)
7546 } else {
7547 let stop_notice = begin_forwarding_drain_if_configured(
7548 spec,
7549 runtime,
7550 registry,
7551 snapshot,
7552 Some(false),
7553 RouteCloseReason::Disable,
7554 )
7555 .await?;
7556 drain_optional_child(
7557 &spec.module_id,
7558 spec.protocol,
7559 stop_notice,
7560 registry,
7561 runtime.forwarding.as_deref(),
7562 snapshot,
7563 &runtime.terminal_ring,
7564 &runtime.spawn_events,
7565 child,
7566 runtime.drain_timeout,
7567 ModuleState::Disabled,
7568 Some(false),
7569 )
7570 .await?;
7571 debug!(module_id = %spec.module_id, "supervised module disabled");
7572 Ok(true)
7573 }
7574}
7575
7576#[allow(clippy::too_many_arguments)]
7577async fn on_child_exit(
7578 spec: &ModuleSpec,
7579 policy: RestartPolicy,
7580 registry: &Registry,
7581 snapshot: &SharedSnapshot,
7582 terminal_ring: &Arc<Mutex<TerminalRing>>,
7583 spawn_events: &SpawnEventFeed,
7584 roster: &ChildRoster,
7585 exit_report: ExitReport,
7586) -> NextAction {
7587 if roster.is_closed() {
7593 return on_child_exit_during_daemon_shutdown(
7594 spec,
7595 registry,
7596 snapshot,
7597 terminal_ring,
7598 spawn_events,
7599 exit_report,
7600 )
7601 .await;
7602 }
7603 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7618 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7619 match exit_report.kind {
7620 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7621 info!(
7622 module_id = %spec.module_id,
7623 exit_code = ?exit_report.code,
7624 exit_signal = ?exit_report.signal,
7625 "supervised module exited cleanly"
7626 );
7627 if let Err(err) = publish_after_terminal_record(
7628 &spec.module_id,
7629 || {
7630 record_terminal(
7631 &spec.module_id,
7632 terminal_ring,
7633 spawn_events,
7634 &exit_report,
7635 TerminalDisposition::Stopped,
7636 )
7637 },
7638 || {
7639 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7640 state.state = ModuleState::Stopped;
7641 clear_current_process_facts(state);
7642 state.last_exit = Some(exit_report.clone());
7643 })
7644 },
7645 ) {
7646 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7647 }
7648 let registration_released = match wait_for_registration_release(
7649 registry,
7650 &spec.module_id,
7651 REGISTRY_RELEASE_TIMEOUT,
7652 )
7653 .await
7654 {
7655 Ok(()) => true,
7656 Err(err) => {
7657 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7658 false
7659 }
7660 };
7661 NextAction::Stop {
7662 registration_released,
7663 }
7664 }
7665 ExitKind::Clean | ExitKind::Crash => {
7666 if unrequested_clean_exit_of_protocol_none {
7667 warn!(
7668 module_id = %spec.module_id,
7669 exit_code = ?exit_report.code,
7670 exit_signal = ?exit_report.signal,
7671 "protocol-none module exited cleanly without a stop request; handling it as a crash"
7672 );
7673 } else {
7674 warn!(
7675 module_id = %spec.module_id,
7676 exit_code = ?exit_report.code,
7677 exit_signal = ?exit_report.signal,
7678 "supervised module exited abnormally (crash)"
7679 );
7680 }
7681 let mut restart_schedule = None;
7682 let mut disposition = TerminalDisposition::Disabled;
7683 let mut final_state = ModuleState::Disabled;
7684 let mut disposition_detail = lock_snapshot(snapshot)
7688 .ok()
7689 .and_then(|mut state| state.spawn_failure.take());
7690 let now = Instant::now();
7691 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7692 if state.enabled {
7693 if let Some(schedule) = state.next_crash_restart(&policy, now) {
7694 state.respawn_pending = true;
7700 final_state = ModuleState::Restarting;
7701 restart_schedule = Some(schedule);
7702 disposition = TerminalDisposition::Restarting;
7703 } else {
7704 final_state = ModuleState::Failed;
7705 disposition = TerminalDisposition::Failed;
7706 let budget = policy.budget_exhausted_detail();
7707 disposition_detail =
7708 Some(disposition_detail.take().map_or_else(
7709 || budget.clone(),
7710 |cause| format!("{cause}; {budget}"),
7711 ));
7712 }
7713 } else {
7714 disposition = TerminalDisposition::Disabled;
7715 }
7716 }) {
7717 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7718 return NextAction::Stop {
7719 registration_released: false,
7720 };
7721 }
7722 if disposition == TerminalDisposition::Failed {
7723 error!(
7728 module_id = %spec.module_id,
7729 max_restarts = policy.max_restarts,
7730 window_secs = policy.window.as_secs(),
7731 "module stopped: {}",
7732 policy.budget_exhausted_detail()
7733 );
7734 }
7735 if let Err(err) = publish_after_terminal_record(
7736 &spec.module_id,
7737 || {
7738 record_terminal_with_detail(
7739 &spec.module_id,
7740 terminal_ring,
7741 spawn_events,
7742 &exit_report,
7743 disposition,
7744 disposition_detail,
7745 )
7746 },
7747 || {
7748 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7749 state.state = final_state;
7750 clear_current_process_facts(state);
7751 state.last_exit = Some(exit_report.clone());
7752 })
7753 },
7754 ) {
7755 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7756 }
7757
7758 if let Some(schedule) = restart_schedule {
7759 NextAction::Restart {
7760 schedule: Some(schedule),
7761 }
7762 } else {
7763 let registration_released = match wait_for_registration_release(
7764 registry,
7765 &spec.module_id,
7766 REGISTRY_RELEASE_TIMEOUT,
7767 )
7768 .await
7769 {
7770 Ok(()) => true,
7771 Err(err) => {
7772 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7773 false
7774 }
7775 };
7776 NextAction::Stop {
7777 registration_released,
7778 }
7779 }
7780 }
7781 ExitKind::DeliberateSeverance => {
7782 warn!(
7783 module_id = %spec.module_id,
7784 exit_code = ?exit_report.code,
7785 exit_signal = ?exit_report.signal,
7786 "supervised module exited after deliberate connection severance"
7787 );
7788 let should_restart = match lock_snapshot(snapshot) {
7789 Ok(state) => state.enabled,
7790 Err(err) => {
7791 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7792 return NextAction::Stop {
7793 registration_released: false,
7794 };
7795 }
7796 };
7797 let final_state = if should_restart {
7798 ModuleState::Restarting
7799 } else {
7800 ModuleState::Disabled
7801 };
7802 if let Err(err) = publish_after_terminal_record(
7803 &spec.module_id,
7804 || {
7805 record_terminal(
7806 &spec.module_id,
7807 terminal_ring,
7808 spawn_events,
7809 &exit_report,
7810 terminal_disposition(final_state),
7811 )
7812 },
7813 || {
7814 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7815 state.state = final_state;
7816 clear_current_process_facts(state);
7817 state.last_exit = Some(exit_report.clone());
7818 state.lifetime_restarts += 1;
7819 })
7820 },
7821 ) {
7822 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7823 return NextAction::Stop {
7824 registration_released: false,
7825 };
7826 }
7827
7828 if should_restart {
7829 NextAction::Restart { schedule: None }
7830 } else {
7831 let registration_released = match wait_for_registration_release(
7832 registry,
7833 &spec.module_id,
7834 REGISTRY_RELEASE_TIMEOUT,
7835 )
7836 .await
7837 {
7838 Ok(()) => true,
7839 Err(err) => {
7840 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7841 false
7842 }
7843 };
7844 NextAction::Stop {
7845 registration_released,
7846 }
7847 }
7848 }
7849 }
7850}
7851
7852async fn on_child_exit_during_daemon_shutdown(
7853 spec: &ModuleSpec,
7854 registry: &Registry,
7855 snapshot: &SharedSnapshot,
7856 terminal_ring: &Arc<Mutex<TerminalRing>>,
7857 spawn_events: &SpawnEventFeed,
7858 exit_report: ExitReport,
7859) -> NextAction {
7860 info!(
7861 module_id = %spec.module_id,
7862 exit_code = ?exit_report.code,
7863 exit_signal = ?exit_report.signal,
7864 exit_kind = ?exit_report.kind,
7865 "supervised module exited during daemon shutdown; not restarting it"
7866 );
7867 if let Err(err) = publish_after_terminal_record(
7868 &spec.module_id,
7869 || {
7870 record_terminal(
7871 &spec.module_id,
7872 terminal_ring,
7873 spawn_events,
7874 &exit_report,
7875 TerminalDisposition::DaemonShutdown,
7876 )
7877 },
7878 || {
7879 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7880 state.state = ModuleState::Stopped;
7881 clear_current_process_facts(state);
7882 state.last_exit = Some(exit_report.clone());
7883 })
7884 },
7885 ) {
7886 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7887 }
7888 let registration_released =
7889 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7890 .await
7891 .is_ok();
7892 NextAction::Stop {
7893 registration_released,
7894 }
7895}
7896
7897fn publish_after_terminal_record<T>(
7902 module_id: &str,
7903 record: impl FnOnce(),
7904 publish: impl FnOnce() -> T,
7905) -> T {
7906 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7907 let published = publish();
7908 if let Err(panic) = recorded {
7909 error!(
7910 module_id,
7911 "terminal recording panicked; published final state before resuming unwind"
7912 );
7913 std::panic::resume_unwind(panic);
7914 }
7915 published
7916}
7917
7918fn fail_after_wait_error(
7919 module_id: &str,
7920 snapshot: &SharedSnapshot,
7921 terminal_ring: &Arc<Mutex<TerminalRing>>,
7922 spawn_events: &SpawnEventFeed,
7923 detail: String,
7924) {
7925 let report = wait_error_exit_report();
7926 publish_after_terminal_record(
7927 module_id,
7928 || {
7929 record_terminal_with_detail(
7930 module_id,
7931 terminal_ring,
7932 spawn_events,
7933 &report,
7934 TerminalDisposition::Failed,
7935 Some(detail),
7936 )
7937 },
7938 || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7939 );
7940}
7941
7942#[cfg(test)]
7943fn record_wait_error_terminal(
7944 module_id: &str,
7945 terminal_ring: &Arc<Mutex<TerminalRing>>,
7946 spawn_events: &SpawnEventFeed,
7947) {
7948 record_terminal(
7949 module_id,
7950 terminal_ring,
7951 spawn_events,
7952 &wait_error_exit_report(),
7953 TerminalDisposition::Failed,
7954 );
7955}
7956
7957fn record_terminal(
7958 module_id: &str,
7959 terminal_ring: &Arc<Mutex<TerminalRing>>,
7960 spawn_events: &SpawnEventFeed,
7961 exit_report: &ExitReport,
7962 disposition: TerminalDisposition,
7963) {
7964 record_terminal_with_detail(
7965 module_id,
7966 terminal_ring,
7967 spawn_events,
7968 exit_report,
7969 disposition,
7970 None,
7971 );
7972}
7973
7974fn durable_terminal_history_of(
7978 terminal_ring: &Mutex<TerminalRing>,
7979 module_id: &str,
7980) -> subc_control::TerminalHistory {
7981 let read = terminal_ring
7982 .lock()
7983 .unwrap_or_else(|p| p.into_inner())
7984 .capture_durable_history();
7985 read.read(module_id)
7986}
7987
7988fn record_terminal_with_detail(
7989 module_id: &str,
7990 terminal_ring: &Arc<Mutex<TerminalRing>>,
7991 spawn_events: &SpawnEventFeed,
7992 exit_report: &ExitReport,
7993 disposition: TerminalDisposition,
7994 disposition_detail: Option<String>,
7995) {
7996 #[cfg(test)]
7997 terminal_history_tests::before_terminal_record(spawn_events);
7998 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7999 let record = TerminalRecord {
8000 exit_code: exit_report.code,
8001 exit_signal: exit_report.signal,
8002 at_ms: exit_report.at_ms,
8003 disposition,
8004 exit_kind: exit_report.kind.into(),
8005 disposition_detail,
8006 };
8007 terminal_ring
8008 .lock()
8009 .unwrap_or_else(|poisoned| poisoned.into_inner())
8010 .record_exit(module_id, record);
8011}
8012
8013fn untrack_if_registration_released(
8014 process_liveness: &SupervisorProcessLiveness,
8015 registry: &Registry,
8016 module_id: &str,
8017 snapshot: &SharedSnapshot,
8018) {
8019 match registry.get_module(module_id) {
8020 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8021 Ok(Some(_)) => {}
8022 Err(err) => {
8023 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8024 }
8025 }
8026}
8027
8028#[cfg(test)]
8042fn apply_wire_spawn_args(
8043 command: &mut Command,
8044 spec: &ModuleSpec,
8045 connection_file_path: Option<&std::path::Path>,
8046 handle: Option<&SupervisorHandle>,
8047) -> Result<Option<NonceHandoff>, SuperviseError> {
8048 apply_wire_spawn_args_for_role(
8049 command,
8050 spec,
8051 connection_file_path,
8052 handle,
8053 SpawnRole::Plain,
8054 )
8055}
8056
8057#[cfg(unix)]
8062type NonceHandoff = subc_os::LaunchNonceHandoff;
8063#[cfg(windows)]
8064type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8065#[cfg(not(any(unix, windows)))]
8066type NonceHandoff = std::convert::Infallible;
8067
8068fn apply_wire_spawn_args_for_role(
8089 command: &mut Command,
8090 spec: &ModuleSpec,
8091 connection_file_path: Option<&std::path::Path>,
8092 handle: Option<&SupervisorHandle>,
8093 role: SpawnRole,
8094) -> Result<Option<NonceHandoff>, SuperviseError> {
8095 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8096 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8101 #[cfg(windows)]
8102 {
8103 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8104 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8105 }
8106 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8108 if spec.protocol == ModuleProtocol::None {
8109 return Ok(None);
8110 }
8111 if let Some(connection_file_path) = connection_file_path {
8112 command.arg(SUBC_ARG).arg(connection_file_path);
8113 }
8114
8115 let nonce = generate_launch_nonce()?;
8119 if let Some(handle) = handle {
8120 match role {
8121 SpawnRole::Plain => {
8122 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8123 if spec.reserved {
8124 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8125 }
8126 }
8127 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8128 }
8129 }
8130 #[cfg(unix)]
8131 let handoff = {
8132 let handoff =
8133 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8134 program: spec.program.clone(),
8135 source,
8136 cgroup_path: None,
8137 })?;
8138 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8139 Some(handoff)
8140 };
8141 #[cfg(windows)]
8142 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8143 #[cfg(not(any(unix, windows)))]
8144 let handoff = None;
8145 #[cfg(not(unix))]
8149 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8150 Ok(handoff)
8151}
8152
8153fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8154 command.env_remove(CK_LOG_ENV);
8155 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8162 for (key, value) in &spec.env {
8163 if matches!(
8167 key.as_str(),
8168 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8169 ) || key == SUBC_SPAWN_ROLE_ENV
8170 {
8171 continue;
8172 }
8173 command.env(key, value);
8174 }
8175}
8176
8177#[cfg(windows)]
8178fn prepare_windows_nonce_pipe(
8179 command: &mut Command,
8180 spec: &ModuleSpec,
8181 nonce: &str,
8182) -> Option<NonceHandoff> {
8183 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8184 Ok(handoff) => {
8185 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8186 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8191 Some(handoff)
8192 }
8193 Err(error) => {
8194 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8195 None
8196 }
8197 }
8198}
8199
8200#[cfg(windows)]
8201fn start_windows_nonce_pipe(
8202 handoff: Option<NonceHandoff>,
8203 pid: u32,
8204 spec: &ModuleSpec,
8205) -> Option<subc_os::LaunchNoncePipeDelivery> {
8206 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8210 Ok(delivery) => Some(delivery),
8211 Err(error) => {
8212 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8213 None
8214 }
8215 })
8216}
8217
8218fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8219 #[cfg(windows)]
8220 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8221 return Some(
8222 if snapshot
8223 .pipe_consumed
8224 .as_ref()
8225 .is_some_and(|consumed| consumed.get().is_some())
8226 {
8227 "pipe"
8228 } else {
8229 "env"
8230 }
8231 .to_owned(),
8232 );
8233 }
8234 let _ = snapshot;
8235 None
8236}
8237
8238#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8241enum SpawnRole {
8242 Plain,
8243 SwapCandidate,
8244}
8245
8246fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8249 if role == SpawnRole::SwapCandidate {
8250 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8251 }
8252}
8253
8254fn previous_instance_error(
8255 spec: &ModuleSpec,
8256 pid: u32,
8257 reason: crate::live_children::RetainedReason,
8258) -> SuperviseError {
8259 SuperviseError::Spawn {
8260 program: spec.program.clone(),
8261 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8262 cgroup_path: None,
8263 }
8264}
8265
8266fn spawn_child(
8267 spec: &ModuleSpec,
8268 connection_file_path: Option<&std::path::Path>,
8269 handle: Option<&SupervisorHandle>,
8270 ring: &Arc<Mutex<StderrRing>>,
8271 capture_logs_dir: Option<&std::path::Path>,
8272 roster: &ChildRoster,
8273 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8274) -> Result<SupervisedChild, SuperviseError> {
8275 spawn_child_in_slot(
8276 spec,
8277 connection_file_path,
8278 handle,
8279 ring,
8280 capture_logs_dir,
8281 roster,
8282 #[cfg(target_os = "linux")]
8283 cgroup_placement,
8284 SpawnRole::Plain,
8285 false,
8286 )
8287}
8288
8289#[allow(clippy::too_many_arguments)]
8302fn spawn_child_in_slot(
8303 spec: &ModuleSpec,
8304 connection_file_path: Option<&std::path::Path>,
8305 handle: Option<&SupervisorHandle>,
8306 ring: &Arc<Mutex<StderrRing>>,
8307 capture_logs_dir: Option<&std::path::Path>,
8308 roster: &ChildRoster,
8309 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8310 role: SpawnRole,
8311 alternate_slot: bool,
8312) -> Result<SupervisedChild, SuperviseError> {
8313 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8314 return Err(previous_instance_error(spec, pid, reason));
8315 }
8316 if roster.is_closed() {
8317 return Err(SuperviseError::Spawn {
8318 program: spec.program.clone(),
8319 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8320 cgroup_path: None,
8321 });
8322 }
8323 #[cfg(target_os = "linux")]
8324 let cgroup_name = {
8325 if cgroup_placement.is_none() {
8330 swap::cgroup_name(&spec.module_id, alternate_slot)
8331 } else {
8332 let nonce = generate_launch_nonce()?;
8333 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8334 let mut end = spec.module_id.len().min(64);
8337 while !spec.module_id.is_char_boundary(end) {
8338 end -= 1;
8339 }
8340 format!(
8341 "{}_{suffix}",
8342 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8343 )
8344 }
8345 };
8346 #[cfg(not(target_os = "linux"))]
8347 let _ = alternate_slot;
8348 #[cfg(target_os = "macos")]
8349 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8350 #[cfg(windows)]
8351 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8352 program: spec.program.clone(),
8353 source,
8354 cgroup_path: None,
8355 })?;
8356 #[cfg(windows)]
8357 let mut command = Command::new(image_capture.path());
8358 #[cfg(not(any(target_os = "macos", windows)))]
8359 let mut command = Command::new(&spec.program);
8360 command.args(&spec.args);
8361 apply_child_env(&mut command, spec);
8391 apply_spawn_role(&mut command, role);
8392 let nonce_handoff =
8393 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8394
8395 #[cfg(target_os = "linux")]
8396 let cgroup_path = cgroup_placement
8397 .map(|placement| placement.module_path(&cgroup_name))
8398 .transpose()
8399 .map_err(|source| SuperviseError::Cgroup {
8400 module_id: spec.module_id.clone(),
8401 source,
8402 })?;
8403 #[cfg(not(target_os = "linux"))]
8404 let cgroup_path: Option<PathBuf> = None;
8405 #[cfg(target_os = "linux")]
8406 if let Some(path) = &cgroup_path {
8407 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8408 if let Some(placement) = cgroup_placement {
8409 remove_module_cgroup(placement, &cgroup_name);
8410 }
8411 return Err(error);
8412 }
8413 }
8414
8415 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8416 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8417 match ChildOutputSink::open(&path, capture_retention(spec)) {
8418 Ok(sink) => sink,
8419 Err(error) => {
8420 warn!(
8421 module_id = %spec.module_id,
8422 path = %path.display(),
8423 error = %error,
8424 "could not open child output capture file; forwarding to stderr"
8425 );
8426 ChildOutputSink::Stderr
8427 }
8428 }
8429 } else {
8430 ChildOutputSink::Stderr
8431 };
8432
8433 command.stdout(Stdio::piped());
8434 command.stderr(Stdio::piped());
8435 command.kill_on_drop(true);
8436 #[cfg(unix)]
8453 command.process_group(0);
8454 command.stdin(Stdio::null());
8455 #[cfg(unix)]
8459 if let Some(handoff) = nonce_handoff {
8460 handoff.install_last(command.as_std_mut());
8461 }
8462 #[cfg(not(any(unix, windows)))]
8463 let _ = nonce_handoff;
8464
8465 #[cfg(windows)]
8470 {
8471 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8472 program: spec.program.clone(),
8473 source,
8474 cgroup_path: None,
8475 })?;
8476 subc_jobobject::suspend_on_create_async(&mut command);
8477 }
8478 let spawned_at = Instant::now();
8482 let mut child = match command.spawn() {
8483 Ok(child) => child,
8484 Err(source) => {
8485 #[cfg(target_os = "linux")]
8486 if let Some(placement) = cgroup_placement {
8487 remove_module_cgroup(placement, &cgroup_name);
8488 }
8489 return Err(SuperviseError::Spawn {
8490 program: spec.program.clone(),
8491 source,
8492 cgroup_path,
8493 });
8494 }
8495 };
8496 #[cfg(test)]
8497 spawn_record_tests::after_create(&child, spec);
8498 #[cfg(target_os = "macos")]
8503 drop(exec_ack);
8504
8505 #[cfg(windows)]
8510 let windows_image = match image_capture.bind_tokio(&child) {
8511 Ok(image) => Arc::new(image),
8512 Err(source) => {
8513 kill_and_reap_rejected_child(&mut child);
8514 return Err(SuperviseError::Spawn {
8515 program: spec.program.clone(),
8516 source,
8517 cgroup_path: None,
8518 });
8519 }
8520 };
8521 #[cfg(windows)]
8524 let job = contain_spawned_child(&mut child, spec)?;
8525 let spawned_at_ms = unix_ms_now();
8526 #[cfg(not(windows))]
8527 let spawned_from = spec.program.clone();
8528 #[cfg(windows)]
8529 let spawned_from = windows_image.path().to_path_buf();
8530 #[cfg(not(windows))]
8531 let spawned_file_identity = spawned_file_identity(&spawned_from);
8532 #[cfg(windows)]
8533 let spawned_file_identity = Some(SpawnedFileIdentity {
8534 device: 0,
8535 inode: 0,
8536 windows_image: Some(Arc::clone(&windows_image)),
8537 });
8538 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8539 program: spec.program.clone(),
8540 source: io::Error::other("spawned child exposed no live pid"),
8541 cgroup_path: cgroup_path.clone(),
8542 })?;
8543 #[cfg(windows)]
8544 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8545 #[cfg(not(windows))]
8546 let process_start_time = crate::provenance::process_start_time(pid);
8547 #[cfg(windows)]
8548 let process_start_time = Some(windows_image.start_time());
8549 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8550 #[cfg(all(test, target_os = "macos"))]
8551 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8552 #[cfg(not(windows))]
8557 let recorded_image = observe_spawned_image(pid);
8558 #[cfg(target_os = "macos")]
8562 let recorded_image = if privacy_exec.is_some() {
8563 None
8564 } else {
8565 recorded_image
8566 };
8567 #[cfg(target_os = "linux")]
8568 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8569 #[cfg(not(target_os = "linux"))]
8570 let recorded_cgroup_name = None;
8571 let roster_guard = roster.admit(
8572 spec.module_id.clone(),
8573 pid,
8574 spec.protocol,
8575 process_start_time,
8576 crate::child_roster::RecordedIdentity {
8577 #[cfg(not(windows))]
8578 start_time: recorded_image.map(|image| image.start_time),
8579 #[cfg(windows)]
8580 start_time: process_start_time,
8581 #[cfg(not(windows))]
8582 executable: recorded_image
8583 .and_then(|image| image.executable)
8584 .map(crate::live_children::ExecutableIdentity::from),
8585 #[cfg(windows)]
8586 executable: Some(crate::live_children::ExecutableIdentity::from(
8587 windows_image.identity(),
8588 )),
8589 cgroup_name: recorded_cgroup_name,
8590 #[cfg(target_os = "linux")]
8591 cgroup_placement: cgroup_placement.cloned(),
8592 },
8593 );
8594 let roster_guard = match roster_guard {
8595 Ok(guard) => guard,
8596 Err(source) => {
8597 #[cfg(windows)]
8598 if let Some(job) = &job {
8599 let _ = job.terminate();
8600 }
8601 #[cfg(target_os = "linux")]
8602 kill_module_cgroup(cgroup_placement, &cgroup_name);
8603 kill_and_reap_rejected_child(&mut child);
8604 #[cfg(target_os = "linux")]
8605 if let Some(placement) = cgroup_placement {
8606 remove_module_cgroup(placement, &cgroup_name);
8607 }
8608 return Err(SuperviseError::Spawn {
8609 program: spec.program.clone(),
8610 source: io::Error::other(format!(
8611 "could not durably record module child: {source}"
8612 )),
8613 cgroup_path,
8614 });
8615 }
8616 };
8617 if roster.is_closed() {
8626 #[cfg(target_os = "linux")]
8628 kill_module_cgroup(cgroup_placement, &cgroup_name);
8629 if let Err(error) = child.start_kill() {
8630 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8631 }
8632 #[cfg(target_os = "linux")]
8633 if let Some(placement) = cgroup_placement {
8634 while matches!(child.try_wait(), Ok(None)) {
8638 std::thread::yield_now();
8639 }
8640 if matches!(
8641 subc_cgroup::kill_module(Some(placement), &cgroup_name),
8642 subc_cgroup::KillOutcome::Killed
8643 ) {
8644 if let Ok(path) = placement.module_path(&cgroup_name) {
8645 while std::fs::read_to_string(path.join("cgroup.events"))
8646 .ok()
8647 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8648 {
8649 std::thread::yield_now();
8650 }
8651 }
8652 }
8653 remove_module_cgroup(placement, &cgroup_name);
8654 }
8655 drop(roster_guard);
8656 return Err(SuperviseError::Spawn {
8657 program: spec.program.clone(),
8658 source: io::Error::other(
8659 "the daemon began shutting down while this process was starting; ended it",
8660 ),
8661 cgroup_path,
8662 });
8663 }
8664
8665 #[cfg(windows)]
8666 if let Err(error) = resume_suspended_child(pid, spec) {
8667 kill_and_reap_rejected_child(&mut child);
8668 return Err(error);
8669 }
8670
8671 let stdout_pump = match child.stdout.take() {
8672 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8673 None => {
8674 warn!(
8675 module_id = %spec.module_id,
8676 "spawned child exposed no stdout pipe; file capture will be incomplete"
8677 );
8678 None
8679 }
8680 };
8681 let stderr_pump = match child.stderr.take() {
8682 Some(stderr) => {
8683 let generation = ring
8684 .lock()
8685 .unwrap_or_else(|poisoned| poisoned.into_inner())
8686 .begin_process();
8687 Some(StderrPump {
8688 task: tokio::spawn(pump_stderr_to(
8689 stderr,
8690 Arc::clone(ring),
8691 generation,
8692 output_sink,
8693 )),
8694 generation,
8695 })
8696 }
8697 None => {
8698 ring.lock()
8702 .unwrap_or_else(|poisoned| poisoned.into_inner())
8703 .mark_not_captured("stderr pipe was not available on spawn");
8704 warn!(
8705 module_id = %spec.module_id,
8706 "spawned child exposed no stderr pipe; tail will be unavailable"
8707 );
8708 None
8709 }
8710 };
8711
8712 let exec = Arc::new(crate::activity::ExecOutcome::new());
8713 #[cfg(target_os = "macos")]
8714 let pending_exec = privacy_exec.is_some();
8715 #[cfg(not(target_os = "macos"))]
8716 let pending_exec = false;
8717 if !pending_exec {
8718 exec.finish(crate::activity::ExecState::CommittedExec);
8719 }
8720 Ok(SupervisedChild {
8721 child,
8722 exec: crate::activity::ExecOwner(exec),
8723 #[cfg(windows)]
8724 nonce_delivery,
8725 protocol: spec.protocol,
8726 #[cfg(target_os = "linux")]
8727 module_id: cgroup_name,
8728 #[cfg(target_os = "linux")]
8729 cgroup_placement: cgroup_placement.cloned(),
8730 #[cfg(windows)]
8731 job,
8732 stdout_pump,
8733 stderr_pump,
8734 stderr_ring: Arc::clone(ring),
8735 spawned_at,
8736 spawned_at_ms,
8737 spawned_from,
8738 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8739 spawned_file_identity,
8740 process_start_time,
8741 process_identity,
8742 pid,
8743 roster_guard: Some(roster_guard),
8744 #[cfg(target_os = "macos")]
8745 privacy_exec,
8746 #[cfg(target_os = "macos")]
8747 report_ready: Arc::new(OnceLock::new()),
8748 spawn_failure: None,
8749 })
8750}
8751
8752#[cfg(test)]
8753mod spawn_record_tests {
8754 use super::*;
8755
8756 thread_local! {
8757 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8758 }
8759
8760 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8761 LAST_PID.set(child.id());
8762 #[cfg(windows)]
8763 if let Some((_, address)) = _spec
8764 .env
8765 .iter()
8766 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8767 {
8768 use std::io::{Read, Write};
8769 let mut stream = std::net::TcpStream::connect(address).unwrap();
8770 stream
8771 .write_all(&child.id().unwrap().to_ne_bytes())
8772 .unwrap();
8773 let mut release = [0];
8774 stream.read_exact(&mut release).unwrap();
8775 }
8776 }
8777
8778 #[cfg(windows)]
8779 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8780 if spec
8781 .env
8782 .iter()
8783 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8784 {
8785 Err(io::Error::other("injected module job assignment failure"))
8786 } else {
8787 Ok(())
8788 }
8789 }
8790
8791 fn spec() -> ModuleSpec {
8792 ModuleSpec {
8793 module_id: "spawn-record-worker".into(),
8794 #[cfg(unix)]
8795 program: "/bin/sleep".into(),
8796 #[cfg(windows)]
8797 program: std::env::current_exe().unwrap(),
8798 #[cfg(unix)]
8799 args: vec!["60".into()],
8800 #[cfg(windows)]
8801 args: vec![
8802 "--exact".into(),
8803 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8804 "--nocapture".into(),
8805 ],
8806 env: vec![],
8807 reserved: false,
8808 reserved_prefixes: vec![],
8809 protocol: ModuleProtocol::None,
8810 overlap: Default::default(),
8811 }
8812 }
8813
8814 #[tokio::test]
8815 async fn record_sync_failure_fails_spawn_and_reaps_child() {
8816 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8817 let supervisor =
8818 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8819 .with_live_children_record(home.join("live-children.json"));
8820 crate::live_children::fail_next_file_sync();
8821 let result = supervisor.spawn(spec());
8822 let failed = result.is_err();
8823 if let Ok(module) = result {
8824 module.stop().await.unwrap();
8825 }
8826 assert!(failed, "record failure must fail spawn");
8827 let pid = LAST_PID.get().expect("the real OS child was created");
8828 assert!(
8829 subc_os::Process::open(pid).unwrap().is_none(),
8830 "child {pid} survived rejected admission or was not reaped"
8831 );
8832 }
8833
8834 #[cfg(target_os = "macos")]
8835 #[test]
8836 #[ignore = "manual on-device latency measurement"]
8837 fn durable_record_write_latency_50() {
8838 let home = cortexkit_test_support::ScratchDir::new("record-latency");
8839 let path = home.join("live-children.json");
8840 let children = vec![crate::live_children::LiveChild {
8841 module_id: "latency-worker".into(),
8842 pid: std::process::id(),
8843 protocol: ModuleProtocol::None,
8844 start_time: Some(1),
8845 executable: None,
8846 cgroup_name: None,
8847 }];
8848 let mut samples = Vec::new();
8849 for _ in 0..50 {
8850 let start = std::time::Instant::now();
8851 crate::live_children::write_record(&path, &children).unwrap();
8852 samples.push(start.elapsed());
8853 }
8854 samples.sort();
8855 eprintln!(
8856 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8857 (samples[24] + samples[25]) / 2,
8858 samples[47]
8859 );
8860 }
8861
8862 #[cfg(windows)]
8863 #[test]
8864 fn windows_store_touch_fixture() {
8865 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8866 std::fs::write(path, b"module ran").unwrap();
8867 }
8868 }
8869
8870 #[cfg(windows)]
8871 #[tokio::test]
8872 async fn windows_daemon_fixture() {
8873 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8874 return;
8875 };
8876 let mut spec = spec();
8877 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8878 let supervisor =
8879 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8880 let _module = supervisor.spawn(spec).unwrap();
8881 }
8882
8883 #[cfg(windows)]
8884 #[test]
8885 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8886 use std::io::Read;
8887 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8888 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8891 command
8892 .args([
8893 "--exact",
8894 "supervise::spawn_record_tests::windows_daemon_fixture",
8895 "--nocapture",
8896 ])
8897 .env(
8898 "SUBC_TEST_DAEMON_BARRIER",
8899 listener.local_addr().unwrap().to_string(),
8900 );
8901 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8902 listener.set_nonblocking(true).unwrap();
8903 let deadline = std::time::Instant::now() + Duration::from_secs(30);
8904 let mut stream = loop {
8905 if let Ok((stream, _)) = listener.accept() {
8906 break stream;
8907 }
8908 assert!(
8909 daemon.child.try_wait().unwrap().is_none(),
8910 "daemon failed before rendezvous"
8911 );
8912 assert!(
8913 std::time::Instant::now() < deadline,
8914 "daemon did not reach CreateProcess failpoint"
8915 );
8916 std::thread::yield_now();
8917 };
8918 stream
8919 .set_read_timeout(Some(Duration::from_secs(30)))
8920 .unwrap();
8921 let mut pid = [0; 4];
8922 stream.read_exact(&mut pid).unwrap();
8923 let pid = u32::from_ne_bytes(pid);
8924 assert!(subc_jobobject::process_exists(pid));
8925 daemon.child.kill().unwrap();
8928 daemon.child.wait().unwrap();
8929 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8930 if !exited {
8931 daemon.job.terminate().unwrap();
8932 }
8933 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8934 }
8935
8936 #[cfg(windows)]
8937 #[tokio::test]
8938 async fn windows_containment_failure_never_resumes() {
8939 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8940 let sentinel = home.join("store-touched");
8941 let mut spec = spec();
8942 spec.env
8943 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8944 spec.env.push((
8945 "SUBC_TEST_STORE_TOUCH".into(),
8946 sentinel.display().to_string(),
8947 ));
8948 let supervisor =
8949 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8950 let result = supervisor.spawn(spec);
8951 if let Ok(module) = result {
8954 let _ = tokio::time::timeout(Duration::from_secs(10), async {
8955 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8956 tokio::task::yield_now().await;
8957 }
8958 })
8959 .await;
8960 let _ = module.stop().await;
8961 panic!(
8962 "assignment failure resumed a module; store touched={}",
8963 sentinel.exists()
8964 );
8965 }
8966 assert!(
8967 !sentinel.exists(),
8968 "refused module executed store-touch sentinel"
8969 );
8970 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8971 }
8972
8973 #[cfg(windows)]
8974 #[tokio::test]
8975 async fn windows_nested_daemon_fixture() {
8976 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8977 return;
8978 };
8979 let mut spec = spec();
8980 spec.env.push((
8981 "SUBC_TEST_STORE_TOUCH".into(),
8982 sentinel.to_str().unwrap().into(),
8983 ));
8984 let supervisor =
8985 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8986 let module = supervisor
8987 .spawn(spec)
8988 .expect("per-module job must nest below daemon and host jobs");
8989 tokio::time::timeout(Duration::from_secs(10), async {
8990 while !PathBuf::from(&sentinel).exists() {
8991 tokio::task::yield_now().await;
8992 }
8993 })
8994 .await
8995 .unwrap();
8996 module.stop().await.unwrap();
8997 }
8998
8999 #[cfg(windows)]
9000 #[test]
9001 fn windows_module_job_nests_under_host_job() {
9002 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9003 let sentinel = home.join("nested-store-touched");
9004 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9005 command
9006 .args([
9007 "--exact",
9008 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9009 "--nocapture",
9010 ])
9011 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9012 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9013 assert!(daemon.child.wait().unwrap().success());
9014 assert!(sentinel.exists(), "nested protocol:none module never ran");
9015 }
9016}
9017
9018#[cfg(target_os = "linux")]
9019pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9020 use subc_cgroup::KillOutcome;
9021 match subc_cgroup::kill_module(placement, module_id) {
9022 KillOutcome::Killed => {}
9023 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9024 debug!(
9025 module_id,
9026 "cgroup tree kill unavailable; using direct-child kill"
9027 );
9028 }
9029 KillOutcome::IoError { path, error } => {
9030 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9031 }
9032 }
9033}
9034
9035#[cfg(windows)]
9043fn contain_spawned_child(
9044 child: &mut Child,
9045 spec: &ModuleSpec,
9046) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9047 let assignment = || {
9048 let job = subc_jobobject::JobObject::new()?;
9049 #[cfg(test)]
9050 spawn_record_tests::assignment_failpoint(spec)?;
9051 job.assign(child)?;
9052 Ok::<_, io::Error>(job)
9053 };
9054 match assignment() {
9055 Ok(job) => Ok(Some(job)),
9056 Err(source) => {
9057 kill_and_reap_rejected_child(child);
9058 Err(SuperviseError::Spawn {
9059 program: spec.program.clone(),
9060 source: io::Error::other(format!(
9061 "Windows module job containment failed: {source}"
9062 )),
9063 cgroup_path: None,
9064 })
9065 }
9066 }
9067}
9068
9069fn kill_and_reap_rejected_child(child: &mut Child) {
9072 #[cfg(unix)]
9073 if let Some(pid) = child
9074 .id()
9075 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9076 {
9077 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9078 }
9079 let _ = child.start_kill();
9080 while matches!(child.try_wait(), Ok(None)) {
9081 std::thread::yield_now();
9082 }
9083}
9084
9085#[cfg(windows)]
9090fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9091 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9092 let _ = std::process::Command::new("taskkill.exe")
9096 .args(["/PID", &pid.to_string(), "/T", "/F"])
9097 .stdin(Stdio::null())
9098 .stdout(Stdio::null())
9099 .stderr(Stdio::null())
9100 .status();
9101 return Err(SuperviseError::Spawn {
9102 program: spec.program.clone(),
9103 source,
9104 cgroup_path: None,
9105 });
9106 }
9107 Ok(())
9108}
9109
9110#[cfg(target_os = "linux")]
9111fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9112 match placement.remove_module(module_id) {
9113 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9114 Err(error) => warn!(
9115 module_id,
9116 error = %error,
9117 "could not remove module cgroup after process exit; continuing teardown"
9118 ),
9119 }
9120}
9121
9122#[cfg(target_os = "linux")]
9123async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9124 if matches!(
9128 subc_cgroup::kill_module(Some(placement), module_id),
9129 subc_cgroup::KillOutcome::Killed
9130 ) {
9131 if let Ok(path) = placement.module_path(module_id) {
9132 while std::fs::read_to_string(path.join("cgroup.events"))
9133 .ok()
9134 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9135 {
9136 sleep(Duration::from_millis(1)).await;
9137 }
9138 }
9139 }
9140 remove_module_cgroup(placement, module_id);
9141}
9142
9143#[cfg(target_os = "linux")]
9144fn apply_cgroup_placement(
9145 command: &mut Command,
9146 spec: &ModuleSpec,
9147 path: &std::path::Path,
9148) -> Result<(), SuperviseError> {
9149 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9150 module_id: spec.module_id.clone(),
9151 source,
9152 })
9153}
9154
9155fn capture_retention(spec: &ModuleSpec) -> Retention {
9156 let defaults = Retention::default();
9157 let value = |name: &str| {
9158 spec.env
9159 .iter()
9160 .rev()
9161 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9162 };
9163 Retention {
9164 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9165 .and_then(|value| value.parse().ok())
9166 .unwrap_or(defaults.max_file_mb),
9167 keep: value(CAPTURE_KEEP_ENV)
9168 .and_then(|value| value.parse().ok())
9169 .unwrap_or(defaults.keep),
9170 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9171 .and_then(|value| value.parse().ok())
9172 .unwrap_or(defaults.max_age_days),
9173 }
9174}
9175
9176fn generate_launch_nonce() -> Result<String, SuperviseError> {
9179 let mut bytes = [0u8; 32];
9180 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9181 reason: source.to_string(),
9182 })?;
9183 let mut hex = String::with_capacity(64);
9184 for b in bytes {
9185 use std::fmt::Write;
9186 let _ = write!(hex, "{b:02x}");
9187 }
9188 Ok(hex)
9189}
9190
9191fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9194 if a.len() != b.len() {
9195 return false;
9196 }
9197 let mut diff = 0u8;
9198 for (x, y) in a.iter().zip(b.iter()) {
9199 diff |= x ^ y;
9200 }
9201 diff == 0
9202}
9203
9204#[cfg(not(windows))]
9208fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9209 let process = subc_os::Process::open(pid).ok().flatten()?;
9210 #[cfg(target_os = "linux")]
9211 return sample_after_linux_exec(
9212 || {
9213 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9214 let committed = linux_exec_committed(&stat);
9215 #[cfg(test)]
9216 spawn_image_tests::report_exec_state(committed);
9217 committed
9218 },
9219 || process.observe(),
9220 );
9221 #[cfg(not(target_os = "linux"))]
9222 process.observe()
9223}
9224
9225#[cfg(target_os = "linux")]
9229fn linux_exec_committed(stat: &str) -> Option<bool> {
9230 const PF_FORKNOEXEC: u64 = 0x40;
9231 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9232 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9233}
9234
9235#[cfg(target_os = "linux")]
9236fn sample_after_linux_exec(
9237 mut exec_committed: impl FnMut() -> Option<bool>,
9238 observe: impl FnOnce() -> Option<subc_os::Observation>,
9239) -> Option<subc_os::Observation> {
9240 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9241 while !exec_committed()? {
9242 if std::time::Instant::now() >= deadline {
9246 return None;
9247 }
9248 std::thread::sleep(Duration::from_micros(50));
9249 }
9250 observe()
9251}
9252
9253#[cfg(all(test, target_os = "linux"))]
9254mod spawn_image_tests {
9255 use super::*;
9256 use std::cell::{Cell, RefCell};
9257
9258 enum ExecEvent {
9259 Uncommitted,
9260 Image(Option<subc_os::Observation>),
9261 }
9262
9263 thread_local! {
9264 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9265 }
9266
9267 pub(super) fn report_exec_state(committed: Option<bool>) {
9268 if committed == Some(false) {
9269 EXEC_EVENTS.with(|events| {
9270 if let Some(events) = events.borrow_mut().take() {
9271 let _ = events.send(ExecEvent::Uncommitted);
9272 }
9273 });
9274 }
9275 }
9276
9277 #[test]
9278 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9279 use std::io::{Read, Write};
9280 use std::os::fd::AsRawFd;
9281
9282 let descriptor = std::fs::File::open("/dev/null").unwrap();
9283 let mut command = std::process::Command::new("/bin/sleep");
9284 command.arg("30");
9285 let (mut ready, mut release) =
9286 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9287 .unwrap();
9288 let spawn = std::thread::spawn(move || command.spawn());
9289 let mut message = [0u8; 8];
9290 ready.read_exact(&mut message).unwrap();
9291 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9292 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9293 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9294 let (events, received) = std::sync::mpsc::channel();
9295 let observer = std::thread::spawn(move || {
9296 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9297 let image = observe_spawned_image(pid);
9298 let _ = events.send(ExecEvent::Image(image));
9299 });
9300 let first = received.recv_timeout(Duration::from_secs(5));
9304 release.write_all(&[1]).unwrap();
9305 let mut child = spawn.join().unwrap().unwrap();
9306 let after = match first {
9307 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9308 Ok(ExecEvent::Image(image)) => image,
9309 _ => None,
9310 },
9311 Ok(ExecEvent::Image(image)) => image,
9312 Err(_) => None,
9313 };
9314 observer.join().unwrap();
9315 child.kill().unwrap();
9316 child.wait().unwrap();
9317
9318 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9319 assert_eq!(
9320 inherited.unwrap().executable,
9321 subc_os::file_identity(&std::env::current_exe().unwrap())
9322 );
9323 assert_eq!(
9324 after.unwrap().executable,
9325 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9326 );
9327 }
9328
9329 #[test]
9330 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9331 let committed = Cell::new(false);
9332 let inherited = subc_os::Observation {
9333 start_time: 23,
9334 executable: Some(subc_os::FileIdentity {
9335 device: 29,
9336 inode: 31,
9337 }),
9338 };
9339 let module = subc_os::Observation {
9340 start_time: 23,
9341 executable: Some(subc_os::FileIdentity {
9342 device: 29,
9343 inode: 37,
9344 }),
9345 };
9346 let polls = Cell::new(0);
9347 let image = sample_after_linux_exec(
9348 || {
9349 polls.set(polls.get() + 1);
9350 committed.set(polls.get() == 2);
9353 Some(committed.get())
9354 },
9355 || Some(if committed.get() { module } else { inherited }),
9356 );
9357 assert_eq!(image, Some(module));
9358 assert_eq!(polls.get(), 2);
9359 }
9360
9361 #[test]
9362 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9363 assert_eq!(
9364 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9365 Some(false)
9366 );
9367 assert_eq!(
9368 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9369 Some(true)
9370 );
9371 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9372 assert_eq!(linux_exec_committed("17 (a) R"), None);
9373 }
9374
9375 #[test]
9376 fn unreadable_exec_state_never_samples_an_inherited_image() {
9377 assert_eq!(
9378 sample_after_linux_exec(
9379 || None,
9380 || panic!("must not sample before exec confirmation")
9381 ),
9382 None
9383 );
9384 }
9385
9386 #[test]
9387 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9388 assert_eq!(
9389 sample_after_linux_exec(
9390 || Some(false),
9391 || panic!("must not sample after an exec timeout")
9392 ),
9393 None
9394 );
9395 }
9396}
9397
9398#[cfg(windows)]
9401fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9402 let mut program = spec.program.clone();
9403 if program.is_absolute() || program.components().count() > 1 {
9404 match subc_os::ExecutableCapture::open(&program) {
9407 Ok(capture) => return Ok(capture),
9408 Err(error)
9409 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9410 Err(error) => return Err(error),
9411 }
9412 program.set_extension("exe");
9413 return subc_os::ExecutableCapture::open(&program);
9414 }
9415 if program.extension().is_none() {
9416 program.set_extension("exe");
9417 }
9418 let custom_path = spec
9419 .env
9420 .iter()
9421 .rev()
9422 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9423 let mut directories = Vec::new();
9424 if let Some(parent) = std::env::current_exe()?.parent() {
9427 directories.push(parent.to_path_buf());
9428 }
9429 directories.push(std::env::current_dir()?);
9430 if let Some(windows) = std::env::var_os("SystemRoot") {
9431 let windows = PathBuf::from(windows);
9432 directories.push(windows.join("System32"));
9433 directories.push(windows.clone());
9434 }
9435 let path = custom_path
9436 .map(|(_, value)| std::ffi::OsString::from(value))
9437 .or_else(|| std::env::var_os("PATH"));
9438 if let Some(path) = path {
9439 directories.extend(std::env::split_paths(&path));
9440 }
9441 for directory in directories {
9442 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9443 Ok(capture) => return Ok(capture),
9444 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9445 Err(error) => return Err(error),
9446 }
9447 }
9448 Err(io::Error::new(
9449 io::ErrorKind::NotFound,
9450 "resolved executable not found",
9451 ))
9452}
9453
9454fn spawn_and_mark_running(
9455 spec: &ModuleSpec,
9456 runtime: &SupervisorRuntimeConfig,
9457 snapshot: &SharedSnapshot,
9458) -> Result<SupervisedChild, SuperviseError> {
9459 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9460 let child = spawn_child(
9461 spec,
9462 runtime.connection_file_path.as_deref(),
9463 runtime.supervisor_handle.as_ref(),
9464 &runtime.stderr_ring,
9465 runtime.capture_logs_dir.as_deref(),
9466 &runtime.child_roster,
9467 #[cfg(target_os = "linux")]
9468 runtime.cgroup_placement.as_ref(),
9469 )?;
9470 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9471 if let Some(spawn) = activity_spawn {
9472 spawn.complete(snapshot);
9473 }
9474 Ok(child)
9475}
9476
9477enum RegistrationWaitOutcome {
9478 Registered,
9479 Exited(ExitReport),
9480 TimedOut,
9481}
9482
9483struct ReloadRegistrationFailure {
9484 exit_report: ExitReport,
9485 reason: String,
9486}
9487
9488#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9489enum BusyGaugeObservation {
9490 Quiescent,
9491 Busy,
9492 Omitted,
9493}
9494
9495#[derive(Debug, Clone, PartialEq, Eq)]
9497enum GaugeVerdict {
9498 Zero,
9500 NonZero(u64),
9502 NotACount(String),
9506 Missing,
9508 ProbeFailed(&'static str),
9511}
9512
9513impl GaugeVerdict {
9514 fn reads_quiet(&self) -> bool {
9515 matches!(self, Self::Zero)
9516 }
9517}
9518
9519impl fmt::Display for GaugeVerdict {
9520 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9521 match self {
9522 Self::Zero => f.write_str("zero"),
9523 Self::NonZero(value) => write!(f, "non_zero({value})"),
9524 Self::NotACount(value) => write!(f, "not_a_count({value})"),
9525 Self::Missing => f.write_str("missing_from_health_body"),
9526 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9527 }
9528 }
9529}
9530
9531const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9534
9535fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9537 let metrics = metrics.and_then(Value::as_object);
9538 gauges
9539 .iter()
9540 .map(
9541 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9542 None => GaugeVerdict::Missing,
9543 Some(value) => match value.as_u64() {
9544 Some(0) => GaugeVerdict::Zero,
9545 Some(count) => GaugeVerdict::NonZero(count),
9546 None => {
9547 let mut text = value.to_string();
9548 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9549 let mut end = GAUGE_VALUE_TEXT_LIMIT;
9550 while !text.is_char_boundary(end) {
9551 end -= 1;
9552 }
9553 text.truncate(end);
9554 text.push_str("...");
9555 }
9556 GaugeVerdict::NotACount(text)
9557 }
9558 },
9559 },
9560 )
9561 .collect()
9562}
9563
9564fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9567 if verdicts
9568 .iter()
9569 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9570 {
9571 BusyGaugeObservation::Omitted
9572 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9573 BusyGaugeObservation::Quiescent
9574 } else {
9575 BusyGaugeObservation::Busy
9576 }
9577}
9578
9579#[derive(Debug, Clone, Default, PartialEq, Eq)]
9584struct GaugeTrace {
9585 latest: Option<(GaugeVerdict, Duration)>,
9586 last_value: Option<(GaugeVerdict, Duration)>,
9587}
9588
9589#[derive(Debug, Clone, Default, PartialEq, Eq)]
9595struct DrainGaugeTrace {
9596 probes: u32,
9598 gauges: Vec<(String, GaugeTrace)>,
9600}
9601
9602impl DrainGaugeTrace {
9603 fn new(gauges: &[String]) -> Self {
9604 Self {
9605 probes: 0,
9606 gauges: gauges
9607 .iter()
9608 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9609 .collect(),
9610 }
9611 }
9612
9613 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9615 self.probes = self.probes.saturating_add(1);
9616 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
9617 trace.latest = Some((verdict.clone(), at));
9618 if matches!(
9619 verdict,
9620 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
9621 ) {
9622 trace.last_value = Some((verdict.clone(), at));
9623 }
9624 }
9625 }
9626}
9627
9628impl fmt::Display for DrainGaugeTrace {
9633 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9634 if self.gauges.is_empty() {
9635 return f.write_str("none declared");
9636 }
9637 for (index, (name, trace)) in self.gauges.iter().enumerate() {
9638 if index > 0 {
9639 f.write_str(" | ")?;
9640 }
9641 write!(f, "{name}: ")?;
9642 match &trace.latest {
9643 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
9644 None => f.write_str("never probed")?,
9645 }
9646 match &trace.last_value {
9647 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
9648 None => f.write_str("; no value ever read")?,
9649 }
9650 }
9651 Ok(())
9652 }
9653}
9654
9655fn declared_busy_gauges(
9656 registry: &Registry,
9657 module_id: &str,
9658) -> Result<Vec<String>, SuperviseError> {
9659 busy_gauges_of(
9660 registry
9661 .get_module(module_id)
9662 .map_err(SuperviseError::Registry)?,
9663 )
9664}
9665
9666fn declared_busy_gauges_for_connection(
9670 registry: &Registry,
9671 connection_id: ConnectionId,
9672) -> Result<Vec<String>, SuperviseError> {
9673 busy_gauges_of(
9674 registry
9675 .get_module_by_connection(connection_id)
9676 .map_err(SuperviseError::Registry)?,
9677 )
9678}
9679
9680fn busy_gauges_of(
9681 registration: Option<crate::registry::ModuleRegistration>,
9682) -> Result<Vec<String>, SuperviseError> {
9683 let Some(registration) = registration else {
9684 return Ok(Vec::new());
9685 };
9686 let Some(self_signals) = registration.manifest.self_signals else {
9687 return Ok(Vec::new());
9688 };
9689
9690 let mut gauges = Vec::new();
9691 for declaration in self_signals {
9692 if declaration.kind != SelfSignalKind::Busy {
9693 continue;
9694 }
9695 match declaration.anchored_to {
9696 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9697 gauges.extend(declared)
9698 }
9699 _ => {
9700 gauges.push(String::new());
9703 }
9704 }
9705 }
9706 Ok(gauges)
9707}
9708
9709#[allow(clippy::too_many_arguments)]
9718async fn wait_for_forwarding_quiescence(
9719 forwarding: &ForwardingTable,
9720 module_id: &str,
9721 runtime: &SupervisorRuntimeConfig,
9722 endpoint: crate::ModuleEndpointId,
9723 deadline: Instant,
9724 busy_gauges: &[String],
9725 scope: DrainScope,
9726 trace: &mut DrainGaugeTrace,
9727) -> Result<bool, SuperviseError> {
9728 let wait_started = Instant::now();
9729 let mut gauges_quiescent = busy_gauges.is_empty();
9730 let mut next_probe_at = wait_started;
9731 let mut omission_reported = false;
9732
9733 loop {
9734 let now = Instant::now();
9735 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9736 let report = match scope {
9737 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9738 DrainScope::Endpoint(endpoint) => {
9739 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9740 }
9741 };
9742 let verdicts = match &report {
9743 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
9744 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
9745 };
9746 trace.record(&verdicts, wait_started.elapsed());
9747 gauges_quiescent = match report {
9748 Ok(_) => match busy_gauge_observation(&verdicts) {
9749 BusyGaugeObservation::Quiescent => true,
9750 BusyGaugeObservation::Busy => false,
9751 BusyGaugeObservation::Omitted => {
9752 if !omission_reported {
9755 forwarding
9756 .counters()
9757 .increment_drains_with_undeclared_gauge();
9758 let missing = busy_gauges
9759 .iter()
9760 .zip(&verdicts)
9761 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
9762 .map(|(gauge, _)| gauge.as_str())
9763 .collect::<Vec<_>>()
9764 .join(",");
9765 warn!(
9766 module_id,
9767 missing_gauges = %missing,
9768 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
9769 );
9770 omission_reported = true;
9771 }
9772 false
9773 }
9774 },
9775 Err(err) => {
9776 warn!(
9777 module_id,
9778 error = %err,
9779 "drain health.check did not produce declared busy gauges; treating module as busy"
9780 );
9781 false
9782 }
9783 };
9784 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9787 }
9788
9789 let in_flight = forwarding
9790 .endpoint_in_flight_count(endpoint)
9791 .map_err(SuperviseError::Forwarding)?;
9792 if in_flight == 0 && gauges_quiescent {
9793 return Ok(true);
9794 }
9795
9796 let now = Instant::now();
9797 if now >= deadline {
9798 return Ok(false);
9799 }
9800 let mut wait = deadline
9801 .saturating_duration_since(now)
9802 .min(REGISTRY_RELEASE_POLL);
9803 if !busy_gauges.is_empty() {
9804 wait = wait.min(next_probe_at.saturating_duration_since(now));
9805 }
9806 sleep(wait).await;
9807 }
9808}
9809
9810fn drain_busy_reprobe_interval() -> Duration {
9814 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9815}
9816
9817fn warn_drain_timed_out(
9832 module_id: &str,
9833 drain_timeout: Duration,
9834 reason: RouteCloseReason,
9835 holdouts: &crate::forwarding::DrainHoldouts,
9836 gauges: &DrainGaugeTrace,
9837) {
9838 warn!(
9839 module_id = %module_id,
9840 waited = ?drain_timeout,
9841 ?reason,
9842 held_requests = holdouts.requests,
9843 held_routes = holdouts.routes,
9844 total_routes = holdouts.total_routes,
9845 top_connections = ?holdouts.top_connections,
9846 held = %holdouts
9849 .held
9850 .iter()
9851 .map(|(channel, corr)| format!("{channel}:{corr}"))
9852 .collect::<Vec<_>>()
9853 .join(","),
9854 gauge_probes = gauges.probes,
9855 busy_gauges = %gauges,
9856 "route drain timed out before request quiescence; forcing teardown"
9857 );
9858}
9859
9860fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9868 match wait_result {
9869 Ok(drained) => *drained,
9870 Err(_) => false,
9871 }
9872}
9873
9874fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9875 for released in released_routes {
9876 let frame = match Frame::build_with_version(
9877 released.negotiated_ver,
9878 FrameType::Goodbye,
9879 control_flags(),
9880 released.channel,
9881 released.epoch,
9882 0,
9883 Vec::new(),
9884 ) {
9885 Ok(frame) => frame,
9886 Err(err) => {
9887 warn!(
9888 route_channel = released.channel,
9889 error = %err,
9890 "failed to build supervisor drain route GOODBYE frame"
9891 );
9892 continue;
9893 }
9894 };
9895 if !released.close_on_delivery_failure() {
9896 crate::forwarding::send_module_route_goodbye(
9897 &forwarding.counters(),
9898 &released.sink,
9899 frame,
9900 released.module_id.as_deref(),
9901 "supervisor drain",
9902 );
9903 continue;
9904 }
9905 if let Err(err) = released.sink.try_send(frame) {
9906 warn!(
9907 target_connection_id = released.connection_id.get(),
9908 route_channel = released.channel,
9909 error = %err,
9910 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9911 );
9912 let _ = forwarding.escalate_client_delivery_failure(
9913 released.connection_id,
9914 released.channel,
9915 released.epoch,
9916 CloseReason::new(
9917 "route_goodbye_delivery_failed",
9918 format!(
9919 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9920 released.channel
9921 ),
9922 ),
9923 crate::forwarding::UndeliveredFrame {
9924 module_id: released.module_id.as_deref(),
9925 sink: &released.sink,
9926 },
9927 );
9928 }
9929 }
9930}
9931
9932fn send_module_draining(
9933 module_id: &str,
9934 reason: RouteCloseReason,
9935 deadline_ms: u64,
9936 target: &ModuleDrainTarget,
9937) {
9938 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9939 reason,
9940 deadline_ms,
9941 }) {
9942 Ok(body) => body,
9943 Err(err) => {
9944 warn!(
9945 module_id,
9946 error = %err,
9947 "failed to encode module draining command"
9948 );
9949 return;
9950 }
9951 };
9952 let frame = match Frame::build_with_version(
9953 target.negotiated_ver,
9954 FrameType::Push,
9955 control_flags(),
9956 0,
9957 0,
9958 0,
9959 body,
9960 ) {
9961 Ok(frame) => frame,
9962 Err(err) => {
9963 warn!(
9964 module_id,
9965 error = %err,
9966 "failed to build module draining command frame"
9967 );
9968 return;
9969 }
9970 };
9971 if let Err(err) = target.sink.try_send(frame) {
9972 warn!(
9973 module_id,
9974 target_connection_id = target.endpoint.connection_id.get(),
9975 error = %err,
9976 "module draining command was not delivered to peer"
9977 );
9978 }
9979}
9980
9981fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9983 match Frame::build_with_version(
9984 negotiated_ver,
9985 FrameType::Goodbye,
9986 control_flags(),
9987 0,
9988 0,
9989 0,
9990 Vec::new(),
9991 ) {
9992 Ok(frame) => Some(frame),
9993 Err(err) => {
9994 warn!(
9995 module_id,
9996 error = %err,
9997 "failed to build module GOODBYE frame"
9998 );
9999 None
10000 }
10001 }
10002}
10003
10004#[cfg(unix)]
10018async fn send_module_goodbyes_for_daemon_shutdown(
10019 forwarding: &Arc<ForwardingTable>,
10020 reason: &CloseReason,
10021 wait_for_flush: bool,
10022) {
10023 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10024 let targets = match forwarding.module_connections() {
10025 Ok(targets) => targets,
10026 Err(err) => {
10027 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10028 return;
10029 }
10030 };
10031 let deadline = Instant::now() + GOODBYE_BUDGET;
10032 let mut sends = tokio::task::JoinSet::new();
10033 for target in targets {
10034 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10035 continue;
10036 };
10037 if !wait_for_flush {
10038 if let Err(err) = target.sink.try_send(frame) {
10039 debug!(
10040 module_id = %target.module_id,
10041 error = %err,
10042 "shutdown module GOODBYE was not queued"
10043 );
10044 }
10045 continue;
10046 }
10047 let forwarding = Arc::clone(forwarding);
10048 let reason = reason.clone();
10049 sends.spawn(async move {
10050 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10051 Ok(Ok(())) => {}
10052 Ok(Err(err)) => debug!(
10053 module_id = %target.module_id,
10054 error = %err,
10055 "module connection closed before its shutdown GOODBYE was written"
10056 ),
10057 Err(_) => warn!(
10058 module_id = %target.module_id,
10059 budget = ?GOODBYE_BUDGET,
10060 "shutdown module GOODBYE was not written within its budget; closing anyway"
10061 ),
10062 }
10063 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10064 });
10065 }
10066 while sends.join_next().await.is_some() {}
10068}
10069
10070fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10071 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10072 return;
10073 };
10074 if let Err(err) = target.sink.try_send(frame) {
10075 warn!(
10076 module_id,
10077 target_connection_id = target.endpoint.connection_id.get(),
10078 error = %err,
10079 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10080 );
10081 forwarding.request_connection_close(
10082 target.endpoint.connection_id,
10083 CloseReason::new(
10084 "module_goodbye_delivery_failed",
10085 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10086 ),
10087 );
10088 }
10089}
10090
10091#[derive(Clone, Copy)]
10092struct ForwardingDrainContext<'a> {
10093 spec: &'a ModuleSpec,
10094 runtime: &'a SupervisorRuntimeConfig,
10095 registry: &'a Registry,
10096 scope: DrainScope,
10097}
10098
10099#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10101enum DrainScope {
10102 Active,
10105 Endpoint(crate::ModuleEndpointId),
10110}
10111
10112#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10120enum StopNotice {
10121 SentOverConnection,
10124 NoConnection,
10128 NotSent,
10132}
10133
10134async fn begin_forwarding_drain(
10135 spec: &ModuleSpec,
10136 runtime: &SupervisorRuntimeConfig,
10137 registry: &Registry,
10138 snapshot: &SharedSnapshot,
10139 enabled: Option<bool>,
10140 reason: RouteCloseReason,
10141) -> Result<StopNotice, SuperviseError> {
10142 let Some(forwarding) = runtime.forwarding.as_ref() else {
10143 return Err(SuperviseError::ReloadUnavailable {
10144 module_id: spec.module_id.clone(),
10145 reason: "supervisor was not configured with a forwarding table".to_string(),
10146 });
10147 };
10148
10149 begin_forwarding_drain_with(
10150 forwarding,
10151 ForwardingDrainContext {
10152 spec,
10153 runtime,
10154 registry,
10155 scope: DrainScope::Active,
10156 },
10157 snapshot,
10158 enabled,
10159 reason,
10160 runtime.drain_timeout,
10161 )
10162 .await
10163}
10164
10165async fn begin_forwarding_drain_if_configured(
10166 spec: &ModuleSpec,
10167 runtime: &SupervisorRuntimeConfig,
10168 registry: &Registry,
10169 snapshot: &SharedSnapshot,
10170 enabled: Option<bool>,
10171 reason: RouteCloseReason,
10172) -> Result<StopNotice, SuperviseError> {
10173 begin_forwarding_drain_with_timeout(
10174 spec,
10175 runtime,
10176 registry,
10177 snapshot,
10178 enabled,
10179 reason,
10180 runtime.drain_timeout,
10181 )
10182 .await
10183}
10184
10185async fn begin_forwarding_drain_with_timeout(
10189 spec: &ModuleSpec,
10190 runtime: &SupervisorRuntimeConfig,
10191 registry: &Registry,
10192 snapshot: &SharedSnapshot,
10193 enabled: Option<bool>,
10194 reason: RouteCloseReason,
10195 drain_timeout: Duration,
10196) -> Result<StopNotice, SuperviseError> {
10197 let Some(forwarding) = runtime.forwarding.as_ref() else {
10198 return Ok(StopNotice::NotSent);
10199 };
10200
10201 begin_forwarding_drain_with(
10202 forwarding,
10203 ForwardingDrainContext {
10204 spec,
10205 runtime,
10206 registry,
10207 scope: DrainScope::Active,
10208 },
10209 snapshot,
10210 enabled,
10211 reason,
10212 drain_timeout,
10213 )
10214 .await
10215}
10216
10217async fn begin_forwarding_drain_with(
10218 forwarding: &ForwardingTable,
10219 context: ForwardingDrainContext<'_>,
10220 snapshot: &SharedSnapshot,
10221 enabled: Option<bool>,
10222 reason: RouteCloseReason,
10223 drain_timeout: Duration,
10224) -> Result<StopNotice, SuperviseError> {
10225 let ForwardingDrainContext {
10226 spec,
10227 runtime,
10228 registry,
10229 scope,
10230 } = context;
10231 debug_assert_ne!(reason, RouteCloseReason::Crash);
10232 let terminal = matches!(reason, RouteCloseReason::Disable);
10233 let drain_started_at = Instant::now();
10234 let drain_deadline = drain_started_at + drain_timeout;
10235 let deadline_ms =
10236 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10237 let busy_gauges = match scope {
10238 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10239 DrainScope::Endpoint(endpoint) => {
10240 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10241 }
10242 };
10243
10244 let gate_started = Instant::now();
10247 let drain_target = match scope {
10248 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10249 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10250 }
10251 .map_err(SuperviseError::Forwarding)?;
10252 info!(
10257 module_id = %spec.module_id,
10258 ?reason,
10259 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10260 connected = drain_target.is_some(),
10261 "module drain began; route admission closed"
10262 );
10263 if scope == DrainScope::Active {
10264 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10265 state.state = ModuleState::Draining;
10266 state.draining_to_replace =
10267 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10268 if let Some(enabled) = enabled {
10269 state.enabled = enabled;
10270 }
10271 })?;
10272 }
10273
10274 let Some(target) = drain_target.as_ref() else {
10275 return Ok(StopNotice::NoConnection);
10279 };
10280 {
10281 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10282 #[cfg(feature = "test-support")]
10283 forwarding.wait_before_route_closing_for_test().await;
10284 let routes = forwarding
10285 .endpoint_routes(target.endpoint)
10286 .map_err(SuperviseError::Forwarding)?;
10287 let routes_notified = routes.len();
10288 crate::control::send_route_control_pushes(
10289 forwarding,
10290 routes.clone(),
10291 ClientControlPush::RouteClosing {
10292 module_id: spec.module_id.clone(),
10293 channels: Vec::new(),
10294 reason,
10295 },
10296 );
10297 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10298
10299 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10305 let wait_result = wait_for_forwarding_quiescence(
10306 forwarding,
10307 &spec.module_id,
10308 runtime,
10309 target.endpoint,
10310 drain_deadline,
10311 &busy_gauges,
10312 scope,
10313 &mut gauge_trace,
10314 )
10315 .await;
10316 let drained = drained_after_quiescence_wait(&wait_result);
10317 if let Err(err) = &wait_result {
10318 error!(
10319 module_id = %spec.module_id,
10320 ?reason,
10321 error = %err,
10322 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10323 );
10324 } else if !drained {
10325 let holdouts = forwarding
10327 .endpoint_drain_holdouts(target.endpoint)
10328 .unwrap_or_default();
10329 warn_drain_timed_out(
10330 &spec.module_id,
10331 drain_timeout,
10332 reason,
10333 &holdouts,
10334 &gauge_trace,
10335 );
10336 }
10337 crate::control::send_route_control_pushes(
10338 forwarding,
10339 routes,
10340 ClientControlPush::RouteClosed {
10341 module_id: spec.module_id.clone(),
10342 channels: Vec::new(),
10343 reason,
10344 drained,
10345 abandoned: target.abandoned_bindings.len() as u32,
10346 excluded_subscriptions: target.excluded_subscriptions,
10347 terminal: Some(terminal),
10348 },
10349 );
10350 wait_result?;
10351
10352 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10358 Ok(routes) => routes,
10359 Err(err) => {
10360 warn!(
10361 module_id = %spec.module_id,
10362 ?reason,
10363 error = %err,
10364 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10365 );
10366 send_module_goodbye(&spec.module_id, forwarding, target);
10367 return Err(SuperviseError::Forwarding(err));
10368 }
10369 };
10370 let route_goodbye_count = released_routes.len();
10371 send_route_goodbyes(forwarding, released_routes);
10372 send_module_goodbye(&spec.module_id, forwarding, target);
10373
10374 info!(
10380 module_id = %spec.module_id,
10381 ?reason,
10382 routes_notified,
10383 route_goodbyes = route_goodbye_count,
10384 abandoned_reservations = target.abandoned_bindings.len(),
10385 excluded_subscriptions = target.excluded_subscriptions,
10386 drained,
10387 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10388 );
10389 }
10390
10391 Ok(StopNotice::SentOverConnection)
10392}
10393
10394async fn wait_for_registration_after_reload(
10397 registry: &Registry,
10398 module_id: &str,
10399 snapshot: &SharedSnapshot,
10400 child: &mut SupervisedChild,
10401 wait: Duration,
10402) -> Result<RegistrationWaitOutcome, SuperviseError> {
10403 wait_for_slot_registration(
10404 registry,
10405 crate::registry::RegistrationSlot::Active(module_id),
10406 module_id,
10407 snapshot,
10408 child,
10409 wait,
10410 )
10411 .await
10412}
10413
10414async fn wait_for_slot_registration(
10422 registry: &Registry,
10423 slot: crate::registry::RegistrationSlot<'_>,
10424 module_id: &str,
10425 snapshot: &SharedSnapshot,
10426 child: &mut SupervisedChild,
10427 wait: Duration,
10428) -> Result<RegistrationWaitOutcome, SuperviseError> {
10429 let deadline = Instant::now() + wait;
10430 loop {
10431 if registry
10432 .registration(slot)
10433 .map_err(SuperviseError::Registry)?
10434 .is_some()
10435 {
10436 return Ok(RegistrationWaitOutcome::Registered);
10437 }
10438
10439 let now = Instant::now();
10440 if now >= deadline {
10441 return Ok(RegistrationWaitOutcome::TimedOut);
10442 }
10443 let remaining = deadline.saturating_duration_since(now);
10444 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10445
10446 tokio::select! {
10447 wait_result = child.wait() => {
10448 let status = wait_result.map_err(|source| SuperviseError::Wait {
10449 module_id: module_id.to_string(),
10450 source,
10451 })?;
10452 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10453 snapshot,
10454 child,
10455 &status,
10456 )));
10457 }
10458 _ = sleep(poll) => {}
10459 }
10460 }
10461}
10462
10463fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10464 if exit_report.kind != ExitKind::DeliberateSeverance {
10467 exit_report.kind = ExitKind::Crash;
10468 }
10469 exit_report
10470}
10471
10472async fn handle_reload_child_registration_failure(
10473 spec: &ModuleSpec,
10474 runtime: &SupervisorRuntimeConfig,
10475 registry: &Registry,
10476 process_liveness: &SupervisorProcessLiveness,
10477 snapshot: &SharedSnapshot,
10478 _child: &mut Option<SupervisedChild>,
10479 failure: ReloadRegistrationFailure,
10480) -> Result<(), SuperviseError> {
10481 let ReloadRegistrationFailure {
10482 exit_report,
10483 reason,
10484 } = failure;
10485 match on_child_exit(
10486 spec,
10487 runtime.restart_policy,
10488 registry,
10489 snapshot,
10490 &runtime.terminal_ring,
10491 &runtime.spawn_events,
10492 &runtime.child_roster,
10493 exit_report,
10494 )
10495 .await
10496 {
10497 NextAction::Stop {
10498 registration_released,
10499 } => {
10500 if registration_released {
10501 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10502 }
10503 }
10504 NextAction::Restart { schedule } => {
10505 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10506 schedule.delay
10507 });
10508 if let Some(schedule) = schedule {
10509 log_crash_respawn(&spec.module_id, schedule);
10510 }
10511 schedule_respawn(
10512 runtime,
10513 snapshot,
10514 &spec.module_id,
10515 delay,
10516 RespawnKind::Spawn,
10517 )?;
10518 }
10519 }
10520 Err(SuperviseError::ReloadFailed {
10521 module_id: spec.module_id.clone(),
10522 reason,
10523 })
10524}
10525
10526async fn handle_reload_spawn_failure(
10527 spec: &ModuleSpec,
10528 runtime: &SupervisorRuntimeConfig,
10529 process_liveness: &SupervisorProcessLiveness,
10530 snapshot: &SharedSnapshot,
10531 _child: &mut Option<SupervisedChild>,
10532 reason: String,
10533) -> Result<(), SuperviseError> {
10534 let now = Instant::now();
10535 let mut schedule = None;
10536 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10537 clear_current_process_facts(state);
10538 if state.enabled {
10539 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10540 state.state = if schedule.is_some() {
10541 ModuleState::Restarting
10542 } else {
10543 ModuleState::Failed
10544 };
10545 } else {
10546 state.state = ModuleState::Disabled;
10547 }
10548 })?;
10549 if let Some(schedule) = schedule {
10550 schedule_respawn(
10551 runtime,
10552 snapshot,
10553 &spec.module_id,
10554 schedule.delay,
10555 RespawnKind::Spawn,
10556 )?;
10557 } else {
10558 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10559 }
10560 Err(SuperviseError::ReloadFailed {
10561 module_id: spec.module_id.clone(),
10562 reason,
10563 })
10564}
10565
10566fn control_flags() -> Flags {
10567 Flags::new(false, Priority::Passive, false)
10568}
10569
10570#[allow(clippy::too_many_arguments)]
10571async fn drain_optional_child(
10572 module_id: &str,
10573 protocol: ModuleProtocol,
10574 stop_notice: StopNotice,
10575 registry: &Registry,
10576 forwarding: Option<&ForwardingTable>,
10577 snapshot: &SharedSnapshot,
10578 terminal_ring: &Arc<Mutex<TerminalRing>>,
10579 spawn_events: &SpawnEventFeed,
10580 child: &mut Option<SupervisedChild>,
10581 drain_timeout: Duration,
10582 final_state: ModuleState,
10583 enabled: Option<bool>,
10584) -> Result<(), SuperviseError> {
10585 if let Some(child) = child.take() {
10586 drain_child_to_state(
10587 module_id,
10588 protocol,
10589 stop_notice,
10590 registry,
10591 forwarding,
10592 snapshot,
10593 terminal_ring,
10594 spawn_events,
10595 child,
10596 drain_timeout,
10597 final_state,
10598 enabled,
10599 )
10600 .await
10601 } else {
10602 update_snapshot(snapshot, Some(module_id), |state| {
10603 state.state = final_state;
10604 if let Some(enabled) = enabled {
10605 state.enabled = enabled;
10606 }
10607 clear_current_process_facts(state);
10608 })?;
10609 release_dead_registration(registry, forwarding, snapshot, module_id).await
10610 }
10611}
10612
10613#[allow(clippy::too_many_arguments)]
10614async fn drain_child_to_state(
10615 module_id: &str,
10616 _protocol: ModuleProtocol,
10617 stop_notice: StopNotice,
10618 registry: &Registry,
10619 forwarding: Option<&ForwardingTable>,
10620 snapshot: &SharedSnapshot,
10621 terminal_ring: &Arc<Mutex<TerminalRing>>,
10622 spawn_events: &SpawnEventFeed,
10623 mut child: SupervisedChild,
10624 drain_timeout: Duration,
10625 final_state: ModuleState,
10626 enabled: Option<bool>,
10627) -> Result<(), SuperviseError> {
10628 let protocol = child.protocol;
10629 update_snapshot(snapshot, Some(module_id), |state| {
10630 state.state = ModuleState::Draining;
10631 state.draining_to_replace = final_state == ModuleState::Restarting;
10632 if let Some(enabled) = enabled {
10633 state.enabled = enabled;
10634 }
10635 })?;
10636
10637 if stop_notice != StopNotice::SentOverConnection {
10648 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10649 info!(
10650 module_id,
10651 pid = child.pid,
10652 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10653 "module has no connection yet; requesting stop by signal"
10654 );
10655 }
10656 request_graceful_stop(module_id, &child);
10657 }
10658
10659 let exit_report = match timeout(drain_timeout, child.wait()).await {
10660 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10661 Ok(Err(source)) => {
10662 fail_after_wait_error(
10663 module_id,
10664 snapshot,
10665 terminal_ring,
10666 spawn_events,
10667 format!("failed to wait during drain: {source}"),
10668 );
10669 return Err(SuperviseError::Wait {
10670 module_id: module_id.to_string(),
10671 source,
10672 });
10673 }
10674 Err(_) => {
10675 warn!(
10688 module_id,
10689 pid = child.pid,
10690 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10691 reason = ?final_state,
10692 ?stop_notice,
10693 "drain budget expired before the module exited; killing it"
10694 );
10695 child.start_kill().map_err(|source| {
10696 fail_after_wait_error(
10697 module_id,
10698 snapshot,
10699 terminal_ring,
10700 spawn_events,
10701 format!("failed to kill during drain: {source}"),
10702 );
10703 SuperviseError::Kill {
10704 module_id: module_id.to_string(),
10705 source,
10706 }
10707 })?;
10708 let status = child.wait().await.map_err(|source| {
10709 fail_after_wait_error(
10710 module_id,
10711 snapshot,
10712 terminal_ring,
10713 spawn_events,
10714 format!("failed to wait after drain kill: {source}"),
10715 );
10716 SuperviseError::Wait {
10717 module_id: module_id.to_string(),
10718 source,
10719 }
10720 })?;
10721 classify_reaped_child_exit(snapshot, &child, &status)
10722 }
10723 };
10724
10725 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10726 publish_after_terminal_record(
10727 module_id,
10728 || {
10729 record_terminal_with_detail(
10730 module_id,
10731 terminal_ring,
10732 spawn_events,
10733 &exit_report,
10734 terminal_disposition(final_state),
10735 detail,
10736 )
10737 },
10738 || {
10739 update_snapshot(snapshot, Some(module_id), |state| {
10740 state.state = final_state;
10741 if let Some(enabled) = enabled {
10742 state.enabled = enabled;
10743 }
10744 clear_current_process_facts(state);
10745 state.last_exit = Some(exit_report.clone());
10746 if exit_report.kind == ExitKind::DeliberateSeverance {
10747 state.lifetime_restarts += 1;
10748 }
10749 })
10750 },
10751 )?;
10752 child.drain_stderr(module_id).await;
10753
10754 release_dead_registration(registry, forwarding, snapshot, module_id).await
10755}
10756
10757#[cfg(unix)]
10777fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10778 let Some(pid) = child
10779 .id()
10780 .and_then(|pid| i32::try_from(pid).ok())
10781 .and_then(rustix::process::Pid::from_raw)
10782 else {
10783 debug!(
10784 module_id,
10785 "no pid to signal for teardown; falling through to the drain wait"
10786 );
10787 return;
10788 };
10789 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10790 Ok(()) => debug!(
10791 module_id,
10792 "sent SIGTERM to a module nothing else asked to stop"
10793 ),
10794 Err(err) => debug!(
10795 module_id,
10796 error = %err,
10797 "SIGTERM to module failed; the drain wait and kill still apply"
10798 ),
10799 }
10800}
10801
10802#[cfg(not(unix))]
10810fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10811 debug!(
10812 module_id,
10813 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10814 );
10815}
10816
10817fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10818 match final_state {
10819 ModuleState::Stopped => TerminalDisposition::Stopped,
10820 ModuleState::Disabled => TerminalDisposition::Disabled,
10821 ModuleState::Restarting => TerminalDisposition::Restarting,
10822 ModuleState::Failed => TerminalDisposition::Failed,
10823 ModuleState::Starting
10824 | ModuleState::Running
10825 | ModuleState::Unresponsive
10826 | ModuleState::Draining => {
10827 unreachable!("terminal exits only finish in terminal or restarting states")
10828 }
10829 }
10830}
10831
10832async fn release_dead_registration(
10840 registry: &Registry,
10841 forwarding: Option<&ForwardingTable>,
10842 snapshot: &SharedSnapshot,
10843 module_id: &str,
10844) -> Result<(), SuperviseError> {
10845 let result = async {
10846 let registration = registry
10847 .get_module(module_id)
10848 .map_err(SuperviseError::Registry)?;
10849 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10850 Ok(()) => return Ok(()),
10851 Err(SuperviseError::RegistrationStillActive { .. }) => {}
10852 Err(err) => return Err(err),
10853 }
10854 let pid = lock_snapshot(snapshot)?.reaped_pid;
10855 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10856 warn!(
10857 module_id,
10858 pid,
10859 connection_id = registration.connection_id.get(),
10860 "reaped module registration outlived release grace; closing dead connection"
10861 );
10862 forwarding.request_connection_close(
10863 registration.connection_id,
10864 CloseReason::new(
10865 "supervised_process_reaped",
10866 format!("module '{module_id}' pid {pid} exited"),
10867 ),
10868 );
10869 wait_for_slot_registration_release(
10870 registry,
10871 crate::registry::RegistrationSlot::Connection(registration.connection_id),
10872 REGISTRY_RELEASE_TIMEOUT,
10873 )
10874 .await?;
10875 }
10876 wait_for_registration_release(registry, module_id, Duration::ZERO).await
10877 }
10878 .await;
10879 if let Err(err) = &result {
10880 fail_snapshot(snapshot, Some(module_id), None);
10881 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10882 }
10883 result
10884}
10885
10886async fn wait_for_registration_release(
10889 registry: &Registry,
10890 module_id: &str,
10891 wait: Duration,
10892) -> Result<(), SuperviseError> {
10893 wait_for_slot_registration_release(
10894 registry,
10895 crate::registry::RegistrationSlot::Active(module_id),
10896 wait,
10897 )
10898 .await
10899}
10900
10901async fn wait_for_slot_registration_release(
10909 registry: &Registry,
10910 slot: crate::registry::RegistrationSlot<'_>,
10911 wait: Duration,
10912) -> Result<(), SuperviseError> {
10913 let deadline = Instant::now() + wait;
10914 let mut release_events = registration_release_events().subscribe();
10915 let still_active = |registration: &crate::registry::ModuleRegistration| {
10916 SuperviseError::RegistrationStillActive {
10917 module_id: registration.manifest.module_id.clone(),
10918 waited: wait,
10919 }
10920 };
10921 loop {
10922 let _observed_generation = *release_events.borrow_and_update();
10923 let Some(registration) = registry
10924 .registration(slot)
10925 .map_err(SuperviseError::Registry)?
10926 else {
10927 return Ok(());
10928 };
10929
10930 let now = Instant::now();
10931 if now >= deadline {
10932 return Err(still_active(®istration));
10933 }
10934
10935 let remaining = deadline.saturating_duration_since(now);
10936 match timeout(remaining, release_events.changed()).await {
10937 Ok(Ok(())) | Ok(Err(_)) => {}
10938 Err(_) => return Err(still_active(®istration)),
10939 }
10940 }
10941}
10942
10943#[cfg(test)]
10944mod slot_registration_wait_tests {
10945 use super::*;
10946 use crate::registry::{ConnectionId, RegistrationSlot};
10947 use subc_protocol::manifest::ModuleManifest;
10948
10949 #[tokio::test]
10950 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10951 let registry = Arc::new(Registry::default());
10952 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
10953 let runtime = supervisor.runtime_config();
10954 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10955 let spec = ModuleSpec {
10956 module_id: "enable-stale-registration".to_string(),
10957 program: PathBuf::from("/missing/enable-retry-test"),
10958 args: Vec::new(),
10959 env: Vec::new(),
10960 reserved: false,
10961 reserved_prefixes: Vec::new(),
10962 protocol: ModuleProtocol::Subc,
10963 overlap: Default::default(),
10964 };
10965 let connection = ConnectionId::new(90);
10966 registry
10967 .register_with_control_ops(
10968 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10969 1,
10970 connection,
10971 Vec::new(),
10972 )
10973 .unwrap();
10974 let mut child = None;
10975 let err = set_child_enabled(
10976 &spec,
10977 &runtime,
10978 ®istry,
10979 &supervisor.process_liveness,
10980 &snapshot,
10981 &mut child,
10982 true,
10983 )
10984 .await
10985 .unwrap_err();
10986 assert!(matches!(
10987 err,
10988 SuperviseError::RegistrationStillActive { .. }
10989 ));
10990 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10991 assert!(child.is_none());
10992 registry.deregister_connection(connection).unwrap();
10993 let err = set_child_enabled(
10994 &spec,
10995 &runtime,
10996 ®istry,
10997 &supervisor.process_liveness,
10998 &snapshot,
10999 &mut child,
11000 true,
11001 )
11002 .await
11003 .unwrap_err();
11004 assert!(
11005 matches!(err, SuperviseError::Spawn { .. }),
11006 "second enable must attempt a spawn: {err}"
11007 );
11008 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11009 }
11010
11011 const INCUMBENT: u64 = 1;
11012 const CANDIDATE: u64 = 2;
11013
11014 fn swapped_registry() -> Arc<Registry> {
11015 let registry = Arc::new(Registry::default());
11016 let manifest = ModuleManifest::builder("m", "0.1.0").build();
11017 registry
11018 .register_with_control_ops(
11019 manifest.clone(),
11020 1,
11021 ConnectionId::new(INCUMBENT),
11022 Vec::new(),
11023 )
11024 .unwrap();
11025 registry
11026 .register_candidate_with_control_ops(
11027 manifest,
11028 1,
11029 ConnectionId::new(CANDIDATE),
11030 Vec::new(),
11031 )
11032 .unwrap();
11033 registry
11034 }
11035
11036 #[tokio::test]
11040 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11041 let registry = swapped_registry();
11042 registry.promote_candidate("m").unwrap().unwrap();
11043
11044 assert!(matches!(
11045 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
11046 Err(SuperviseError::RegistrationStillActive { .. })
11047 ));
11048
11049 assert!(matches!(
11051 wait_for_slot_registration_release(
11052 ®istry,
11053 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11054 Duration::from_millis(50),
11055 )
11056 .await,
11057 Err(SuperviseError::RegistrationStillActive { .. })
11058 ));
11059
11060 let releaser = Arc::clone(®istry);
11061 let release = tokio::spawn(async move {
11062 sleep(Duration::from_millis(20)).await;
11063 releaser
11064 .deregister_connection(ConnectionId::new(INCUMBENT))
11065 .unwrap();
11066 notify_registration_release();
11067 });
11068 wait_for_slot_registration_release(
11069 ®istry,
11070 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11071 Duration::from_secs(5),
11072 )
11073 .await
11074 .expect("the incumbent's own registration is released");
11075 release.await.unwrap();
11076 assert!(registry.get_module("m").unwrap().is_some());
11077 }
11078
11079 #[tokio::test]
11082 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11083 let registry = swapped_registry();
11084 assert!(matches!(
11085 wait_for_slot_registration_release(
11086 ®istry,
11087 RegistrationSlot::Candidate("m"),
11088 Duration::from_millis(50),
11089 )
11090 .await,
11091 Err(SuperviseError::RegistrationStillActive { .. })
11092 ));
11093 registry
11094 .deregister_connection(ConnectionId::new(CANDIDATE))
11095 .unwrap();
11096 wait_for_slot_registration_release(
11097 ®istry,
11098 RegistrationSlot::Candidate("m"),
11099 Duration::from_millis(50),
11100 )
11101 .await
11102 .expect("a candidate slot with no candidate is released");
11103 assert!(registry
11104 .registration(RegistrationSlot::Active("m"))
11105 .unwrap()
11106 .is_some());
11107 }
11108}
11109
11110fn classify_exit(status: &ExitStatus) -> ExitReport {
11111 ExitReport {
11112 kind: if status.success() {
11113 ExitKind::Clean
11114 } else {
11115 ExitKind::Crash
11116 },
11117 code: status.code(),
11118 signal: exit_signal(status),
11119 at_ms: unix_ms_now(),
11120 }
11121}
11122
11123fn wait_error_exit_report() -> ExitReport {
11129 ExitReport {
11130 kind: ExitKind::Crash,
11131 code: None,
11132 signal: None,
11133 at_ms: unix_ms_now(),
11134 }
11135}
11136
11137#[cfg(unix)]
11138fn exit_signal(status: &ExitStatus) -> Option<i32> {
11139 use std::os::unix::process::ExitStatusExt;
11140
11141 status.signal()
11142}
11143
11144#[cfg(not(unix))]
11145fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11146 None
11147}
11148
11149fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11155 update_snapshot(snapshot, Some(module_id), |state| {
11156 state.clear_crash_restarts();
11157 })
11158}
11159
11160fn set_running(
11161 snapshot: &SharedSnapshot,
11162 child: &SupervisedChild,
11163 module_id: &str,
11164 spawn_events: &SpawnEventFeed,
11165) -> Result<(), SuperviseError> {
11166 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11167 module_id: Some(module_id.to_string()),
11168 })?;
11169 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11170 if std::mem::take(&mut state.coalesced_restart_pending) {
11171 let generation = state.spawn_generation;
11172 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11173 }
11174 state.drain_disposition_detail = None;
11175 state.spawn_failure = None;
11176 state.waiting_for_previous_pid = None;
11177 state.waiting_for_previous_detail = None;
11178 state.in_alternate_slot = false;
11181 state.configuration_updated_since_spawn = false;
11182 state.spawned_protocol = Some(child.protocol);
11183 state.state = ModuleState::Running;
11184 state.enabled = true;
11185 state.process_alive = true;
11186 state.pid = child.id();
11187 #[cfg(windows)]
11188 {
11189 state.pipe_consumed = child
11190 .nonce_delivery
11191 .as_ref()
11192 .map(|delivery| delivery.consumed());
11193 }
11194 #[cfg(target_os = "macos")]
11195 {
11196 state.report_ready = Some(Arc::clone(&child.report_ready));
11197 }
11198 state.spawned_at_ms = Some(child.spawned_at_ms);
11199 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11200 state.spawned_from = Some(child.spawned_from.clone());
11201 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11202 state.spawned_file_identity = child.spawned_file_identity.clone();
11203 state.process_start_time = child.process_start_time;
11204 Ok(())
11205}
11206
11207fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11208 state.waiting_for_previous_pid = None;
11209 state.waiting_for_previous_detail = None;
11210 state.process_alive = false;
11211 state.spawned_protocol = None;
11212 state.pid = None;
11213 #[cfg(windows)]
11214 {
11215 state.pipe_consumed = None;
11216 }
11217 #[cfg(target_os = "macos")]
11218 {
11219 state.report_ready = None;
11220 }
11221 state.spawned_at_ms = None;
11222 state.exec_outcome = None;
11223 state.spawned_from = None;
11224 state.launch_spec_digest = None;
11225 state.spawned_file_identity = None;
11226 state.process_start_time = None;
11227 state.deliberate_severance = None;
11228}
11229
11230#[cfg(test)]
11231fn record_deliberate_severance(
11232 snapshot: &SharedSnapshot,
11233 identity: ProcessIdentity,
11234) -> Result<(), SuperviseError> {
11235 update_snapshot(snapshot, None, |state| {
11236 state.deliberate_severance = Some(identity);
11237 })
11238}
11239
11240fn apply_deliberate_severance_marker(
11241 snapshot: &SharedSnapshot,
11242 exited_identity: Option<ProcessIdentity>,
11243 mut exit_report: ExitReport,
11244) -> ExitReport {
11245 let marker = lock_snapshot(snapshot)
11246 .ok()
11247 .and_then(|mut state| state.deliberate_severance.take());
11248 if marker.is_some() && marker == exited_identity {
11249 exit_report.kind = ExitKind::DeliberateSeverance;
11250 }
11251 exit_report
11252}
11253
11254fn classify_reaped_child_exit(
11255 snapshot: &SharedSnapshot,
11256 child: &SupervisedChild,
11257 status: &ExitStatus,
11258) -> ExitReport {
11259 let _ = update_snapshot(snapshot, None, |state| {
11260 state.reaped_pid = Some(child.pid);
11261 state.spawn_failure = child.spawn_failure.clone();
11262 });
11263 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11264}
11265
11266fn fail_snapshot(
11267 snapshot: &SharedSnapshot,
11268 module_id: Option<&str>,
11269 last_exit: Option<ExitReport>,
11270) {
11271 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11272 state.state = ModuleState::Failed;
11273 clear_current_process_facts(state);
11274 if let Some(last_exit) = last_exit {
11275 state.last_exit = Some(last_exit);
11276 }
11277 }) {
11278 error!(error = %err, "failed to mark supervisor state failed");
11279 }
11280}
11281
11282fn update_snapshot(
11283 snapshot: &SharedSnapshot,
11284 module_id: Option<&str>,
11285 update: impl FnOnce(&mut SupervisorSnapshot),
11286) -> Result<(), SuperviseError> {
11287 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11288 module_id: module_id.map(ToOwned::to_owned),
11289 })?;
11290 update(&mut state);
11291 Ok(())
11292}
11293
11294const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11295
11296fn lock_snapshot_for_control<'a>(
11297 snapshot: &'a SharedSnapshot,
11298 module_id: &str,
11299 caller: &'static str,
11300) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11301 let started_at = Instant::now();
11302 let guard = lock_snapshot(snapshot)?;
11303 let waited = started_at.elapsed();
11304 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11305 warn!(
11306 module_id = %module_id,
11307 waited_ms = waited.as_millis() as u64,
11308 caller = %caller,
11309 "slow snapshot lock"
11310 );
11311 }
11312 Ok(guard)
11313}
11314
11315fn lock_snapshot(
11316 snapshot: &SharedSnapshot,
11317) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11318 snapshot
11319 .lock()
11320 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11321}
11322
11323#[cfg(test)]
11324mod terminal_history_tests {
11325 use std::{
11326 path::PathBuf,
11327 sync::Arc,
11328 time::{Duration, Instant},
11329 };
11330
11331 use tokio::{sync::mpsc, time::sleep};
11332
11333 use super::{
11334 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11335 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11336 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11337 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11338 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11339 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11340 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11341 SupervisorHealthStatus, SupervisorSnapshot,
11342 };
11343 use super::Instant as ClockInstant;
11348 use crate::{
11349 registry::Registry,
11350 terminal_ring::{TerminalRing, TerminalRingConfig},
11351 };
11352 use std::sync::Mutex;
11353 use subc_control::TerminalDisposition;
11354
11355 pub(super) fn fake_aft_stub_path() -> PathBuf {
11360 let mut path = std::env::current_exe().expect("current_exe available in tests");
11361 path.pop();
11362 path.pop();
11363 path.push(if cfg!(windows) {
11364 "fake-aft-stub.exe"
11365 } else {
11366 "fake-aft-stub"
11367 });
11368 assert!(
11369 path.exists(),
11370 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11371 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11372 path.display()
11373 );
11374 path
11375 }
11376
11377 #[test]
11378 fn reserved_never_spawned_refuses_every_hello() {
11379 let supervisor = SupervisorHandle::default();
11384 supervisor.apply_identity_configuration(&ModuleSpec {
11385 module_id: "never-spawned".to_string(),
11386 program: PathBuf::from("/usr/bin/false"),
11387 args: Vec::new(),
11388 env: Vec::new(),
11389 reserved: true,
11390 reserved_prefixes: Vec::new(),
11391 protocol: ModuleProtocol::Subc,
11392 overlap: Default::default(),
11393 });
11394 assert!(
11395 supervisor
11396 .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11397 .is_some(),
11398 "forged nonce must refuse on a reserved never-spawned id"
11399 );
11400 assert!(
11401 supervisor
11402 .reserved_hello_rejection("never-spawned", None)
11403 .is_some(),
11404 "absent nonce must refuse on a reserved never-spawned id"
11405 );
11406 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11408 supervisor.apply_identity_configuration(&ModuleSpec {
11409 module_id: "never-spawned".to_string(),
11410 program: PathBuf::from("/usr/bin/false"),
11411 args: Vec::new(),
11412 env: Vec::new(),
11413 reserved: true,
11414 reserved_prefixes: Vec::new(),
11415 protocol: ModuleProtocol::Subc,
11416 overlap: Default::default(),
11417 });
11418 assert!(supervisor
11419 .reserved_hello_rejection("never-spawned", Some("minted"))
11420 .is_none());
11421 assert!(supervisor
11422 .reserved_hello_rejection("never-spawned", Some("forged"))
11423 .is_some());
11424 }
11425
11426 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11429 let now = ClockInstant::now();
11430 for _ in 0..count {
11431 state.crash_restarts.push_back(now);
11432 }
11433 }
11434
11435 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11439 let aged = state
11440 .crash_restarts
11441 .front()
11442 .expect("a crash restart must be recorded before it can be aged")
11443 .checked_sub(window + Duration::from_secs(1))
11444 .expect("the test clock is far enough from its origin to age an instant");
11445 state.crash_restarts[0] = aged;
11446 }
11447
11448 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11449 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11450 seed_crash_restarts(&mut state, count);
11451 state
11452 }
11453
11454 #[test]
11455 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11456 let policy = RestartPolicy::new(3, Duration::ZERO);
11457 let now = ClockInstant::now();
11458 assert!(daemon_will_restart(
11459 &mut snapshot_with_restarts(true, 2),
11460 &policy,
11461 now
11462 ));
11463 assert!(!daemon_will_restart(
11464 &mut snapshot_with_restarts(true, 3),
11465 &policy,
11466 now
11467 ));
11468 assert!(!daemon_will_restart(
11469 &mut snapshot_with_restarts(false, 0),
11470 &policy,
11471 now
11472 ));
11473 }
11474
11475 #[test]
11476 fn crash_restart_backoff_escalates_with_in_window_count() {
11477 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11478 .with_max_backoff(Duration::from_secs(30));
11479 let now = ClockInstant::now();
11480 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11481 let schedules = (0..4)
11482 .map(|_| {
11483 state
11484 .next_crash_restart(&policy, now)
11485 .expect("the test policy allows four crash restarts")
11486 })
11487 .collect::<Vec<_>>();
11488
11489 assert_eq!(
11490 schedules
11491 .iter()
11492 .map(|schedule| schedule.restart_in_window)
11493 .collect::<Vec<_>>(),
11494 vec![0, 1, 2, 3]
11495 );
11496 assert_eq!(
11497 schedules
11498 .iter()
11499 .map(|schedule| schedule.delay)
11500 .collect::<Vec<_>>(),
11501 vec![
11502 Duration::from_millis(100),
11503 Duration::from_secs(1),
11504 Duration::from_secs(10),
11505 Duration::from_secs(30),
11506 ]
11507 );
11508 }
11509
11510 #[test]
11511 fn crash_restart_backoff_resets_after_ring_clear() {
11512 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11513 let now = ClockInstant::now();
11514 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11515 assert_eq!(
11516 state.next_crash_restart(&policy, now).unwrap().delay,
11517 Duration::from_millis(100)
11518 );
11519 assert_eq!(
11520 state.next_crash_restart(&policy, now).unwrap().delay,
11521 Duration::from_secs(1)
11522 );
11523
11524 state.clear_crash_restarts();
11525 let schedule = state
11526 .next_crash_restart(&policy, now)
11527 .expect("a cleared ring must allow another restart");
11528 assert_eq!(schedule.restart_in_window, 0);
11529 assert_eq!(schedule.delay, Duration::from_millis(100));
11530 }
11531
11532 #[test]
11533 fn crash_restart_backoff_ignores_aged_restarts() {
11534 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11535 let now = ClockInstant::now();
11536 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11537 state
11538 .next_crash_restart(&policy, now)
11539 .expect("the first restart is allowed");
11540 state
11541 .next_crash_restart(&policy, now)
11542 .expect("the second restart is allowed");
11543 state.crash_restarts[0] = now
11544 .checked_sub(policy.window + Duration::from_secs(1))
11545 .expect("the fake clock can age a restart past the window");
11546
11547 let schedule = state
11548 .next_crash_restart(&policy, now)
11549 .expect("an aged restart must release its slot");
11550 assert_eq!(schedule.restart_in_window, 1);
11551 assert_eq!(schedule.delay, Duration::from_secs(1));
11552 assert_eq!(state.crash_restarts.len(), 2);
11553 }
11554
11555 #[test]
11559 fn a_budget_spent_before_the_window_no_longer_refuses() {
11560 let policy = RestartPolicy::new(3, Duration::ZERO);
11561 let mut state = snapshot_with_restarts(true, 3);
11562 let now = ClockInstant::now();
11563 assert!(!daemon_will_restart(&mut state, &policy, now));
11564
11565 assert!(daemon_will_restart(
11566 &mut state,
11567 &policy,
11568 now + policy.window + Duration::from_secs(1)
11569 ));
11570 assert!(
11571 state.crash_restarts.is_empty(),
11572 "reading the budget must drop the instants that left the window"
11573 );
11574 }
11575
11576 fn module_with_recovery_snapshot(
11577 state: ModuleState,
11578 enabled: bool,
11579 restart_count: u32,
11580 ) -> SupervisedModule {
11581 let registry = Arc::new(Registry::default());
11582 let supervisor =
11583 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
11584 let runtime = supervisor.runtime_config();
11585 let spec = ModuleSpec {
11586 module_id: "recovery-snapshot".to_string(),
11587 program: fake_aft_stub_path(),
11588 args: Vec::new(),
11589 env: Vec::new(),
11590 reserved: false,
11591 reserved_prefixes: Vec::new(),
11592 protocol: ModuleProtocol::Subc,
11593 overlap: Default::default(),
11594 };
11595 let mut snapshot = SupervisorSnapshot::new(state, enabled);
11596 seed_crash_restarts(&mut snapshot, restart_count);
11597 let (commands, _rx) = mpsc::channel(4);
11600 SupervisedModule {
11601 inner: Arc::new(SupervisedModuleInner {
11602 module_id: spec.module_id.clone(),
11603 registry,
11604 snapshot: Arc::new(Mutex::new(snapshot)),
11605 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11606 spec,
11607 health: runtime.health,
11608 })),
11609 stderr_ring: runtime.stderr_ring,
11610 terminal_ring: runtime.terminal_ring,
11611 commands,
11612 monitor: Mutex::new(None),
11613 restart_policy: runtime.restart_policy,
11614 effective_drain_timeout: runtime.effective_drain_timeout,
11615 provenance_probe: supervisor.provenance_probe.clone(),
11616 }),
11617 }
11618 }
11619
11620 #[cfg(target_os = "linux")]
11621 #[tokio::test]
11622 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11623 let supervisor =
11624 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11625 .with_cgroup_placement(None);
11626 let result = supervisor.spawn(ModuleSpec {
11627 module_id: "no-cgroup-placement".to_string(),
11628 program: fake_aft_stub_path(),
11629 args: Vec::new(),
11630 env: Vec::new(),
11631 reserved: false,
11632 reserved_prefixes: Vec::new(),
11633 protocol: ModuleProtocol::Subc,
11634 overlap: Default::default(),
11635 });
11636
11637 assert!(
11638 result.is_ok(),
11639 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11640 );
11641 }
11642
11643 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11644 async fn undecided_snapshot_uses_shared_restart_predicate() {
11645 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11646 .will_recover_after_connection_loss()
11647 .unwrap());
11648 assert!(
11649 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11650 .will_recover_after_connection_loss()
11651 .unwrap()
11652 );
11653 }
11654
11655 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11656 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11657 assert!(
11658 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11659 .will_recover_after_connection_loss()
11660 .unwrap()
11661 );
11662 }
11663
11664 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11665 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11666 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11667 .will_recover_after_connection_loss()
11668 .unwrap());
11669 assert!(
11670 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11671 .will_recover_after_connection_loss()
11672 .unwrap()
11673 );
11674 }
11675
11676 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11677 async fn warming_snapshot_is_limited_to_startup_phases() {
11678 for state in [
11679 ModuleState::Starting,
11680 ModuleState::Running,
11681 ModuleState::Restarting,
11682 ] {
11683 assert!(
11684 module_with_recovery_snapshot(state, true, 0)
11685 .is_warming()
11686 .unwrap(),
11687 "{state:?} should be warming"
11688 );
11689 }
11690 for state in [
11691 ModuleState::Unresponsive,
11692 ModuleState::Draining,
11693 ModuleState::Stopped,
11694 ModuleState::Failed,
11695 ModuleState::Disabled,
11696 ] {
11697 assert!(
11698 !module_with_recovery_snapshot(state, true, 0)
11699 .is_warming()
11700 .unwrap(),
11701 "{state:?} should not be warming"
11702 );
11703 }
11704 }
11705
11706 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11707 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11708 let registry = Arc::new(Registry::default());
11709 let supervisor =
11710 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
11711 let module = supervisor
11712 .spawn(ModuleSpec {
11713 module_id: "terminal-history".to_string(),
11714 program: fake_aft_stub_path(),
11715 args: Vec::new(),
11716 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11717 reserved: false,
11718 reserved_prefixes: Vec::new(),
11719 protocol: ModuleProtocol::Subc,
11720 overlap: Default::default(),
11721 })
11722 .unwrap();
11723
11724 let deadline = Instant::now() + Duration::from_secs(5);
11725 loop {
11726 let history = module.terminal_history();
11727 if history.entries.len() == 2 {
11728 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11729 assert_eq!(history.dropped, 0);
11730 assert_eq!(
11731 history
11732 .entries
11733 .iter()
11734 .map(|entry| entry.exit_code)
11735 .collect::<Vec<_>>(),
11736 vec![Some(23), Some(23)]
11737 );
11738 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11739 return;
11740 }
11741 assert!(
11742 Instant::now() < deadline,
11743 "module did not retain two terminal exits: {history:?}"
11744 );
11745 sleep(Duration::from_millis(10)).await;
11746 }
11747 }
11748
11749 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11753 async fn disable_during_crash_backoff_cancels_pending_respawn() {
11754 let backoff = Duration::from_secs(2);
11755 let supervisor = Supervisor::new_for_test(
11756 Arc::new(Registry::default()),
11757 RestartPolicy::new(10, backoff),
11758 );
11759 let module = supervisor
11760 .spawn(ModuleSpec {
11761 module_id: "disable-during-backoff".to_string(),
11762 program: fake_aft_stub_path(),
11763 args: Vec::new(),
11764 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11765 reserved: false,
11766 reserved_prefixes: Vec::new(),
11767 protocol: ModuleProtocol::Subc,
11768 overlap: Default::default(),
11769 })
11770 .unwrap();
11771
11772 let deadline = Instant::now() + Duration::from_secs(5);
11774 loop {
11775 if module.status().unwrap().state == ModuleState::Restarting {
11776 break;
11777 }
11778 assert!(
11779 Instant::now() < deadline,
11780 "module never entered the crash backoff"
11781 );
11782 sleep(Duration::from_millis(10)).await;
11783 }
11784
11785 let started = Instant::now();
11786 module.set_enabled(false).await.unwrap();
11787 let waited = started.elapsed();
11788
11789 assert!(
11790 waited < backoff / 2,
11791 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11792 );
11793 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11794
11795 sleep(backoff + Duration::from_millis(500)).await;
11797 let status = module.status().unwrap();
11798 assert_eq!(status.state, ModuleState::Disabled);
11799 assert_eq!(
11800 status.spawn_generation, 1,
11801 "module respawned after the operator disabled it"
11802 );
11803 }
11804
11805 #[cfg(unix)]
11808 fn protocol_none_sigterm_exits_clean_spec(
11809 module_id: &str,
11810 dir: &std::path::Path,
11811 ) -> (ModuleSpec, PathBuf, PathBuf) {
11812 let ready = dir.join("ready");
11813 let marker = dir.join("sigterm");
11814 let spec = ModuleSpec {
11815 module_id: module_id.to_string(),
11816 program: fake_aft_stub_path(),
11817 args: Vec::new(),
11818 env: vec![
11819 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11820 (
11821 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11822 marker.display().to_string(),
11823 ),
11824 (
11825 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11826 ready.display().to_string(),
11827 ),
11828 ],
11829 reserved: false,
11830 reserved_prefixes: Vec::new(),
11831 protocol: ModuleProtocol::None,
11832 overlap: Default::default(),
11833 };
11834 (spec, ready, marker)
11835 }
11836
11837 #[cfg(unix)]
11841 async fn wait_for_file(path: &std::path::Path) {
11842 let deadline = Instant::now() + Duration::from_secs(10);
11843 while !path.exists() {
11844 assert!(
11845 Instant::now() < deadline,
11846 "{} never appeared",
11847 path.display()
11848 );
11849 sleep(Duration::from_millis(10)).await;
11850 }
11851 }
11852
11853 #[cfg(unix)]
11857 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11858 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11859 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11860 let (spec, ready, marker) =
11861 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11862 let supervisor = Supervisor::new_for_test(
11863 Arc::new(Registry::default()),
11864 RestartPolicy::new(3, Duration::ZERO),
11865 );
11866 let module = supervisor.spawn(spec).unwrap();
11867 wait_for_file(&ready).await;
11868 let deadline = Instant::now() + Duration::from_secs(10);
11872 let first_pid = loop {
11873 let status = module.status().unwrap();
11874 if status.state == ModuleState::Running {
11875 if let Some(pid) = status.pid {
11876 break pid;
11877 }
11878 }
11879 assert!(
11880 Instant::now() < deadline,
11881 "a running module must report its pid after exec confirmation: {status:?}"
11882 );
11883 sleep(Duration::from_millis(10)).await;
11884 };
11885
11886 rustix::process::kill_process(
11887 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11888 rustix::process::Signal::TERM,
11889 )
11890 .unwrap();
11891
11892 let deadline = Instant::now() + Duration::from_secs(10);
11893 let respawned = loop {
11894 let status = module.status().unwrap();
11895 if status.state == ModuleState::Running
11896 && status.pid.is_some_and(|pid| pid != first_pid)
11897 {
11898 break status;
11899 }
11900 assert!(
11901 Instant::now() < deadline,
11902 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11903 );
11904 sleep(Duration::from_millis(10)).await;
11905 };
11906 assert_eq!(respawned.spawn_generation, 2);
11907 assert!(
11908 marker.exists(),
11909 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11910 );
11911
11912 let history = module.terminal_history();
11913 assert_eq!(history.entries.len(), 1, "{history:?}");
11914 let entry = &history.entries[0];
11915 assert_eq!(entry.exit_code, Some(0));
11916 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11917 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11918
11919 module.stop().await.unwrap();
11920 }
11921
11922 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11926 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11927 let supervisor = Supervisor::new_for_test(
11928 Arc::new(Registry::default()),
11929 RestartPolicy::new(1, Duration::ZERO),
11930 );
11931 let module = supervisor
11932 .spawn(ModuleSpec {
11933 module_id: "none-clean-exit-budget".to_string(),
11934 program: fake_aft_stub_path(),
11935 args: Vec::new(),
11936 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11937 reserved: false,
11938 reserved_prefixes: Vec::new(),
11939 protocol: ModuleProtocol::None,
11940 overlap: Default::default(),
11941 })
11942 .unwrap();
11943
11944 let deadline = Instant::now() + Duration::from_secs(10);
11947 loop {
11948 let status = module.status().unwrap();
11949 if status.state == ModuleState::Failed {
11950 break;
11951 }
11952 assert!(
11953 Instant::now() < deadline,
11954 "module never exhausted its budget: {status:?} {:?}",
11955 module.terminal_history()
11956 );
11957 sleep(Duration::from_millis(10)).await;
11958 }
11959 let history = module.terminal_history();
11960 assert_eq!(
11961 history
11962 .entries
11963 .iter()
11964 .map(|entry| (entry.exit_code, entry.disposition.clone()))
11965 .collect::<Vec<_>>(),
11966 vec![
11967 (Some(0), TerminalDisposition::Restarting),
11968 (Some(0), TerminalDisposition::Failed),
11969 ]
11970 );
11971 let detail = history.entries[1]
11972 .disposition_detail
11973 .as_deref()
11974 .expect("a budget failure names the budget");
11975 assert!(detail.contains("max_restarts=1"), "{detail}");
11976 assert_eq!(module.status().unwrap().spawn_generation, 2);
11977 }
11978
11979 #[test]
11980 fn restart_budget_failure_is_published_after_its_terminal_record() {
11981 use std::io::{Read, Write};
11982 let supervisor = Supervisor::new_for_test(
11983 Arc::new(Registry::default()),
11984 RestartPolicy::new(0, Duration::ZERO),
11985 );
11986 let runtime = supervisor.runtime_config();
11987 let spec = ModuleSpec {
11988 protocol: ModuleProtocol::None,
11989 ..windowed_crash_spec("budget-publication-order")
11990 };
11991 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11992 ModuleState::Running,
11993 true,
11994 )));
11995 lock_snapshot(&snapshot).unwrap().process_alive = true;
11996 let reaped_snapshot = snapshot.clone();
11997 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11998 let history = runtime.terminal_ring.clone();
11999 let reap = std::thread::spawn(move || {
12000 tokio::runtime::Builder::new_current_thread()
12001 .enable_all()
12002 .build()
12003 .unwrap()
12004 .block_on(on_child_exit(
12005 &spec,
12006 runtime.restart_policy,
12007 &supervisor.registry,
12008 &reaped_snapshot,
12009 &runtime.terminal_ring,
12010 &runtime.spawn_events,
12011 &runtime.child_roster,
12012 ExitReport {
12013 kind: ExitKind::Clean,
12014 code: Some(0),
12015 signal: None,
12016 at_ms: 1,
12017 },
12018 ))
12019 });
12020 let mut reached = [0];
12021 barrier.read_exact(&mut reached).unwrap();
12022 assert_eq!(&reached, b"R");
12023 let (before_state, before_alive) = {
12024 let state = lock_snapshot(&snapshot).unwrap();
12025 (state.state, state.process_alive)
12026 };
12027 let before_history = history.lock().unwrap().snapshot();
12028 barrier.write_all(b"X").unwrap();
12029 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12030 assert!(before_history.entries.is_empty());
12031 assert_ne!(
12032 before_state,
12033 ModuleState::Failed,
12034 "Failed was visible before its terminal record could be written"
12035 );
12036 assert!(before_alive);
12037 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12038 let history = history.lock().unwrap().snapshot();
12039 assert_eq!(history.entries.len(), 1);
12040 assert_eq!(history.entries[0].exit_code, Some(0));
12041 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12042 }
12043
12044 #[test]
12045 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12046 use std::io::{Read, Write};
12047
12048 let supervisor = Supervisor::new_for_test(
12049 Arc::new(Registry::default()),
12050 RestartPolicy::new(3, Duration::ZERO),
12051 );
12052 let mut runtime = supervisor.runtime_config();
12053 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12054 let snapshot = module.inner.snapshot.clone();
12055 lock_snapshot(&snapshot).unwrap().process_alive = true;
12056 runtime.terminal_ring = module.inner.terminal_ring.clone();
12057 let spec = windowed_crash_spec(module.module_id());
12058 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12059 let reaped_snapshot = snapshot.clone();
12060 let reap = std::thread::spawn(move || {
12061 tokio::runtime::Builder::new_current_thread()
12062 .enable_all()
12063 .build()
12064 .unwrap()
12065 .block_on(on_child_exit(
12066 &spec,
12067 runtime.restart_policy,
12068 &supervisor.registry,
12069 &reaped_snapshot,
12070 &runtime.terminal_ring,
12071 &runtime.spawn_events,
12072 &runtime.child_roster,
12073 ExitReport {
12074 kind: ExitKind::Crash,
12075 code: Some(2),
12076 signal: None,
12077 at_ms: 1,
12078 },
12079 ))
12080 });
12081
12082 let mut reached = [0];
12086 barrier.read_exact(&mut reached).unwrap();
12087 assert_eq!(&reached, b"R");
12088 let before_status = module.status().unwrap();
12089 let before_history = module.terminal_history();
12090 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12091 barrier.write_all(b"X").unwrap();
12092 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12093
12094 assert_eq!(before_status.state, ModuleState::Running);
12095 assert!(before_status.process_alive);
12096 assert_eq!(before_status.restart_count, 3);
12097 assert!(before_history.entries.is_empty());
12098 assert!(
12099 before_recovers,
12100 "connection cleanup would report a terminal close for a scheduled crash restart"
12101 );
12102 let restarting = module.status().unwrap();
12103 assert_eq!(restarting.state, ModuleState::Restarting);
12104 assert_eq!(restarting.restart_count, 3);
12105 assert!(module.will_recover_after_connection_loss().unwrap());
12106 let history = module.terminal_history();
12107 assert_eq!(history.entries.len(), 1);
12108 assert_eq!(
12109 history.entries[0].disposition,
12110 TerminalDisposition::Restarting
12111 );
12112
12113 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12116 state.state = ModuleState::Running;
12117 state.respawn_pending = false;
12118 })
12119 .unwrap();
12120 assert!(!module.will_recover_after_connection_loss().unwrap());
12121 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12122 state.enabled = false;
12123 state.respawn_pending = true;
12124 })
12125 .unwrap();
12126 assert!(!module.will_recover_after_connection_loss().unwrap());
12127 }
12128
12129 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12130 use std::net::{TcpListener, TcpStream};
12131 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12132 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12133 let (barrier, _) = listener.accept().unwrap();
12134 for stream in [&writer, &barrier] {
12135 stream
12136 .set_read_timeout(Some(Duration::from_secs(10)))
12137 .unwrap();
12138 stream
12139 .set_write_timeout(Some(Duration::from_secs(10)))
12140 .unwrap();
12141 }
12142 *events.1.lock().unwrap() = Some(writer);
12143 barrier
12144 }
12145
12146 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12147 use std::io::{Read, Write};
12148 let barrier = events.1.lock().unwrap().take();
12149 if let Some(mut barrier) = barrier {
12150 barrier.write_all(b"R").unwrap();
12151 let mut release = [0];
12152 barrier.read_exact(&mut release).unwrap();
12153 assert_eq!(&release, b"X");
12154 }
12155 }
12156
12157 #[test]
12158 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12159 use std::io::{Read, Write};
12160
12161 let supervisor = Supervisor::new_for_test(
12162 Arc::new(Registry::default()),
12163 RestartPolicy::new(0, Duration::ZERO),
12164 );
12165 let mut runtime = supervisor.runtime_config();
12166 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12167 let snapshot = module.inner.snapshot.clone();
12168 runtime.terminal_ring = module.inner.terminal_ring.clone();
12169 let spec = ModuleSpec {
12170 module_id: module.module_id().into(),
12171 program: fake_aft_stub_path(),
12172 protocol: ModuleProtocol::None,
12173 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12174 ..windowed_crash_spec(module.module_id())
12175 };
12176 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12177 let reap = std::thread::spawn(move || {
12178 tokio::runtime::Builder::new_current_thread()
12179 .enable_all()
12180 .build()
12181 .unwrap()
12182 .block_on(async {
12183 let mut child =
12184 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12185 health_restart_child(
12186 &spec,
12187 &runtime,
12188 &supervisor.registry,
12189 &supervisor.process_liveness,
12190 &snapshot,
12191 &mut child,
12192 SupervisorHealthStatus::Failing,
12193 Some("health probe did not answer"),
12194 1,
12195 )
12196 .await
12197 .unwrap();
12198 assert!(child.is_none());
12199 });
12200 });
12201 let mut reached = [0];
12206 barrier.read_exact(&mut reached).unwrap();
12207 assert_eq!(&reached, b"R");
12208 let before_status = module.status().unwrap();
12209 let before_history = module.terminal_history();
12210 barrier.write_all(b"X").unwrap();
12211 reap.join().unwrap();
12212 assert!(before_history.entries.is_empty());
12213 assert_ne!(
12214 before_status.state,
12215 ModuleState::Failed,
12216 "Failed was visible before its terminal record could be written"
12217 );
12218 assert!(
12219 before_status.process_alive,
12220 "process death was visible before its terminal record could be written"
12221 );
12222 let failed = module.status().unwrap();
12223 assert_eq!(failed.state, ModuleState::Failed);
12224 assert!(!failed.process_alive);
12225 let history = module.terminal_history();
12226 assert_eq!(history.entries.len(), 1);
12227 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12228 assert_eq!(
12229 history.entries[0].disposition_detail.as_deref(),
12230 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12231 );
12232 }
12233
12234 #[test]
12235 fn observed_exits_publish_process_death_after_terminal_history() {
12236 use std::io::{Read, Write};
12237 for (kind, enabled, shutdown, wait_error, expected_state) in [
12238 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12239 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12240 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12241 (
12242 ExitKind::DeliberateSeverance,
12243 true,
12244 false,
12245 false,
12246 ModuleState::Restarting,
12247 ),
12248 #[cfg(unix)]
12249 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12250 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12251 ] {
12252 let supervisor = Supervisor::new_for_test(
12253 Arc::new(Registry::default()),
12254 RestartPolicy::new(1, Duration::ZERO),
12255 );
12256 let runtime = supervisor.runtime_config();
12257 #[cfg(unix)]
12258 if shutdown {
12259 runtime.child_roster.close();
12260 }
12261 #[cfg(not(unix))]
12262 let _ = shutdown;
12263 let spec = windowed_crash_spec("exit-publication-order");
12264 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12265 ModuleState::Running,
12266 enabled,
12267 )));
12268 lock_snapshot(&snapshot).unwrap().process_alive = true;
12269 let reaped_snapshot = snapshot.clone();
12270 let history = runtime.terminal_ring.clone();
12271 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12272 let reap = std::thread::spawn(move || {
12273 if wait_error {
12274 super::fail_after_wait_error(
12275 &spec.module_id,
12276 &reaped_snapshot,
12277 &runtime.terminal_ring,
12278 &runtime.spawn_events,
12279 "wait failed".into(),
12280 );
12281 } else {
12282 tokio::runtime::Builder::new_current_thread()
12283 .enable_all()
12284 .build()
12285 .unwrap()
12286 .block_on(on_child_exit(
12287 &spec,
12288 runtime.restart_policy,
12289 &supervisor.registry,
12290 &reaped_snapshot,
12291 &runtime.terminal_ring,
12292 &runtime.spawn_events,
12293 &runtime.child_roster,
12294 ExitReport {
12295 kind,
12296 code: Some(0),
12297 signal: None,
12298 at_ms: 1,
12299 },
12300 ));
12301 }
12302 });
12303 let mut reached = [0];
12304 barrier.read_exact(&mut reached).unwrap();
12305 assert_eq!(&reached, b"R");
12306 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12307 let before_history = history.lock().unwrap().snapshot();
12308 barrier.write_all(b"X").unwrap();
12309 reap.join().unwrap();
12310 assert!(
12311 before_alive,
12312 "{kind:?}: death was published before recording"
12313 );
12314 assert!(before_history.entries.is_empty());
12315 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12316 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12317 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12318 }
12319 }
12320
12321 #[tokio::test]
12322 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12323 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12324 let path = dir.join("terminals.jsonl");
12325 std::fs::create_dir(&path).unwrap();
12326 let supervisor = Supervisor::new_for_test(
12327 Arc::new(Registry::default()),
12328 RestartPolicy::new(0, Duration::ZERO),
12329 )
12330 .with_terminal_journal(path, "health-budget-journal-failure".into());
12331 let runtime = supervisor.runtime_config();
12332 let spec = ModuleSpec {
12333 program: fake_aft_stub_path(),
12334 protocol: ModuleProtocol::None,
12335 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12336 ..windowed_crash_spec("health-budget-journal-failure")
12337 };
12338 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12339 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12340 health_restart_child(
12341 &spec,
12342 &runtime,
12343 &supervisor.registry,
12344 &supervisor.process_liveness,
12345 &snapshot,
12346 &mut child,
12347 SupervisorHealthStatus::Failing,
12348 None,
12349 1,
12350 )
12351 .await
12352 .unwrap();
12353 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12354 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12355 let history = runtime
12356 .terminal_ring
12357 .lock()
12358 .unwrap()
12359 .durable_history(&spec.module_id);
12360 assert!(history.journal_write_failures > 0);
12361 assert_eq!(history.entries.len(), 1);
12362 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12363 }
12364
12365 #[test]
12366 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12367 let supervisor = Supervisor::new_for_test(
12368 Arc::new(Registry::default()),
12369 RestartPolicy::new(0, Duration::ZERO),
12370 );
12371 let runtime = supervisor.runtime_config();
12372 let spec = ModuleSpec {
12373 program: fake_aft_stub_path(),
12374 protocol: ModuleProtocol::None,
12375 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12376 ..windowed_crash_spec("health-budget-recording-panic")
12377 };
12378 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12379 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12380 tokio::runtime::Builder::new_current_thread()
12381 .enable_all()
12382 .build()
12383 .unwrap()
12384 .block_on(async {
12385 let mut child =
12386 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12387 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12388 health_restart_child(
12389 &spec,
12390 &runtime,
12391 &supervisor.registry,
12392 &supervisor.process_liveness,
12393 &snapshot,
12394 &mut child,
12395 SupervisorHealthStatus::Failing,
12396 None,
12397 1,
12398 )
12399 .await
12400 .unwrap();
12401 });
12402 }));
12403 let panic = result.expect_err("recording must still unwind");
12404 assert_eq!(
12405 panic.downcast_ref::<String>().map(String::as_str),
12406 Some("spawn event sequence exhausted")
12407 );
12408 let state = lock_snapshot(&snapshot).unwrap();
12409 assert_eq!(state.state, ModuleState::Failed);
12410 assert!(!state.process_alive);
12411 assert!(state.last_exit.is_some());
12412 }
12413
12414 #[tokio::test]
12415 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12416 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12417 let path = dir.join("terminals.jsonl");
12418 std::fs::create_dir(&path).unwrap();
12419 let supervisor = Supervisor::new_for_test(
12420 Arc::new(Registry::default()),
12421 RestartPolicy::new(0, Duration::ZERO),
12422 )
12423 .with_terminal_journal(path, "budget-journal-failure".into());
12424 let runtime = supervisor.runtime_config();
12425 let spec = windowed_crash_spec("budget-journal-failure");
12426 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12427 assert!(matches!(
12428 on_child_exit(
12429 &spec,
12430 runtime.restart_policy,
12431 &supervisor.registry,
12432 &snapshot,
12433 &runtime.terminal_ring,
12434 &runtime.spawn_events,
12435 &runtime.child_roster,
12436 crash_exit_report(1),
12437 )
12438 .await,
12439 NextAction::Stop { .. }
12440 ));
12441 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12442 let history = runtime
12443 .terminal_ring
12444 .lock()
12445 .unwrap()
12446 .durable_history(&spec.module_id);
12447 assert!(history.journal_write_failures > 0);
12448 assert_eq!(history.entries.len(), 1);
12449 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12450 }
12451
12452 #[test]
12453 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12454 let supervisor = Supervisor::new_for_test(
12455 Arc::new(Registry::default()),
12456 RestartPolicy::new(0, Duration::ZERO),
12457 );
12458 let runtime = supervisor.runtime_config();
12459 let spec = windowed_crash_spec("budget-recording-panic");
12460 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12461 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12462 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12465 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12466 tokio::runtime::Builder::new_current_thread()
12467 .enable_all()
12468 .build()
12469 .unwrap()
12470 .block_on(on_child_exit(
12471 &spec,
12472 runtime.restart_policy,
12473 &supervisor.registry,
12474 &snapshot,
12475 &runtime.terminal_ring,
12476 &runtime.spawn_events,
12477 &runtime.child_roster,
12478 crash_exit_report(1),
12479 ))
12480 }));
12481 let panic = result.err().expect("recording must still unwind");
12482 assert_eq!(
12483 panic.downcast_ref::<String>().map(String::as_str),
12484 Some("spawn event sequence exhausted")
12485 );
12486 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12487 }
12488
12489 #[cfg(unix)]
12492 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12493 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12494 for disable in [false, true] {
12495 let label = if disable {
12496 "none-requested-disable"
12497 } else {
12498 "none-requested-stop"
12499 };
12500 let dir = cortexkit_test_support::ScratchDir::new(label);
12501 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12502 let supervisor = Supervisor::new_for_test(
12503 Arc::new(Registry::default()),
12504 RestartPolicy::new(3, Duration::ZERO),
12505 );
12506 let module = supervisor.spawn(spec).unwrap();
12507 wait_for_file(&ready).await;
12508
12509 if disable {
12510 module.set_enabled(false).await.unwrap();
12511 } else {
12512 module.stop().await.unwrap();
12513 }
12514 assert!(
12515 marker.exists(),
12516 "{label}: the child must have left through its SIGTERM handler with exit 0"
12517 );
12518
12519 sleep(Duration::from_millis(500)).await;
12522 let status = module.status().unwrap();
12523 let expected = if disable {
12524 ModuleState::Disabled
12525 } else {
12526 ModuleState::Stopped
12527 };
12528 assert_eq!(status.state, expected, "{label}");
12529 assert_eq!(
12530 status.spawn_generation, 1,
12531 "{label}: respawned after a requested stop"
12532 );
12533 let history = module.terminal_history();
12534 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12535 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12536 assert_ne!(
12537 history.entries[0].disposition,
12538 TerminalDisposition::Restarting,
12539 "{label}"
12540 );
12541 }
12542 }
12543
12544 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12547 async fn subc_wire_clean_exit_is_still_a_stop() {
12548 let supervisor = Supervisor::new_for_test(
12549 Arc::new(Registry::default()),
12550 RestartPolicy::new(3, Duration::ZERO),
12551 );
12552 let module = supervisor
12553 .spawn(ModuleSpec {
12554 module_id: "wire-clean-exit".to_string(),
12555 program: fake_aft_stub_path(),
12556 args: Vec::new(),
12557 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12558 reserved: false,
12559 reserved_prefixes: Vec::new(),
12560 protocol: ModuleProtocol::Subc,
12561 overlap: Default::default(),
12562 })
12563 .unwrap();
12564
12565 let deadline = Instant::now() + Duration::from_secs(10);
12566 while module.terminal_history().entries.is_empty() {
12567 assert!(Instant::now() < deadline, "module never exited");
12568 sleep(Duration::from_millis(10)).await;
12569 }
12570 sleep(Duration::from_millis(500)).await;
12572 let status = module.status().unwrap();
12573 assert_eq!(status.state, ModuleState::Stopped);
12574 assert_eq!(status.spawn_generation, 1);
12575 let history = module.terminal_history();
12576 assert_eq!(history.entries.len(), 1, "{history:?}");
12577 assert_eq!(history.entries[0].exit_code, Some(0));
12578 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12579 }
12580
12581 #[cfg(unix)]
12582 #[tokio::test]
12583 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12584 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12585 let record = dir.join("live-children.json");
12586 let supervisor = Supervisor::new_for_test(
12587 Arc::new(Registry::default()),
12588 RestartPolicy::new(0, Duration::ZERO),
12589 );
12590 let mut runtime = supervisor.runtime_config();
12591 runtime.child_roster.record_to(record.clone());
12592 let gate = Arc::new(super::ReloadExitRecordGate::default());
12593 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12594 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12595 let spec = ModuleSpec {
12596 module_id: "reload-exit-roster".into(),
12597 program: fake_aft_stub_path(),
12598 args: Vec::new(),
12599 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12600 reserved: false,
12601 reserved_prefixes: Vec::new(),
12602 protocol: ModuleProtocol::Subc,
12603 overlap: Default::default(),
12604 };
12605 let mut child = None;
12606 let reload = super::finish_reload_child(
12607 &spec,
12608 &runtime,
12609 &supervisor.registry,
12610 &supervisor.process_liveness,
12611 &snapshot,
12612 &mut child,
12613 );
12614 tokio::pin!(reload);
12615 tokio::select! {
12616 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12617 _ = gate.reached.notified() => {}
12618 }
12619 assert!(runtime
12620 .terminal_ring
12621 .lock()
12622 .unwrap()
12623 .snapshot()
12624 .entries
12625 .is_empty());
12626 assert_eq!(
12627 crate::live_children::read_record(&record).unwrap().len(),
12628 1,
12629 "shutdown must still wait for the reaped child until its terminal record exists"
12630 );
12631 runtime.child_roster.close();
12632 gate.resume.notify_one();
12633 assert!(reload.await.is_err());
12634 assert!(crate::live_children::read_record(&record)
12635 .unwrap()
12636 .is_empty());
12637 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12638 assert_eq!(history.entries.len(), 1);
12639 assert_eq!(
12640 history.entries[0].disposition,
12641 TerminalDisposition::DaemonShutdown
12642 );
12643 }
12644
12645 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12649 async fn every_restart_increment_path_advances_lifetime_count() {
12650 let supervisor = Supervisor::new_for_test(
12651 Arc::new(Registry::default()),
12652 RestartPolicy::new(1, Duration::ZERO),
12653 );
12654 let runtime = supervisor.runtime_config();
12655 let spec = ModuleSpec {
12656 module_id: "lifetime-increment-path".to_string(),
12657 program: PathBuf::from("/unused/lifetime-increment-path"),
12658 args: Vec::new(),
12659 env: Vec::new(),
12660 reserved: false,
12661 reserved_prefixes: Vec::new(),
12662 protocol: ModuleProtocol::Subc,
12663 overlap: Default::default(),
12664 };
12665
12666 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12667 assert!(matches!(
12668 on_child_exit(
12669 &spec,
12670 runtime.restart_policy,
12671 &supervisor.registry,
12672 &crash_snapshot,
12673 &runtime.terminal_ring,
12674 &runtime.spawn_events,
12675 &runtime.child_roster,
12676 ExitReport {
12677 kind: ExitKind::Crash,
12678 code: Some(1),
12679 signal: None,
12680 at_ms: 1,
12681 },
12682 )
12683 .await,
12684 NextAction::Restart { schedule: _ }
12685 ));
12686 let (crash_restarts, crash_lifetime) = {
12687 let state = lock_snapshot(&crash_snapshot).unwrap();
12688 (state.crash_restarts.len(), state.lifetime_restarts)
12689 };
12690 assert_eq!(crash_restarts, 1);
12691 assert_eq!(crash_lifetime, 1);
12692
12693 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12694 let mut health_child = None;
12695 assert!(matches!(
12696 health_restart_child(
12697 &spec,
12698 &runtime,
12699 &supervisor.registry,
12700 &supervisor.process_liveness,
12701 &health_snapshot,
12702 &mut health_child,
12703 SupervisorHealthStatus::Failing,
12704 None,
12705 2,
12706 )
12707 .await,
12708 Ok(())
12709 ));
12710 assert!(health_child.is_none());
12711 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12712 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12713 let (health_restarts, health_lifetime) = {
12714 let state = lock_snapshot(&health_snapshot).unwrap();
12715 (state.crash_restarts.len(), state.lifetime_restarts)
12716 };
12717 assert_eq!(health_restarts, 1);
12718 assert_eq!(health_lifetime, 1);
12719
12720 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12721 let mut reload_child = None;
12722 assert!(matches!(
12723 handle_reload_spawn_failure(
12724 &spec,
12725 &runtime,
12726 &supervisor.process_liveness,
12727 &reload_snapshot,
12728 &mut reload_child,
12729 "forced reload spawn failure".to_string(),
12730 )
12731 .await,
12732 Err(SuperviseError::ReloadFailed { .. })
12733 ));
12734 let (reload_restarts, reload_lifetime) = {
12735 let state = lock_snapshot(&reload_snapshot).unwrap();
12736 (state.crash_restarts.len(), state.lifetime_restarts)
12737 };
12738 assert_eq!(reload_restarts, 1);
12739 assert_eq!(reload_lifetime, 1);
12740 }
12741
12742 #[tokio::test]
12743 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12744 let supervisor = Supervisor::new_for_test(
12745 Arc::new(Registry::default()),
12746 RestartPolicy::new(3, Duration::ZERO),
12747 );
12748 let runtime = supervisor.runtime_config();
12749 let spec = ModuleSpec {
12750 module_id: "deliberately-severed".to_string(),
12751 program: PathBuf::from("/unused/deliberately-severed"),
12752 args: Vec::new(),
12753 env: Vec::new(),
12754 reserved: false,
12755 reserved_prefixes: Vec::new(),
12756 protocol: ModuleProtocol::Subc,
12757 overlap: Default::default(),
12758 };
12759 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12760 let process = ProcessIdentity {
12761 pid: 41,
12762 start_time: 101,
12763 };
12764 record_deliberate_severance(&snapshot, process).unwrap();
12765 let exit_report = apply_deliberate_severance_marker(
12766 &snapshot,
12767 Some(process),
12768 ExitReport {
12769 kind: ExitKind::Crash,
12770 code: Some(1),
12771 signal: None,
12772 at_ms: 1,
12773 },
12774 );
12775 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12776
12777 assert!(matches!(
12778 on_child_exit(
12779 &spec,
12780 runtime.restart_policy,
12781 &supervisor.registry,
12782 &snapshot,
12783 &runtime.terminal_ring,
12784 &runtime.spawn_events,
12785 &runtime.child_roster,
12786 exit_report,
12787 )
12788 .await,
12789 NextAction::Restart { schedule: _ }
12790 ));
12791 let state = lock_snapshot(&snapshot).unwrap();
12792 assert_eq!(state.lifetime_restarts, 1);
12793 assert_eq!(state.crash_restarts.len(), 0);
12794 }
12795
12796 #[tokio::test]
12797 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12798 let supervisor = Supervisor::new_for_test(
12799 Arc::new(Registry::default()),
12800 RestartPolicy::new(3, Duration::ZERO),
12801 );
12802 let runtime = supervisor.runtime_config();
12803 let spec = ModuleSpec {
12804 module_id: "genuine-crash".to_string(),
12805 program: PathBuf::from("/unused/genuine-crash"),
12806 args: Vec::new(),
12807 env: Vec::new(),
12808 reserved: false,
12809 reserved_prefixes: Vec::new(),
12810 protocol: ModuleProtocol::Subc,
12811 overlap: Default::default(),
12812 };
12813 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12814
12815 assert!(matches!(
12816 on_child_exit(
12817 &spec,
12818 runtime.restart_policy,
12819 &supervisor.registry,
12820 &snapshot,
12821 &runtime.terminal_ring,
12822 &runtime.spawn_events,
12823 &runtime.child_roster,
12824 ExitReport {
12825 kind: ExitKind::Crash,
12826 code: Some(1),
12827 signal: None,
12828 at_ms: 1,
12829 },
12830 )
12831 .await,
12832 NextAction::Restart { schedule: _ }
12833 ));
12834 let state = lock_snapshot(&snapshot).unwrap();
12835 assert_eq!(state.lifetime_restarts, 1);
12836 assert_eq!(state.crash_restarts.len(), 1);
12837 }
12838
12839 fn crash_exit_report(at_ms: u64) -> ExitReport {
12840 ExitReport {
12841 kind: ExitKind::Crash,
12842 code: Some(1),
12843 signal: None,
12844 at_ms,
12845 }
12846 }
12847
12848 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12849 ModuleSpec {
12850 module_id: module_id.to_string(),
12851 program: PathBuf::from("/unused").join(module_id),
12852 args: Vec::new(),
12853 env: Vec::new(),
12854 reserved: false,
12855 reserved_prefixes: Vec::new(),
12856 protocol: ModuleProtocol::Subc,
12857 overlap: Default::default(),
12858 }
12859 }
12860
12861 #[tokio::test]
12867 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12868 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12869 let supervisor = Supervisor::new_for_test(
12870 Arc::new(Registry::default()),
12871 RestartPolicy::new(2, Duration::ZERO),
12872 );
12873 let runtime = supervisor.runtime_config();
12874 let spec = windowed_crash_spec("crash-loop-in-window");
12875 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12876
12877 for attempt in 1..=2 {
12878 assert!(
12879 matches!(
12880 on_child_exit(
12881 &spec,
12882 runtime.restart_policy,
12883 &supervisor.registry,
12884 &snapshot,
12885 &runtime.terminal_ring,
12886 &runtime.spawn_events,
12887 &runtime.child_roster,
12888 crash_exit_report(attempt),
12889 )
12890 .await,
12891 NextAction::Restart { schedule: _ }
12892 ),
12893 "crash {attempt} is inside the budget and must respawn"
12894 );
12895 }
12896
12897 assert!(matches!(
12898 on_child_exit(
12899 &spec,
12900 runtime.restart_policy,
12901 &supervisor.registry,
12902 &snapshot,
12903 &runtime.terminal_ring,
12904 &runtime.spawn_events,
12905 &runtime.child_roster,
12906 crash_exit_report(3),
12907 )
12908 .await,
12909 NextAction::Stop { .. }
12910 ));
12911
12912 {
12913 let state = lock_snapshot(&snapshot).unwrap();
12914 assert_eq!(state.state, ModuleState::Failed);
12915 assert_eq!(state.crash_restarts.len(), 2);
12916 assert_eq!(state.lifetime_restarts, 2);
12917 }
12918
12919 let history = runtime
12920 .terminal_ring
12921 .lock()
12922 .expect("terminal ring is not poisoned")
12923 .snapshot();
12924 let last = history
12925 .entries
12926 .last()
12927 .expect("the refused crash is retained");
12928 assert_eq!(last.disposition, TerminalDisposition::Failed);
12929 assert_eq!(
12930 last.disposition_detail.as_deref(),
12931 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12932 );
12933
12934 let captured = crate::router::test_log::captured_logs(&logs);
12935 assert!(
12936 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12937 "the stop must be logged with its window: {captured}"
12938 );
12939 }
12940
12941 #[tokio::test]
12949 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12950 let supervisor = Supervisor::new_for_test(
12951 Arc::new(Registry::default()),
12952 RestartPolicy::new(2, Duration::ZERO),
12953 );
12954 let runtime = supervisor.runtime_config();
12955 let spec = windowed_crash_spec("crash-across-windows");
12956 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12957
12958 for attempt in 1..=2 {
12959 assert!(matches!(
12960 on_child_exit(
12961 &spec,
12962 runtime.restart_policy,
12963 &supervisor.registry,
12964 &snapshot,
12965 &runtime.terminal_ring,
12966 &runtime.spawn_events,
12967 &runtime.child_roster,
12968 crash_exit_report(attempt),
12969 )
12970 .await,
12971 NextAction::Restart { schedule: _ }
12972 ));
12973 }
12974
12975 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12978 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12979 })
12980 .unwrap();
12981
12982 assert!(
12983 matches!(
12984 on_child_exit(
12985 &spec,
12986 runtime.restart_policy,
12987 &supervisor.registry,
12988 &snapshot,
12989 &runtime.terminal_ring,
12990 &runtime.spawn_events,
12991 &runtime.child_roster,
12992 crash_exit_report(3),
12993 )
12994 .await,
12995 NextAction::Restart { schedule: _ }
12996 ),
12997 "a crash older than the window must not hold a budget slot"
12998 );
12999
13000 let state = lock_snapshot(&snapshot).unwrap();
13001 assert_eq!(state.state, ModuleState::Restarting);
13002 assert_eq!(
13003 state.crash_restarts.len(),
13004 2,
13005 "the aged instant is dropped and the new one takes its place"
13006 );
13007 assert_eq!(
13008 state.lifetime_restarts, 3,
13009 "the ledger counts every restart, including the ones the window forgot"
13010 );
13011 }
13012
13013 #[tokio::test]
13018 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13019 let supervisor = Supervisor::new_for_test(
13020 Arc::new(Registry::default()),
13021 RestartPolicy::new(2, Duration::ZERO),
13022 );
13023 let runtime = supervisor.runtime_config();
13024 let spec = windowed_crash_spec("operator-cleared-budget");
13025 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13026
13027 for attempt in 1..=2 {
13028 assert!(matches!(
13029 on_child_exit(
13030 &spec,
13031 runtime.restart_policy,
13032 &supervisor.registry,
13033 &snapshot,
13034 &runtime.terminal_ring,
13035 &runtime.spawn_events,
13036 &runtime.child_roster,
13037 crash_exit_report(attempt),
13038 )
13039 .await,
13040 NextAction::Restart { schedule: _ }
13041 ));
13042 }
13043
13044 reset_restart_count(&snapshot, &spec.module_id).unwrap();
13045 {
13046 let state = lock_snapshot(&snapshot).unwrap();
13047 assert!(
13048 state.crash_restarts.is_empty(),
13049 "an operator restart returns the full budget"
13050 );
13051 assert_eq!(
13052 state.lifetime_restarts, 2,
13053 "clearing the budget must not unmake the crashes"
13054 );
13055 }
13056
13057 assert!(
13058 matches!(
13059 on_child_exit(
13060 &spec,
13061 runtime.restart_policy,
13062 &supervisor.registry,
13063 &snapshot,
13064 &runtime.terminal_ring,
13065 &runtime.spawn_events,
13066 &runtime.child_roster,
13067 crash_exit_report(3),
13068 )
13069 .await,
13070 NextAction::Restart { schedule: _ }
13071 ),
13072 "the cleared budget must be spendable again"
13073 );
13074 let state = lock_snapshot(&snapshot).unwrap();
13075 assert_eq!(state.crash_restarts.len(), 1);
13076 assert_eq!(state.lifetime_restarts, 3);
13077 }
13078
13079 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13080 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13081 let severed = ProcessIdentity {
13082 pid: 41,
13083 start_time: 101,
13084 };
13085 let successor = ProcessIdentity {
13086 pid: 41,
13087 start_time: 202,
13088 };
13089 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13090 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13091 state.pid = Some(successor.pid);
13092 state.process_start_time = Some(successor.start_time);
13093 })
13094 .unwrap();
13095 assert!(!module.record_deliberate_severance(severed).unwrap());
13096
13097 let exit_report = apply_deliberate_severance_marker(
13098 &module.inner.snapshot,
13099 Some(successor),
13100 ExitReport {
13101 kind: ExitKind::Crash,
13102 code: Some(1),
13103 signal: None,
13104 at_ms: 1,
13105 },
13106 );
13107
13108 assert_eq!(exit_report.kind, ExitKind::Crash);
13109 }
13110
13111 #[tokio::test]
13112 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13113 let registry = Registry::default();
13114 let supervisor = Supervisor::new_for_test(
13115 Arc::new(Registry::default()),
13116 RestartPolicy::new(3, Duration::ZERO),
13117 );
13118 let runtime = supervisor.runtime_config();
13119 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13120 let spec = ModuleSpec {
13121 module_id: "drain-deliberate-severance".to_string(),
13122 program: fake_aft_stub_path(),
13123 args: Vec::new(),
13124 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13125 reserved: false,
13126 reserved_prefixes: Vec::new(),
13127 protocol: ModuleProtocol::Subc,
13128 overlap: Default::default(),
13129 };
13130 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13131 let process = ProcessIdentity {
13132 pid: 41,
13133 start_time: 101,
13134 };
13135 child.process_identity = Some(process);
13136 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13137 state.pid = Some(process.pid);
13138 state.process_start_time = Some(process.start_time);
13139 })
13140 .unwrap();
13141 record_deliberate_severance(&snapshot, process).unwrap();
13142
13143 drain_child_to_state(
13144 &spec.module_id,
13145 spec.protocol,
13146 StopNotice::SentOverConnection,
13149 ®istry,
13150 None,
13151 &snapshot,
13152 &runtime.terminal_ring,
13153 &runtime.spawn_events,
13154 child,
13155 Duration::from_secs(1),
13156 ModuleState::Stopped,
13157 Some(false),
13158 )
13159 .await
13160 .unwrap();
13161
13162 let state = lock_snapshot(&snapshot).unwrap();
13163 assert_eq!(
13164 state.last_exit.as_ref().map(|exit| exit.kind),
13165 Some(ExitKind::DeliberateSeverance)
13166 );
13167 assert_eq!(state.lifetime_restarts, 1);
13168 assert_eq!(state.crash_restarts.len(), 0);
13169 drop(state);
13170 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13171 assert_eq!(
13172 history.entries[0].exit_kind,
13173 subc_control::TerminalExitKind::DeliberateSeverance
13174 );
13175 }
13176
13177 #[tokio::test]
13178 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13179 let registry = Registry::default();
13180 let supervisor = Supervisor::new_for_test(
13181 Arc::new(Registry::default()),
13182 RestartPolicy::new(3, Duration::ZERO),
13183 );
13184 let runtime = supervisor.runtime_config();
13185 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13186 let spec = ModuleSpec {
13187 module_id: "ordinary-drain".to_string(),
13188 program: fake_aft_stub_path(),
13189 args: Vec::new(),
13190 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13191 reserved: false,
13192 reserved_prefixes: Vec::new(),
13193 protocol: ModuleProtocol::Subc,
13194 overlap: Default::default(),
13195 };
13196 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13197
13198 drain_child_to_state(
13199 &spec.module_id,
13200 spec.protocol,
13201 StopNotice::SentOverConnection,
13204 ®istry,
13205 None,
13206 &snapshot,
13207 &runtime.terminal_ring,
13208 &runtime.spawn_events,
13209 child,
13210 Duration::from_secs(1),
13211 ModuleState::Stopped,
13212 Some(false),
13213 )
13214 .await
13215 .unwrap();
13216
13217 let state = lock_snapshot(&snapshot).unwrap();
13218 assert_eq!(
13219 state.last_exit.as_ref().map(|exit| exit.kind),
13220 Some(ExitKind::Crash)
13221 );
13222 assert_eq!(state.lifetime_restarts, 0);
13223 assert_eq!(state.crash_restarts.len(), 0);
13224 }
13225
13226 #[test]
13227 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13228 assert!(!include_str!("server.rs")
13234 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13235 }
13236
13237 #[test]
13244 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13245 assert!(drained_after_quiescence_wait(&Ok(true)));
13246 assert!(!drained_after_quiescence_wait(&Ok(false)));
13247 assert!(!drained_after_quiescence_wait(&Err(
13248 SuperviseError::StatePoisoned { module_id: None }
13249 )));
13250 }
13251
13252 #[test]
13261 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13262 let ring = Arc::new(Mutex::new(TerminalRing::new(
13263 TerminalRingConfig::default(),
13264 0,
13265 )));
13266 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13267
13268 let snapshot = ring.lock().unwrap().snapshot();
13269 assert_eq!(snapshot.entries.len(), 1);
13270 let entry = &snapshot.entries[0];
13271 assert_eq!(entry.exit_code, None);
13272 assert_eq!(entry.exit_signal, None);
13273 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13274 }
13275
13276 #[test]
13277 fn wait_error_exit_path_preserves_spawn_event_density() {
13278 let feed = super::SpawnEventFeed::default();
13279 feed.configure_incarnation("wait-error-density".to_string());
13280 feed.emit_spawned("wait-error", 41, 1);
13281 let ring = Arc::new(Mutex::new(TerminalRing::new(
13282 TerminalRingConfig::default(),
13283 0,
13284 )));
13285
13286 record_wait_error_terminal("wait-error", &ring, &feed);
13287 feed.emit_spawned("after-wait-error", 42, 2);
13288
13289 let state = feed.0.lock().unwrap();
13290 let sequences = state
13291 .events
13292 .iter()
13293 .map(|event| event.cursor.seq)
13294 .collect::<Vec<_>>();
13295 assert_eq!(sequences, vec![1, 2, 3]);
13296 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13297 assert_eq!(state.events[1].exit_code, None);
13298 assert_eq!(state.events[1].exit_signal, None);
13299 }
13300
13301 #[test]
13305 fn wait_error_exit_report_is_classified_as_a_crash() {
13306 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13307 }
13308}
13309
13310#[cfg(all(test, unix))]
13311mod restart_ordering_tests {
13312 use super::*;
13313 use std::io::Write;
13314 use tokio::io::AsyncReadExt;
13315
13316 #[test]
13317 #[ignore = "child process fixture"]
13318 fn serving_child() {
13319 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13320 return;
13321 };
13322 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13323 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13324 for stream in listener.incoming() {
13325 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13326 }
13327 }
13328
13329 async fn restart_before_publication(requests: usize) {
13330 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13331 let address_file = home.path().join("address");
13332 let spec = ModuleSpec {
13333 module_id: "restart-serving-child".into(),
13334 program: std::env::current_exe().unwrap(),
13335 args: vec![
13336 "--exact".into(),
13337 "supervise::restart_ordering_tests::serving_child".into(),
13338 "--ignored".into(),
13339 "--nocapture".into(),
13340 ],
13341 env: vec![(
13342 "RESTART_TEST_ADDRESS_FILE".into(),
13343 address_file.display().to_string(),
13344 )],
13345 reserved: false,
13346 reserved_prefixes: vec![],
13347 protocol: ModuleProtocol::None,
13348 overlap: Default::default(),
13349 };
13350 let mut supervisor =
13351 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13352 let (spawned_tx, spawned_rx) = oneshot::channel();
13353 let spawned_tx = Mutex::new(Some(spawned_tx));
13354 let (release_tx, release_rx) = std::sync::mpsc::channel();
13355 let release_rx = Mutex::new(release_rx);
13356 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13359 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13360 release_rx
13361 .lock()
13362 .unwrap()
13363 .recv_timeout(Duration::from_secs(20))
13364 .unwrap();
13365 })));
13366 let registered = supervisor
13367 .register_configured_with_health(
13368 spec.clone(),
13369 true,
13370 HealthConfig::default(),
13371 None,
13372 RestartPolicy::new(3, Duration::ZERO),
13373 )
13374 .unwrap();
13375 let module = registered.module.clone();
13376 let startup = tokio::task::spawn_blocking(move || {
13377 supervisor.start_registered_configured(registered, None)
13378 });
13379 spawned_rx.await.unwrap();
13380 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13381 let address = loop {
13382 if let Ok(address) = std::fs::read_to_string(&address_file) {
13383 if !address.is_empty() {
13384 break address;
13385 }
13386 }
13387 tokio::time::sleep(Duration::from_millis(1)).await;
13388 };
13389 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13390 let mut reply = String::new();
13391 stream.read_to_string(&mut reply).await.unwrap();
13392 reply.trim().parse::<u32>().unwrap()
13393 })
13394 .await
13395 .expect("child must serve before generation publication");
13396 assert_eq!(
13397 lock_snapshot(&module.inner.snapshot)
13398 .unwrap()
13399 .spawn_generation,
13400 0
13401 );
13402 let mut replies = Vec::new();
13403 for _ in 0..requests {
13404 let mut restart = Box::pin(module.restart(Some(0)));
13405 std::future::poll_fn(|cx| {
13408 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13409 std::task::Poll::Ready(())
13410 })
13411 .await;
13412 replies.push(restart);
13413 }
13414 release_tx.send(()).unwrap();
13415 startup.await.unwrap().unwrap();
13416 for reply in replies {
13417 reply.await.unwrap();
13418 }
13419 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13422 loop {
13423 if lock_snapshot(&module.inner.snapshot)
13424 .unwrap()
13425 .spawn_generation
13426 > 1
13427 {
13428 break;
13429 }
13430 tokio::time::sleep(Duration::from_millis(1)).await;
13431 }
13432 })
13433 .await;
13434 module.update_spec_for_test(spec).await.unwrap();
13438 let (generation, pid) = {
13439 let state = lock_snapshot(&module.inner.snapshot).unwrap();
13440 (state.spawn_generation, state.pid)
13441 };
13442 module.stop().await.unwrap();
13443 replacement.expect("restart must publish a replacement for the already-serving child");
13444 assert_ne!(
13445 pid,
13446 Some(old_pid),
13447 "a serving child predating the restart must be replaced"
13448 );
13449 assert_eq!(generation, 2, "queued restarts must share one replacement");
13450 }
13451
13452 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13453 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13454 restart_before_publication(1).await;
13455 }
13456
13457 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13458 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13459 restart_before_publication(2).await;
13460 }
13461
13462 #[tokio::test]
13463 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13464 let supervisor =
13465 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13466 let registered = supervisor
13467 .register_configured_with_health(
13468 ModuleSpec {
13469 module_id: "queued-before-replacement".into(),
13470 program: "/bin/sh".into(),
13471 args: vec!["-c".into(), "exec sleep 600".into()],
13472 env: vec![],
13473 reserved: false,
13474 reserved_prefixes: vec![],
13475 protocol: ModuleProtocol::None,
13476 overlap: Default::default(),
13477 },
13478 true,
13479 HealthConfig::default(),
13480 None,
13481 RestartPolicy::default(),
13482 )
13483 .unwrap();
13484 let RegisteredConfiguredModule {
13485 mut spec,
13486 mut runtime,
13487 module,
13488 mut commands,
13489 } = registered;
13490 let snapshot = &module.inner.snapshot;
13491 let mut restart = Box::pin(module.restart(Some(0)));
13492 std::future::poll_fn(|cx| {
13493 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13494 std::task::Poll::Ready(())
13495 })
13496 .await;
13497 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13500 let replacement_pid = child.as_ref().unwrap().id();
13501 let command = commands.recv().await.unwrap();
13502 assert!(
13503 handle_supervisor_command(
13504 command,
13505 &mut spec,
13506 &mut runtime,
13507 &supervisor.registry,
13508 &supervisor.process_liveness,
13509 snapshot,
13510 &mut child,
13511 &mut commands,
13512 &mut VecDeque::new(),
13513 )
13514 .await
13515 );
13516 restart.await.unwrap();
13517 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13518 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13519 if let Some(mut child) = child {
13520 child.start_kill().unwrap();
13521 child.wait().await.unwrap();
13522 }
13523 assert_eq!(
13524 kept_pid, replacement_pid,
13525 "a newer replacement must survive"
13526 );
13527 assert!(
13528 !pending,
13529 "a satisfied restart must not schedule another spawn"
13530 );
13531 }
13532}
13533
13534#[cfg(all(test, unix))]
13535mod health_event_tests {
13536 use super::*;
13537 use cortexkit_test_support::ScratchDir;
13538 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13539
13540 struct Actor {
13541 module: SupervisedModule,
13542 registry: Arc<Registry>,
13543 forwarding: Arc<ForwardingTable>,
13544 spec: ModuleSpec,
13545 health: HealthConfig,
13546 rx: mpsc::Receiver<crate::router::OutboundFrame>,
13547 _home: ScratchDir,
13548 }
13549
13550 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13551 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13555 let tick = Instant::now();
13556 loop {
13557 if let Some(value) = observe() {
13558 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13559 return value;
13560 }
13561 assert!(
13562 std::time::Instant::now() < deadline,
13563 "timed out waiting for {reason}"
13564 );
13565 tokio::task::yield_now().await;
13566 }
13567 }
13568
13569 impl Actor {
13570 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13571 let home = ScratchDir::new("health-events");
13572 let registry = Arc::new(Registry::default());
13573 let forwarding = Arc::new(ForwardingTable::default());
13574 let health = HealthConfig {
13575 cadence,
13576 deadline: Duration::from_secs(3600),
13577 ..HealthConfig::default()
13578 };
13579 let supervisor =
13580 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13581 .with_forwarding(forwarding.clone())
13582 .with_health_config(health.clone());
13583 let spec = ModuleSpec {
13584 module_id: "event-health-child".into(),
13585 program: PathBuf::from("/bin/sh"),
13586 args: vec!["-c".into(), "exec sleep 600".into()],
13589 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13590 .into_iter()
13591 .map(|key| (key.into(), home.path().display().to_string()))
13592 .collect(),
13593 reserved: false,
13594 reserved_prefixes: vec![],
13595 protocol,
13596 overlap: Default::default(),
13597 };
13598 let module = supervisor.spawn(spec.clone()).unwrap();
13599 let (_, rx) = mpsc::channel(8);
13600 let actor = Self {
13601 module,
13602 registry,
13603 forwarding,
13604 spec,
13605 health,
13606 rx,
13607 _home: home,
13608 };
13609 actor
13610 .wait_for_select(|checkpoint| checkpoint.generation > 0)
13611 .await;
13612 actor
13613 }
13614
13615 fn turns(&self) -> u64 {
13616 lock_snapshot(&self.module.inner.snapshot)
13617 .unwrap()
13618 .actor_turns
13619 }
13620
13621 async fn wait_for_select(
13622 &self,
13623 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13624 ) -> ActorSelectCheckpoint {
13625 wait_for("actor select checkpoint", || {
13626 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13627 state.actor_select.filter(|checkpoint| {
13628 checkpoint.generation == state.spawn_generation
13629 && state.state == ModuleState::Running
13630 && state.process_alive
13631 && state.reported_pid().is_some()
13632 && expected(checkpoint)
13633 })
13634 })
13635 .await
13636 }
13637
13638 fn hello(&mut self, connection: u64, health: bool) {
13639 let manifest =
13640 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13641 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13642 .build();
13643 let connection = ConnectionId::new(connection);
13644 let (tx, rx) = mpsc::channel(8);
13645 self.rx = rx;
13646 self.forwarding
13647 .register_module_connection(
13648 connection,
13649 self.spec.module_id.clone(),
13650 subc_protocol::PROTOCOL_VERSION,
13651 Concurrency::ModuleManaged,
13652 FrameSink::new(tx),
13653 )
13654 .unwrap();
13655 self.registry
13656 .register_with_control_ops(
13657 manifest,
13658 subc_protocol::PROTOCOL_VERSION,
13659 connection,
13660 if health {
13661 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13662 } else {
13663 vec![]
13664 },
13665 )
13666 .unwrap();
13667 }
13668
13669 async fn probe(&mut self) -> crate::router::OutboundFrame {
13670 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13671 Ok(frame) => Some(frame),
13672 Err(mpsc::error::TryRecvError::Empty) => None,
13673 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13674 })
13675 .await;
13676 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13677 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13678 frame
13679 }
13680
13681 fn no_probe(&mut self) {
13682 assert!(matches!(
13683 self.rx.try_recv(),
13684 Err(mpsc::error::TryRecvError::Empty)
13685 ));
13686 }
13687
13688 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13689 self.forwarding
13690 .complete_module_control_rpc(
13691 ConnectionId::new(connection),
13692 frame.header.corr,
13693 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13694 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13695 status: HealthStatus::Ok,
13696 detail: None,
13697 metrics: None,
13698 }),
13699 )
13700 .unwrap();
13701 }
13702 }
13703
13704 #[tokio::test(start_paused = true)]
13705 async fn no_health_children_park_for_sixty_seconds() {
13706 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13707 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13708 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13709 unadvertised.hello(1, false);
13710 for _ in 0..6000 {
13715 tokio::time::advance(Duration::from_millis(10)).await;
13716 tokio::task::yield_now().await;
13717 }
13718 for actor in [&none, &unregistered, &unadvertised] {
13719 assert!(
13720 actor.turns() <= 5,
13721 "no-health actor woke {} times",
13722 actor.turns()
13723 );
13724 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13725 }
13726 }
13727
13728 #[tokio::test(start_paused = true)]
13729 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13730 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13734 let tick = Instant::now();
13735 actor.hello(1, true);
13736 actor.probe().await;
13737 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13738 }
13739
13740 #[tokio::test(start_paused = true)]
13741 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13742 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13743 let hello_tick = Instant::now();
13744 actor.hello(1, true);
13745 let armed = actor
13746 .wait_for_select(|checkpoint| {
13747 checkpoint.registered_connection == Some(ConnectionId::new(1))
13748 && checkpoint.next_probe_at.is_some()
13749 })
13750 .await;
13751 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13752 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13753 let first_deadline = hello_tick + delay;
13754 assert_eq!(armed.next_probe_at, Some(first_deadline));
13755 tokio::time::advance(Duration::from_secs(29)).await;
13756 let turns = actor.turns();
13757 actor
13758 .registry
13759 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13760 .unwrap();
13761 let updated = actor
13762 .wait_for_select(|checkpoint| checkpoint.turn > turns)
13763 .await;
13764 assert_eq!(
13765 updated.next_probe_at,
13766 Some(first_deadline),
13767 "catalog update must not reset cadence"
13768 );
13769 actor.no_probe();
13770 tokio::time::advance(first_deadline - Instant::now()).await;
13771 let frame = actor.probe().await;
13772 actor.answer(1, frame);
13773 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13774 let next_deadline = Instant::now() + next;
13775 let rearmed = actor
13776 .wait_for_select(|checkpoint| {
13777 checkpoint
13778 .next_probe_at
13779 .is_some_and(|deadline| deadline > first_deadline)
13780 })
13781 .await;
13782 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13783 tokio::time::advance(next - Duration::from_nanos(1)).await;
13784 actor.no_probe();
13785 tokio::time::advance(Duration::from_nanos(1)).await;
13786 actor.probe().await;
13787 }
13788
13789 #[tokio::test(start_paused = true)]
13790 async fn disconnect_disarms_and_reconnect_rearms_health() {
13791 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13792 actor.hello(1, true);
13793 actor
13794 .wait_for_select(|checkpoint| {
13795 checkpoint.registered_connection == Some(ConnectionId::new(1))
13796 && checkpoint.next_probe_at.is_some()
13797 })
13798 .await;
13799 let turns = actor.turns();
13800 let tick = Instant::now();
13801 actor
13802 .registry
13803 .deregister_connection(ConnectionId::new(1))
13804 .unwrap();
13805 let disarmed = actor
13806 .wait_for_select(|checkpoint| {
13807 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13808 })
13809 .await;
13810 assert_eq!(disarmed.next_probe_at, None);
13811 assert_eq!(disarmed.wake_after, None);
13812 assert_eq!(
13813 Instant::now(),
13814 tick,
13815 "disconnect must disarm in the notification tick"
13816 );
13817 tokio::time::advance(Duration::from_secs(60)).await;
13818 actor.no_probe();
13819 let reconnect_tick = Instant::now();
13820 actor.hello(2, true);
13821 let rearmed = actor
13822 .wait_for_select(|checkpoint| {
13823 checkpoint.registered_connection == Some(ConnectionId::new(2))
13824 && checkpoint.next_probe_at.is_some()
13825 })
13826 .await;
13827 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13828 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13829 tokio::time::advance(delay).await;
13830 actor.probe().await;
13831 }
13832
13833 #[tokio::test(start_paused = true)]
13834 async fn reregistration_without_health_disarms_the_old_schedule() {
13835 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13836 actor.hello(1, true);
13837 actor
13838 .wait_for_select(|checkpoint| {
13839 checkpoint.registered_connection == Some(ConnectionId::new(1))
13840 && checkpoint.next_probe_at.is_some()
13841 })
13842 .await;
13843 actor
13844 .registry
13845 .deregister_connection(ConnectionId::new(1))
13846 .unwrap();
13847 actor.hello(2, false);
13848 let disarmed = actor
13849 .wait_for_select(|checkpoint| {
13850 checkpoint.registered_connection == Some(ConnectionId::new(2))
13851 })
13852 .await;
13853 assert_eq!(disarmed.next_probe_at, None);
13854 assert_eq!(disarmed.wake_after, None);
13855 tokio::time::advance(Duration::from_secs(60)).await;
13856 actor.no_probe();
13857 }
13858
13859 #[tokio::test(start_paused = true)]
13860 async fn restart_waits_for_its_own_hello_before_rearming_health() {
13861 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13862 actor.hello(1, true);
13863 let armed = actor
13864 .wait_for_select(|checkpoint| {
13865 checkpoint.registered_connection == Some(ConnectionId::new(1))
13866 && checkpoint.next_probe_at.is_some()
13867 })
13868 .await;
13869 actor
13873 .registry
13874 .deregister_connection(ConnectionId::new(1))
13875 .unwrap();
13876 actor.module.restart(Some(0)).await.unwrap();
13877 let parked = actor
13878 .wait_for_select(|checkpoint| {
13879 checkpoint.generation > armed.generation
13880 && checkpoint.registered_connection.is_none()
13881 && checkpoint.next_probe_at.is_none()
13882 })
13883 .await;
13884 assert_eq!(parked.wake_after, None);
13885 actor.rx.close();
13886 while actor.rx.try_recv().is_ok() {}
13889 let turns = actor.turns();
13890 tokio::time::advance(Duration::from_secs(60)).await;
13891 let after = actor.turns();
13892 eprintln!(
13893 "restart park turns: before={turns} after={after} delta={}",
13894 after - turns
13895 );
13896 assert_eq!(after, turns, "replacement must park before HELLO");
13897 let hello_tick = Instant::now();
13898 actor.hello(2, true);
13899 let rearmed = actor
13900 .wait_for_select(|checkpoint| {
13901 checkpoint.registered_connection == Some(ConnectionId::new(2))
13902 && checkpoint.next_probe_at.is_some()
13903 })
13904 .await;
13905 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13906 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13907 tokio::time::advance(delay).await;
13908 actor.probe().await;
13909 }
13910
13911 #[tokio::test(start_paused = true)]
13912 async fn rescan_adds_and_removes_http_health_without_polling() {
13913 use tokio::{
13914 io::{AsyncReadExt, AsyncWriteExt},
13915 net::TcpListener,
13916 };
13917 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13918 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13919 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13920 actor.health.cadence = Duration::from_millis(1);
13922 let (started_tx, mut started_rx) = mpsc::channel(8);
13923 let peer = tokio::spawn(async move {
13924 loop {
13925 let (mut socket, _) = listener.accept().await.unwrap();
13926 let mut request = [0; 1024];
13927 assert!(socket.read(&mut request).await.unwrap() > 0);
13928 started_tx.send(()).await.unwrap();
13929 socket
13930 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13931 .await
13932 .unwrap();
13933 }
13934 });
13935 let tick = Instant::now();
13936 let turns = actor.turns();
13937 actor
13938 .module
13939 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13940 .await
13941 .unwrap();
13942 let armed = actor
13943 .wait_for_select(|checkpoint| {
13944 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13945 })
13946 .await;
13947 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13948 assert_eq!(Instant::now(), tick);
13949 tokio::time::advance(Duration::from_millis(1)).await;
13950 let mut started = false;
13951 wait_for("successful HTTP probe", || {
13952 started |= started_rx.try_recv().is_ok();
13953 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13954 .then_some(())
13955 })
13956 .await;
13957 assert!(started, "added HTTP check must start at its first deadline");
13958 assert_eq!(
13959 actor.module.status().unwrap().health.status,
13960 SupervisorHealthStatus::Ok
13961 );
13962 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13963 actor.health.http = None;
13964 let turns = actor.turns();
13965 actor
13966 .module
13967 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13968 .await
13969 .unwrap();
13970 let parked = actor
13971 .wait_for_select(|checkpoint| {
13972 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13973 })
13974 .await;
13975 assert_eq!(parked.wake_after, None);
13976 let turns = actor.turns();
13977 tokio::time::advance(Duration::from_secs(60)).await;
13978 assert_eq!(
13979 actor.turns(),
13980 turns,
13981 "removed HTTP check must park the actor"
13982 );
13983 assert!(
13984 started_rx.try_recv().is_err(),
13985 "removed HTTP check must stay disarmed"
13986 );
13987 assert_eq!(
13988 actor.module.status().unwrap().health,
13989 ModuleHealthStatus::default()
13990 );
13991 peer.abort();
13992 }
13993}
13994
13995#[cfg(test)]
13996mod drain_busy_reprobe_tests {
13997 use super::*;
14001 use std::collections::BTreeMap;
14002 use std::sync::Mutex as StdMutex;
14003 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14004
14005 const MODULE_ID: &str = "drain-reprobe-module";
14006 const CONNECTION: u64 = 7;
14007 const GAUGE: &str = "runs_in_flight";
14008 const CADENCE: Duration = Duration::from_secs(30);
14011 const CEILING: Duration = Duration::from_secs(30);
14012
14013 #[derive(Clone, Copy)]
14015 enum Answer {
14016 BusyUntil { until: Instant, latency: Duration },
14019 OmitsGauge,
14021 Silent,
14023 }
14024
14025 struct Drain {
14026 forwarding: Arc<ForwardingTable>,
14027 runtime: SupervisorRuntimeConfig,
14028 endpoint: crate::ModuleEndpointId,
14029 probes: Arc<StdMutex<Vec<Instant>>>,
14031 responder: tokio::task::JoinHandle<()>,
14032 }
14033
14034 impl Drain {
14035 fn start(answer: Answer) -> Self {
14036 let registry = Arc::new(Registry::default());
14037 let forwarding = Arc::new(ForwardingTable::default());
14038 let health = HealthConfig {
14039 cadence: CADENCE,
14040 ..HealthConfig::default()
14041 };
14042 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14043 .with_forwarding(forwarding.clone())
14044 .with_health_config(health)
14045 .runtime_config();
14046 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14047 let endpoint = forwarding
14048 .register_module_connection(
14049 ConnectionId::new(CONNECTION),
14050 MODULE_ID.to_string(),
14051 subc_protocol::PROTOCOL_VERSION,
14052 Concurrency::ModuleManaged,
14053 FrameSink::new(tx),
14054 )
14055 .unwrap();
14056 let probes = Arc::new(StdMutex::new(Vec::new()));
14057 let responder = tokio::spawn({
14058 let forwarding = forwarding.clone();
14059 let probes = probes.clone();
14060 async move {
14061 while let Some(frame) = rx.recv().await {
14062 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14063 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14064 probes.lock().unwrap().push(Instant::now());
14065 let metrics = match answer {
14066 Answer::BusyUntil { until, latency } => {
14067 sleep(latency).await;
14068 let busy = u64::from(Instant::now() < until);
14069 serde_json::json!({ GAUGE: busy })
14070 }
14071 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14072 Answer::Silent => continue,
14073 };
14074 let _ = forwarding.complete_module_control_rpc(
14077 ConnectionId::new(CONNECTION),
14078 frame.header.corr,
14079 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14080 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14081 status: HealthStatus::Ok,
14082 detail: None,
14083 metrics: Some(metrics),
14084 }),
14085 );
14086 }
14087 }
14088 });
14089 Self {
14090 forwarding,
14091 runtime,
14092 endpoint,
14093 probes,
14094 responder,
14095 }
14096 }
14097
14098 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14101 let (drained, elapsed, _) = self.run_traced(ceiling).await;
14102 (drained, elapsed)
14103 }
14104
14105 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14107 let started = Instant::now();
14108 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14109 let drained = wait_for_forwarding_quiescence(
14110 &self.forwarding,
14111 MODULE_ID,
14112 &self.runtime,
14113 self.endpoint,
14114 started + ceiling,
14115 &[GAUGE.to_string()],
14116 DrainScope::Active,
14117 &mut trace,
14118 )
14119 .await
14120 .unwrap();
14121 (drained, started.elapsed(), trace)
14122 }
14123
14124 fn probes(&self) -> Vec<Instant> {
14125 self.probes.lock().unwrap().clone()
14126 }
14127 }
14128
14129 impl Drop for Drain {
14130 fn drop(&mut self) {
14131 self.responder.abort();
14132 }
14133 }
14134
14135 #[tokio::test(start_paused = true)]
14136 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14137 let drain = Drain::start(Answer::BusyUntil {
14138 until: Instant::now() + Duration::from_secs(1),
14139 latency: Duration::ZERO,
14140 });
14141 let (drained, elapsed) = drain.run(CEILING).await;
14142 assert!(
14143 drained,
14144 "gauges that read zero during the drain must let it finish drained, \
14145 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14146 );
14147 assert!(
14148 elapsed < Duration::from_secs(3),
14149 "drain took {elapsed:?} after the gauges went quiet at 1s"
14150 );
14151 let probes = drain.probes();
14152 assert!(
14153 probes.len() >= 2,
14154 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14155 );
14156 }
14157
14158 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14163 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14164 let drain = Drain::start(Answer::BusyUntil {
14165 until: Instant::now() + Duration::from_millis(600),
14166 latency: Duration::ZERO,
14167 });
14168 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14169 assert!(drained, "the gauge fell at 600ms: {trace}");
14170 assert!(
14171 elapsed < Duration::from_secs(3),
14172 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14173 );
14174 let probes = drain.probes();
14175 assert!(
14176 probes.len() >= 2 && trace.probes as usize == probes.len(),
14177 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14178 );
14179 }
14180
14181 #[tokio::test(start_paused = true)]
14182 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14183 let started = Instant::now();
14184 let drain = Drain::start(Answer::BusyUntil {
14185 until: started + Duration::from_secs(3600),
14186 latency: Duration::ZERO,
14187 });
14188 let (drained, elapsed) = drain.run(CEILING).await;
14189 assert!(!drained, "a module busy throughout must not drain");
14190 assert!(
14191 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14192 "a busy module is held until the ceiling and no longer: {elapsed:?}"
14193 );
14194 let probes = drain.probes();
14197 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14198 assert!(
14199 probes.len() as u128 <= max_probes,
14200 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14201 probes.len()
14202 );
14203 for pair in probes.windows(2) {
14204 assert!(
14205 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14206 "consecutive probes {:?} apart",
14207 pair[1] - pair[0]
14208 );
14209 }
14210 }
14211
14212 #[tokio::test(start_paused = true)]
14213 async fn a_slow_probe_is_never_overlapped_by_the_next() {
14214 let latency = Duration::from_secs(1);
14217 let drain = Drain::start(Answer::BusyUntil {
14218 until: Instant::now() + Duration::from_secs(3600),
14219 latency,
14220 });
14221 let (drained, _) = drain.run(Duration::from_secs(5)).await;
14222 assert!(!drained);
14223 let probes = drain.probes();
14224 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14225 for pair in probes.windows(2) {
14226 assert!(
14227 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14228 "a probe started {:?} after the previous one, before it could finish",
14229 pair[1] - pair[0]
14230 );
14231 }
14232 }
14233
14234 #[tokio::test(start_paused = true)]
14235 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14236 let drain = Drain::start(Answer::Silent);
14237 let ceiling = Duration::from_secs(2);
14238 let (drained, elapsed) = drain.run(ceiling).await;
14239 assert!(!drained, "an unanswered probe counts as busy");
14240 assert!(
14241 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14242 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14243 );
14244 assert_eq!(
14245 drain.probes().len(),
14246 1,
14247 "the unanswered probe holds the only slot until the ceiling"
14248 );
14249 }
14250
14251 #[derive(Clone, Default)]
14253 struct LogCapture {
14254 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14255 }
14256
14257 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14258 fn on_event(
14259 &self,
14260 event: &tracing::Event<'_>,
14261 _context: tracing_subscriber::layer::Context<'_, S>,
14262 ) {
14263 struct Fields(BTreeMap<String, String>);
14264 impl tracing::field::Visit for Fields {
14265 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14266 self.0
14267 .insert(field.name().to_string(), format!("{value:?}"));
14268 }
14269 }
14270 let mut fields = Fields(BTreeMap::new());
14271 event.record(&mut fields);
14272 self.events.lock().unwrap().push(fields.0);
14273 }
14274 }
14275
14276 impl LogCapture {
14277 fn install(&self) -> tracing::subscriber::DefaultGuard {
14278 use tracing_subscriber::layer::SubscriberExt;
14279 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14280 }
14281
14282 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14283 self.events
14284 .lock()
14285 .unwrap()
14286 .iter()
14287 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14288 .cloned()
14289 .collect()
14290 }
14291 }
14292
14293 #[tokio::test(start_paused = true)]
14298 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14299 let capture = LogCapture::default();
14300 let _guard = capture.install();
14301 let drain = Drain::start(Answer::OmitsGauge);
14302 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14303 assert!(!drained, "a gauge never reported cannot read quiet");
14304 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14305
14306 let missing = capture.with_message("lacks a declared busy gauge");
14307 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14308 assert_eq!(
14309 missing[0].get("missing_gauges").map(String::as_str),
14310 Some(GAUGE)
14311 );
14312
14313 warn_drain_timed_out(
14314 MODULE_ID,
14315 Duration::from_secs(2),
14316 RouteCloseReason::Restart,
14317 &Default::default(),
14318 &trace,
14319 );
14320 let timeouts = capture.with_message("route drain timed out");
14321 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14322 let gauges = &timeouts[0]["busy_gauges"];
14323 assert!(
14324 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14325 "the timeout line must name the gauge and why it read busy: {gauges}"
14326 );
14327 assert!(
14328 gauges.ends_with("; no value ever read"),
14329 "no value was ever read for the gauge: {gauges}"
14330 );
14331 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14332 }
14333
14334 #[tokio::test(start_paused = true)]
14338 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14339 let busy = Drain::start(Answer::BusyUntil {
14340 until: Instant::now() + Duration::from_secs(3600),
14341 latency: Duration::ZERO,
14342 });
14343 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14344 let rendered = trace.to_string();
14345 assert!(
14346 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14347 && rendered.contains("; last value non_zero(1) @"),
14348 "{rendered}"
14349 );
14350
14351 let silent = Drain::start(Answer::Silent);
14352 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14353 assert_eq!(
14356 trace.to_string(),
14357 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14358 );
14359 assert_eq!(trace.probes, 1);
14360 }
14361
14362 #[test]
14364 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14365 let gauges = [GAUGE.to_string()];
14366 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14367 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14368 assert_eq!(
14369 busy_gauge_observation(&verdicts),
14370 BusyGaugeObservation::Busy
14371 );
14372 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14373 assert_eq!(
14374 busy_gauge_observation(&zero),
14375 BusyGaugeObservation::Quiescent
14376 );
14377 let none = gauge_verdicts(None, &gauges);
14378 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14379 }
14380}
14381
14382#[cfg(test)]
14383mod health_evidence_tests {
14384 use super::{HealthProbeError, HealthProbeEvidence};
14385 use std::collections::HashSet;
14386
14387 #[test]
14395 fn only_a_dead_lane_is_proof_of_death() {
14396 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14397 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14401 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14402 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14403 }
14404
14405 #[test]
14411 fn every_evidence_class_has_a_distinct_label() {
14412 let labels = [
14413 HealthProbeError::lane_dead("").label(),
14414 HealthProbeError::no_answer("").label(),
14415 HealthProbeError::bad_answer("").label(),
14416 HealthProbeError::misconfigured("").label(),
14417 ];
14418 let unique: HashSet<_> = labels.iter().collect();
14419 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14420 }
14421
14422 #[test]
14428 fn classification_preserves_the_original_message() {
14429 let err = HealthProbeError::no_answer("module did not answer within 5s");
14430 assert_eq!(err.to_string(), "module did not answer within 5s");
14431 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14432 }
14433}
14434
14435#[cfg(test)]
14436mod health_tombstone_tests {
14437 use std::{path::PathBuf, sync::Arc, time::Duration};
14438
14439 use subc_protocol::{
14440 manifest::Concurrency,
14441 session::{HealthStatus, ModuleControlResponse},
14442 };
14443 use tokio::sync::mpsc;
14444
14445 use super::{
14446 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14447 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14448 };
14449 use crate::{
14450 control::ControlHandler,
14451 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14452 registry::{ConnectionId, Registry},
14453 router::FrameSink,
14454 };
14455
14456 struct ProbeHarness {
14457 spec: ModuleSpec,
14458 runtime: SupervisorRuntimeConfig,
14459 forwarding: Arc<ForwardingTable>,
14460 module_connection: ConnectionId,
14461 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14462 handler: ControlHandler,
14463 module: super::SupervisedModule,
14464 }
14465
14466 fn probe_harness() -> ProbeHarness {
14467 let registry = Arc::new(Registry::default());
14468 let forwarding = Arc::new(ForwardingTable::default());
14469 let supervisor_handle = super::SupervisorHandle::new();
14470 let health = HealthConfig {
14471 http: None,
14472 cadence: Duration::from_secs(30),
14473 deadline: Duration::from_secs(5),
14474 failure_threshold: 3,
14475 on_degraded: HealthAction::Report,
14476 on_failing: HealthAction::Report,
14477 critical: false,
14478 };
14479 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14480 .with_forwarding(Arc::clone(&forwarding))
14481 .with_handle(supervisor_handle.clone())
14482 .with_health_config(health);
14483 let spec = ModuleSpec {
14484 module_id: "late-health-module".to_string(),
14485 program: PathBuf::from("disabled-module"),
14486 args: Vec::new(),
14487 env: Vec::new(),
14488 reserved: false,
14489 reserved_prefixes: Vec::new(),
14490 protocol: ModuleProtocol::Subc,
14491 overlap: Default::default(),
14492 };
14493 let module = supervisor
14494 .supervise_configured(spec.clone(), false)
14495 .unwrap();
14496 let runtime = supervisor.runtime_config();
14497 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14498 .with_supervisor(supervisor_handle);
14499 let module_connection = ConnectionId::new(700);
14500 let (module_tx, module_rx) = mpsc::channel(8);
14501 forwarding
14502 .register_module_connection(
14503 module_connection,
14504 spec.module_id.clone(),
14505 subc_protocol::PROTOCOL_VERSION,
14506 Concurrency::ModuleManaged,
14507 FrameSink::new(module_tx),
14508 )
14509 .unwrap();
14510
14511 ProbeHarness {
14512 spec,
14513 runtime,
14514 forwarding,
14515 module_connection,
14516 module_rx,
14517 handler,
14518 module,
14519 }
14520 }
14521
14522 async fn finish_after(
14523 harness: &mut ProbeHarness,
14524 stall: Duration,
14525 ) -> ModuleControlRpcCompletion {
14526 assert!(stall > harness.runtime.health.deadline);
14527 let deadline = harness.runtime.health.deadline;
14528 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14529 let answer = async {
14530 let frame = harness.module_rx.recv().await.expect("health.check frame");
14531 tokio::time::advance(deadline).await;
14532 tokio::task::yield_now().await;
14533 tokio::time::advance(stall - deadline).await;
14534 harness
14535 .forwarding
14536 .complete_module_control_rpc(
14537 harness.module_connection,
14538 frame.header.corr,
14539 Some("health.check"),
14540 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14541 status: HealthStatus::Ok,
14542 detail: None,
14543 metrics: None,
14544 }),
14545 )
14546 .unwrap()
14547 };
14548 let (probe_result, completion) = tokio::join!(probe, answer);
14549 let err = probe_result.expect_err("probe must miss its deadline");
14550 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14551 completion
14552 }
14553
14554 async fn time_out_without_answer(harness: &mut ProbeHarness) {
14555 let deadline = harness.runtime.health.deadline;
14556 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14557 let exhaust_deadline = async {
14558 let _frame = harness.module_rx.recv().await.expect("health.check frame");
14559 tokio::time::advance(deadline).await;
14560 tokio::task::yield_now().await;
14561 };
14562 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14563 let err = probe_result.expect_err("probe must miss its deadline");
14564 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14565 }
14566
14567 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14568 let registry = Arc::clone(&harness.module.inner.registry);
14569 let snapshot = Arc::clone(&harness.module.inner.snapshot);
14570 let process_liveness = super::SupervisorProcessLiveness::default();
14571 let mut child = None;
14572 let cycle = super::run_health_probe_cycle(
14573 &harness.spec,
14574 &harness.runtime,
14575 ®istry,
14576 &process_liveness,
14577 &snapshot,
14578 &mut child,
14579 );
14580 let peer = async {
14581 let frame = harness.module_rx.recv().await.expect("health.check frame");
14582 if answer {
14583 harness
14584 .forwarding
14585 .complete_module_control_rpc(
14586 harness.module_connection,
14587 frame.header.corr,
14588 Some("health.check"),
14589 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14590 status: HealthStatus::Ok,
14591 detail: None,
14592 metrics: Some(serde_json::json!({"ready": true})),
14593 }),
14594 )
14595 .unwrap();
14596 } else {
14597 tokio::time::advance(harness.runtime.health.deadline).await;
14598 tokio::task::yield_now().await;
14599 }
14600 };
14601 tokio::join!(cycle, peer);
14602 }
14603
14604 #[tokio::test(start_paused = true)]
14605 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14606 let mut harness = probe_harness();
14607 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14611 monitor.abort();
14612 let _ = monitor.await;
14613 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14614 state.enabled = true;
14615 state.state = super::ModuleState::Running;
14616 state.process_alive = true;
14617 })
14618 .unwrap();
14619
14620 run_probe_cycle(&mut harness, true).await;
14621 assert_eq!(
14622 harness.module.status().unwrap().health.status,
14623 super::SupervisorHealthStatus::Ok
14624 );
14625
14626 for failures in 1..harness.runtime.health.failure_threshold {
14627 run_probe_cycle(&mut harness, false).await;
14628 let status = harness.module.status().unwrap();
14629 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14630 assert_eq!(status.health.consecutive_failures, failures);
14631 assert!(status.health.last_probe_ms.is_some());
14632 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14633 assert!(status.health.metrics.is_none());
14634 assert_eq!(status.state, super::ModuleState::Running);
14635 assert!(status.process_alive);
14636 assert_eq!(status.restart_count, 0);
14637 assert_eq!(status.lifetime_restarts, 0);
14638 assert!(status.health.last_action.is_none());
14639 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14640 }
14641
14642 run_probe_cycle(&mut harness, true).await;
14643 let recovered = harness.module.status().unwrap();
14644 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14645 assert_eq!(recovered.health.consecutive_failures, 0);
14646 assert!(recovered.health.detail.is_none());
14647 assert_eq!(
14648 recovered.health.metrics,
14649 Some(serde_json::json!({"ready": true}))
14650 );
14651 assert_eq!(recovered.lifetime_restarts, 0);
14652
14653 for failures in 1..=harness.runtime.health.failure_threshold {
14654 run_probe_cycle(&mut harness, false).await;
14655 let status = harness.module.status().unwrap();
14656 assert_eq!(status.health.consecutive_failures, failures);
14657 if failures < harness.runtime.health.failure_threshold {
14658 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14659 assert_eq!(status.state, super::ModuleState::Running);
14660 assert_eq!(status.lifetime_restarts, 0);
14661 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14662 } else {
14663 assert_eq!(
14664 status.health.status,
14665 super::SupervisorHealthStatus::Unresponsive
14666 );
14667 assert_eq!(status.state, super::ModuleState::Restarting);
14668 assert_eq!(status.restart_count, 1);
14669 assert_eq!(status.lifetime_restarts, 1);
14670 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14671 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14672 }
14673 }
14674 }
14675
14676 #[tokio::test(start_paused = true)]
14677 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14678 let mut harness = probe_harness();
14679
14680 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14681 let first_latency = match &first {
14682 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14683 other => panic!("late answer was not retained: {other:?}"),
14684 };
14685 assert!(harness.handler.observe_module_control_completion(first));
14686
14687 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14688 let second_latency = match &second {
14689 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14690 other => panic!("late answer was not retained: {other:?}"),
14691 };
14692 assert!(harness.handler.observe_module_control_completion(second));
14693
14694 assert_eq!(first_latency, Duration::from_secs(8));
14695 assert_eq!(
14696 second_latency - first_latency,
14697 Duration::from_secs(3),
14698 "latency must grow linearly with the additional stall"
14699 );
14700 let health = harness.module.status().unwrap().health;
14701 assert_eq!(health.late_answer_count, 2);
14702 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14703 }
14704
14705 #[tokio::test(start_paused = true)]
14713 async fn late_answer_clears_the_consecutive_failure_streak() {
14714 let mut harness = probe_harness();
14715
14716 time_out_without_answer(&mut harness).await;
14718 harness
14719 .module
14720 .record_health_probe_failure_for_test("[no-answer] test miss")
14721 .unwrap();
14722 assert_eq!(
14723 harness.module.status().unwrap().health.consecutive_failures,
14724 1,
14725 "precondition: the miss must be on the streak before the late answer"
14726 );
14727
14728 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14730 assert!(matches!(
14731 late,
14732 ModuleControlRpcCompletion::LateHealthAnswer { .. }
14733 ));
14734 assert!(harness.handler.observe_module_control_completion(late));
14735
14736 let health = harness.module.status().unwrap().health;
14737 assert_eq!(
14738 health.consecutive_failures, 0,
14739 "a late answer is an answer: the streak must reset"
14740 );
14741 assert_eq!(health.late_answer_count, 1);
14742 }
14743
14744 #[tokio::test(start_paused = true)]
14745 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14746 let mut harness = probe_harness();
14747
14748 for _ in 0..20 {
14749 time_out_without_answer(&mut harness).await;
14750 assert_eq!(
14751 harness.forwarding.health_probe_tombstone_count().unwrap(),
14752 1
14753 );
14754 }
14755 }
14756}
14757
14758#[cfg(test)]
14759mod child_env_tests {
14760 #[cfg(windows)]
14761 use super::{
14762 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14763 };
14764 #[cfg(windows)]
14765 use std::sync::{Arc, OnceLock};
14766 #[cfg(windows)]
14767 #[test]
14768 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14769 let spec = spec(Vec::new());
14770 let mut command = Command::new("unused");
14771 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14772 .unwrap()
14773 .unwrap();
14774 let name = command
14775 .as_std()
14776 .get_envs()
14777 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14778 .and_then(|(_, value)| value)
14779 .unwrap();
14780 assert_eq!(name, OsStr::new(handoff.name()));
14781 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14782 assert!(command.as_std().get_envs().any(|(key, value)| key
14783 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14784 && value == Some(OsStr::new("env"))));
14785 assert!(command
14786 .as_std()
14787 .get_envs()
14788 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14789 }
14790
14791 #[cfg(windows)]
14792 #[test]
14793 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14794 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14795 snapshot.process_alive = true;
14796 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14797 assert_eq!(
14798 observed_launch_nonce_source(&snapshot).as_deref(),
14799 Some("env")
14800 );
14801 let consumed = Arc::new(OnceLock::new());
14802 snapshot.pipe_consumed = Some(consumed.clone());
14803 assert_eq!(
14804 observed_launch_nonce_source(&snapshot).as_deref(),
14805 Some("env")
14806 );
14807 consumed.set(()).unwrap();
14808 assert_eq!(
14809 observed_launch_nonce_source(&snapshot).as_deref(),
14810 Some("pipe")
14811 );
14812 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14813 assert_eq!(
14814 observed_launch_nonce_source(&snapshot).as_deref(),
14815 Some("env")
14816 );
14817 clear_current_process_facts(&mut snapshot);
14818 assert_eq!(observed_launch_nonce_source(&snapshot), None);
14819 }
14820 use super::{
14821 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
14822 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
14823 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14824 };
14825 use std::{ffi::OsStr, path::PathBuf};
14826 use tokio::process::Command;
14827
14828 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14829 ModuleSpec {
14830 module_id: "env-plan".to_string(),
14831 program: PathBuf::from("/nonexistent"),
14832 args: Vec::new(),
14833 env,
14834 reserved: false,
14835 reserved_prefixes: Vec::new(),
14836 protocol: ModuleProtocol::Subc,
14837 overlap: Default::default(),
14838 }
14839 }
14840
14841 #[test]
14842 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
14843 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
14844 let digest = LaunchSpecDigest::from_spec(&base);
14845 let mut reordered = base.clone();
14846 reordered.env.reverse();
14847 reordered.module_id = "different-id".into();
14848 reordered.reserved = true;
14849 reordered.overlap = Default::default();
14850 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
14851 let mut delimiter_collision = base.clone();
14852 delimiter_collision.env[0] = ("A=B".into(), "C".into());
14853 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
14854 let mut changed = base.clone();
14855 changed.protocol = ModuleProtocol::None;
14856 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14857 changed = base.clone();
14858 changed.args.push("--flag".into());
14859 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14860 changed = base.clone();
14861 changed.program = PathBuf::from("/other");
14862 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14863 changed = base.clone();
14864 changed.env[1].1 = "other".into();
14865 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14866 assert!(!format!("{digest:?}").contains("B=C"));
14867 assert!(!format!("{digest:?}").contains("[u8"));
14868 }
14869
14870 #[cfg(unix)]
14871 #[test]
14872 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
14873 use std::os::unix::ffi::OsStringExt;
14874 let mut left = spec(Vec::new());
14875 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
14876 let mut right = left.clone();
14877 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
14878 assert_eq!(
14879 left.program.to_string_lossy(),
14880 right.program.to_string_lossy()
14881 );
14882 assert_ne!(
14883 LaunchSpecDigest::from_spec(&left),
14884 LaunchSpecDigest::from_spec(&right)
14885 );
14886 }
14887
14888 #[test]
14902 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14903 let mut command = Command::new("/nonexistent");
14904 apply_child_env(&mut command, &spec(Vec::new()));
14905 let removed = command
14906 .as_std()
14907 .get_envs()
14908 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14909 assert!(
14910 removed,
14911 "ambient CK_LOG must be explicitly removed for an unconfigured module"
14912 );
14913
14914 let mut configured = Command::new("/nonexistent");
14915 apply_child_env(
14916 &mut configured,
14917 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14918 );
14919 let effective = configured
14920 .as_std()
14921 .get_envs()
14922 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14923 .last()
14924 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14925 assert_eq!(
14926 effective,
14927 Some(Some("debug".to_string())),
14928 "a module's configured CK_LOG must survive the ambient removal"
14929 );
14930 }
14931
14932 #[test]
14941 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14942 let connection_file = std::path::Path::new("/run/subc-connection.json");
14943 let handle = SupervisorHandle::new();
14944
14945 let mut none_spec = spec(Vec::new());
14946 none_spec.protocol = ModuleProtocol::None;
14947 let mut none = Command::new("/nonexistent");
14948 let none_handoff =
14949 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14950 .expect("protocol-none spawn args apply");
14951 assert!(
14952 none_handoff.is_none(),
14953 "protocol:none spawn must not receive a nonce descriptor"
14954 );
14955 #[cfg(windows)]
14956 assert!(!none.as_std().get_envs().any(|(key, value)| key
14957 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14958 && value.is_some()));
14959 assert!(
14960 !none.as_std().get_envs().any(|(key, value)| key
14961 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14962 && value.is_some()),
14963 "protocol:none spawn must not name a nonce descriptor"
14964 );
14965 let none_args: Vec<String> = none
14966 .as_std()
14967 .get_args()
14968 .map(|a| a.to_string_lossy().into_owned())
14969 .collect();
14970 assert!(
14971 !none_args.iter().any(|a| a == SUBC_ARG),
14972 "protocol:none argv must not carry --subc; got {none_args:?}"
14973 );
14974 let none_has_nonce = none
14975 .as_std()
14976 .get_envs()
14977 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
14978 assert!(
14979 !none_has_nonce,
14980 "protocol:none spawn must not receive a launch nonce"
14981 );
14982 let none_has_module_id = none
14983 .as_std()
14984 .get_envs()
14985 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
14986 assert!(
14987 none_has_module_id,
14988 "SUBC_MODULE_ID is inert and stays on every path"
14989 );
14990 assert!(
14991 handle.spawn_nonce(&none_spec.module_id).is_none(),
14992 "no nonce record for a process that will never present one"
14993 );
14994
14995 let wire_spec = spec(Vec::new());
14997 let mut wire = Command::new("/nonexistent");
14998 let wire_handoff =
14999 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15000 .expect("subc-wire spawn args apply");
15001 let wire_fd_env = wire
15002 .as_std()
15003 .get_envs()
15004 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15005 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15006 #[cfg(unix)]
15007 assert_eq!(
15008 wire_fd_env,
15009 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15010 "a subc-wire spawn names the pipe it will receive at descriptor 3"
15011 );
15012 #[cfg(windows)]
15013 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15014 #[cfg(not(any(unix, windows)))]
15015 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15016 let wire_args: Vec<String> = wire
15017 .as_std()
15018 .get_args()
15019 .map(|a| a.to_string_lossy().into_owned())
15020 .collect();
15021 assert_eq!(
15022 wire_args,
15023 vec![
15024 SUBC_ARG.to_string(),
15025 connection_file.to_string_lossy().into_owned()
15026 ],
15027 "a subc-wire spawn still carries --subc <path>"
15028 );
15029 assert_eq!(
15030 wire.as_std()
15031 .get_envs()
15032 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15033 !cfg!(unix),
15034 "only Windows supplies the environment nonce"
15035 );
15036 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15037 }
15038
15039 #[test]
15049 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15050 let role = |command: &Command| {
15051 command
15052 .as_std()
15053 .get_envs()
15054 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15055 .last()
15056 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15057 };
15058 let forged = spec(vec![(
15059 SUBC_SPAWN_ROLE_ENV.to_string(),
15060 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15061 )]);
15062
15063 let mut plain = Command::new("/nonexistent");
15064 apply_child_env(&mut plain, &forged);
15065 apply_spawn_role(&mut plain, SpawnRole::Plain);
15066 assert_eq!(
15067 role(&plain),
15068 Some(None),
15069 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15070 );
15071
15072 let mut candidate = Command::new("/nonexistent");
15073 apply_child_env(&mut candidate, &spec(Vec::new()));
15074 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15075 assert_eq!(
15076 role(&candidate),
15077 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15078 );
15079 }
15080
15081 #[test]
15087 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15088 let mut command = Command::new("/nonexistent");
15089 apply_child_env(
15090 &mut command,
15091 &spec(vec![
15092 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15093 ("KEPT".to_string(), "yes".to_string()),
15094 ]),
15095 );
15096 let keys: Vec<String> = command
15097 .as_std()
15098 .get_envs()
15099 .filter(|(_, value)| value.is_some())
15100 .map(|(key, _)| key.to_string_lossy().into_owned())
15101 .collect();
15102 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15103 assert!(
15104 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15105 "daemon-private capture key leaked to the child: {keys:?}"
15106 );
15107 }
15108}
15109
15110#[cfg(test)]
15111mod jitter_tests {
15112 use super::jittered_health_delay;
15113 use std::{collections::HashSet, time::Duration};
15114
15115 const FLEET: [&str; 14] = [
15124 "aft",
15125 "alfonso-core",
15126 "magic-context",
15127 "broca",
15128 "thalamus",
15129 "quota",
15130 "engram",
15131 "plexus",
15132 "cerebellum",
15133 "astrocyte",
15134 "synapse",
15135 "subc-mcp",
15136 "cortexkit-credentials",
15137 "subc-federation",
15138 ];
15139
15140 #[test]
15148 fn probe_delays_disperse_across_the_fleet() {
15149 let cadence = Duration::from_secs(30);
15150 let delays: HashSet<Duration> = FLEET
15151 .iter()
15152 .map(|id| jittered_health_delay(id, 0, cadence))
15153 .collect();
15154 assert_eq!(
15155 delays.len(),
15156 FLEET.len(),
15157 "every supervised module must land on its own probe offset"
15158 );
15159 }
15160
15161 #[test]
15167 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15168 let cadence = Duration::from_secs(30);
15169 let span = cadence / 10;
15170 for id in FLEET {
15171 for probe_index in 0..8 {
15172 let delay = jittered_health_delay(id, probe_index, cadence);
15173 assert!(
15174 delay >= cadence,
15175 "{id}#{probe_index}: jitter must not shorten the cadence"
15176 );
15177 assert!(
15178 delay < cadence + span,
15179 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15180 );
15181 }
15182 }
15183 }
15184
15185 #[test]
15191 fn a_module_offset_is_stable_across_restarts() {
15192 let cadence = Duration::from_secs(30);
15193 for id in FLEET {
15194 assert_eq!(
15195 jittered_health_delay(id, 0, cadence),
15196 jittered_health_delay(id, 0, cadence),
15197 "{id}: the same module and probe index must produce the same offset"
15198 );
15199 }
15200 }
15201
15202 #[test]
15204 fn zero_cadence_yields_zero_delay() {
15205 assert_eq!(
15206 jittered_health_delay("aft", 0, Duration::ZERO),
15207 Duration::ZERO
15208 );
15209 }
15210}
15211
15212#[cfg(all(test, target_os = "linux"))]
15213mod cgroup_placement_tests {
15214 use super::{
15215 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15216 SuperviseError, SupervisedChild,
15217 };
15218 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15219 use cortexkit_test_support::ScratchDir;
15220 use std::{
15221 fs, io,
15222 path::{Path, PathBuf},
15223 sync::{Arc, Mutex},
15224 };
15225 use tokio::process::Command;
15226
15227 #[tokio::test]
15228 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15229 use super::*;
15230 let dir = ScratchDir::new("unique-spawn-cgroups");
15231 let root = PathBuf::from(format!(
15232 "/sys/fs/cgroup/subc-unique-test-{}-{}",
15233 std::process::id(),
15234 unix_ms_now()
15235 ));
15236 if let Err(error) = fs::create_dir(&root) {
15237 assert!(
15238 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15239 "required cgroup test cannot execute: {error}"
15240 );
15241 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15242 return;
15243 }
15244 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15245 let group_count = || {
15246 fs::read_dir(root.join("subc-modules"))
15247 .unwrap()
15248 .map(|entry| entry.unwrap().file_type().unwrap())
15249 .filter(|kind| kind.is_dir())
15250 .count()
15251 };
15252 let supervisor =
15253 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15254 .with_cgroup_placement(Some(placement.clone()));
15255 let runtime = supervisor.runtime_config();
15256 let mut spec = ModuleSpec {
15257 module_id: "unique-spawn".into(),
15258 program: PathBuf::from("/bin/sleep"),
15259 args: vec!["60".into()],
15260 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15261 .into_iter()
15262 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15263 .collect(),
15264 reserved: false,
15265 reserved_prefixes: vec![],
15266 protocol: ModuleProtocol::None,
15267 overlap: Default::default(),
15268 };
15269 let spawn = |spec: &ModuleSpec| {
15270 spawn_child(
15271 spec,
15272 None,
15273 None,
15274 &runtime.stderr_ring,
15275 None,
15276 &runtime.child_roster,
15277 Some(&placement),
15278 )
15279 .unwrap()
15280 };
15281 let mut live = spawn(&spec);
15282 for _ in 0..3 {
15283 let next = spawn(&spec);
15285 assert_ne!(live.module_id, next.module_id);
15286 live.start_kill().unwrap();
15287 live.wait().await.unwrap();
15288 live = next;
15289 assert!(
15290 live.child.try_wait().unwrap().is_none(),
15291 "retiring the old slot must not kill the replacement"
15292 );
15293 assert_eq!(
15294 group_count(),
15295 1,
15296 "only the live spawn's cgroup should remain"
15297 );
15298 }
15299 supervisor.begin_daemon_shutdown();
15300 let reap = tokio::spawn(async move {
15301 live.wait().await.unwrap();
15302 });
15303 supervisor
15304 .end_children_for_daemon_shutdown(false, std::future::pending())
15305 .await;
15306 reap.await.unwrap();
15307 assert_eq!(group_count(), 0);
15308 spec.program = PathBuf::from("/bin/true");
15310 spec.args.clear();
15311 let fresh_roster = ChildRoster::default();
15312 let mut short = spawn_child(
15313 &spec,
15314 None,
15315 None,
15316 &runtime.stderr_ring,
15317 None,
15318 &fresh_roster,
15319 Some(&placement),
15320 )
15321 .unwrap();
15322 short.wait().await.unwrap();
15323 assert_eq!(group_count(), 0);
15324 spec.module_id = "_".repeat(255);
15325 let mut long_id = spawn_child(
15326 &spec,
15327 None,
15328 None,
15329 &runtime.stderr_ring,
15330 None,
15331 &fresh_roster,
15332 Some(&placement),
15333 )
15334 .unwrap();
15335 long_id.wait().await.unwrap();
15336 assert_eq!(
15337 group_count(),
15338 0,
15339 "valid long module IDs must not exceed cgroup NAME_MAX"
15340 );
15341 fs::remove_dir(root.join("subc-modules")).unwrap();
15342 fs::remove_dir(root).unwrap();
15343 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15344 }
15345
15346 #[test]
15347 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15348 let path = Path::new("/definitely-missing-subc-cgroup");
15349 let mut command = Command::new("true");
15350 let error = apply_cgroup_placement(
15351 &mut command,
15352 &ModuleSpec {
15353 module_id: "broken-cgroup".to_string(),
15354 program: PathBuf::from("true"),
15355 args: Vec::new(),
15356 env: Vec::new(),
15357 reserved: false,
15358 reserved_prefixes: Vec::new(),
15359 protocol: ModuleProtocol::Subc,
15360 overlap: Default::default(),
15361 },
15362 path,
15363 )
15364 .expect_err("a parent cgroup open failure must reject the supervised spawn");
15365 let reason = error.to_string();
15366
15367 assert!(
15368 matches!(error, SuperviseError::Cgroup { .. }),
15369 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15370 );
15371 assert!(
15372 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15373 "parent cgroup open failure must name cgroup.procs: {reason}"
15374 );
15375 }
15376
15377 #[tokio::test]
15378 async fn reaping_a_child_removes_its_empty_module_cgroup() {
15379 let root = ScratchDir::new("supervisor-reap-cgroup");
15380 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15381 let placement = subc_cgroup::prepare_at(&root)
15382 .expect("prepare scratch cgroup root")
15383 .expect("scratch root has a cgroup.procs marker");
15384 let module_id = "reaped-module";
15385 let module = placement
15386 .module_path(module_id)
15387 .expect("create scratch module cgroup");
15388 let child = Command::new("true")
15389 .env("XDG_DATA_HOME", root.path())
15390 .env("XDG_RUNTIME_DIR", root.path())
15391 .env("XDG_CONFIG_HOME", root.path())
15392 .spawn()
15393 .expect("spawn short-lived child");
15394 let pid = child.id().expect("spawned child has pid");
15395 let mut child = SupervisedChild {
15396 child,
15397 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15398 protocol: ModuleProtocol::Subc,
15399 module_id: module_id.to_string(),
15400 cgroup_placement: Some(placement),
15401 stdout_pump: None,
15402 stderr_pump: None,
15403 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15404 spawned_at: tokio::time::Instant::now(),
15405 spawned_at_ms: 0,
15406 spawned_from: PathBuf::from("true"),
15407 launch_spec_digest: LaunchSpecDigest([0; 32]),
15408 spawned_file_identity: None,
15409 process_start_time: None,
15410 process_identity: None,
15411 pid,
15412 roster_guard: None,
15413 #[cfg(target_os = "macos")]
15414 privacy_exec: None,
15415 spawn_failure: None,
15416 };
15417
15418 child.wait().await.expect("reap short-lived child");
15419
15420 assert!(
15421 !module.exists(),
15422 "reaping the supervised child must remove its empty cgroup"
15423 );
15424 }
15425
15426 #[test]
15427 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15428 let root = ScratchDir::new("supervisor-non-empty-cgroup");
15429 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15430 let placement = subc_cgroup::prepare_at(&root)
15431 .expect("prepare scratch cgroup root")
15432 .expect("scratch root has a cgroup.procs marker");
15433 let module = placement
15434 .module_path("surviving-module")
15435 .expect("create scratch module cgroup");
15436 fs::write(module.join("surviving-process"), b"still present")
15437 .expect("make scratch cgroup non-empty");
15438 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15439
15440 remove_module_cgroup(&placement, "surviving-module");
15441
15442 let logs = crate::router::test_log::captured_logs(&logs);
15443 assert!(
15444 module.exists(),
15445 "failed removal must leave the cgroup intact"
15446 );
15447 assert!(
15448 logs.contains("could not remove module cgroup after process exit; continuing teardown")
15449 && logs.contains("surviving-module"),
15450 "best-effort removal must report the failure without returning it: {logs}"
15451 );
15452 }
15453
15454 #[test]
15455 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15456 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15457 let reason = SuperviseError::Spawn {
15458 program: PathBuf::from("/bin/true"),
15459 source: io::Error::from_raw_os_error(13),
15460 cgroup_path: Some(cgroup_path.clone()),
15461 }
15462 .to_string();
15463
15464 assert!(
15465 reason.contains(&cgroup_path.display().to_string()),
15466 "a pre_exec spawn failure must name the cgroup path: {reason}"
15467 );
15468 }
15469}
15470
15471#[cfg(test)]
15472mod spawn_subscriber_lag_tests {
15473 use super::*;
15474
15475 #[tokio::test]
15480 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15481 let feed = SpawnEventFeed::default();
15482 feed.configure_incarnation("lag-incarnation".to_string());
15483 let (tx, mut rx) = mpsc::channel(1);
15486 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15487 .expect("subscribe");
15488 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15489 for index in 0..emitted {
15490 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15491 tokio::task::yield_now().await;
15494 }
15495 assert_eq!(
15496 feed.subscriber_count(),
15497 0,
15498 "the lagged subscriber must be removed"
15499 );
15500
15501 let mut data = Vec::new();
15502 let mut last = None;
15503 loop {
15504 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15505 .await
15506 .expect("the forwarder must finish once the subscriber is dropped");
15507 let Some(outbound) = next else { break };
15508 let frame = outbound.frame;
15509 if frame.header.ty == FrameType::StreamData {
15510 assert!(last.is_none(), "no data may follow the terminal frame");
15511 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15512 data.push(event.cursor.seq);
15513 } else {
15514 assert!(last.is_none(), "exactly one terminal frame");
15515 last = Some(frame);
15516 }
15517 }
15518 assert!(!data.is_empty(), "queued frames drain before the terminal");
15519 for pair in data.windows(2) {
15520 assert_eq!(
15521 pair[1],
15522 pair[0] + 1,
15523 "queued frames arrive dense and in order"
15524 );
15525 }
15526 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15527 assert_eq!(terminal.header.ty, FrameType::Error);
15528 assert_eq!(terminal.header.corr, 7);
15529 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15530 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15531 let detail = body.detail.expect("lagged error carries detail");
15532 assert_eq!(
15533 detail["first_undelivered_cursor"]["seq"],
15534 data.last().unwrap() + 1,
15535 "the named cursor is the first event the subscriber did not receive"
15536 );
15537 assert_eq!(
15538 detail["first_undelivered_cursor"]["daemon_incarnation"],
15539 "lag-incarnation"
15540 );
15541 }
15542}
15543
15544#[cfg(test)]
15545mod terminal_history_read_concurrency_tests {
15546 use super::*;
15547 use crate::terminal_journal::read_pause;
15548 use cortexkit_test_support::ScratchDir;
15549 use std::sync::mpsc as std_mpsc;
15550
15551 fn journaled_ring(
15552 journal: &Arc<crate::terminal_journal::TerminalJournal>,
15553 ) -> Arc<Mutex<TerminalRing>> {
15554 Arc::new(Mutex::new(
15555 TerminalRing::new(TerminalRingConfig::default(), 1)
15556 .with_journal(Some(Arc::clone(journal))),
15557 ))
15558 }
15559
15560 fn crash(at_ms: u64) -> ExitReport {
15561 ExitReport {
15562 kind: ExitKind::Crash,
15563 code: Some(1),
15564 signal: None,
15565 at_ms,
15566 }
15567 }
15568
15569 fn record_within(
15572 module_id: &'static str,
15573 ring: &Arc<Mutex<TerminalRing>>,
15574 at_ms: u64,
15575 bound: Duration,
15576 ) -> bool {
15577 let ring = Arc::clone(ring);
15578 let (done, done_rx) = std_mpsc::channel();
15579 std::thread::spawn(move || {
15580 record_terminal(
15581 module_id,
15582 &ring,
15583 &SpawnEventFeed::default(),
15584 &crash(at_ms),
15585 TerminalDisposition::Restarting,
15586 );
15587 let _ = done.send(());
15588 });
15589 done_rx.recv_timeout(bound).is_ok()
15590 }
15591
15592 #[test]
15597 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15598 let dir = ScratchDir::new("terminal-history-concurrent-read");
15599 let path = dir.join("terminals.jsonl");
15600 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15601 path.clone(),
15602 "daemon".into(),
15603 ));
15604 let reader_ring = journaled_ring(&journal);
15605 let other_ring = journaled_ring(&journal);
15606 assert!(record_within(
15607 "reader-module",
15608 &reader_ring,
15609 10,
15610 Duration::from_secs(5)
15611 ));
15612
15613 let (started, release) = read_pause::install(&path);
15614 let reading = {
15615 let ring = Arc::clone(&reader_ring);
15616 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15617 };
15618 started
15619 .recv_timeout(Duration::from_secs(5))
15620 .expect("the history read reached its pause");
15621
15622 let bound = Duration::from_secs(1);
15623 assert!(
15624 record_within("other-module", &other_ring, 20, bound),
15625 "another module's exit waited on a history read (journal writer held)"
15626 );
15627 assert!(
15628 record_within("reader-module", &reader_ring, 30, bound),
15629 "the read module's own exit waited on its history read (ring held)"
15630 );
15631
15632 drop(release);
15633 let paused = reading.join().unwrap();
15634 assert_eq!(
15635 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15636 vec![10],
15637 "an exit recorded after the read began lands in neither half of it"
15638 );
15639 assert_eq!(paused.journal_skipped_lines, 0);
15640 assert_eq!(paused.journal_read_errors, 0);
15641
15642 let after = durable_terminal_history_of(&reader_ring, "reader-module");
15643 assert_eq!(
15644 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15645 vec![10, 30],
15646 "the next read merges ring and journal with no duplicate"
15647 );
15648 assert_eq!(after.journal_skipped_lines, 0);
15649 }
15650}
15651
15652#[cfg(test)]
15657mod stderr_settle_tests {
15658 use std::{
15659 future::Future,
15660 io,
15661 pin::Pin,
15662 sync::{Arc, Mutex},
15663 task::{Context, Poll},
15664 time::Duration,
15665 };
15666
15667 use tokio::{
15668 io::{AsyncRead, ReadBuf},
15669 sync::oneshot,
15670 time::Instant,
15671 };
15672
15673 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15674 use crate::stderr_tail::{
15675 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15676 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15677 };
15678
15679 const BOUND: Duration = Duration::from_millis(250);
15680
15681 struct HeldReader {
15685 before: Option<Vec<u8>>,
15686 gate: Option<oneshot::Receiver<()>>,
15687 after: io::Cursor<Vec<u8>>,
15688 }
15689
15690 impl AsyncRead for HeldReader {
15691 fn poll_read(
15692 mut self: Pin<&mut Self>,
15693 cx: &mut Context<'_>,
15694 buf: &mut ReadBuf<'_>,
15695 ) -> Poll<io::Result<()>> {
15696 if let Some(bytes) = self.before.take() {
15697 buf.put_slice(&bytes);
15698 return Poll::Ready(Ok(()));
15699 }
15700 if let Some(gate) = self.gate.as_mut() {
15701 match Pin::new(gate).poll(cx) {
15702 Poll::Pending => return Poll::Pending,
15703 Poll::Ready(_) => self.gate = None,
15704 }
15705 }
15706 Pin::new(&mut self.after).poll_read(cx, buf)
15707 }
15708 }
15709
15710 struct DiscardSink;
15711
15712 impl OutputSink for DiscardSink {
15713 fn write_line(&mut self, _line: &[u8]) {}
15714 }
15715
15716 fn line(text: &str) -> TailEntry {
15717 TailEntry::Line {
15718 text: text.to_string(),
15719 truncated: false,
15720 at_ms: None,
15721 }
15722 }
15723
15724 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
15725 ring.lock().unwrap()
15726 }
15727
15728 fn held_pump(
15732 ring: &Arc<Mutex<StderrRing>>,
15733 before: &str,
15734 after: &str,
15735 ) -> (StderrPump, oneshot::Sender<()>) {
15736 let generation = lock(ring).begin_process();
15737 let (release, gate) = oneshot::channel();
15738 let reader = HeldReader {
15739 before: Some(before.as_bytes().to_vec()),
15740 gate: Some(gate),
15741 after: io::Cursor::new(after.as_bytes().to_vec()),
15742 };
15743 let task = tokio::spawn(pump_stderr_to(
15744 reader,
15745 Arc::clone(ring),
15746 generation,
15747 DiscardSink,
15748 ));
15749 (StderrPump { task, generation }, release)
15750 }
15751
15752 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
15753 for _ in 0..1000 {
15754 if done(&lock(ring)) {
15755 return;
15756 }
15757 tokio::time::sleep(Duration::from_millis(1)).await;
15758 }
15759 panic!(
15760 "ring never reached the expected state: {:?}",
15761 lock(ring).snapshot(None, None)
15762 );
15763 }
15764
15765 #[tokio::test(start_paused = true)]
15766 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
15767 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15768 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
15769
15770 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
15771 let before_release = lock(&ring).snapshot(None, None);
15772 assert!(
15773 matches!(before_release.capture, CaptureState::Incomplete { .. }),
15774 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
15775 );
15776
15777 let next = lock(&ring).begin_process();
15780 lock(&ring).push_line_from(next, "next process booting");
15781 release.send(()).unwrap();
15782 wait_until(&ring, |ring| {
15783 ring.snapshot(None, None).capture == CaptureState::Captured
15784 })
15785 .await;
15786
15787 assert_eq!(
15788 untimed(lock(&ring).snapshot(None, None).entries),
15789 vec![
15790 line("booting"),
15791 line("config error: missing storage"),
15792 TailEntry::ProcessStart,
15793 line("next process booting"),
15794 ],
15795 "the crash's last line must survive a slow reader and stay in the crashed process's section"
15796 );
15797 }
15798
15799 #[tokio::test(start_paused = true)]
15800 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15801 ) {
15802 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15803 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15806
15807 let started = Instant::now();
15808 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15809 assert_eq!(
15810 started.elapsed(),
15811 BOUND,
15812 "the restart must wait exactly the bound for a pipe that stays open, no longer"
15813 );
15814
15815 let next = lock(&ring).begin_process();
15816 lock(&ring).push_line_from(next, "next process booting");
15817 tokio::time::sleep(Duration::from_secs(60)).await;
15818
15819 let snapshot = lock(&ring).snapshot(None, None);
15820 match &snapshot.capture {
15821 CaptureState::Incomplete { reason } => assert!(
15822 reason.contains("had not reached EOF") && reason.contains("250ms"),
15823 "the reason must say what is missing and after how long: {reason}"
15824 ),
15825 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15826 }
15827 assert_eq!(
15828 untimed(snapshot.entries),
15829 vec![
15830 line("parent exiting"),
15831 TailEntry::ProcessStart,
15832 line("next process booting"),
15833 ]
15834 );
15835 }
15836
15837 #[tokio::test(start_paused = true)]
15838 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15839 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15840 let (pump, release) = held_pump(&ring, "one\n", "two\n");
15841 release.send(()).unwrap();
15842
15843 settle_stderr_pump("clean", &ring, pump, BOUND).await;
15844
15845 let snapshot = lock(&ring).snapshot(None, None);
15846 assert_eq!(snapshot.capture, CaptureState::Captured);
15847 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15848 }
15849
15850 #[tokio::test(start_paused = true)]
15851 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15852 const BURST: usize = 150;
15853 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15854 let path = capture.join("two-pipe.stderr.log");
15855 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15856 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15857 let generation = lock(&ring).begin_process();
15858 let stdout = (0..BURST)
15859 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15860 .collect::<String>();
15861 let stderr = (0..BURST)
15862 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15863 .collect::<String>();
15864 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15869 let (release, gate) = oneshot::channel();
15870 let reader = HeldReader {
15871 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15872 gate: Some(gate),
15873 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15874 };
15875 let (finished, completion) = oneshot::channel();
15876 let stdout_sink = sink.clone();
15877 let pump = tokio::spawn(async move {
15878 pump_stdout_to(reader, stdout_sink).await;
15879 finished.send(()).unwrap();
15880 });
15881 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15882
15883 let started = Instant::now();
15884 settle_stdout_pump("two-pipe", pump, BOUND).await;
15885 assert_eq!(
15886 started.elapsed(),
15887 BOUND,
15888 "a late reader must not hold up restart"
15889 );
15890 assert_eq!(
15891 std::fs::read_to_string(&path).unwrap().lines().count(),
15892 BURST + 1
15893 );
15894
15895 release
15896 .send(())
15897 .expect("stdout reader must survive the exit drain bound");
15898 tokio::time::timeout(Duration::from_secs(1), completion)
15899 .await
15900 .expect("released stdout reader must reach EOF")
15901 .expect("stdout capture must finish at EOF, not be cancelled");
15902 let contents = std::fs::read_to_string(&path).unwrap();
15903 let mut lines = contents
15904 .lines()
15905 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15906 .collect::<Vec<_>>();
15907 let mut expected = (0..BURST)
15908 .flat_map(|i| {
15909 [
15910 format!("out-{i:04}-{}", "o".repeat(64)),
15911 format!("err-{i:04}-{}", "e".repeat(64)),
15912 ]
15913 })
15914 .collect::<Vec<_>>();
15915 lines.sort_unstable();
15916 expected.sort_unstable();
15917 assert_eq!(
15918 lines, expected,
15919 "both pipes must retain every complete line exactly once"
15920 );
15921 }
15922}
15923
15924#[cfg(all(test, windows))]
15938mod job_containment_tests {
15939 use super::*;
15940 use cortexkit_test_support::ScratchDir;
15941 use std::{
15942 path::{Path, PathBuf},
15943 sync::{Arc, Mutex},
15944 time::{Duration, Instant},
15945 };
15946
15947 fn stub_path() -> PathBuf {
15953 let mut path = std::env::current_exe().expect("current_exe available in tests");
15954 path.pop();
15955 path.pop();
15956 path.push("fake-aft-stub.exe");
15957 assert!(
15958 path.exists(),
15959 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15960 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15961 path.display()
15962 );
15963 path
15964 }
15965
15966 fn read_grandchild_pid(path: &Path) -> u32 {
15968 let deadline = Instant::now() + Duration::from_secs(10);
15969 loop {
15970 if let Ok(contents) = std::fs::read_to_string(path) {
15971 if let Ok(pid) = contents.trim().parse() {
15972 return pid;
15973 }
15974 }
15975 assert!(
15976 Instant::now() < deadline,
15977 "the stub never recorded a grandchild pid at {}",
15978 path.display()
15979 );
15980 std::thread::sleep(Duration::from_millis(10));
15981 }
15982 }
15983
15984 struct Fixture {
15987 _dir: ScratchDir,
15988 module_id: String,
15989 grandchild: u32,
15990 child: Option<SupervisedChild>,
15991 registry: Arc<Registry>,
15992 snapshot: Arc<Mutex<SupervisorSnapshot>>,
15993 terminal_ring: Arc<Mutex<TerminalRing>>,
15994 spawn_events: SpawnEventFeed,
15995 }
15996
15997 fn fixture(label: &str, module_id: &str) -> Fixture {
15998 let dir = ScratchDir::new(label);
15999 let pid_file = dir.join("grandchild.pid");
16000 let supervisor = Supervisor::new_for_test(
16001 Arc::new(Registry::default()),
16002 RestartPolicy::new(3, Duration::ZERO),
16003 );
16004 let runtime = supervisor.runtime_config();
16005 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16006 let spec = ModuleSpec {
16007 module_id: module_id.to_string(),
16008 program: stub_path(),
16009 args: Vec::new(),
16013 env: vec![
16014 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16015 (
16016 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16017 pid_file.display().to_string(),
16018 ),
16019 ],
16020 reserved: false,
16021 reserved_prefixes: Vec::new(),
16022 protocol: ModuleProtocol::Subc,
16023 overlap: Default::default(),
16024 };
16025 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16026 .expect("spawn the supervised fixture");
16027 let grandchild = read_grandchild_pid(&pid_file);
16028 Fixture {
16029 _dir: dir,
16030 module_id: module_id.to_string(),
16031 grandchild,
16032 child: Some(child),
16033 registry: Arc::new(Registry::default()),
16034 snapshot,
16035 terminal_ring: Arc::clone(&runtime.terminal_ring),
16036 spawn_events: SpawnEventFeed::default(),
16037 }
16038 }
16039
16040 impl Fixture {
16041 async fn drain(&mut self) {
16043 let child = self
16044 .child
16045 .take()
16046 .expect("the fixture child is still present");
16047 drain_child_to_state(
16048 &self.module_id,
16049 ModuleProtocol::Subc,
16050 StopNotice::NotSent,
16053 &self.registry,
16054 None,
16055 &self.snapshot,
16056 &self.terminal_ring,
16057 &self.spawn_events,
16058 child,
16059 Duration::from_millis(500),
16060 ModuleState::Stopped,
16061 Some(false),
16062 )
16063 .await
16064 .expect("drain the supervised fixture");
16065 }
16066 }
16067
16068 #[tokio::test]
16074 async fn teardown_reaps_the_grandchild() {
16075 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16076 let grandchild = fixture.grandchild;
16077
16078 assert!(
16079 subc_jobobject::process_exists(grandchild),
16080 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16081 );
16082
16083 fixture.drain().await;
16084
16085 assert!(
16086 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16087 "grandchild {grandchild} outlived module teardown: the tree was not contained"
16088 );
16089 }
16090
16091 #[test]
16104 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16105 let dir = ScratchDir::new("teardown-uncontained");
16106 let pid_file = dir.join("grandchild.pid");
16107 let mut child = std::process::Command::new(stub_path())
16108 .env("FAKE_AFT_NEVER_CONNECT", "1")
16109 .env(
16110 "FAKE_AFT_GRANDCHILD_PID_FILE",
16111 pid_file.display().to_string(),
16112 )
16113 .stdin(std::process::Stdio::null())
16114 .stdout(std::process::Stdio::null())
16115 .stderr(std::process::Stdio::null())
16116 .spawn()
16117 .expect("spawn the uncontained fixture");
16118 let grandchild = read_grandchild_pid(&pid_file);
16119
16120 child.kill().expect("kill the direct child");
16122 let _ = child.wait();
16123
16124 assert!(
16125 subc_jobobject::process_exists(grandchild),
16126 "grandchild {grandchild} died with the direct child, so this control no longer \
16127 distinguishes contained from uncontained teardown and the regression test is \
16128 passing vacuously"
16129 );
16130
16131 kill_tree(grandchild);
16134 }
16135
16136 #[tokio::test]
16145 async fn dropping_containment_reaps_the_grandchild() {
16146 let mut fixture = fixture("drop-containment", "tree-drop");
16147 let grandchild = fixture.grandchild;
16148
16149 assert!(subc_jobobject::process_exists(grandchild));
16150
16151 fixture.child.as_mut().expect("child present").job = None;
16153
16154 assert!(
16155 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16156 "grandchild {grandchild} survived the containment handle closing, so a daemon \
16157 crash would leave the tree behind"
16158 );
16159 }
16160
16161 fn kill_tree(pid: u32) {
16163 let _ = std::process::Command::new("taskkill.exe")
16164 .args(["/PID", &pid.to_string(), "/T", "/F"])
16165 .stdin(std::process::Stdio::null())
16166 .stdout(std::process::Stdio::null())
16167 .stderr(std::process::Stdio::null())
16168 .status();
16169 assert!(
16170 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16171 "could not clean up grandchild {pid}"
16172 );
16173 }
16174}
16175
16176#[cfg(test)]
16177mod privacy_trampoline_configuration_tests {
16178 #[tokio::test]
16179 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16180 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16181 #[cfg(target_os = "macos")]
16182 {
16183 let supervisor = super::Supervisor::new(
16184 std::sync::Arc::new(crate::Registry::default()),
16185 super::RestartPolicy::default(),
16186 );
16187 let error = supervisor.spawn(spec()).unwrap_err();
16188 assert!(
16189 error
16190 .to_string()
16191 .contains("no privacy trampoline configured"),
16192 "{error}"
16193 );
16194 }
16195 }
16196
16197 #[tokio::test]
16198 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16199 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16200 #[cfg(target_os = "macos")]
16201 {
16202 let supervisor = super::Supervisor::new(
16203 std::sync::Arc::new(crate::Registry::default()),
16204 super::RestartPolicy::default(),
16205 )
16206 .with_privacy_trampoline(std::env::current_exe().unwrap());
16207 let error = supervisor.spawn(spec()).unwrap_err();
16208 assert!(
16209 error
16210 .to_string()
16211 .contains("binary does not implement the privacy trampoline protocol"),
16212 "{error}"
16213 );
16214 }
16215 }
16216
16217 #[cfg(target_os = "macos")]
16218 fn spec() -> super::ModuleSpec {
16219 super::ModuleSpec {
16220 module_id: "privacy-configuration".into(),
16221 program: "/bin/sleep".into(),
16222 args: vec!["30".into()],
16223 env: vec![],
16224 reserved: false,
16225 reserved_prefixes: vec![],
16226 protocol: subc_control::ModuleProtocol::None,
16227 overlap: super::ModuleOverlap::Exclusive,
16228 }
16229 }
16230}
16231
16232#[cfg(test)]
16233mod privacy_exec_boundary_tests {
16234 #[cfg(target_os = "macos")]
16235 use super::*;
16236 #[cfg(target_os = "macos")]
16237 use std::{
16238 io::{Read, Write},
16239 net::{TcpListener, TcpStream},
16240 };
16241
16242 #[cfg(target_os = "macos")]
16245 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16246 if let Some((_, path)) = spec
16247 .env
16248 .iter()
16249 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16250 {
16251 let mut barrier = TcpStream::connect(path).unwrap();
16252 barrier
16253 .set_read_timeout(Some(Duration::from_secs(30)))
16254 .unwrap();
16255 barrier.write_all(&pid.to_ne_bytes()).unwrap();
16256 let mut release = [0];
16257 barrier.read_exact(&mut release).unwrap();
16258 assert_eq!(&release, b"X");
16259 }
16260 }
16261
16262 #[cfg(target_os = "macos")]
16263 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16264 ModuleSpec {
16265 module_id: "privacy-boundary".into(),
16266 program: program.into(),
16267 args: args.iter().map(|arg| (*arg).into()).collect(),
16268 env: vec![],
16269 reserved: false,
16270 reserved_prefixes: vec![],
16271 protocol: ModuleProtocol::None,
16272 overlap: ModuleOverlap::Exclusive,
16273 }
16274 }
16275
16276 #[cfg(target_os = "macos")]
16277 async fn accept(listener: TcpListener) -> TcpStream {
16278 let listener = tokio::net::TcpListener::from_std({
16280 listener.set_nonblocking(true).unwrap();
16281 listener
16282 })
16283 .unwrap();
16284 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16285 .await
16286 .unwrap()
16287 .unwrap();
16288 let stream = stream.into_std().unwrap();
16289 stream.set_nonblocking(false).unwrap();
16290 stream
16291 .set_read_timeout(Some(Duration::from_secs(30)))
16292 .unwrap();
16293 stream
16294 }
16295
16296 #[tokio::test]
16297 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16298 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16299 #[cfg(target_os = "macos")]
16300 {
16301 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16302 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16305 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16306 let record = root.join("live-children.json");
16307 let supervisor =
16308 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16309 .with_live_children_record(&record);
16310 let runtime = supervisor.runtime_config();
16311 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16312 let mut spec = spec("/bin/sleep", &["30"]);
16313 spec.env = vec![
16314 (
16315 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16316 exec_listener.local_addr().unwrap().to_string(),
16317 ),
16318 (
16319 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16320 sample_listener.local_addr().unwrap().to_string(),
16321 ),
16322 ];
16323 let spawn = tokio::task::spawn_blocking(move || {
16324 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16325 });
16326 let mut sample = accept(sample_listener).await;
16327 let mut pid = [0; 4];
16328 sample.read_exact(&mut pid).unwrap();
16329 let pid = u32::from_ne_bytes(pid);
16330 let mut exec = accept(exec_listener).await;
16331 let mut ready = [0];
16332 exec.read_exact(&mut ready).unwrap();
16333 assert_eq!(&ready, b"R");
16334 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16337 assert_eq!(
16338 observe_spawned_image(pid).unwrap().executable,
16339 Some(trampoline)
16340 );
16341 sample.write_all(b"X").unwrap();
16342 let mut child = spawn.await.unwrap();
16343 let early = crate::live_children::read_record(&record).unwrap();
16344 assert_eq!(early.len(), 1);
16345 assert_eq!(early[0].pid, pid);
16346 assert_eq!(
16347 early[0].executable, None,
16348 "unconfirmed trampoline image entered the roster"
16349 );
16350 assert!(child.report_ready.get().is_none());
16351 child.privacy_exec.as_mut().unwrap().deadline =
16354 tokio::time::Instant::now() + Duration::from_secs(30);
16355 exec.write_all(b"X").unwrap();
16356 child.confirm_privacy_exec().await;
16357 assert_eq!(child.spawn_failure, None);
16358 assert!(child.report_ready.get().is_some());
16359 let confirmed = crate::live_children::read_record(&record).unwrap();
16360 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16361 assert_ne!(module, trampoline);
16362 assert_eq!(confirmed[0].executable, Some(module.into()));
16363 child.start_kill().unwrap();
16364 child.wait().await.unwrap();
16365 child.release_roster();
16366 }
16367 }
16368
16369 #[tokio::test]
16370 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16371 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16372 #[cfg(target_os = "macos")]
16373 {
16374 let registry = Arc::new(Registry::default());
16375 let policy = RestartPolicy::new(0, Duration::ZERO);
16376 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
16377 let runtime = supervisor.runtime_config();
16378 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16379 let spec = spec("/bin/sh", &["-c", "exit 121"]);
16380 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16384 let pid = child.pid;
16385 tokio::task::spawn_blocking(move || {
16386 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16387 })
16388 .await
16389 .unwrap()
16390 .unwrap();
16391 child.privacy_exec.as_mut().unwrap().deadline =
16392 tokio::time::Instant::now() + Duration::from_secs(30);
16393 let status = child.wait().await.unwrap();
16394 assert_eq!(status.code(), Some(121));
16395 assert!(child.privacy_exec.is_none());
16396 assert!(
16397 child.report_ready.get().is_none(),
16398 "an exited module must not publish a live pid"
16399 );
16400 let report = classify_reaped_child_exit(&snapshot, &child, &status);
16401 on_child_exit(
16402 &spec,
16403 policy,
16404 ®istry,
16405 &snapshot,
16406 &runtime.terminal_ring,
16407 &runtime.spawn_events,
16408 &runtime.child_roster,
16409 report,
16410 )
16411 .await;
16412 let state = lock_snapshot(&snapshot).unwrap();
16413 assert_eq!(state.state, ModuleState::Failed);
16414 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16415 assert_eq!(state.reported_pid(), None);
16416 drop(state);
16417 let history = runtime.terminal_ring.lock().unwrap().snapshot();
16418 assert_eq!(history.entries.len(), 1);
16419 let terminal = &history.entries[0];
16420 assert_eq!(terminal.exit_code, Some(121));
16421 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16422 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16423 assert_eq!(
16424 terminal.disposition_detail.as_deref(),
16425 Some(policy.budget_exhausted_detail().as_str()),
16426 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16427 );
16428 assert_eq!(child.spawn_failure, None);
16429 child.release_roster();
16430 }
16431 }
16432}
16433
16434#[cfg(all(test, unix))]
16438mod launch_nonce_descriptor_tests {
16439 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16440 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16441 use cortexkit_test_support::ScratchDir;
16442 use std::{
16443 path::PathBuf,
16444 sync::{Arc, Mutex},
16445 time::{Duration, Instant},
16446 };
16447
16448 async fn probe(role: super::SpawnRole) {
16449 let scratch = ScratchDir::new("launch-nonce-descriptor");
16450 let fd_copy = scratch.join("from-descriptor");
16451 let env_copy = scratch.join("environment");
16452 let script = format!(
16453 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16454 fd = fd_copy.display(), env = env_copy.display(),
16455 );
16456 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16457 let spec = ModuleSpec {
16458 module_id: "nonce-descriptor-probe".to_string(),
16459 program: PathBuf::from("/bin/sh"),
16460 args: vec!["-c".to_string(), script],
16461 env: vec![
16462 xdg("XDG_DATA_HOME"),
16463 xdg("XDG_RUNTIME_DIR"),
16464 xdg("XDG_CONFIG_HOME"),
16465 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16466 ],
16467 reserved: true,
16468 reserved_prefixes: Vec::new(),
16469 protocol: ModuleProtocol::Subc,
16470 overlap: Default::default(),
16471 };
16472 let handle = SupervisorHandle::new();
16473 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16474 let roster = ChildRoster::default();
16475 #[cfg(target_os = "macos")]
16476 {
16477 let path = super::test_privacy_trampoline();
16478 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16479 }
16480 let child = super::spawn_child_in_slot(
16481 &spec,
16482 None,
16483 Some(&handle),
16484 &ring,
16485 None,
16486 &roster,
16487 #[cfg(target_os = "linux")]
16488 None,
16489 role,
16490 matches!(role, super::SpawnRole::SwapCandidate),
16491 )
16492 .expect("spawn probe");
16493 let deadline = Instant::now() + Duration::from_secs(10);
16494 while !(fd_copy.exists() && env_copy.exists()) {
16495 assert!(Instant::now() < deadline, "probe never wrote its copies");
16496 tokio::time::sleep(Duration::from_millis(20)).await;
16497 }
16498 let nonce = std::fs::read_to_string(fd_copy).unwrap();
16499 assert!(!nonce.is_empty());
16500 let environment = std::fs::read_to_string(env_copy).unwrap();
16501 assert!(environment
16502 .lines()
16503 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16504 let copy = environment
16505 .lines()
16506 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16507 assert_eq!(
16508 copy, None,
16509 "Unix children must never receive the environment nonce"
16510 );
16511 if matches!(role, super::SpawnRole::Plain) {
16512 assert_eq!(
16513 handle.spawn_nonce(&spec.module_id).as_deref(),
16514 Some(nonce.as_str())
16515 );
16516 }
16517 drop(child);
16518 }
16519
16520 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16521 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16522 probe(super::SpawnRole::Plain).await;
16523 }
16524
16525 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16526 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16527 probe(super::SpawnRole::SwapCandidate).await;
16528 }
16529}
16530
16531#[cfg(all(test, target_os = "linux"))]
16532mod cgroup_containment_tests {
16533 use super::*;
16534 use cortexkit_test_support::ScratchDir;
16535
16536 fn running(pid: u32) -> bool {
16537 std::fs::read_to_string(format!("/proc/{pid}/stat"))
16539 .ok()
16540 .and_then(|stat| {
16541 stat.rsplit_once(") ")
16542 .map(|(_, rest)| rest.starts_with('Z'))
16543 })
16544 .is_some_and(|zombie| !zombie)
16545 }
16546
16547 #[tokio::test]
16548 async fn linux_teardown_reaps_the_grandchild() {
16549 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16550 }
16551
16552 #[tokio::test]
16553 async fn linux_shutdown_straggler_reaps_the_grandchild() {
16554 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16555 }
16556
16557 async fn teardown_tree(test_name: &str, shutdown: bool) {
16558 let dir = ScratchDir::new(test_name);
16559 let root = PathBuf::from(format!(
16560 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16561 std::process::id(),
16562 unix_ms_now()
16563 ));
16564 if let Err(error) = std::fs::create_dir(&root) {
16565 assert!(
16566 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16567 "required cgroup test cannot execute: {error}"
16568 );
16569 eprintln!(
16570 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16571 root.display()
16572 );
16573 return;
16574 }
16575 let placement = subc_cgroup::prepare_at(&root)
16576 .expect("prepare isolated kernel cgroup")
16577 .expect("isolated cgroup is delegated");
16578 let module_id = "tree-teardown";
16579 let module = placement
16580 .module_path(module_id)
16581 .expect("create isolated module cgroup");
16582 if !module.join("cgroup.kill").exists() {
16583 std::fs::remove_dir(&module).unwrap();
16584 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16585 std::fs::remove_dir(&root).unwrap();
16586 assert!(
16587 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16588 "required cgroup.kill interface unavailable"
16589 );
16590 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16591 return;
16592 }
16593 let supervisor = Supervisor::new_for_test(
16594 Arc::new(Registry::default()),
16595 RestartPolicy::new(3, Duration::ZERO),
16596 )
16597 .with_cgroup_placement(Some(placement));
16598 let mut runtime = supervisor.runtime_config();
16599 runtime.child_roster = runtime
16600 .child_roster
16601 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16602 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16603 let pid_file = dir.join("grandchild.pid");
16604 let spec = ModuleSpec {
16605 module_id: module_id.to_string(),
16606 program: PathBuf::from("/bin/sh"),
16607 args: vec![
16608 "-c".into(),
16609 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16610 "fixture".into(),
16611 pid_file.display().to_string(),
16612 ],
16613 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16614 .into_iter()
16615 .map(|key| (key.to_string(), dir.display().to_string()))
16616 .collect(),
16617 reserved: false,
16618 reserved_prefixes: Vec::new(),
16619 protocol: ModuleProtocol::None,
16620 overlap: Default::default(),
16621 };
16622 let child =
16623 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16624 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16625 let grandchild: u32 = loop {
16626 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16627 if let Ok(pid) = contents.trim().parse() {
16628 break pid;
16629 }
16630 }
16631 assert!(
16632 tokio::time::Instant::now() < deadline,
16633 "grandchild pid was not recorded"
16634 );
16635 tokio::time::sleep(Duration::from_millis(10)).await;
16636 };
16637 assert!(
16638 running(grandchild),
16639 "grandchild must be alive before teardown"
16640 );
16641 if shutdown {
16642 let mut child = child;
16643 crate::child_roster::end_children_for_daemon_shutdown(
16644 &runtime.child_roster,
16645 false,
16646 std::future::pending(),
16647 )
16648 .await;
16649 child.wait().await.expect("reap shutdown straggler");
16650 } else {
16651 drain_child_to_state(
16652 module_id,
16653 ModuleProtocol::None,
16654 StopNotice::NotSent,
16655 &Registry::default(),
16656 None,
16657 &snapshot,
16658 &runtime.terminal_ring,
16659 &SpawnEventFeed::default(),
16660 child,
16661 Duration::from_millis(100),
16662 ModuleState::Stopped,
16663 Some(false),
16664 )
16665 .await
16666 .expect("real supervisor teardown");
16667 }
16668 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16669 while running(grandchild) && tokio::time::Instant::now() < deadline {
16670 tokio::time::sleep(Duration::from_millis(10)).await;
16671 }
16672 let survived = running(grandchild);
16673 if survived {
16675 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16676 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16677 tokio::time::sleep(Duration::from_millis(100)).await;
16678 }
16679 if module.exists() {
16680 std::fs::remove_dir(&module).expect("remove empty module cgroup");
16681 }
16682 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16683 std::fs::remove_dir(&root).unwrap();
16684 assert!(
16685 !survived,
16686 "grandchild {grandchild} outlived module teardown"
16687 );
16688 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16689 }
16690}