1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136 deadline: tokio::time::Instant,
137 expected: Option<subc_os::FileIdentity>,
138 trampoline: Option<subc_os::FileIdentity>,
139 script: bool,
140 module_id: String,
141 #[cfg(test)]
142 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147 let path = std::env::current_exe()
150 .unwrap()
151 .parent()
152 .unwrap()
153 .parent()
154 .unwrap()
155 .join("privacy-trampoline-fixture");
156 assert!(
160 path.exists(),
161 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162 --bins --features test-support` or `cargo test -p subc-daemon` first",
163 path.display()
164 );
165 path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170 use std::io::Read;
171 let mut probe = std::process::Command::new(path)
172 .args(["__disclaim-exec", "--probe"])
173 .stdin(Stdio::null())
174 .stdout(Stdio::piped())
175 .stderr(Stdio::piped())
176 .spawn()
177 .map_err(|error| {
178 format!(
179 "privacy trampoline probe failed for {}: {error}",
180 path.display()
181 )
182 })?;
183 let deadline = std::time::Instant::now() + Duration::from_secs(5);
184 let status = loop {
185 match probe.try_wait() {
186 Ok(Some(status)) => break status,
187 Ok(None) if std::time::Instant::now() < deadline => {
188 std::thread::sleep(Duration::from_millis(5))
189 }
190 result => {
191 let _ = probe.kill();
192 let _ = probe.wait();
193 return Err(format!(
194 "privacy trampoline probe failed or timed out for {}: {result:?}",
195 path.display()
196 ));
197 }
198 }
199 };
200 let mut answer = String::new();
201 if let Some(stdout) = probe.stdout.take() {
202 let _ = stdout.take(256).read_to_string(&mut answer);
203 }
204 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205 return Ok(());
206 }
207 let mut diagnostic = String::new();
208 if let Some(stderr) = probe.stderr.take() {
209 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210 }
211 let cause = diagnostic
212 .trim()
213 .strip_prefix("ck-subc: own privacy identity refused: ")
214 .unwrap_or("binary does not implement the privacy trampoline protocol");
215 Err(format!(
216 "{cause}: probe of {} exited {status}",
217 path.display()
218 ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223 spec: &ModuleSpec,
224 roster: &ChildRoster,
225) -> Result<
226 (
227 Command,
228 Option<PrivacyExec>,
229 subc_os::privacy_identity::ExecAcknowledgement,
230 ),
231 SuperviseError,
232> {
233 let failure = |cause: String| {
234 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235 SuperviseError::Spawn {
236 program: spec.program.clone(),
237 source: io::Error::other(cause),
238 cgroup_path: None,
239 }
240 };
241 let trampoline = roster.privacy_trampoline().map_err(failure)?;
242 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247 let path = spec
248 .env
249 .iter()
250 .find(|(key, _)| key == "PATH")
251 .map(|(_, value)| std::ffi::OsString::from(value))
252 .or_else(|| std::env::var_os("PATH"))
253 .unwrap_or_else(|| "/usr/bin:/bin".into());
254 std::env::split_paths(&path)
255 .map(|dir| dir.join(&spec.program))
256 .find(|path| path.is_file())
257 .unwrap_or_else(|| spec.program.clone())
258 } else {
259 spec.program.clone()
260 };
261 let expected = subc_os::file_identity(&program);
262 let trampoline_image = subc_os::file_identity(&trampoline);
263 let script = {
264 use std::io::Read;
265 let mut prefix = [0u8; 2];
266 std::fs::File::open(&program)
267 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268 };
269 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270 return Err(failure(
271 "privacy identity module executable is missing or is the trampoline itself".to_string(),
272 ));
273 }
274 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275 .map_err(|error| failure(error.to_string()))?;
276 let reader =
277 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278 let mut command = Command::new(&trampoline);
279 command
280 .arg("__disclaim-exec")
281 .arg(ack.fd().to_string())
282 .arg(&program);
283 ack.install(command.as_std_mut());
284 Ok((
285 command,
286 Some(PrivacyExec {
287 reader,
288 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289 expected,
290 trampoline: trampoline_image,
291 script,
292 module_id: spec.module_id.clone(),
293 #[cfg(test)]
294 confirmation_pause: spec
295 .env
296 .iter()
297 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298 .map(|(_, address)| (address.clone(), None)),
299 }),
300 ack,
301 ))
302}
303
304struct SupervisedChild {
305 child: Child,
306 exec: crate::activity::ExecOwner,
307 #[cfg(windows)]
308 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309 #[cfg(target_os = "macos")]
310 privacy_exec: Option<PrivacyExec>,
311 #[cfg(target_os = "macos")]
318 report_ready: Arc<OnceLock<()>>,
319 spawn_failure: Option<String>,
321 protocol: ModuleProtocol,
326 #[cfg(target_os = "linux")]
332 module_id: String,
333 #[cfg(target_os = "linux")]
334 cgroup_placement: Option<subc_cgroup::Placement>,
335 #[cfg(windows)]
357 job: Option<subc_jobobject::JobObject>,
358 stdout_pump: Option<JoinHandle<()>>,
359 stderr_pump: Option<StderrPump>,
360 stderr_ring: Arc<Mutex<StderrRing>>,
361 spawned_at_ms: u64,
362 spawned_from: PathBuf,
363 launch_spec_digest: LaunchSpecDigest,
364 spawned_file_identity: Option<SpawnedFileIdentity>,
365 process_start_time: Option<u64>,
366 process_identity: Option<ProcessIdentity>,
367 pid: u32,
368 roster_guard: Option<crate::child_roster::RosterGuard>,
371}
372
373impl SupervisedChild {
374 fn id(&self) -> Option<u32> {
375 Some(self.pid)
376 }
377
378 fn process_identity(&self) -> Option<ProcessIdentity> {
379 self.process_identity
380 }
381
382 async fn wait(&mut self) -> io::Result<ExitStatus> {
383 #[cfg(target_os = "macos")]
384 self.confirm_privacy_exec().await;
385 let result = self.child.wait().await;
393 #[cfg(target_os = "linux")]
394 if result.is_ok() {
395 if let Some(placement) = self.cgroup_placement.as_ref() {
396 cleanup_reaped_cgroup(placement, &self.module_id).await;
397 self.cgroup_placement = None;
400 }
401 }
402 result
403 }
404
405 #[cfg(target_os = "macos")]
406 async fn confirm_privacy_exec(&mut self) {
407 let Some(pending) = &mut self.privacy_exec else {
408 return;
409 };
410 let result = tokio::time::timeout_at(pending.deadline, async {
411 let mut record = Vec::new();
412 loop {
413 let mut ready = pending.reader.readable().await?;
414 let read = ready.try_io(|reader| {
415 use std::io::Read;
416 let mut reader = reader.get_ref();
417 let mut buffer = [0u8; 256];
418 reader.read(&mut buffer).map(|count| (count, buffer))
419 });
420 match read {
421 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
422 Ok(Ok((count, buffer))) => {
423 if record.len() + count > 1024 {
424 return Err(io::Error::other(
425 "privacy exec refusal record is too long",
426 ));
427 }
428 record.extend_from_slice(&buffer[..count]);
429 }
430 Ok(Err(error)) => return Err(error),
431 Err(_) => continue,
432 }
433 }
434 })
435 .await;
436 #[cfg(test)]
437 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
438 use tokio::io::{AsyncReadExt, AsyncWriteExt};
439 if stream.is_none() {
440 *stream = Some(
441 tokio::net::TcpStream::connect(address.as_str())
442 .await
443 .unwrap(),
444 );
445 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
446 stream
447 .as_mut()
448 .unwrap()
449 .write_all(&self.pid.to_ne_bytes())
450 .await
451 .unwrap();
452 }
453 let mut reply = [0];
454 stream
455 .as_mut()
456 .unwrap()
457 .read_exact(&mut reply)
458 .await
459 .unwrap();
460 pending.confirmation_pause = None;
461 if &reply == b"F" {
462 Ok(Err(io::Error::other("injected late exec refusal")))
463 } else {
464 assert_eq!(&reply, b"X");
465 result
466 }
467 } else {
468 result
469 };
470 let pending = self.privacy_exec.as_ref().expect("pending exec");
473 let cause = match result {
474 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
475 Ok(Err(error)) => Some(format!(
476 "privacy identity exec acknowledgement failed: {error}"
477 )),
478 Ok(Ok(record)) if !record.is_empty() => Some(
479 std::str::from_utf8(&record)
480 .ok()
481 .and_then(|record| {
482 record
483 .trim()
484 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
485 })
486 .filter(|cause| !cause.is_empty())
487 .unwrap_or("invalid privacy exec refusal record")
488 .to_string(),
489 ),
490 Ok(Ok(_)) => match self.child.try_wait() {
491 Ok(Some(_status)) => None,
495 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
496 Ok(None) => {
497 let image = observe_spawned_image(self.pid);
498 if let Some(image) = image.filter(|image| {
499 image.executable.is_some()
500 && image.executable != pending.trampoline
501 && (image.executable == pending.expected || pending.script)
502 }) {
503 if let Err(error) = self
504 .roster_guard
505 .as_ref()
506 .map(|guard| guard.confirm_executable(image))
507 .transpose()
508 {
509 Some(format!(
510 "could not durably record confirmed module image: {error}"
511 ))
512 } else {
513 let _ = self.report_ready.set(());
514 info!(module_id = %pending.module_id, pid = self.pid,
515 "module spawned with own privacy identity (responsibility disclaimed)");
516 None
517 }
518 } else if image.is_none()
519 || image.is_some_and(|image| image.executable.is_none())
520 {
521 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
528 Ok(Ok(_status)) => None,
529 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
530 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
531 }
532 } else {
533 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
534 }
535 }
536 },
537 };
538 let pending = self.privacy_exec.take().expect("pending exec");
539 self.exec.0.finish(if cause.is_some() {
542 crate::activity::ExecState::Refused
543 } else {
544 crate::activity::ExecState::CommittedExec
545 });
546 if let Some(cause) = cause {
547 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
548 self.spawn_failure = Some(cause);
549 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
552 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
553 }
554 let _ = self.child.start_kill();
555 }
556 }
557
558 fn release_roster(&mut self) {
562 self.roster_guard = None;
563 }
564
565 fn start_kill(&mut self) -> io::Result<()> {
578 #[cfg(windows)]
579 if let Some(job) = &self.job {
580 if let Err(error) = job.terminate() {
581 debug!(
582 error = %error,
583 "job termination failed; the direct-child kill still owns the outcome"
584 );
585 }
586 }
587 #[cfg(target_os = "linux")]
588 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
589 self.child.start_kill()
590 }
591
592 async fn drain_stderr(&mut self, module_id: &str) {
593 if let Some(pump) = self.stdout_pump.take() {
594 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
595 }
596
597 let Some(pump) = self.stderr_pump.take() else {
598 return;
599 };
600 settle_stderr_pump(
601 module_id,
602 &self.stderr_ring,
603 pump,
604 STDERR_PUMP_DRAIN_TIMEOUT,
605 )
606 .await;
607 }
608}
609
610async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
614 match timeout(bound, &mut pump).await {
615 Ok(Ok(())) => {}
616 Ok(Err(error)) => {
617 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
618 }
619 Err(_) => {
620 drop(pump);
621 warn!(
622 module_id,
623 waited = ?bound,
624 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
625 );
626 }
627 }
628}
629
630struct StderrPump {
633 task: JoinHandle<()>,
634 generation: u64,
635}
636
637async fn settle_stderr_pump(
643 module_id: &str,
644 ring: &Arc<Mutex<StderrRing>>,
645 pump: StderrPump,
646 bound: Duration,
647) {
648 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
649 let StderrPump {
650 mut task,
651 generation,
652 } = pump;
653 lock().retire_pump(generation);
654 match timeout(bound, &mut task).await {
655 Ok(Ok(())) => {}
656 Ok(Err(err)) => {
657 let mut ring = lock();
658 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
659 ring.finish_pump(generation);
660 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
661 }
662 Err(_) => {
663 drop(task);
665 lock().mark_pump_late(
666 generation,
667 format!(
668 "stderr of the exited process had not reached EOF {bound:?} after it was \
669 retired (a descendant may still hold the pipe open); lines it still \
670 writes are kept in that process's section"
671 ),
672 );
673 warn!(
674 module_id,
675 waited = ?bound,
676 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
677 );
678 }
679 }
680}
681
682fn registration_release_events() -> &'static watch::Sender<u64> {
683 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
684 EVENTS.get_or_init(|| {
685 let (sender, _receiver) = watch::channel(0);
686 sender
687 })
688}
689
690pub(crate) fn notify_registration_release() {
691 let events = registration_release_events();
692 let next_generation = (*events.borrow()).wrapping_add(1);
693 events.send_replace(next_generation);
694}
695
696#[derive(Debug, Clone, PartialEq, Eq)]
698pub struct ModuleSpec {
699 pub module_id: String,
700 pub program: PathBuf,
701 pub args: Vec<String>,
702 pub env: Vec<(String, String)>,
703 pub reserved: bool,
708 pub reserved_prefixes: Vec<String>,
713 pub protocol: ModuleProtocol,
732 pub overlap: ModuleOverlap,
737}
738
739#[derive(Clone, PartialEq, Eq)]
742pub(crate) struct LaunchSpecDigest([u8; 32]);
743
744impl fmt::Debug for LaunchSpecDigest {
745 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
746 f.write_str("LaunchSpecDigest([redacted])")
747 }
748}
749
750impl LaunchSpecDigest {
751 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
752 fn field(hash: &mut Sha256, bytes: &[u8]) {
753 hash.update((bytes.len() as u64).to_le_bytes());
754 hash.update(bytes);
755 }
756 let mut hash = Sha256::new();
757 if let Some(program) = spec.program.to_str() {
758 field(&mut hash, b"utf8-program");
759 field(&mut hash, program.as_bytes());
760 } else {
761 #[cfg(unix)]
764 {
765 use std::os::unix::ffi::OsStrExt;
766 field(&mut hash, b"unix-program");
767 field(&mut hash, spec.program.as_os_str().as_bytes());
768 }
769 #[cfg(windows)]
770 {
771 use std::os::windows::ffi::OsStrExt;
772 field(&mut hash, b"windows-program");
773 let bytes = spec
774 .program
775 .as_os_str()
776 .encode_wide()
777 .flat_map(u16::to_le_bytes)
778 .collect::<Vec<_>>();
779 field(&mut hash, &bytes);
780 }
781 #[cfg(not(any(unix, windows)))]
782 field(&mut hash, spec.program.to_string_lossy().as_bytes());
783 }
784 hash.update((spec.args.len() as u64).to_le_bytes());
785 for arg in &spec.args {
786 field(&mut hash, arg.as_bytes());
787 }
788 let mut env = spec.env.iter().collect::<Vec<_>>();
789 env.sort_unstable();
790 hash.update((env.len() as u64).to_le_bytes());
791 for (key, value) in env {
792 field(&mut hash, key.as_bytes());
793 field(&mut hash, value.as_bytes());
794 }
795 field(
796 &mut hash,
797 match spec.protocol {
798 ModuleProtocol::Subc => b"subc",
799 ModuleProtocol::None => b"none",
800 },
801 );
802 Self(hash.finalize().into())
803 }
804}
805
806#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
813pub enum ModuleOverlap {
814 #[default]
816 Exclusive,
817 Safe,
829}
830
831impl ModuleOverlap {
832 pub fn as_str(self) -> &'static str {
833 match self {
834 Self::Exclusive => "exclusive",
835 Self::Safe => "safe",
836 }
837 }
838}
839
840pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
850pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
852pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
857
858#[derive(Debug, Clone, Copy, PartialEq, Eq)]
876pub struct RestartPolicy {
877 pub max_restarts: u32,
878 pub backoff: Duration,
881 pub max_backoff: Duration,
883 pub window: Duration,
887}
888
889impl RestartPolicy {
890 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
894 Self {
895 max_restarts,
896 backoff,
897 max_backoff: DEFAULT_MAX_BACKOFF,
898 window: DEFAULT_RESTART_WINDOW,
899 }
900 }
901
902 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
903 self.max_backoff = max_backoff;
904 self
905 }
906
907 pub fn with_window(mut self, window: Duration) -> Self {
908 self.window = window;
909 self
910 }
911
912 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
917 if self.backoff.is_zero() || self.max_backoff.is_zero() {
918 return Duration::ZERO;
919 }
920
921 let mut delay = self.backoff;
922 for _ in 0..restart_in_window {
923 if delay >= self.max_backoff {
924 return self.max_backoff;
925 }
926 delay = delay
927 .checked_mul(10)
928 .unwrap_or(self.max_backoff)
929 .min(self.max_backoff);
930 }
931 delay.min(self.max_backoff)
932 }
933
934 fn budget_exhausted_detail(&self) -> String {
939 format!(
940 "crash budget exhausted: max_restarts={} within window_secs={}",
941 self.max_restarts,
942 self.window.as_secs()
943 )
944 }
945}
946
947impl Default for RestartPolicy {
948 fn default() -> Self {
949 Self {
950 max_restarts: DEFAULT_MAX_RESTARTS,
951 backoff: DEFAULT_BACKOFF,
952 max_backoff: DEFAULT_MAX_BACKOFF,
953 window: DEFAULT_RESTART_WINDOW,
954 }
955 }
956}
957
958#[derive(Debug, Clone, Copy, PartialEq, Eq)]
959struct CrashRestartSchedule {
960 restart_in_window: u32,
961 delay: Duration,
962}
963
964fn daemon_will_restart(
971 state: &mut SupervisorSnapshot,
972 policy: &RestartPolicy,
973 now: Instant,
974) -> bool {
975 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
976}
977
978const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
979const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
980const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
981const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
982
983#[derive(Debug, Clone, Copy, PartialEq, Eq)]
984pub enum HealthAction {
985 Report,
986 Restart,
987 Alert,
988}
989
990impl fmt::Display for HealthAction {
991 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
992 f.write_str(match self {
993 Self::Report => "report",
994 Self::Restart => "restart",
995 Self::Alert => "alert",
996 })
997 }
998}
999
1000#[derive(Debug, Clone, PartialEq, Eq)]
1001pub struct HealthConfig {
1002 pub http: Option<String>,
1006 pub cadence: Duration,
1007 pub deadline: Duration,
1008 pub failure_threshold: u32,
1009 pub on_degraded: HealthAction,
1010 pub on_failing: HealthAction,
1011 pub critical: bool,
1012}
1013
1014impl Default for HealthConfig {
1015 fn default() -> Self {
1016 Self {
1017 http: None,
1018 cadence: DEFAULT_HEALTH_CADENCE,
1019 deadline: DEFAULT_HEALTH_DEADLINE,
1020 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1021 on_degraded: HealthAction::Report,
1022 on_failing: HealthAction::Report,
1023 critical: false,
1024 }
1025 }
1026}
1027
1028#[derive(Debug, Clone, PartialEq)]
1046pub struct ModuleHealthStatus {
1047 pub status: SupervisorHealthStatus,
1048 pub last_probe_ms: Option<u64>,
1049 pub detail: Option<String>,
1050 pub metrics: Option<Value>,
1051 pub consecutive_failures: u32,
1052 pub late_answer_count: u64,
1055 pub last_late_answer_latency_ms: Option<u64>,
1057 pub last_action: Option<String>,
1058 pub last_action_ms: Option<u64>,
1062}
1063
1064impl Default for ModuleHealthStatus {
1065 fn default() -> Self {
1066 Self {
1067 status: SupervisorHealthStatus::Unknown,
1068 last_probe_ms: None,
1069 detail: None,
1070 metrics: None,
1071 consecutive_failures: 0,
1072 late_answer_count: 0,
1073 last_late_answer_latency_ms: None,
1074 last_action: None,
1075 last_action_ms: None,
1076 }
1077 }
1078}
1079
1080#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1082pub enum ModuleState {
1083 Starting,
1084 Running,
1085 Unresponsive,
1086 Restarting,
1087 Draining,
1088 Stopped,
1089 Failed,
1090 Disabled,
1091}
1092
1093impl fmt::Display for ModuleState {
1094 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1095 f.write_str(match self {
1096 Self::Starting => "starting",
1097 Self::Running => "running",
1098 Self::Unresponsive => "unresponsive",
1099 Self::Restarting => "restarting",
1100 Self::Draining => "draining",
1101 Self::Stopped => "stopped",
1102 Self::Failed => "failed",
1103 Self::Disabled => "disabled",
1104 })
1105 }
1106}
1107
1108#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1110pub enum ExitKind {
1111 Clean,
1112 Crash,
1113 DeliberateSeverance,
1114}
1115
1116impl From<ExitKind> for TerminalExitKind {
1117 fn from(kind: ExitKind) -> Self {
1118 match kind {
1119 ExitKind::Clean => Self::Clean,
1120 ExitKind::Crash => Self::Crash,
1121 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1122 }
1123 }
1124}
1125
1126#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1129pub(crate) struct ProcessIdentity {
1130 pub(crate) pid: u32,
1131 pub(crate) start_time: u64,
1132}
1133
1134#[derive(Debug, Clone, PartialEq, Eq)]
1136pub struct ExitReport {
1137 pub kind: ExitKind,
1138 pub code: Option<i32>,
1139 pub signal: Option<i32>,
1140 pub at_ms: u64,
1141}
1142
1143#[derive(Debug, Clone, PartialEq)]
1146pub struct ModuleStatus {
1147 pub module_id: String,
1148 pub waiting_for_previous_pid: Option<u32>,
1150 pub waiting_for_previous_detail: Option<String>,
1152 pub launch_nonce_source: Option<String>,
1157 pub state: ModuleState,
1158 pub enabled: bool,
1159 pub process_alive: bool,
1160 pub registration_active: bool,
1161 pub protocol: ModuleProtocol,
1166 pub live: bool,
1177 pub restart_count: u32,
1181 pub lifetime_restarts: u32,
1185 pub spawn_generation: u64,
1186 pub max_restarts: u32,
1191 pub restart_window: Duration,
1195 pub drain_timeout: Duration,
1199 pub restart_backoff: Duration,
1200 pub restart_max_backoff: Duration,
1201 pub pid: Option<u32>,
1206 pub spawned_at_ms: Option<u64>,
1207 pub spawned_from: Option<PathBuf>,
1208 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1209 pub process_start_time: Option<u64>,
1210 pub last_exit: Option<ExitReport>,
1211 pub health: ModuleHealthStatus,
1212}
1213
1214#[cfg(test)]
1215#[derive(Debug, Clone, Copy, PartialEq)]
1216struct ActorSelectCheckpoint {
1217 generation: u64,
1218 turn: u64,
1219 registered_connection: Option<ConnectionId>,
1220 next_probe_at: Option<Instant>,
1221 wake_after: Option<Duration>,
1222}
1223
1224#[derive(Debug, Clone, PartialEq)]
1225struct SupervisorSnapshot {
1226 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1227 #[cfg(windows)]
1228 pipe_consumed: Option<Arc<OnceLock<()>>>,
1229 #[cfg(test)]
1231 actor_turns: u64,
1232 #[cfg(test)]
1235 actor_select: Option<ActorSelectCheckpoint>,
1236 state: ModuleState,
1237 enabled: bool,
1238 process_alive: bool,
1239 spawned_protocol: Option<ModuleProtocol>,
1240 spawn_failure: Option<String>,
1241 waiting_for_previous_pid: Option<u32>,
1242 waiting_for_previous_detail: Option<String>,
1243 crash_restarts: VecDeque<Instant>,
1249 lifetime_restarts: u32,
1250 spawn_generation: u64,
1259 pid: Option<u32>,
1260 #[cfg(target_os = "macos")]
1261 report_ready: Option<Arc<OnceLock<()>>>,
1262 reaped_pid: Option<u32>,
1264 respawn_pending: bool,
1266 coalesced_restart_pending: bool,
1268 spawned_at_ms: Option<u64>,
1269 spawned_from: Option<PathBuf>,
1270 launch_spec_digest: Option<LaunchSpecDigest>,
1271 spawned_file_identity: Option<SpawnedFileIdentity>,
1272 process_start_time: Option<u64>,
1273 deliberate_severance: Option<ProcessIdentity>,
1274 last_exit: Option<ExitReport>,
1275 drain_disposition_detail: Option<String>,
1277 health: ModuleHealthStatus,
1278 in_alternate_slot: bool,
1283 draining_to_replace: bool,
1290 configuration_updated_since_spawn: bool,
1296}
1297
1298impl SupervisorSnapshot {
1299 fn reported_pid(&self) -> Option<u32> {
1304 #[cfg(target_os = "macos")]
1305 if self
1306 .report_ready
1307 .as_ref()
1308 .is_some_and(|ready| ready.get().is_none())
1309 {
1310 return None;
1311 }
1312 self.pid
1313 }
1314
1315 fn starting() -> Self {
1316 Self::new(ModuleState::Starting, true)
1317 }
1318
1319 fn disabled() -> Self {
1320 Self::new(ModuleState::Disabled, false)
1321 }
1322
1323 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1327 while let Some(oldest) = self.crash_restarts.front() {
1328 if now.duration_since(*oldest) > window {
1329 self.crash_restarts.pop_front();
1330 } else {
1331 break;
1332 }
1333 }
1334 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1335 }
1336
1337 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1343 self.crash_restarts.push_back(now);
1344 while self.crash_restarts.len() > policy.max_restarts as usize {
1345 self.crash_restarts.pop_front();
1346 }
1347 self.lifetime_restarts += 1;
1348 }
1349
1350 fn next_crash_restart(
1354 &mut self,
1355 policy: &RestartPolicy,
1356 now: Instant,
1357 ) -> Option<CrashRestartSchedule> {
1358 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1359 if restart_in_window >= policy.max_restarts {
1360 return None;
1361 }
1362 self.record_crash_restart(policy, now);
1363 Some(CrashRestartSchedule {
1364 restart_in_window,
1365 delay: policy.delay_for_restart(restart_in_window),
1366 })
1367 }
1368
1369 fn clear_crash_restarts(&mut self) {
1374 self.crash_restarts.clear();
1375 }
1376
1377 fn new(state: ModuleState, enabled: bool) -> Self {
1378 Self {
1379 exec_outcome: None,
1380 #[cfg(test)]
1381 actor_turns: 0,
1382 #[cfg(test)]
1383 actor_select: None,
1384 state,
1385 enabled,
1386 process_alive: false,
1387 #[cfg(windows)]
1388 pipe_consumed: None,
1389 spawned_protocol: None,
1390 spawn_failure: None,
1391 waiting_for_previous_pid: None,
1392 waiting_for_previous_detail: None,
1393 crash_restarts: VecDeque::new(),
1394 lifetime_restarts: 0,
1395 spawn_generation: 0,
1396 pid: None,
1397 #[cfg(target_os = "macos")]
1398 report_ready: None,
1399 reaped_pid: None,
1400 respawn_pending: false,
1401 coalesced_restart_pending: false,
1402 spawned_at_ms: None,
1403 spawned_from: None,
1404 launch_spec_digest: None,
1405 spawned_file_identity: None,
1406 process_start_time: None,
1407 deliberate_severance: None,
1408 last_exit: None,
1409 drain_disposition_detail: None,
1410 health: ModuleHealthStatus::default(),
1411 in_alternate_slot: false,
1412 draining_to_replace: false,
1413 configuration_updated_since_spawn: false,
1414 }
1415 }
1416}
1417
1418type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1419
1420type SpawnSubscriberKey = (ConnectionId, u64);
1421
1422#[derive(Debug)]
1423struct SpawnSubscriber {
1424 version: u8,
1425 frames: mpsc::Sender<Frame>,
1426 lagged: Option<oneshot::Sender<SpawnCursor>>,
1430}
1431
1432#[derive(Debug)]
1433struct SpawnEventState {
1434 daemon_incarnation: String,
1435 seq: u64,
1436 capacity: usize,
1437 live: HashMap<String, LiveSpawn>,
1438 generations: HashMap<String, u64>,
1439 events: VecDeque<SpawnEvent>,
1440 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1441}
1442
1443impl Default for SpawnEventState {
1444 fn default() -> Self {
1445 Self {
1446 daemon_incarnation: "unconfigured".to_string(),
1447 seq: 0,
1448 capacity: SPAWN_EVENT_RING_CAPACITY,
1449 live: HashMap::new(),
1450 generations: HashMap::new(),
1451 events: VecDeque::new(),
1452 subscribers: HashMap::new(),
1453 }
1454 }
1455}
1456
1457#[derive(Debug, Clone, Default)]
1458struct SpawnEventFeed(
1459 Arc<Mutex<SpawnEventState>>,
1460 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1461);
1462
1463#[derive(Debug, Clone, PartialEq, Eq)]
1464pub(crate) enum SpawnSubscribeRefusal {
1465 ForeignIncarnation { current: String },
1466 TooOld { oldest: SpawnCursor },
1467 Frame(String),
1468}
1469
1470impl SpawnEventFeed {
1471 fn configure_incarnation(&self, daemon_incarnation: String) {
1472 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1473 state.daemon_incarnation = daemon_incarnation;
1474 state.seq = 0;
1475 state.live.clear();
1476 state.generations.clear();
1477 state.events.clear();
1478 state.subscribers.clear();
1479 }
1480
1481 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1482 SpawnCursor {
1483 daemon_incarnation: state.daemon_incarnation.clone(),
1484 seq: state.seq,
1485 }
1486 }
1487
1488 fn snapshot(&self) -> SpawnSnapshot {
1489 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1490 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1491 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1492 SpawnSnapshot {
1493 cursor: Self::cursor(&state),
1494 ring_bound: state.capacity as u64,
1495 live,
1496 }
1497 }
1498
1499 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1500 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1501 let generation = state
1502 .generations
1503 .get(module_id)
1504 .copied()
1505 .unwrap_or(0)
1506 .checked_add(1)
1507 .expect("spawn generation exhausted");
1508 state.generations.insert(module_id.to_string(), generation);
1509 let live = LiveSpawn {
1510 module_id: module_id.to_string(),
1511 spawn_generation: generation,
1512 pid,
1513 spawned_at_ms,
1514 };
1515 state.live.insert(module_id.to_string(), live);
1516 Self::emit_locked(
1517 &mut state,
1518 SpawnEventKind::Spawned,
1519 module_id.to_string(),
1520 generation,
1521 pid,
1522 None,
1523 None,
1524 );
1525 generation
1526 }
1527
1528 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1529 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1530 let Some(live) = state.live.remove(module_id) else {
1531 warn!(
1532 module_id,
1533 "terminal record had no live spawn event identity"
1534 );
1535 return;
1536 };
1537 Self::emit_locked(
1538 &mut state,
1539 SpawnEventKind::Exited,
1540 module_id.to_string(),
1541 live.spawn_generation,
1542 live.pid,
1543 exit_code,
1544 exit_signal,
1545 );
1546 }
1547
1548 fn emit_superseded_exited(
1555 &self,
1556 module_id: &str,
1557 spawn_generation: u64,
1558 pid: u32,
1559 exit_code: Option<i32>,
1560 exit_signal: Option<i32>,
1561 ) {
1562 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1563 if state
1564 .live
1565 .get(module_id)
1566 .is_some_and(|live| live.spawn_generation == spawn_generation)
1567 {
1568 state.live.remove(module_id);
1569 }
1570 Self::emit_locked(
1571 &mut state,
1572 SpawnEventKind::Exited,
1573 module_id.to_string(),
1574 spawn_generation,
1575 pid,
1576 exit_code,
1577 exit_signal,
1578 );
1579 }
1580
1581 #[allow(clippy::too_many_arguments)]
1582 fn emit_locked(
1583 state: &mut SpawnEventState,
1584 kind: SpawnEventKind,
1585 module_id: String,
1586 spawn_generation: u64,
1587 pid: u32,
1588 exit_code: Option<i32>,
1589 exit_signal: Option<i32>,
1590 ) {
1591 state.seq = state
1592 .seq
1593 .checked_add(1)
1594 .expect("spawn event sequence exhausted");
1595 let event = SpawnEvent {
1596 cursor: Self::cursor(state),
1597 kind,
1598 module_id,
1599 spawn_generation,
1600 pid,
1601 exit_code,
1602 exit_signal,
1603 };
1604 state.events.push_back(event.clone());
1605 while state.events.len() > state.capacity {
1606 state.events.pop_front();
1607 }
1608 let body = match serde_json::to_vec(&event) {
1609 Ok(body) => body,
1610 Err(error) => {
1611 error!(%error, "failed to serialize supervisor spawn event");
1612 return;
1613 }
1614 };
1615 state.subscribers.retain(|(connection_id, corr), subscriber| {
1616 let frame = Frame::build_with_version(
1617 subscriber.version,
1618 FrameType::StreamData,
1619 control_flags(),
1620 0,
1621 0,
1622 *corr,
1623 body.clone(),
1624 );
1625 match frame {
1626 Ok(frame) => {
1627 if subscriber.frames.try_send(frame).is_ok() {
1628 true
1629 } else {
1630 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1631 if let Some(lagged) = subscriber.lagged.take() {
1632 let _ = lagged.send(event.cursor.clone());
1633 }
1634 false
1635 }
1636 }
1637 Err(error) => {
1638 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1639 false
1640 }
1641 }
1642 });
1643 }
1644
1645 fn subscribe(
1646 &self,
1647 connection_id: ConnectionId,
1648 corr: u64,
1649 version: u8,
1650 since: Option<SpawnCursor>,
1651 sink: FrameSink,
1652 ) -> Result<(), SpawnSubscribeRefusal> {
1653 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1654 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1655 {
1656 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1657 let replay = if let Some(since) = since {
1658 if since.daemon_incarnation != state.daemon_incarnation {
1659 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1660 current: state.daemon_incarnation.clone(),
1661 });
1662 }
1663 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1664 if since.seq < oldest.seq.saturating_sub(1) {
1665 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1666 }
1667 }
1668 state
1669 .events
1670 .iter()
1671 .filter(|event| event.cursor.seq > since.seq)
1672 .cloned()
1673 .collect::<Vec<_>>()
1674 } else {
1675 Vec::new()
1676 };
1677 for event in replay {
1678 let body = serde_json::to_vec(&event)
1679 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1680 let frame = Frame::build_with_version(
1681 version,
1682 FrameType::StreamData,
1683 control_flags(),
1684 0,
1685 0,
1686 corr,
1687 body,
1688 )
1689 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1690 frames
1691 .try_send(frame)
1692 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693 }
1694 state.subscribers.insert(
1695 (connection_id, corr),
1696 SpawnSubscriber {
1697 version,
1698 frames,
1699 lagged: Some(lagged),
1700 },
1701 );
1702 }
1703 tokio::spawn(async move {
1714 while let Some(frame) = receiver.recv().await {
1715 if sink.send(frame).await.is_err() {
1716 return;
1717 }
1718 }
1719 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1720 return;
1721 };
1722 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1723 Ok(frame) => {
1724 let _ = sink.send(frame).await;
1725 }
1726 Err(error) => {
1727 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1728 }
1729 }
1730 });
1731 Ok(())
1732 }
1733
1734 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1735 let Some(subscriber) = self
1736 .0
1737 .lock()
1738 .unwrap_or_else(|p| p.into_inner())
1739 .subscribers
1740 .remove(&(connection_id, corr))
1741 else {
1742 return false;
1743 };
1744 if let Ok(frame) = Frame::build_with_version(
1745 subscriber.version,
1746 FrameType::StreamEnd,
1747 control_flags(),
1748 0,
1749 0,
1750 corr,
1751 Vec::new(),
1752 ) {
1753 tokio::spawn(async move {
1754 let _ = subscriber.frames.send(frame).await;
1755 });
1756 }
1757 true
1758 }
1759
1760 fn remove_connection(&self, connection_id: ConnectionId) {
1761 self.0
1762 .lock()
1763 .unwrap_or_else(|p| p.into_inner())
1764 .subscribers
1765 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1766 }
1767
1768 #[cfg(any(test, feature = "test-support"))]
1769 fn set_capacity(&self, capacity: usize) {
1770 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1771 }
1772
1773 #[cfg(any(test, feature = "test-support"))]
1774 fn subscriber_count(&self) -> usize {
1775 self.0
1776 .lock()
1777 .unwrap_or_else(|p| p.into_inner())
1778 .subscribers
1779 .len()
1780 }
1781}
1782
1783fn spawn_subscriber_lagged_frame(
1786 version: u8,
1787 corr: u64,
1788 first_undelivered: SpawnCursor,
1789) -> Result<Frame, String> {
1790 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1791 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1792 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1793 .to_string(),
1794 detail: Some(serde_json::json!({
1795 "first_undelivered_cursor": first_undelivered
1796 })),
1797 })
1798 .map_err(|error| error.to_string())?;
1799 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1800 .map_err(|error| error.to_string())
1801}
1802
1803pub trait ModuleProcessLiveness: Send + Sync {
1804 fn process_live(&self, module_id: &str) -> Option<bool>;
1805
1806 fn process_replacing(&self, _module_id: &str) -> bool {
1812 false
1813 }
1814}
1815
1816#[derive(Debug, Clone, Default)]
1818pub struct SupervisorProcessLiveness {
1819 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1820}
1821
1822impl SupervisorProcessLiveness {
1823 pub fn new() -> Self {
1824 Self::default()
1825 }
1826
1827 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1828 let mut snapshots = self
1829 .snapshots
1830 .lock()
1831 .unwrap_or_else(|poisoned| poisoned.into_inner());
1832 snapshots.insert(module_id, snapshot);
1833 }
1834
1835 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1836 let mut snapshots = self
1837 .snapshots
1838 .lock()
1839 .unwrap_or_else(|poisoned| poisoned.into_inner());
1840 let is_current = snapshots
1841 .get(module_id)
1842 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1843 .unwrap_or(false);
1844 if is_current {
1845 snapshots.remove(module_id);
1846 }
1847 }
1848}
1849
1850impl ModuleProcessLiveness for SupervisorProcessLiveness {
1851 fn process_live(&self, module_id: &str) -> Option<bool> {
1852 let snapshot = {
1853 let snapshots = self
1854 .snapshots
1855 .lock()
1856 .unwrap_or_else(|poisoned| poisoned.into_inner());
1857 snapshots.get(module_id).cloned()
1858 }?;
1859 let snapshot = snapshot
1860 .lock()
1861 .unwrap_or_else(|poisoned| poisoned.into_inner());
1862 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1863 }
1864
1865 fn process_replacing(&self, module_id: &str) -> bool {
1866 let Some(snapshot) = self
1867 .snapshots
1868 .lock()
1869 .unwrap_or_else(|poisoned| poisoned.into_inner())
1870 .get(module_id)
1871 .cloned()
1872 else {
1873 return false;
1874 };
1875 let snapshot = snapshot
1876 .lock()
1877 .unwrap_or_else(|poisoned| poisoned.into_inner());
1878 snapshot.enabled
1879 && match snapshot.state {
1880 ModuleState::Restarting => true,
1881 ModuleState::Draining => snapshot.draining_to_replace,
1882 ModuleState::Starting
1883 | ModuleState::Running
1884 | ModuleState::Unresponsive
1885 | ModuleState::Stopped
1886 | ModuleState::Failed
1887 | ModuleState::Disabled => false,
1888 }
1889 }
1890}
1891
1892#[cfg(test)]
1893#[derive(Debug, Default)]
1894struct ReloadExitRecordGate {
1895 reached: tokio::sync::Notify,
1896 resume: tokio::sync::Notify,
1897}
1898
1899#[derive(Debug, Clone, Copy)]
1900enum RespawnKind {
1901 Spawn,
1902 Reload,
1903}
1904
1905#[derive(Debug, Clone, Copy)]
1906struct PendingRespawn {
1907 deadline: Instant,
1908 kind: RespawnKind,
1909}
1910
1911type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1912
1913#[derive(Debug, Clone)]
1914struct SupervisorRuntimeConfig {
1915 start_requester: Option<Principal>,
1916 pending_activity_restart: bool,
1917 provenance_probe: ExecutableIdentityProbe,
1918 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1920 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1922 restart_policy: RestartPolicy,
1923 drain_timeout: Duration,
1926 effective_drain_timeout: Arc<Mutex<Duration>>,
1929 default_drain_timeout: Duration,
1932 health: HealthConfig,
1933 connection_file_path: Option<PathBuf>,
1934 capture_logs_dir: Option<PathBuf>,
1935 forwarding: Option<Arc<ForwardingTable>>,
1936 supervisor_handle: Option<SupervisorHandle>,
1939 stderr_ring: Arc<Mutex<StderrRing>>,
1946 terminal_ring: Arc<Mutex<TerminalRing>>,
1947 spawn_events: SpawnEventFeed,
1948 child_roster: ChildRoster,
1949 #[cfg(target_os = "linux")]
1950 cgroup_placement: Option<subc_cgroup::Placement>,
1951 #[cfg(test)]
1952 test_seed_stale_facts_before_enable_spawn: bool,
1953 #[cfg(test)]
1954 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1955}
1956
1957#[derive(Debug, Clone, PartialEq, Eq)]
1958struct SupervisedConfiguration {
1959 spec: ModuleSpec,
1960 health: HealthConfig,
1961}
1962
1963#[derive(Debug, Clone, Default)]
1969pub struct SupervisorHandle {
1970 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1971 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1972 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1973 configured_ids: Arc<Mutex<HashSet<String>>>,
1985 spawn_events: SpawnEventFeed,
1986 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
1997 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2003 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2007 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2015 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2021 promotion_observer: PromotionObserverSlot,
2023 operation_lock: Arc<AsyncMutex<()>>,
2027}
2028
2029pub(crate) trait SwapPromotionObserver: Send + Sync {
2038 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2039}
2040
2041#[derive(Clone, Default)]
2045struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2046
2047impl fmt::Debug for PromotionObserverSlot {
2048 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2049 f.write_str("PromotionObserverSlot")
2050 }
2051}
2052
2053#[derive(Debug, Clone)]
2055struct OpenSwap {
2056 candidate_nonce: String,
2059 incumbent_nonce: Option<String>,
2064 candidate_admitted: bool,
2068}
2069
2070#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2073pub(crate) enum SwapHelloAdmission {
2074 NotSwapping,
2077 Candidate,
2079 Refused,
2082}
2083
2084#[derive(Debug, Clone, PartialEq, Eq)]
2085pub(crate) enum ReservedHelloRejection {
2086 Exact {
2087 module_id: String,
2088 },
2089 Prefix {
2090 prefix: String,
2091 owner_module_id: String,
2092 },
2093}
2094
2095impl SupervisorHandle {
2096 pub fn new() -> Self {
2097 Self::default()
2098 }
2099
2100 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2101 self.spawn_events.snapshot()
2102 }
2103
2104 pub(crate) fn subscribe_spawns(
2105 &self,
2106 connection_id: ConnectionId,
2107 corr: u64,
2108 version: u8,
2109 since: Option<SpawnCursor>,
2110 sink: FrameSink,
2111 ) -> Result<(), SpawnSubscribeRefusal> {
2112 self.spawn_events
2113 .subscribe(connection_id, corr, version, since, sink)
2114 }
2115
2116 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2117 self.spawn_events.cancel(connection_id, corr)
2118 }
2119
2120 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2121 self.spawn_events.remove_connection(connection_id);
2122 }
2123
2124 #[cfg(any(test, feature = "test-support"))]
2125 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2126 assert!(capacity > 0, "spawn event capacity must be non-zero");
2127 self.spawn_events.set_capacity(capacity);
2128 }
2129
2130 #[cfg(any(test, feature = "test-support"))]
2131 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2132 self.spawn_events.subscriber_count()
2133 }
2134
2135 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2138 self.spawn_nonces
2139 .lock()
2140 .unwrap_or_else(|poisoned| poisoned.into_inner())
2141 .insert(module_id.to_string(), nonce);
2142 }
2143
2144 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2147 self.reserved_nonces
2148 .lock()
2149 .unwrap_or_else(|poisoned| poisoned.into_inner())
2150 .insert(module_id.to_string(), Some(nonce));
2151 }
2152
2153 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2155 let mut owners = self
2156 .reserved_prefix_owners
2157 .lock()
2158 .unwrap_or_else(|poisoned| poisoned.into_inner());
2159 owners.retain(|_, owner| owner != owner_module_id);
2160 for prefix in prefixes {
2161 owners.insert(prefix.clone(), owner_module_id.to_string());
2162 }
2163 }
2164
2165 #[cfg(test)]
2167 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2168 self.spawn_nonces
2169 .lock()
2170 .unwrap_or_else(|poisoned| poisoned.into_inner())
2171 .get(module_id)
2172 .cloned()
2173 }
2174
2175 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2176 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2177 let spawn_nonce = self
2178 .spawn_nonces
2179 .lock()
2180 .unwrap_or_else(|poisoned| poisoned.into_inner())
2181 .get(&spec.module_id)
2182 .cloned();
2183 let mut reserved_nonces = self
2184 .reserved_nonces
2185 .lock()
2186 .unwrap_or_else(|poisoned| poisoned.into_inner());
2187 if spec.reserved {
2188 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2193 }
2194 drop(reserved_nonces);
2195 self.removal_tombstones
2199 .lock()
2200 .unwrap_or_else(|poisoned| poisoned.into_inner())
2201 .remove(&spec.module_id);
2202 }
2203
2204 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2209 self.reserved_hello_rejection(module_id, presented)
2210 .is_none()
2211 }
2212
2213 pub(crate) fn reserved_hello_rejection(
2214 &self,
2215 module_id: &str,
2216 presented: Option<&str>,
2217 ) -> Option<ReservedHelloRejection> {
2218 let nonces = self
2219 .reserved_nonces
2220 .lock()
2221 .unwrap_or_else(|poisoned| poisoned.into_inner());
2222 if let Some(expected) = nonces.get(module_id) {
2223 let authorized = match expected {
2227 Some(expected) => {
2228 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2229 }
2230 None => false,
2231 };
2232 if authorized {
2233 return None;
2234 }
2235 return Some(ReservedHelloRejection::Exact {
2236 module_id: module_id.to_string(),
2237 });
2238 }
2239 drop(nonces);
2240
2241 let matched_prefix = self
2242 .reserved_prefix_owners
2243 .lock()
2244 .unwrap_or_else(|poisoned| poisoned.into_inner())
2245 .iter()
2246 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2247 .max_by_key(|(prefix, _)| prefix.len())
2248 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2249 let (prefix, owner_module_id) = matched_prefix?;
2250
2251 let authorized = presented.is_some_and(|presented| {
2252 self.spawn_nonces
2253 .lock()
2254 .unwrap_or_else(|poisoned| poisoned.into_inner())
2255 .get(&owner_module_id)
2256 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2257 || self.swap_nonce_matches(&owner_module_id, presented)
2260 });
2261 if authorized {
2262 None
2263 } else {
2264 Some(ReservedHelloRejection::Prefix {
2265 prefix,
2266 owner_module_id,
2267 })
2268 }
2269 }
2270
2271 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2276 if presented.is_empty() {
2277 return false;
2278 }
2279 let nonces = self
2280 .spawn_nonces
2281 .lock()
2282 .unwrap_or_else(|poisoned| poisoned.into_inner());
2283 let current = nonces
2284 .get(module_id)
2285 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2286 drop(nonces);
2287 current || self.swap_nonce_matches(module_id, presented)
2292 }
2293
2294 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2296 let swaps = self
2297 .swaps
2298 .lock()
2299 .unwrap_or_else(|poisoned| poisoned.into_inner());
2300 swaps.get(module_id).is_some_and(|swap| {
2301 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2302 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2303 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2304 })
2305 })
2306 }
2307
2308 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2311 let incumbent_nonce = self
2312 .spawn_nonces
2313 .lock()
2314 .unwrap_or_else(|poisoned| poisoned.into_inner())
2315 .get(module_id)
2316 .cloned();
2317 self.swaps
2318 .lock()
2319 .unwrap_or_else(|poisoned| poisoned.into_inner())
2320 .insert(
2321 module_id.to_string(),
2322 OpenSwap {
2323 candidate_nonce,
2324 incumbent_nonce,
2325 candidate_admitted: false,
2326 },
2327 );
2328 }
2329
2330 pub(crate) fn close_swap(&self, module_id: &str) {
2333 self.swaps
2334 .lock()
2335 .unwrap_or_else(|poisoned| poisoned.into_inner())
2336 .remove(module_id);
2337 }
2338
2339 pub(crate) fn set_swap_promotion_observer(
2342 &self,
2343 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2344 ) {
2345 *self
2346 .promotion_observer
2347 .0
2348 .lock()
2349 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2350 }
2351
2352 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2355 let observer = self
2356 .promotion_observer
2357 .0
2358 .lock()
2359 .unwrap_or_else(|poisoned| poisoned.into_inner())
2360 .as_ref()
2361 .and_then(std::sync::Weak::upgrade);
2362 if let Some(observer) = observer {
2363 observer.swap_promoted(registration);
2364 }
2365 }
2366
2367 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2369 self.swaps
2370 .lock()
2371 .unwrap_or_else(|poisoned| poisoned.into_inner())
2372 .contains_key(module_id)
2373 }
2374
2375 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2380 let candidate_nonce = self
2381 .swaps
2382 .lock()
2383 .unwrap_or_else(|poisoned| poisoned.into_inner())
2384 .get(module_id)
2385 .map(|swap| swap.candidate_nonce.clone());
2386 let Some(nonce) = candidate_nonce else {
2387 return;
2388 };
2389 self.set_spawn_nonce(module_id, nonce.clone());
2390 if reserved {
2391 self.set_reserved_nonce(module_id, nonce);
2392 }
2393 }
2394
2395 pub(crate) fn swap_hello_admission(
2410 &self,
2411 module_id: &str,
2412 presented: Option<&str>,
2413 ) -> SwapHelloAdmission {
2414 let swaps = self
2415 .swaps
2416 .lock()
2417 .unwrap_or_else(|poisoned| poisoned.into_inner());
2418 let Some(swap) = swaps.get(module_id) else {
2419 return SwapHelloAdmission::NotSwapping;
2420 };
2421 let Some(presented) = presented else {
2422 return SwapHelloAdmission::Refused;
2423 };
2424 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2425 return if swap.candidate_admitted {
2426 SwapHelloAdmission::Refused
2427 } else {
2428 SwapHelloAdmission::Candidate
2429 };
2430 }
2431 if swap
2432 .incumbent_nonce
2433 .as_deref()
2434 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2435 {
2436 return SwapHelloAdmission::NotSwapping;
2437 }
2438 SwapHelloAdmission::Refused
2439 }
2440
2441 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2444 if let Some(swap) = self
2445 .swaps
2446 .lock()
2447 .unwrap_or_else(|poisoned| poisoned.into_inner())
2448 .get_mut(module_id)
2449 {
2450 swap.candidate_admitted = true;
2451 }
2452 }
2453
2454 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2456 self.spawn_nonces
2457 .lock()
2458 .unwrap_or_else(|poisoned| poisoned.into_inner())
2459 .get(module_id)
2460 .cloned()
2461 }
2462
2463 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2465 self.reserved_nonces
2466 .lock()
2467 .unwrap_or_else(|poisoned| poisoned.into_inner())
2468 .get(module_id)
2469 .cloned()
2470 .flatten()
2471 }
2472
2473 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2474 self.mark_configured(module.module_id());
2478 let mut modules = self
2479 .modules
2480 .lock()
2481 .unwrap_or_else(|poisoned| poisoned.into_inner());
2482 modules.insert(module.module_id().to_string(), module)
2483 }
2484
2485 fn mark_configured(&self, module_id: &str) {
2489 self.configured_ids
2490 .lock()
2491 .unwrap_or_else(|poisoned| poisoned.into_inner())
2492 .insert(module_id.to_string());
2493 }
2494
2495 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2499 let modules = self
2500 .modules
2501 .lock()
2502 .unwrap_or_else(|poisoned| poisoned.into_inner());
2503 if !modules.contains_key(module_id) {
2504 self.configured_ids
2505 .lock()
2506 .unwrap_or_else(|poisoned| poisoned.into_inner())
2507 .remove(module_id);
2508 }
2509 }
2510
2511 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2519 self.configured_ids
2520 .lock()
2521 .unwrap_or_else(|poisoned| poisoned.into_inner())
2522 .contains(module_id)
2523 }
2524
2525 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2526 let modules = self
2527 .modules
2528 .lock()
2529 .unwrap_or_else(|poisoned| poisoned.into_inner());
2530 modules.get(module_id).cloned()
2531 }
2532
2533 pub(crate) fn record_late_health_answer(
2534 &self,
2535 module_id: &str,
2536 latency_ms: u64,
2537 ) -> Result<bool, SuperviseError> {
2538 let Some(module) = self.get(module_id) else {
2539 return Ok(false);
2540 };
2541 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2542 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2543 state.health.last_late_answer_latency_ms = Some(latency_ms);
2544 state.health.consecutive_failures = 0;
2552 })?;
2553 Ok(true)
2554 }
2555
2556 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2562 let Some(module) = self.get(module_id) else {
2563 return Ok(false);
2564 };
2565 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2566 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2567 return Ok(false);
2568 };
2569 drop(snapshot);
2570 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2571 }
2572
2573 pub fn list(&self) -> Vec<SupervisedModule> {
2574 let modules = self
2575 .modules
2576 .lock()
2577 .unwrap_or_else(|poisoned| poisoned.into_inner());
2578 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2579 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2580 modules
2581 }
2582
2583 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2584 self.activity_images
2585 .lock()
2586 .unwrap_or_else(|p| p.into_inner())
2587 .remove(module_id);
2588 self.spawn_nonces
2589 .lock()
2590 .unwrap_or_else(|poisoned| poisoned.into_inner())
2591 .remove(module_id);
2592 self.close_swap(module_id);
2593 let mut reserved_nonces = self
2594 .reserved_nonces
2595 .lock()
2596 .unwrap_or_else(|poisoned| poisoned.into_inner());
2597 if reserved_nonces.contains_key(module_id) {
2598 reserved_nonces.insert(module_id.to_string(), None);
2601 }
2602 drop(reserved_nonces);
2603 self.reserved_prefix_owners
2604 .lock()
2605 .unwrap_or_else(|poisoned| poisoned.into_inner())
2606 .retain(|_, owner| owner != module_id);
2607 let removed = self
2608 .modules
2609 .lock()
2610 .unwrap_or_else(|poisoned| poisoned.into_inner())
2611 .remove(module_id);
2612 self.configured_ids
2613 .lock()
2614 .unwrap_or_else(|poisoned| poisoned.into_inner())
2615 .remove(module_id);
2616 removed
2617 }
2618
2619 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2622 self.removal_tombstones
2623 .lock()
2624 .unwrap_or_else(|poisoned| poisoned.into_inner())
2625 .insert(module_id.to_string(), unix_ms_now());
2626 }
2627
2628 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2630 self.removal_tombstones
2631 .lock()
2632 .unwrap_or_else(|poisoned| poisoned.into_inner())
2633 .get(module_id)
2634 .copied()
2635 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2636 }
2637
2638 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2643 if self.get(module_id).is_some() {
2644 return false;
2645 }
2646 let mut reserved_nonces = self
2647 .reserved_nonces
2648 .lock()
2649 .unwrap_or_else(|poisoned| poisoned.into_inner());
2650 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2651 return false;
2652 }
2653 reserved_nonces.remove(module_id);
2654 true
2655 }
2656
2657 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2658 Arc::clone(&self.operation_lock)
2659 }
2660}
2661
2662#[derive(Debug, Clone)]
2664pub struct Supervisor {
2665 registry: Arc<Registry>,
2666 restart_policy: RestartPolicy,
2667 drain_timeout: Duration,
2668 connection_file_path: Option<PathBuf>,
2669 capture_logs_dir: Option<PathBuf>,
2670 forwarding: Option<Arc<ForwardingTable>>,
2671 process_liveness: Arc<SupervisorProcessLiveness>,
2672 supervisor_handle: Option<SupervisorHandle>,
2673 health: HealthConfig,
2674 daemon_start_clock: crate::clock::StartClock,
2675 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2676 spawn_events: SpawnEventFeed,
2677 provenance_probe: ExecutableIdentityProbe,
2678 child_roster: ChildRoster,
2681 #[cfg(target_os = "linux")]
2682 cgroup_placement: Option<subc_cgroup::Placement>,
2683 #[cfg(test)]
2684 test_after_first_spawn: AfterFirstSpawnHook,
2685}
2686
2687pub(crate) struct RegisteredConfiguredModule {
2691 spec: ModuleSpec,
2692 runtime: SupervisorRuntimeConfig,
2693 module: SupervisedModule,
2694 commands: mpsc::Receiver<SupervisorCommand>,
2695}
2696
2697#[cfg(test)]
2703#[derive(Clone, Default)]
2704struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2705
2706#[cfg(test)]
2707type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2708
2709#[cfg(test)]
2710impl fmt::Debug for AfterFirstSpawnHook {
2711 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2712 f.write_str("AfterFirstSpawnHook")
2713 }
2714}
2715
2716#[cfg(test)]
2717impl AfterFirstSpawnHook {
2718 fn run(&self, module_id: &str) {
2719 if let Some(hook) = &self.0 {
2720 hook(module_id);
2721 }
2722 }
2723}
2724
2725impl Supervisor {
2726 #[cfg(test)]
2727 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2728 let supervisor = Self::new(registry, policy);
2729 #[cfg(target_os = "macos")]
2730 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2731 supervisor
2732 }
2733 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2738 let path = path.into();
2739 #[cfg(target_os = "macos")]
2740 {
2741 let result = probe_privacy_trampoline(&path).map(|()| path);
2742 if let Err(cause) = &result {
2743 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2744 }
2745 self.child_roster.set_privacy_trampoline(result);
2746 }
2747 #[cfg(not(target_os = "macos"))]
2748 let _ = path;
2749 self
2750 }
2751 #[cfg(unix)]
2762 pub(crate) fn begin_daemon_shutdown(&self) {
2763 self.child_roster.close();
2764 if let Some(journal) = &self.terminal_journal {
2765 journal.stamp_shutdown();
2766 }
2767 }
2768
2769 #[cfg(unix)]
2773 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2774 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2775 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2776 let Some(forwarding) = &self.forwarding else {
2777 return Ok(());
2778 };
2779 let modules = forwarding
2780 .begin_daemon_drain()
2781 .map_err(SuperviseError::Forwarding)?;
2782 let deadline_ms =
2783 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2784 let mut notices = tokio::task::JoinSet::new();
2785 let mut drains = Vec::new();
2786 for (module_id, endpoint) in modules {
2787 let Some(target) = forwarding
2788 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2789 .map_err(SuperviseError::Forwarding)?
2790 else {
2791 continue;
2792 };
2793 let routes = forwarding
2794 .endpoint_routes(target.endpoint)
2795 .map_err(SuperviseError::Forwarding)?;
2796 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2802 reason: RouteCloseReason::Restart,
2803 deadline_ms,
2804 })
2805 .expect("module draining serializes");
2806 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2807 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2808 for route in routes {
2809 let client = route.goodbye_target;
2810 if let Some((_, channels)) = clients
2811 .iter_mut()
2812 .find(|(existing, _)| existing.connection_id == client.connection_id)
2813 {
2814 channels.push(client.channel);
2815 } else {
2816 let channel = client.channel;
2817 clients.push((client, vec![channel]));
2818 }
2819 }
2820 for (client, mut channels) in clients {
2821 channels.sort_unstable();
2822 channels.dedup();
2823 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2824 module_id: module_id.clone(),
2825 channels,
2826 reason: RouteCloseReason::Restart,
2827 })
2828 .expect("route closing serializes");
2829 recipients.push((client.sink, client.negotiated_ver, closing));
2830 }
2831 for (sink, version, body) in recipients {
2832 notices.spawn(async move {
2833 let frame = Frame::build_with_version(
2834 version,
2835 FrameType::Push,
2836 control_flags(),
2837 0,
2838 0,
2839 0,
2840 body,
2841 )
2842 .expect("bounded lifecycle notice frame builds");
2843 sink.send_flushed(frame).await
2844 });
2845 }
2846 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2847 drains.push((module_id, target.endpoint, gauges));
2848 }
2849 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2852 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2853 if !matches!(result, Ok(Ok(()))) {
2854 warn!(?result, "daemon shutdown notice delivery failed");
2855 }
2856 }
2857 notices.abort_all();
2858 let deadline = Instant::now() + DRAIN_BUDGET;
2859 let mut waits = tokio::task::JoinSet::new();
2860 for (module_id, endpoint, gauges) in drains {
2861 let forwarding = Arc::clone(forwarding);
2862 let mut runtime = self.runtime_config();
2863 runtime.health.cadence = Duration::from_millis(100);
2864 waits.spawn(async move {
2865 let wait_result = wait_for_forwarding_quiescence(
2866 &forwarding,
2867 &module_id,
2868 &runtime,
2869 endpoint,
2870 deadline,
2871 &gauges,
2872 DrainScope::Endpoint(endpoint),
2873 )
2874 .await;
2875 forwarding
2879 .record_daemon_drain_result(
2880 endpoint,
2881 drained_after_quiescence_wait(&wait_result),
2882 )
2883 .map_err(SuperviseError::Forwarding)?;
2884 wait_result
2885 });
2886 }
2887 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2888 if !matches!(result, Ok(Ok(true))) {
2889 warn!(?result, "daemon shutdown drain did not reach quiescence");
2890 }
2891 }
2892 Ok(())
2893 }
2894
2895 #[cfg(unix)]
2907 pub(crate) async fn end_children_for_daemon_shutdown(
2908 &self,
2909 already_escalated: bool,
2910 escalate: impl std::future::Future<Output = ()>,
2911 ) {
2912 tokio::pin!(escalate);
2913 let mut escalated = already_escalated;
2914 if let Some(forwarding) = &self.forwarding {
2915 let reason = CloseReason::new(
2916 "daemon_shutdown",
2917 "the daemon is exiting after its shutdown notice and drain",
2918 );
2919 if escalated {
2920 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2923 } else {
2924 tokio::select! {
2925 biased;
2926 _ = escalate.as_mut() => {
2927 info!("second SIGTERM: abandoning module GOODBYE delivery");
2928 escalated = true;
2929 }
2930 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2931 }
2932 }
2933 let closed = forwarding.close_all_connections(&reason);
2934 debug!(closed, "closed established connections for daemon shutdown");
2935 }
2936 let escalated_here = escalated && !already_escalated;
2940 let remaining_escalate = async move {
2941 if escalated_here {
2942 std::future::pending::<()>().await;
2943 } else {
2944 escalate.await;
2945 }
2946 };
2947 crate::child_roster::end_children_for_daemon_shutdown(
2948 &self.child_roster,
2949 escalated,
2950 remaining_escalate,
2951 )
2952 .await;
2953 }
2954
2955 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2956 Self {
2957 registry,
2958 restart_policy,
2959 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2960 connection_file_path: None,
2961 capture_logs_dir: None,
2962 forwarding: None,
2963 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2964 supervisor_handle: None,
2965 health: HealthConfig::default(),
2966 daemon_start_clock: crate::clock::StartClock::capture(),
2967 terminal_journal: None,
2968 spawn_events: SpawnEventFeed::default(),
2969 provenance_probe: ExecutableIdentityProbe::default(),
2970 child_roster: ChildRoster::default(),
2971 #[cfg(target_os = "linux")]
2972 cgroup_placement: None,
2973 #[cfg(test)]
2974 test_after_first_spawn: AfterFirstSpawnHook::default(),
2975 }
2976 }
2977
2978 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2979 self.drain_timeout = drain_timeout;
2980 self
2981 }
2982
2983 pub fn with_process_liveness(
2984 mut self,
2985 process_liveness: Arc<SupervisorProcessLiveness>,
2986 ) -> Self {
2987 self.process_liveness = process_liveness;
2988 self
2989 }
2990
2991 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2992 self.connection_file_path = Some(connection_file_path.into());
2993 self
2994 }
2995
2996 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
2998 self.capture_logs_dir = Some(logs_dir.into());
2999 self
3000 }
3001
3002 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3005 self.spawn_events.configure_incarnation(daemon_incarnation);
3009 self
3010 }
3011
3012 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3015 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3016 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3017 path,
3018 daemon_incarnation,
3019 )));
3020 this
3021 }
3022
3023 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3024 self.forwarding = Some(forwarding);
3025 self
3026 }
3027
3028 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3029 self.spawn_events = supervisor_handle.spawn_events.clone();
3030 self.supervisor_handle = Some(supervisor_handle);
3031 self
3032 }
3033
3034 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3035 self.health = health;
3036 self
3037 }
3038
3039 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3043 self.child_roster.record_to(path.into());
3044 self
3045 }
3046
3047 #[cfg(target_os = "linux")]
3048 pub fn with_cgroup_placement(
3049 mut self,
3050 cgroup_placement: Option<subc_cgroup::Placement>,
3051 ) -> Self {
3052 self.cgroup_placement = cgroup_placement;
3053 self
3054 }
3055
3056 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3062 validate_spec(&spec)?;
3063 self.establish_identity(&spec);
3064
3065 let runtime = self.runtime_config();
3066 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3067 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3068 let spawned = spawn_child(
3069 &spec,
3070 runtime.connection_file_path.as_deref(),
3071 self.supervisor_handle.as_ref(),
3072 &runtime.stderr_ring,
3073 runtime.capture_logs_dir.as_deref(),
3074 &runtime.child_roster,
3075 #[cfg(target_os = "linux")]
3076 runtime.cgroup_placement.as_ref(),
3077 );
3078 #[cfg(test)]
3079 self.test_after_first_spawn.run(&spec.module_id);
3080 let child = match spawned {
3081 Ok(child) => child,
3082 Err(err) => {
3083 self.abandon_unrostered(&spec.module_id);
3086 return Err(err);
3087 }
3088 };
3089 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3090 if let Some(spawn) = activity_spawn {
3091 spawn.complete(&snapshot);
3092 }
3093
3094 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3095 }
3096
3097 fn establish_identity(&self, spec: &ModuleSpec) {
3110 if let Some(supervisor_handle) = &self.supervisor_handle {
3111 supervisor_handle.apply_identity_configuration(spec);
3112 supervisor_handle.mark_configured(&spec.module_id);
3113 }
3114 }
3115
3116 fn abandon_unrostered(&self, module_id: &str) {
3119 if let Some(supervisor_handle) = &self.supervisor_handle {
3120 supervisor_handle.unmark_configured_unless_rostered(module_id);
3121 }
3122 }
3123
3124 fn mark_first_process_running(
3127 &self,
3128 spec: &ModuleSpec,
3129 runtime: &SupervisorRuntimeConfig,
3130 snapshot: &SharedSnapshot,
3131 child: &SupervisedChild,
3132 ) -> Result<(), SuperviseError> {
3133 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3134 self.abandon_unrostered(&spec.module_id);
3135 return Err(err);
3136 }
3137 self.process_liveness
3138 .track(spec.module_id.clone(), Arc::clone(snapshot));
3139 Ok(())
3140 }
3141
3142 pub fn supervise_configured(
3148 &self,
3149 spec: ModuleSpec,
3150 enabled: bool,
3151 ) -> Result<SupervisedModule, SuperviseError> {
3152 self.supervise_configured_requested(
3153 spec,
3154 enabled,
3155 self.health.clone(),
3156 None,
3157 self.restart_policy,
3158 None,
3159 )
3160 }
3161
3162 pub fn supervise_configured_with_health(
3168 &self,
3169 spec: ModuleSpec,
3170 enabled: bool,
3171 health: HealthConfig,
3172 drain_timeout_ms: Option<u64>,
3173 restart_policy: RestartPolicy,
3174 ) -> Result<SupervisedModule, SuperviseError> {
3175 self.supervise_configured_requested(
3176 spec,
3177 enabled,
3178 health,
3179 drain_timeout_ms,
3180 restart_policy,
3181 None,
3182 )
3183 }
3184
3185 pub(crate) fn supervise_configured_requested(
3186 &self,
3187 spec: ModuleSpec,
3188 enabled: bool,
3189 health: HealthConfig,
3190 drain_timeout_ms: Option<u64>,
3191 restart_policy: RestartPolicy,
3192 requester: Option<Principal>,
3193 ) -> Result<SupervisedModule, SuperviseError> {
3194 let registered = self.register_configured_with_health(
3195 spec,
3196 enabled,
3197 health,
3198 drain_timeout_ms,
3199 restart_policy,
3200 )?;
3201 self.start_registered_configured(registered, requester)
3202 }
3203
3204 pub(crate) fn register_configured_with_health(
3209 &self,
3210 spec: ModuleSpec,
3211 enabled: bool,
3212 health: HealthConfig,
3213 drain_timeout_ms: Option<u64>,
3214 restart_policy: RestartPolicy,
3215 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3216 validate_spec(&spec)?;
3217 self.establish_identity(&spec);
3218
3219 let mut runtime = self.runtime_config();
3220 runtime.health = health;
3221 runtime.restart_policy = restart_policy;
3222 if let Some(ms) = drain_timeout_ms {
3223 runtime.drain_timeout = Duration::from_millis(ms);
3224 *runtime
3225 .effective_drain_timeout
3226 .lock()
3227 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3228 }
3229 let snapshot = if enabled {
3230 SupervisorSnapshot::starting()
3231 } else {
3232 SupervisorSnapshot::disabled()
3233 };
3234 let snapshot = {
3235 let mut snapshot = snapshot;
3236 if enabled {
3237 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3238 {
3239 snapshot.waiting_for_previous_pid = Some(pid);
3240 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3241 }
3242 }
3243 snapshot
3244 };
3245 let (module, commands) =
3246 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3247 Ok(RegisteredConfiguredModule {
3248 spec,
3249 runtime,
3250 module,
3251 commands,
3252 })
3253 }
3254
3255 pub(crate) fn start_registered_configured(
3258 &self,
3259 registered: RegisteredConfiguredModule,
3260 requester: Option<Principal>,
3261 ) -> Result<SupervisedModule, SuperviseError> {
3262 let RegisteredConfiguredModule {
3263 spec,
3264 mut runtime,
3265 module,
3266 commands,
3267 } = registered;
3268 runtime.start_requester = requester;
3269 let snapshot = &module.inner.snapshot;
3270 let should_spawn = {
3271 let state = lock_snapshot(snapshot)?;
3272 state.enabled && state.waiting_for_previous_pid.is_none()
3273 };
3274 let child = if should_spawn {
3275 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3276 let spawned = spawn_child(
3277 &spec,
3278 runtime.connection_file_path.as_deref(),
3279 self.supervisor_handle.as_ref(),
3280 &runtime.stderr_ring,
3281 runtime.capture_logs_dir.as_deref(),
3282 &runtime.child_roster,
3283 #[cfg(target_os = "linux")]
3284 runtime.cgroup_placement.as_ref(),
3285 );
3286 #[cfg(test)]
3287 self.test_after_first_spawn.run(&spec.module_id);
3288 match spawned {
3289 Ok(child) => {
3290 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3291 if let Some(spawn) = activity_spawn {
3292 spawn.complete(snapshot);
3293 }
3294 Some(child)
3295 }
3296 Err(err) => {
3297 if runtime.health.critical {
3298 error!(
3299 module_id = %spec.module_id,
3300 program = %spec.program.display(),
3301 error = %err,
3302 "critical configured module failed to spawn; marking failed and alerting"
3303 );
3304 } else {
3305 error!(
3306 module_id = %spec.module_id,
3307 program = %spec.program.display(),
3308 error = %err,
3309 "configured module failed to spawn; marking failed and continuing"
3310 );
3311 }
3312 update_snapshot(snapshot, Some(&spec.module_id), |state| {
3313 state.state = ModuleState::Failed;
3314 clear_current_process_facts(state);
3315 })?;
3316 None
3317 }
3318 }
3319 } else {
3320 None
3321 };
3322 self.start_monitor(spec, runtime, &module, child, commands);
3323 Ok(module)
3324 }
3325
3326 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3327 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3328 SupervisorRuntimeConfig {
3329 start_requester: None,
3330 pending_activity_restart: false,
3331 provenance_probe: self.provenance_probe.clone(),
3332 scheduled_respawn: Arc::default(),
3333 deferred_reload_reply: Arc::default(),
3334 restart_policy: self.restart_policy,
3335 drain_timeout: self.drain_timeout,
3336 child_roster: self
3339 .child_roster
3340 .for_module(Arc::clone(&effective_drain_timeout)),
3341 effective_drain_timeout,
3342 default_drain_timeout: self.drain_timeout,
3343 health: self.health.clone(),
3344 connection_file_path: self.connection_file_path.clone(),
3345 capture_logs_dir: self.capture_logs_dir.clone(),
3346 forwarding: self.forwarding.clone(),
3347 supervisor_handle: self.supervisor_handle.clone(),
3348 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3349 terminal_ring: Arc::new(Mutex::new(
3350 TerminalRing::new(
3351 TerminalRingConfig::default(),
3352 self.daemon_start_clock.started_at_ms(),
3353 )
3354 .with_start_clock(self.daemon_start_clock)
3355 .with_journal(self.terminal_journal.clone())
3356 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3357 )),
3358 spawn_events: self.spawn_events.clone(),
3359 #[cfg(target_os = "linux")]
3360 cgroup_placement: self.cgroup_placement.clone(),
3361 #[cfg(test)]
3362 test_seed_stale_facts_before_enable_spawn: false,
3363 #[cfg(test)]
3364 test_reload_exit_record_gate: None,
3365 }
3366 }
3367
3368 fn supervised_module(
3369 &self,
3370 spec: ModuleSpec,
3371 runtime: SupervisorRuntimeConfig,
3372 snapshot: SharedSnapshot,
3373 child: Option<SupervisedChild>,
3374 ) -> SupervisedModule {
3375 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3376 self.start_monitor(spec, runtime, &module, child, commands);
3377 module
3378 }
3379
3380 fn register_module(
3381 &self,
3382 spec: &ModuleSpec,
3383 runtime: &SupervisorRuntimeConfig,
3384 snapshot: SharedSnapshot,
3385 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3386 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3387 spec: spec.clone(),
3388 health: runtime.health.clone(),
3389 }));
3390 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3391 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3392 let restart_policy = runtime.restart_policy;
3396 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3397 let (tx, rx) = mpsc::channel(4);
3398 let module_id = spec.module_id.clone();
3399 let module = SupervisedModule {
3400 inner: Arc::new(SupervisedModuleInner {
3401 module_id: module_id.clone(),
3402 registry: Arc::clone(&self.registry),
3403 snapshot,
3404 configuration,
3405 stderr_ring,
3406 terminal_ring,
3407 commands: tx,
3408 monitor: Mutex::new(None),
3409 restart_policy,
3410 effective_drain_timeout,
3411 provenance_probe: self.provenance_probe.clone(),
3412 }),
3413 };
3414 if let Some(supervisor_handle) = &self.supervisor_handle {
3417 supervisor_handle.insert(module.clone());
3418 }
3419 (module, rx)
3420 }
3421
3422 fn start_monitor(
3423 &self,
3424 spec: ModuleSpec,
3425 runtime: SupervisorRuntimeConfig,
3426 module: &SupervisedModule,
3427 child: Option<SupervisedChild>,
3428 commands: mpsc::Receiver<SupervisorCommand>,
3429 ) {
3430 let monitor = tokio::spawn(supervise_loop(
3431 spec,
3432 runtime,
3433 Arc::clone(&self.registry),
3434 Arc::clone(&self.process_liveness),
3435 Arc::clone(&module.inner.snapshot),
3436 child,
3437 commands,
3438 ));
3439 *module
3440 .inner
3441 .monitor
3442 .lock()
3443 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3444 }
3445}
3446
3447impl Default for Supervisor {
3448 fn default() -> Self {
3449 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3450 }
3451}
3452
3453#[derive(Clone)]
3455pub struct SupervisedModule {
3456 inner: Arc<SupervisedModuleInner>,
3457}
3458
3459struct SupervisedModuleInner {
3460 module_id: String,
3461 registry: Arc<Registry>,
3462 snapshot: SharedSnapshot,
3463 configuration: Arc<Mutex<SupervisedConfiguration>>,
3464 stderr_ring: Arc<Mutex<StderrRing>>,
3465 terminal_ring: Arc<Mutex<TerminalRing>>,
3466 commands: mpsc::Sender<SupervisorCommand>,
3467 monitor: Mutex<Option<JoinHandle<()>>>,
3468 restart_policy: RestartPolicy,
3472 effective_drain_timeout: Arc<Mutex<Duration>>,
3473 provenance_probe: ExecutableIdentityProbe,
3474}
3475
3476impl fmt::Debug for SupervisedModule {
3477 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3478 f.debug_struct("SupervisedModule")
3479 .field("module_id", &self.inner.module_id)
3480 .field("status", &self.status())
3481 .finish_non_exhaustive()
3482 }
3483}
3484
3485impl SupervisedModule {
3486 pub fn module_id(&self) -> &str {
3487 &self.inner.module_id
3488 }
3489
3490 #[cfg(test)]
3494 pub(crate) fn record_health_probe_failure_for_test(
3495 &self,
3496 detail: &str,
3497 ) -> Result<(), SuperviseError> {
3498 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3499 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3500 state.health.detail = Some(detail.to_string());
3501 })
3502 }
3503
3504 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3505 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3506 }
3507
3508 pub(crate) fn registration_end_reason(
3509 &self,
3510 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3511 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3512 Ok(match snapshot.state {
3513 ModuleState::Draining if snapshot.draining_to_replace => {
3514 Some(RegistrationEndReason::SupervisorRestart)
3515 }
3516 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3517 ModuleState::Restarting if snapshot.draining_to_replace => {
3518 Some(RegistrationEndReason::SupervisorRestart)
3519 }
3520 _ => None,
3521 })
3522 }
3523
3524 pub fn stderr_tail(
3531 &self,
3532 max_lines: Option<usize>,
3533 max_bytes: Option<usize>,
3534 ) -> StderrTailSnapshot {
3535 self.inner
3536 .stderr_ring
3537 .lock()
3538 .unwrap_or_else(|poisoned| poisoned.into_inner())
3539 .snapshot(max_lines, max_bytes)
3540 }
3541
3542 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3547 self.inner
3548 .terminal_ring
3549 .lock()
3550 .unwrap_or_else(|poisoned| poisoned.into_inner())
3551 .snapshot()
3552 }
3553
3554 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3559 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3560 }
3561
3562 pub(crate) async fn read_durable_terminal_history(
3567 &self,
3568 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3569 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3570 let module_id = self.inner.module_id.clone();
3571 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3572 .await
3573 }
3574
3575 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3576 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3577 .map(|(status, _)| status)
3578 }
3579
3580 pub(crate) fn record_deliberate_severance(
3581 &self,
3582 identity: ProcessIdentity,
3583 ) -> Result<bool, SuperviseError> {
3584 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3585 if snapshot.pid != Some(identity.pid)
3586 || snapshot.process_start_time != Some(identity.start_time)
3587 {
3588 return Ok(false);
3589 }
3590 snapshot.deliberate_severance = Some(identity);
3591 Ok(true)
3592 }
3593
3594 pub(crate) fn status_for_control(
3599 &self,
3600 caller: &'static str,
3601 ) -> Result<ModuleStatus, SuperviseError> {
3602 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3603 .map(|(status, _)| status)
3604 }
3605
3606 fn status_with_snapshot_lock(
3607 &self,
3608 snapshot: &SharedSnapshot,
3609 caller: Option<&'static str>,
3610 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3611 let mut guard = match caller {
3612 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3613 None => lock_snapshot(snapshot)?,
3614 };
3615 let restart_count =
3618 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3619 let snapshot = guard.clone();
3620 drop(guard);
3621 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3622 SuperviseError::StatePoisoned {
3623 module_id: Some(self.inner.module_id.clone()),
3624 }
3625 })?;
3626 let registration_active = self
3627 .inner
3628 .registry
3629 .get_module(&self.inner.module_id)
3630 .map_err(SuperviseError::Registry)?
3631 .is_some();
3632 let protocol = snapshot
3633 .spawned_protocol
3634 .unwrap_or(self.declared_protocol()?);
3635 let running_process =
3636 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3637 let live = match protocol {
3643 ModuleProtocol::Subc => running_process && registration_active,
3644 ModuleProtocol::None => running_process,
3645 };
3646
3647 Ok((
3648 ModuleStatus {
3649 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3650 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3651 module_id: self.inner.module_id.clone(),
3652 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3653 state: snapshot.state,
3654 enabled: snapshot.enabled,
3655 process_alive: snapshot.process_alive,
3656 registration_active,
3657 protocol,
3658 live,
3659 restart_count,
3660 lifetime_restarts: snapshot.lifetime_restarts,
3661 spawn_generation: snapshot.spawn_generation,
3662 max_restarts: self.inner.restart_policy.max_restarts,
3663 restart_window: self.inner.restart_policy.window,
3664 drain_timeout,
3665 restart_backoff: self.inner.restart_policy.backoff,
3666 restart_max_backoff: self.inner.restart_policy.max_backoff,
3667 pid: snapshot.reported_pid(),
3668 spawned_at_ms: snapshot.spawned_at_ms,
3669 spawned_from: snapshot.spawned_from,
3670 launch_spec_digest: snapshot.launch_spec_digest,
3671 process_start_time: snapshot.process_start_time,
3672 last_exit: snapshot.last_exit,
3673 health: snapshot.health,
3674 },
3675 snapshot.spawned_file_identity.clone(),
3676 ))
3677 }
3678
3679 #[cfg(test)]
3680 pub(crate) fn hold_snapshot_for_test(
3681 &self,
3682 acquired: std::sync::mpsc::Sender<()>,
3683 hold: Duration,
3684 ) -> std::thread::JoinHandle<()> {
3685 let snapshot = Arc::clone(&self.inner.snapshot);
3686 std::thread::spawn(move || {
3687 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3688 acquired
3689 .send(())
3690 .expect("test receiver waits for snapshot lock");
3691 std::thread::sleep(hold);
3692 })
3693 }
3694
3695 pub(crate) async fn status_and_running_image_agreement(
3701 &self,
3702 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3703 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3704 let image = self
3705 .inner
3706 .provenance_probe
3707 .observe(
3708 status.pid,
3709 status.spawned_from.as_deref(),
3710 identity,
3711 status.process_start_time,
3712 )
3713 .await;
3714 Ok((status, image))
3715 }
3716
3717 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3718 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3719 Ok(snapshot) => snapshot.clone(),
3720 Err(_) => {
3721 return subc_control::RunningImageAgreement::Unavailable {
3722 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3723 };
3724 }
3725 };
3726 self.inner
3727 .provenance_probe
3728 .observe(
3729 snapshot.reported_pid(),
3730 snapshot.spawned_from.as_deref(),
3731 snapshot.spawned_file_identity.clone(),
3732 snapshot.process_start_time,
3733 )
3734 .await
3735 }
3736
3737 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3740 #[cfg(windows)]
3741 {
3742 let image = lock_snapshot(&self.inner.snapshot)
3743 .ok()
3744 .and_then(|snapshot| {
3745 snapshot
3746 .spawned_file_identity
3747 .as_ref()
3748 .and_then(|identity| identity.windows_image.clone())
3749 });
3750 match image {
3751 Some(image) => {
3752 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3753 }
3754 None => subc_control::ChildResourceUsage::Unavailable {
3755 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3756 },
3757 }
3758 }
3759 #[cfg(not(windows))]
3760 {
3761 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3762 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3763 Err(_) => {
3764 return subc_control::ChildResourceUsage::Unavailable {
3765 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3766 }
3767 }
3768 };
3769 crate::child_resources::read(pid, start_time)
3770 }
3771 }
3772
3773 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3774 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3775 Ok(match snapshot.state {
3776 ModuleState::Restarting => true,
3777 ModuleState::Failed | ModuleState::Disabled => false,
3778 _ if snapshot.enabled && snapshot.respawn_pending => true,
3779 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3780 })
3781 }
3782
3783 #[cfg(test)]
3784 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3785 self.is_warming_with_snapshot_lock(None)
3786 }
3787
3788 pub(crate) fn is_warming_for_control(
3789 &self,
3790 caller: &'static str,
3791 ) -> Result<bool, SuperviseError> {
3792 self.is_warming_with_snapshot_lock(Some(caller))
3793 }
3794
3795 fn is_warming_with_snapshot_lock(
3796 &self,
3797 caller: Option<&'static str>,
3798 ) -> Result<bool, SuperviseError> {
3799 let snapshot = match caller {
3800 Some(caller) => {
3801 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3802 }
3803 None => lock_snapshot(&self.inner.snapshot)?,
3804 }
3805 .clone();
3806 Ok(matches!(
3807 snapshot.state,
3808 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3809 ))
3810 }
3811
3812 pub async fn drain(&self) -> Result<(), SuperviseError> {
3814 self.stop().await
3815 }
3816
3817 pub(crate) async fn retire_requested(
3818 &self,
3819 requester: Option<Principal>,
3820 ) -> Result<(), SuperviseError> {
3821 match self.state()? {
3822 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3823 ModuleState::Starting
3824 | ModuleState::Running
3825 | ModuleState::Unresponsive
3826 | ModuleState::Restarting
3827 | ModuleState::Draining
3828 | ModuleState::Disabled => {}
3829 }
3830
3831 let (reply_tx, reply_rx) = oneshot::channel();
3832 self.inner
3833 .commands
3834 .send(SupervisorCommand::Retire {
3835 requester,
3836 reply: reply_tx,
3837 })
3838 .await
3839 .map_err(|_| SuperviseError::CommandClosed {
3840 module_id: self.inner.module_id.clone(),
3841 })?;
3842 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3843 module_id: self.inner.module_id.clone(),
3844 })?
3845 }
3846
3847 pub async fn stop(&self) -> Result<(), SuperviseError> {
3848 self.stop_requested(None).await
3849 }
3850
3851 pub(crate) async fn stop_requested(
3852 &self,
3853 requester: Option<Principal>,
3854 ) -> Result<(), SuperviseError> {
3855 match self.state()? {
3856 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3857 ModuleState::Starting
3858 | ModuleState::Running
3859 | ModuleState::Unresponsive
3860 | ModuleState::Restarting
3861 | ModuleState::Draining
3862 | ModuleState::Disabled => {}
3863 }
3864
3865 let (reply_tx, reply_rx) = oneshot::channel();
3866 self.inner
3867 .commands
3868 .send(SupervisorCommand::Drain {
3869 requester,
3870 reply: reply_tx,
3871 })
3872 .await
3873 .map_err(|_| SuperviseError::CommandClosed {
3874 module_id: self.inner.module_id.clone(),
3875 })?;
3876 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3877 module_id: self.inner.module_id.clone(),
3878 })?
3879 }
3880
3881 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3882 self.restart_requested(drain_timeout_ms, None).await
3883 }
3884
3885 pub(crate) async fn restart_requested(
3886 &self,
3887 drain_timeout_ms: Option<u64>,
3888 requester: Option<Principal>,
3889 ) -> Result<(), SuperviseError> {
3890 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3891 let (reply_tx, reply_rx) = oneshot::channel();
3892 self.inner
3893 .commands
3894 .send(SupervisorCommand::Restart {
3895 requester,
3896 drain_timeout_ms,
3897 received_at_generation,
3898 queued_at: Instant::now(),
3899 reply: reply_tx,
3900 })
3901 .await
3902 .map_err(|_| SuperviseError::CommandClosed {
3903 module_id: self.inner.module_id.clone(),
3904 })?;
3905 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3906 module_id: self.inner.module_id.clone(),
3907 })?
3908 }
3909
3910 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3915 self.swap_requested(ready_timeout, None).await
3916 }
3917
3918 pub(crate) async fn swap_requested(
3919 &self,
3920 ready_timeout: Option<Duration>,
3921 requester: Option<Principal>,
3922 ) -> Result<(), SuperviseError> {
3923 let (reply_tx, reply_rx) = oneshot::channel();
3924 self.inner
3925 .commands
3926 .send(SupervisorCommand::Swap {
3927 requester,
3928 ready_timeout,
3929 reply: reply_tx,
3930 })
3931 .await
3932 .map_err(|_| SuperviseError::CommandClosed {
3933 module_id: self.inner.module_id.clone(),
3934 })?;
3935 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3936 module_id: self.inner.module_id.clone(),
3937 })?
3938 }
3939
3940 pub async fn reload(&self) -> Result<(), SuperviseError> {
3941 self.reload_requested(None).await
3942 }
3943
3944 pub(crate) async fn reload_requested(
3945 &self,
3946 requester: Option<Principal>,
3947 ) -> Result<(), SuperviseError> {
3948 let (reply_tx, reply_rx) = oneshot::channel();
3949 self.inner
3950 .commands
3951 .send(SupervisorCommand::Reload {
3952 requester,
3953 reply: reply_tx,
3954 })
3955 .await
3956 .map_err(|_| SuperviseError::CommandClosed {
3957 module_id: self.inner.module_id.clone(),
3958 })?;
3959 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3960 module_id: self.inner.module_id.clone(),
3961 })?
3962 }
3963
3964 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3965 self.set_enabled_requested(enabled, None).await
3966 }
3967
3968 pub(crate) async fn set_enabled_requested(
3969 &self,
3970 enabled: bool,
3971 requester: Option<Principal>,
3972 ) -> Result<bool, SuperviseError> {
3973 let (reply_tx, reply_rx) = oneshot::channel();
3974 self.inner
3975 .commands
3976 .send(SupervisorCommand::SetEnabled {
3977 requester,
3978 enabled,
3979 reply: reply_tx,
3980 })
3981 .await
3982 .map_err(|_| SuperviseError::CommandClosed {
3983 module_id: self.inner.module_id.clone(),
3984 })?;
3985 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3986 module_id: self.inner.module_id.clone(),
3987 })?
3988 }
3989
3990 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3994 let configured = self
3995 .inner
3996 .configuration
3997 .lock()
3998 .map_err(|_| SuperviseError::StatePoisoned {
3999 module_id: Some(self.inner.module_id.clone()),
4000 })?
4001 .spec
4002 .protocol;
4003 let state = lock_snapshot(&self.inner.snapshot)?;
4004 Ok(state.spawned_protocol.unwrap_or(configured))
4005 }
4006
4007 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4008 let configuration =
4009 self.inner
4010 .configuration
4011 .lock()
4012 .map_err(|_| SuperviseError::StatePoisoned {
4013 module_id: Some(self.inner.module_id.clone()),
4014 })?;
4015 Ok((configuration.spec.clone(), configuration.health.clone()))
4016 }
4017
4018 #[cfg(any(test, feature = "test-support"))]
4022 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4023 let (_, health) = self.configuration()?;
4024 let drain_timeout_ms = u64::try_from(
4025 self.inner
4026 .effective_drain_timeout
4027 .lock()
4028 .unwrap_or_else(|poisoned| poisoned.into_inner())
4029 .as_millis(),
4030 )
4031 .ok();
4032 self.update_configuration(spec, health, drain_timeout_ms)
4033 .await
4034 }
4035
4036 pub(crate) async fn update_configuration(
4037 &self,
4038 spec: ModuleSpec,
4039 health: HealthConfig,
4040 drain_timeout_ms: Option<u64>,
4041 ) -> Result<(), SuperviseError> {
4042 if spec.module_id != self.inner.module_id {
4043 return Err(SuperviseError::InvalidSpec {
4044 reason: "a supervised module's module_id cannot be changed".to_string(),
4045 });
4046 }
4047 validate_spec(&spec)?;
4048 let (reply_tx, reply_rx) = oneshot::channel();
4049 self.inner
4050 .commands
4051 .send(SupervisorCommand::UpdateConfiguration {
4052 spec: spec.clone(),
4053 health: health.clone(),
4054 drain_timeout_ms,
4055 reply: reply_tx,
4056 })
4057 .await
4058 .map_err(|_| SuperviseError::CommandClosed {
4059 module_id: self.inner.module_id.clone(),
4060 })?;
4061 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4062 module_id: self.inner.module_id.clone(),
4063 })?;
4064 let mut configuration =
4065 self.inner
4066 .configuration
4067 .lock()
4068 .map_err(|_| SuperviseError::StatePoisoned {
4069 module_id: Some(self.inner.module_id.clone()),
4070 })?;
4071 configuration.spec = spec;
4072 configuration.health = health;
4073 Ok(())
4074 }
4075}
4076
4077impl Drop for SupervisedModuleInner {
4078 fn drop(&mut self) {
4079 let Ok(mut monitor) = self.monitor.lock() else {
4080 return;
4081 };
4082 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4083 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4084 state.state = ModuleState::Stopped;
4085 clear_current_process_facts(state);
4086 });
4087 monitor.abort();
4088 }
4089 let _ = monitor.take();
4090 }
4091}
4092
4093#[derive(Debug)]
4094enum SupervisorCommand {
4095 Drain {
4096 requester: Option<Principal>,
4097 reply: oneshot::Sender<Result<(), SuperviseError>>,
4098 },
4099 Retire {
4100 requester: Option<Principal>,
4101 reply: oneshot::Sender<Result<(), SuperviseError>>,
4102 },
4103 Restart {
4104 requester: Option<Principal>,
4105 drain_timeout_ms: Option<u64>,
4110 received_at_generation: u64,
4114 queued_at: Instant,
4117 reply: oneshot::Sender<Result<(), SuperviseError>>,
4118 },
4119 Reload {
4120 requester: Option<Principal>,
4121 reply: oneshot::Sender<Result<(), SuperviseError>>,
4122 },
4123 SetEnabled {
4124 requester: Option<Principal>,
4125 enabled: bool,
4126 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4127 },
4128 UpdateConfiguration {
4129 spec: ModuleSpec,
4130 health: HealthConfig,
4131 drain_timeout_ms: Option<u64>,
4134 reply: oneshot::Sender<()>,
4135 },
4136 Swap {
4137 requester: Option<Principal>,
4138 ready_timeout: Option<Duration>,
4141 reply: oneshot::Sender<Result<(), SuperviseError>>,
4143 },
4144}
4145
4146#[derive(Debug)]
4147pub enum SuperviseError {
4148 InvalidSpec {
4149 reason: String,
4150 },
4151 Spawn {
4152 program: PathBuf,
4153 source: io::Error,
4154 cgroup_path: Option<PathBuf>,
4155 },
4156 Cgroup {
4157 module_id: String,
4158 source: io::Error,
4159 },
4160 LaunchNonce {
4163 reason: String,
4164 },
4165 Wait {
4166 module_id: String,
4167 source: io::Error,
4168 },
4169 Kill {
4170 module_id: String,
4171 source: io::Error,
4172 },
4173 Forwarding(ForwardingError),
4174 Registry(RegistryError),
4175 ReloadUnavailable {
4176 module_id: String,
4177 reason: String,
4178 },
4179 Disabled {
4184 module_id: String,
4185 },
4186 ReloadFailed {
4187 module_id: String,
4188 reason: String,
4189 },
4190 RegistrationStillActive {
4191 module_id: String,
4192 waited: Duration,
4193 },
4194 StatePoisoned {
4195 module_id: Option<String>,
4196 },
4197 CommandClosed {
4198 module_id: String,
4199 },
4200 SwapInProgress {
4204 module_id: String,
4205 },
4206 SwapRefused {
4208 module_id: String,
4209 reason: SwapRefusal,
4210 },
4211 SwapFailed {
4215 module_id: String,
4216 arm: SwapFailureArm,
4217 detail: String,
4218 candidate_exit: Option<ExitReport>,
4221 },
4222}
4223
4224#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4226pub enum SwapRefusal {
4227 OverlapExclusive,
4229 NotRegistered,
4232 ProtocolNone,
4235 NotConfigured,
4238 AlreadySwapping,
4240}
4241
4242impl SwapRefusal {
4243 pub fn as_str(self) -> &'static str {
4244 match self {
4245 Self::OverlapExclusive => "overlap_exclusive",
4246 Self::NotRegistered => "not_registered",
4247 Self::ProtocolNone => "protocol_none",
4248 Self::NotConfigured => "not_configured",
4249 Self::AlreadySwapping => "already_swapping",
4250 }
4251 }
4252}
4253
4254#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4257pub enum SwapFailureArm {
4258 SpawnFailed,
4260 NeverRegistered,
4262 NeverReady,
4264 CandidateExited,
4266 CandidateUnhealthy,
4268 Interrupted,
4272 CutoverLost,
4277}
4278
4279impl SwapFailureArm {
4280 pub fn as_str(self) -> &'static str {
4281 match self {
4282 Self::SpawnFailed => "spawn_failed",
4283 Self::NeverRegistered => "never_registered",
4284 Self::NeverReady => "never_ready",
4285 Self::CandidateExited => "candidate_exited",
4286 Self::CandidateUnhealthy => "candidate_unhealthy",
4287 Self::Interrupted => "interrupted",
4288 Self::CutoverLost => "cutover_lost",
4289 }
4290 }
4291}
4292
4293impl fmt::Display for SuperviseError {
4294 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4295 match self {
4296 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4297 Self::Spawn {
4298 program,
4299 source,
4300 cgroup_path: Some(cgroup_path),
4301 } => write!(
4302 f,
4303 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4304 cgroup_path.display(),
4305 program.display()
4306 ),
4307 Self::Spawn {
4308 program,
4309 source,
4310 cgroup_path: None,
4311 } => write!(
4312 f,
4313 "failed to spawn module '{}': {source}",
4314 program.display()
4315 ),
4316 Self::Cgroup { module_id, source } => {
4317 write!(
4318 f,
4319 "failed to prepare cgroup for module '{module_id}': {source}"
4320 )
4321 }
4322 Self::LaunchNonce { reason } => {
4323 write!(
4324 f,
4325 "failed to generate reserved-module launch nonce: {reason}"
4326 )
4327 }
4328 Self::Wait { module_id, source } => {
4329 write!(f, "failed to wait for module '{module_id}': {source}")
4330 }
4331 Self::Kill { module_id, source } => {
4332 write!(f, "failed to kill module '{module_id}': {source}")
4333 }
4334 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4335 Self::Registry(err) => write!(f, "registry error: {err}"),
4336 Self::ReloadUnavailable { module_id, reason } => {
4337 write!(f, "reload unavailable for module '{module_id}': {reason}")
4338 }
4339 Self::Disabled { module_id } => {
4340 write!(
4341 f,
4342 "module '{module_id}' is disabled; enable it before restart or reload"
4343 )
4344 }
4345 Self::ReloadFailed { module_id, reason } => {
4346 write!(f, "reload failed for module '{module_id}': {reason}")
4347 }
4348 Self::RegistrationStillActive { module_id, waited } => write!(
4349 f,
4350 "module '{module_id}' registration remained active after waiting {waited:?}"
4351 ),
4352 Self::StatePoisoned { module_id } => match module_id {
4353 Some(module_id) => {
4354 write!(f, "supervisor state for module '{module_id}' was poisoned")
4355 }
4356 None => write!(f, "supervisor state was poisoned"),
4357 },
4358 Self::CommandClosed { module_id } => {
4359 write!(
4360 f,
4361 "supervisor command channel for module '{module_id}' is closed"
4362 )
4363 }
4364 Self::SwapInProgress { module_id } => write!(
4365 f,
4366 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4367 ),
4368 Self::SwapRefused { module_id, reason } => match reason {
4369 SwapRefusal::OverlapExclusive => write!(
4370 f,
4371 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4372 ),
4373 SwapRefusal::NotRegistered => write!(
4374 f,
4375 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4376 ),
4377 SwapRefusal::ProtocolNone => write!(
4378 f,
4379 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4380 ),
4381 SwapRefusal::NotConfigured => write!(
4382 f,
4383 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4384 ),
4385 SwapRefusal::AlreadySwapping => {
4386 write!(f, "module '{module_id}' is already being swapped")
4387 }
4388 },
4389 Self::SwapFailed {
4390 module_id,
4391 arm,
4392 detail,
4393 ..
4394 } => write!(
4395 f,
4396 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4397 arm.as_str()
4398 ),
4399 }
4400 }
4401}
4402
4403impl Error for SuperviseError {
4404 fn source(&self) -> Option<&(dyn Error + 'static)> {
4405 match self {
4406 Self::Spawn { source, .. }
4407 | Self::Cgroup { source, .. }
4408 | Self::Wait { source, .. }
4409 | Self::Kill { source, .. } => Some(source),
4410 Self::Forwarding(err) => Some(err),
4411 Self::Registry(err) => Some(err),
4412 Self::LaunchNonce { .. }
4413 | Self::InvalidSpec { .. }
4414 | Self::ReloadUnavailable { .. }
4415 | Self::Disabled { .. }
4416 | Self::ReloadFailed { .. }
4417 | Self::RegistrationStillActive { .. }
4418 | Self::StatePoisoned { .. }
4419 | Self::CommandClosed { .. }
4420 | Self::SwapInProgress { .. }
4421 | Self::SwapRefused { .. }
4422 | Self::SwapFailed { .. } => None,
4423 }
4424 }
4425}
4426
4427pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4428 if spec.module_id.trim().is_empty() {
4429 return Err(SuperviseError::InvalidSpec {
4430 reason: "module_id must not be empty".to_string(),
4431 });
4432 }
4433
4434 Ok(())
4435}
4436
4437#[derive(Debug, Default)]
4438struct HealthProbeRuntime {
4439 configured_health: Option<HealthConfig>,
4440 registered_connection: Option<crate::ConnectionId>,
4441 advertised: bool,
4442 next_probe_at: Option<Instant>,
4443 probe_index: u64,
4444}
4445
4446fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4447 lock_snapshot(snapshot)
4448 .ok()
4449 .and_then(|state| state.spawned_protocol)
4450 .unwrap_or(spec.protocol)
4451}
4452
4453impl HealthProbeRuntime {
4454 fn refresh_registration(
4455 &mut self,
4456 spec: &ModuleSpec,
4457 runtime: &SupervisorRuntimeConfig,
4458 registry: &Registry,
4459 snapshot: &SharedSnapshot,
4460 ) {
4461 if self.configured_health.as_ref() != Some(&runtime.health) {
4462 self.configured_health = Some(runtime.health.clone());
4463 self.next_probe_at = None;
4464 self.registered_connection = None;
4465 self.probe_index = 0;
4466 }
4467 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4471 self.registered_connection = None;
4472 self.advertised = runtime.health.http.is_some();
4473 if !self.advertised {
4474 self.next_probe_at = None;
4475 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4476 let unknown = ModuleHealthStatus::default();
4477 if state.health != unknown {
4478 state.health = unknown;
4479 }
4480 });
4481 } else if self.next_probe_at.is_none() {
4482 self.next_probe_at = Some(
4483 Instant::now()
4484 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4485 );
4486 }
4487 return;
4488 }
4489
4490 let registration = match registry.get_module(&spec.module_id) {
4491 Ok(registration) => registration,
4492 Err(err) => {
4493 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4494 self.advertised = false;
4495 self.next_probe_at = None;
4496 return;
4497 }
4498 };
4499
4500 let Some(registration) = registration else {
4501 self.registered_connection = None;
4502 self.advertised = false;
4503 self.next_probe_at = None;
4504 return;
4505 };
4506
4507 let advertised = registration
4508 .control_ops
4509 .iter()
4510 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4511 if !advertised {
4512 self.registered_connection = Some(registration.connection_id);
4513 self.advertised = false;
4514 self.next_probe_at = None;
4515 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4516 if state.health.status != SupervisorHealthStatus::Unknown
4517 || state.health.consecutive_failures != 0
4518 || state.health.last_probe_ms.is_some()
4519 || state.health.detail.is_some()
4520 || state.health.metrics.is_some()
4521 {
4522 state.health.status = SupervisorHealthStatus::Unknown;
4523 state.health.consecutive_failures = 0;
4524 state.health.last_probe_ms = None;
4525 state.health.detail = None;
4526 state.health.metrics = None;
4527 }
4528 });
4529 return;
4530 }
4531
4532 let reregistered = self.registered_connection != Some(registration.connection_id);
4533 self.registered_connection = Some(registration.connection_id);
4534 self.advertised = true;
4535 if reregistered || self.next_probe_at.is_none() {
4536 self.probe_index = 0;
4537 self.next_probe_at = Some(
4538 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4539 );
4540 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4541 state.health.status = SupervisorHealthStatus::Unknown;
4542 state.health.consecutive_failures = 0;
4543 state.health.detail = None;
4544 state.health.metrics = None;
4545 });
4546 }
4547 }
4548
4549 fn wake_after(&self) -> Option<Duration> {
4550 if !self.advertised {
4551 return None;
4552 }
4553 self.next_probe_at
4554 .map(|next| next.saturating_duration_since(Instant::now()))
4555 }
4556
4557 fn due(&self) -> bool {
4558 self.advertised
4559 && self
4560 .next_probe_at
4561 .is_some_and(|next| Instant::now() >= next)
4562 }
4563
4564 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4565 self.probe_index = self.probe_index.wrapping_add(1);
4566 self.next_probe_at = Some(
4567 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4568 );
4569 }
4570}
4571
4572#[derive(Debug)]
4607enum HealthProbeEvidence {
4608 LaneDead,
4610 NoAnswer,
4612 BadAnswer,
4614 Misconfigured,
4616}
4617
4618#[derive(Debug)]
4619struct HealthProbeError {
4620 evidence: HealthProbeEvidence,
4621 message: String,
4622}
4623
4624impl HealthProbeError {
4625 fn lane_dead(message: impl Into<String>) -> Self {
4626 Self::with(HealthProbeEvidence::LaneDead, message)
4627 }
4628
4629 fn no_answer(message: impl Into<String>) -> Self {
4630 Self::with(HealthProbeEvidence::NoAnswer, message)
4631 }
4632
4633 fn bad_answer(message: impl Into<String>) -> Self {
4634 Self::with(HealthProbeEvidence::BadAnswer, message)
4635 }
4636
4637 fn misconfigured(message: impl Into<String>) -> Self {
4638 Self::with(HealthProbeEvidence::Misconfigured, message)
4639 }
4640
4641 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4642 Self {
4643 evidence,
4644 message: message.into(),
4645 }
4646 }
4647
4648 #[allow(dead_code)]
4662 fn is_proof_of_death(&self) -> bool {
4663 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4664 }
4665
4666 fn label(&self) -> &'static str {
4674 match self.evidence {
4675 HealthProbeEvidence::LaneDead => "lane-dead",
4676 HealthProbeEvidence::NoAnswer => "no-answer",
4677 HealthProbeEvidence::BadAnswer => "bad-answer",
4678 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4679 }
4680 }
4681}
4682
4683impl fmt::Display for HealthProbeError {
4684 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4685 f.write_str(&self.message)
4686 }
4687}
4688
4689async fn run_health_probe_cycle(
4690 spec: &ModuleSpec,
4691 runtime: &SupervisorRuntimeConfig,
4692 registry: &Registry,
4693 process_liveness: &SupervisorProcessLiveness,
4694 snapshot: &SharedSnapshot,
4695 child: &mut Option<SupervisedChild>,
4696) {
4697 let now_ms = unix_ms_now();
4698 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4699 .then_some(runtime.health.http.as_deref())
4700 .flatten();
4701 let result = match http {
4702 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4703 None => probe_module_health(&spec.module_id, runtime, None).await,
4704 };
4705 match result {
4706 Ok(report) => {
4707 handle_health_report(
4708 spec,
4709 runtime,
4710 registry,
4711 process_liveness,
4712 snapshot,
4713 child,
4714 report,
4715 now_ms,
4716 )
4717 .await;
4718 }
4719 Err(err) => {
4720 if http.is_some() {
4721 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4722 state.health.status = SupervisorHealthStatus::Failing;
4723 });
4724 }
4725 handle_health_probe_failure(
4726 spec,
4727 runtime,
4728 registry,
4729 process_liveness,
4730 snapshot,
4731 child,
4732 err,
4733 now_ms,
4734 )
4735 .await;
4736 }
4737 }
4738}
4739
4740pub(crate) struct HttpProbeTarget<'a> {
4741 address: std::net::SocketAddr,
4742 localhost: bool,
4743 authority: &'a str,
4744 path: String,
4745}
4746
4747pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4750 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4751 return Err("must not contain whitespace, controls, or a fragment".into());
4752 }
4753 let rest = url
4754 .strip_prefix("http://")
4755 .ok_or("must use plain http://")?;
4756 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4757 let (authority, suffix) = rest.split_at(split);
4758 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4759 ("::1", rest)
4760 } else {
4761 let split = authority.find(':').unwrap_or(authority.len());
4762 authority.split_at(split)
4763 };
4764 let ip: std::net::IpAddr = match host {
4765 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4766 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4767 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4768 };
4769 let port = if port.is_empty() {
4770 80
4771 } else {
4772 port.strip_prefix(':')
4773 .and_then(|p| p.parse::<u16>().ok())
4774 .filter(|p| *p > 0)
4775 .ok_or("must have a valid nonzero TCP port")?
4776 };
4777 let path = if suffix.is_empty() {
4778 "/".into()
4779 } else if suffix.starts_with('?') {
4780 format!("/{suffix}")
4781 } else {
4782 suffix.into()
4783 };
4784 Ok(HttpProbeTarget {
4785 address: std::net::SocketAddr::new(ip, port),
4786 localhost: host == "localhost",
4787 authority,
4788 path,
4789 })
4790}
4791
4792async fn probe_http_health(
4793 url: &str,
4794 deadline: Duration,
4795) -> Result<HealthReport, HealthProbeError> {
4796 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4797 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4798 let mut response_status = String::new();
4801 let mut body = Vec::new();
4802 let probe = async {
4803 let connection = match tokio::net::TcpStream::connect(target.address).await {
4806 Err(_) if target.localhost => {
4807 tokio::net::TcpStream::connect((
4808 std::net::Ipv6Addr::LOCALHOST,
4809 target.address.port(),
4810 ))
4811 .await
4812 }
4813 result => result,
4814 };
4815 let mut stream = connection.map_err(|error| {
4816 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4817 })?;
4818 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4819 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4820 let mut reader = BufReader::new(stream);
4821 let mut budget = 16 * 1024;
4822 let status = http_line(&mut reader, &mut budget).await?;
4823 let mut words = status.split_ascii_whitespace();
4824 let version = words.next();
4825 let code = words
4826 .next()
4827 .filter(|word| word.len() == 3)
4828 .and_then(|word| word.parse::<u16>().ok());
4829 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4830 || !code.is_some_and(|code| (100..600).contains(&code))
4831 {
4832 return Err(HealthProbeError::bad_answer(format!(
4833 "invalid HTTP status: {status}"
4834 )));
4835 }
4836 let code = code.expect("validated status code");
4837 response_status = status.clone();
4838 let mut length = None;
4839 let mut chunked = false;
4840 loop {
4841 let line = http_line(&mut reader, &mut budget).await?;
4842 if line.is_empty() {
4843 break;
4844 }
4845 if let Some((name, value)) = line.split_once(':') {
4846 if name.eq_ignore_ascii_case("content-length") {
4847 length = Some(value.trim().parse::<u64>().map_err(|_| {
4848 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4849 })?);
4850 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4851 chunked = value.trim().eq_ignore_ascii_case("chunked");
4852 }
4853 }
4854 }
4855 if chunked {
4856 while body.len() < 200 {
4857 let line = http_line(&mut reader, &mut budget).await?;
4858 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4859 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4860 if size == 0 {
4861 break;
4862 }
4863 let count = size.min((200 - body.len()) as u64) as usize;
4864 let start = body.len();
4865 (&mut reader)
4866 .take(count as u64)
4867 .read_to_end(&mut body)
4868 .await
4869 .map_err(|error| {
4870 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4871 })?;
4872 if body.len() - start != count {
4873 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4874 }
4875 if size > count as u64 || body.len() == 200 {
4876 break;
4877 }
4878 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4879 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4880 }
4881 }
4882 } else {
4883 reader
4884 .take(length.unwrap_or(200).min(200))
4885 .read_to_end(&mut body)
4886 .await
4887 .map_err(|error| {
4888 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4889 })?;
4890 }
4891 if (200..300).contains(&code) {
4892 Ok(HealthReport::ok())
4893 } else {
4894 Err(HealthProbeError::bad_answer(
4895 "HTTP health endpoint returned non-2xx",
4896 ))
4897 }
4898 };
4899 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4900 Err(HealthProbeError::no_answer(format!(
4901 "HTTP probe timed out after {deadline:?}"
4902 )))
4903 });
4904 if let Err(error) = &mut result {
4905 if !response_status.is_empty() {
4906 error.message = format!(
4907 "{}; {response_status}: {}",
4908 error.message,
4909 String::from_utf8_lossy(&body)
4910 );
4911 }
4912 }
4913 result
4914}
4915
4916async fn http_line(
4917 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4918 remaining: &mut usize,
4919) -> Result<String, HealthProbeError> {
4920 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4921 let mut line = Vec::new();
4922 (&mut *reader)
4923 .take(*remaining as u64)
4924 .read_until(b'\n', &mut line)
4925 .await
4926 .map_err(|error| {
4927 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4928 })?;
4929 *remaining -= line.len();
4930 if !line.ends_with(b"\r\n") {
4931 return Err(HealthProbeError::bad_answer(
4932 "HTTP headers are incomplete or exceed 16 KiB",
4933 ));
4934 }
4935 line.truncate(line.len() - 2);
4936 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4937}
4938
4939async fn probe_module_health(
4940 module_id: &str,
4941 runtime: &SupervisorRuntimeConfig,
4942 drain_deadline: Option<Instant>,
4943) -> Result<HealthReport, HealthProbeError> {
4944 let Some(forwarding) = runtime.forwarding.as_ref() else {
4945 return Err(HealthProbeError::misconfigured(
4946 "supervisor was not configured with a forwarding table",
4947 ));
4948 };
4949 let probe_started_at = Instant::now();
4950 let mut deadline = probe_started_at + runtime.health.deadline;
4951 if let Some(drain_deadline) = drain_deadline {
4952 deadline = deadline.min(drain_deadline);
4953 }
4954 let pending = if drain_deadline.is_some() {
4955 forwarding.begin_drain_health_probe_rpc_for(
4956 module_id,
4957 MODULE_CONTROL_OP_HEALTH_CHECK,
4958 probe_started_at,
4959 deadline,
4960 )
4961 } else {
4962 forwarding.begin_health_probe_rpc_for(
4963 module_id,
4964 MODULE_CONTROL_OP_HEALTH_CHECK,
4965 probe_started_at,
4966 deadline,
4967 )
4968 }
4969 .map_err(|err| {
4970 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4973 })?;
4974 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4975}
4976
4977async fn probe_endpoint_health(
4984 endpoint: crate::ModuleEndpointId,
4985 runtime: &SupervisorRuntimeConfig,
4986 deadline_cap: Option<Instant>,
4987) -> Result<HealthReport, HealthProbeError> {
4988 let Some(forwarding) = runtime.forwarding.as_ref() else {
4989 return Err(HealthProbeError::misconfigured(
4990 "supervisor was not configured with a forwarding table",
4991 ));
4992 };
4993 let probe_started_at = Instant::now();
4994 let mut deadline = probe_started_at + runtime.health.deadline;
4995 if let Some(cap) = deadline_cap {
4996 deadline = deadline.min(cap);
4997 }
4998 let pending = forwarding
4999 .begin_endpoint_health_probe_rpc_for(
5000 endpoint,
5001 MODULE_CONTROL_OP_HEALTH_CHECK,
5002 probe_started_at,
5003 deadline,
5004 )
5005 .map_err(|err| {
5006 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5007 })?;
5008 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5009}
5010
5011async fn await_health_probe(
5013 forwarding: &ForwardingTable,
5014 pending: PendingModuleControlRpc,
5015 deadline: Instant,
5016 probe_budget: Duration,
5017) -> Result<HealthReport, HealthProbeError> {
5018 let PendingModuleControlRpc {
5019 endpoint,
5020 module_sink,
5021 negotiated_ver,
5022 corr,
5023 receiver,
5024 } = pending;
5025 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5026 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5027 })?;
5028 let frame = Frame::build_with_version(
5029 negotiated_ver,
5030 FrameType::Request,
5031 control_flags(),
5032 0,
5033 0,
5034 corr,
5035 body,
5036 )
5037 .map_err(|err| {
5038 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5039 })?;
5040
5041 match timeout_at(deadline, module_sink.send(frame)).await {
5047 Ok(Ok(())) => {}
5048 Ok(Err(err)) => {
5049 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5050 return Err(HealthProbeError::lane_dead(format!(
5053 "failed to send health.check: {err}"
5054 )));
5055 }
5056 Err(_elapsed) => {
5057 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5058 return Err(HealthProbeError::no_answer(
5062 "health.check send timed out before enqueue (module egress full)",
5063 ));
5064 }
5065 }
5066
5067 match timeout_at(deadline, receiver).await {
5068 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5072 response.health_report().ok_or_else(|| {
5073 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5074 })
5075 }
5076 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5077 format!("health.check rejected: {}", body.message),
5078 )),
5079 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5080 Err(HealthProbeError::lane_dead(message))
5081 }
5082 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5083 Err(HealthProbeError::bad_answer(message))
5084 }
5085 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5086 Err(HealthProbeError::bad_answer(format!(
5087 "expected module-control op '{expected}', got '{actual}'"
5088 )))
5089 }
5090 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5094 "module answered health.check after its daemon deadline",
5095 )),
5096 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5097 "health.check waiter was canceled before the module responded",
5098 )),
5099 Err(_) => {
5100 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5101 Err(HealthProbeError::no_answer(format!(
5102 "module did not answer health.check within {probe_budget:?}"
5103 )))
5104 }
5105 }
5106}
5107
5108#[allow(clippy::too_many_arguments)]
5109async fn handle_health_report(
5110 spec: &ModuleSpec,
5111 runtime: &SupervisorRuntimeConfig,
5112 registry: &Registry,
5113 process_liveness: &SupervisorProcessLiveness,
5114 snapshot: &SharedSnapshot,
5115 child: &mut Option<SupervisedChild>,
5116 report: HealthReport,
5117 now_ms: u64,
5118) {
5119 let status = supervisor_health_status(report.status);
5120 let detail = report.detail.clone();
5121 let metrics = truncate_health_metrics(report.metrics);
5122 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5123 state.health.status = status;
5124 state.health.last_probe_ms = Some(now_ms);
5125 state.health.detail = detail.clone();
5126 state.health.metrics = metrics.clone();
5127 state.health.consecutive_failures = 0;
5128 });
5129
5130 let action = match report.status {
5131 HealthStatus::Ok => return,
5132 HealthStatus::Degraded => runtime.health.on_degraded,
5133 HealthStatus::Failing => runtime.health.on_failing,
5134 };
5135 apply_l3_health_action(
5136 spec,
5137 runtime,
5138 registry,
5139 process_liveness,
5140 snapshot,
5141 child,
5142 status,
5143 detail.as_deref(),
5144 action,
5145 now_ms,
5146 )
5147 .await;
5148}
5149
5150#[allow(clippy::too_many_arguments)]
5151async fn handle_health_probe_failure(
5152 spec: &ModuleSpec,
5153 runtime: &SupervisorRuntimeConfig,
5154 registry: &Registry,
5155 process_liveness: &SupervisorProcessLiveness,
5156 snapshot: &SharedSnapshot,
5157 child: &mut Option<SupervisedChild>,
5158 err: HealthProbeError,
5159 now_ms: u64,
5160) {
5161 let threshold = runtime.health.failure_threshold.max(1);
5162 let mut failures = 0;
5163 let detail = format!("[{}] {err}", err.label());
5168 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5169 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5172 state.health.status = SupervisorHealthStatus::Unknown;
5173 }
5174 state.health.last_probe_ms = Some(now_ms);
5175 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5176 state.health.detail = Some(detail.clone());
5177 state.health.metrics = None;
5178 failures = state.health.consecutive_failures;
5179 });
5180
5181 if failures < threshold {
5182 warn!(
5183 module_id = %spec.module_id,
5184 consecutive_failures = failures,
5185 threshold,
5186 evidence = err.label(),
5187 detail = %detail,
5188 "health.check probe failed"
5189 );
5190 return;
5191 }
5192
5193 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5194 state.state = ModuleState::Unresponsive;
5195 state.health.status = SupervisorHealthStatus::Unresponsive;
5196 });
5197 if runtime.health.critical {
5201 error!(
5202 module_id = %spec.module_id,
5203 status = "unresponsive",
5204 evidence = err.label(),
5205 detail = %detail,
5206 "critical module health alert"
5207 );
5208 } else {
5209 warn!(
5210 module_id = %spec.module_id,
5211 status = "unresponsive",
5212 evidence = err.label(),
5213 detail = %detail,
5214 "module health threshold breached"
5215 );
5216 }
5217 if let Err(err) = health_restart_child(
5218 spec,
5219 runtime,
5220 registry,
5221 process_liveness,
5222 snapshot,
5223 child,
5224 SupervisorHealthStatus::Unresponsive,
5225 Some(&detail),
5226 now_ms,
5227 )
5228 .await
5229 {
5230 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5231 }
5232}
5233
5234#[allow(clippy::too_many_arguments)]
5235async fn apply_l3_health_action(
5236 spec: &ModuleSpec,
5237 runtime: &SupervisorRuntimeConfig,
5238 registry: &Registry,
5239 process_liveness: &SupervisorProcessLiveness,
5240 snapshot: &SharedSnapshot,
5241 child: &mut Option<SupervisedChild>,
5242 status: SupervisorHealthStatus,
5243 detail: Option<&str>,
5244 action: HealthAction,
5245 now_ms: u64,
5246) {
5247 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5248 match action {
5249 HealthAction::Report => {
5250 info!(
5251 module_id = %spec.module_id,
5252 status = ?status,
5253 detail,
5254 "module reported non-ok health"
5255 );
5256 }
5257 HealthAction::Alert => {
5258 error!(
5259 module_id = %spec.module_id,
5260 status = ?status,
5261 detail,
5262 "module health alert"
5263 );
5264 }
5265 HealthAction::Restart => {
5266 if let Err(err) = health_restart_child(
5267 spec,
5268 runtime,
5269 registry,
5270 process_liveness,
5271 snapshot,
5272 child,
5273 status,
5274 detail,
5275 now_ms,
5276 )
5277 .await
5278 {
5279 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5280 }
5281 }
5282 }
5283}
5284
5285#[allow(clippy::too_many_arguments)]
5286async fn health_restart_child(
5287 spec: &ModuleSpec,
5288 runtime: &SupervisorRuntimeConfig,
5289 registry: &Registry,
5290 process_liveness: &SupervisorProcessLiveness,
5291 snapshot: &SharedSnapshot,
5292 child: &mut Option<SupervisedChild>,
5293 status: SupervisorHealthStatus,
5294 detail: Option<&str>,
5295 now_ms: u64,
5296) -> Result<(), SuperviseError> {
5297 let (enabled, schedule) = {
5298 let mut state = lock_snapshot(snapshot)?;
5299 let enabled = state.enabled;
5300 let schedule = if enabled {
5301 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5302 } else {
5303 None
5304 };
5305 (enabled, schedule)
5306 };
5307
5308 if !enabled {
5309 return Err(SuperviseError::Disabled {
5310 module_id: spec.module_id.clone(),
5311 });
5312 }
5313
5314 if schedule.is_none() {
5315 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5316 error!(
5317 module_id = %spec.module_id,
5318 status = ?status,
5319 detail,
5320 max_restarts = runtime.restart_policy.max_restarts,
5321 window_secs = runtime.restart_policy.window.as_secs(),
5322 reason = %runtime.restart_policy.budget_exhausted_detail(),
5323 "health restart budget exhausted; marking module failed"
5324 );
5325 let stop_notice = begin_forwarding_drain_if_configured(
5326 spec,
5327 runtime,
5328 registry,
5329 snapshot,
5330 Some(true),
5331 RouteCloseReason::Disable,
5332 )
5333 .await?;
5334 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5335 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5336 })?;
5337 drain_optional_child(
5338 &spec.module_id,
5339 spec.protocol,
5340 stop_notice,
5341 registry,
5342 runtime.forwarding.as_deref(),
5343 snapshot,
5344 &runtime.terminal_ring,
5345 &runtime.spawn_events,
5346 child,
5347 runtime.drain_timeout,
5348 ModuleState::Failed,
5349 Some(true),
5350 )
5351 .await?;
5352 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5353 return Ok(());
5354 }
5355
5356 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5357 let mut restart_count = 0;
5358 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5359 restart_count = state.crash_restarts.len();
5360 state.state = ModuleState::Unresponsive;
5361 state.health.status = status;
5362 state.health.last_action = Some(HealthAction::Restart.to_string());
5363 state.health.last_action_ms = Some(now_ms);
5364 })?;
5365 warn!(
5366 module_id = %spec.module_id,
5367 status = ?status,
5368 detail,
5369 restart_count,
5370 restart_in_window = schedule.restart_in_window,
5371 delay_ms = schedule.delay.as_millis() as u64,
5372 "health-triggered module restart"
5373 );
5374
5375 let stop_notice = begin_forwarding_drain_if_configured(
5376 spec,
5377 runtime,
5378 registry,
5379 snapshot,
5380 Some(true),
5381 RouteCloseReason::Restart,
5382 )
5383 .await?;
5384 drain_optional_child(
5385 &spec.module_id,
5386 spec.protocol,
5387 stop_notice,
5388 registry,
5389 runtime.forwarding.as_deref(),
5390 snapshot,
5391 &runtime.terminal_ring,
5392 &runtime.spawn_events,
5393 child,
5394 runtime.drain_timeout,
5395 ModuleState::Restarting,
5396 Some(true),
5397 )
5398 .await?;
5399 schedule_respawn(
5400 runtime,
5401 snapshot,
5402 &spec.module_id,
5403 schedule.delay,
5404 RespawnKind::Spawn,
5405 )
5406}
5407
5408fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5409 if let Some(reply) = runtime
5410 .deferred_reload_reply
5411 .lock()
5412 .unwrap_or_else(|p| p.into_inner())
5413 .take()
5414 {
5415 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5416 module_id: module_id.to_string(),
5417 reason: reason.to_string(),
5418 }));
5419 }
5420}
5421
5422fn schedule_respawn(
5423 runtime: &SupervisorRuntimeConfig,
5424 snapshot: &SharedSnapshot,
5425 module_id: &str,
5426 delay: Duration,
5427 kind: RespawnKind,
5428) -> Result<(), SuperviseError> {
5429 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5430 update_snapshot(snapshot, Some(module_id), |state| {
5431 state.respawn_pending = true
5432 })?;
5433 *runtime
5434 .scheduled_respawn
5435 .lock()
5436 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5437 deadline: Instant::now() + delay,
5438 kind,
5439 });
5440 Ok(())
5441}
5442
5443fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5444 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5445 state.health.last_action = Some(action);
5446 state.health.last_action_ms = Some(now_ms);
5447 });
5448}
5449
5450fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5451 match status {
5452 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5453 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5454 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5455 }
5456}
5457
5458fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5470 let metrics = metrics?;
5471 match serde_json::to_vec(&metrics) {
5472 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5473 "truncated": true,
5474 "original_bytes": encoded.len(),
5475 })),
5476 Ok(_) | Err(_) => Some(metrics),
5477 }
5478}
5479
5480fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5486 if cadence.is_zero() {
5487 return Duration::ZERO;
5488 }
5489 let cadence_ms = cadence.as_millis() as u64;
5490 if cadence_ms == 0 {
5506 return cadence;
5507 }
5508 let jitter_span = (cadence_ms / 10).max(1);
5523 let hash = module_id.as_bytes().iter().fold(
5524 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5525 |acc, byte| {
5526 acc.wrapping_mul(1099511628211)
5527 .wrapping_add(u64::from(*byte))
5528 },
5529 );
5530 cadence + Duration::from_millis(hash % jitter_span)
5531}
5532
5533#[cfg(test)]
5534mod tests {
5535 use super::*;
5536
5537 #[test]
5538 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5539 let handle = SupervisorHandle::new();
5540 let module_id = "readded-tombstone";
5541 handle.record_rescan_removal(module_id);
5542 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5543
5544 handle.apply_identity_configuration(&ModuleSpec {
5545 module_id: module_id.to_string(),
5546 program: PathBuf::from("/test/module"),
5547 args: Vec::new(),
5548 env: Vec::new(),
5549 reserved: false,
5550 reserved_prefixes: Vec::new(),
5551 protocol: ModuleProtocol::Subc,
5552 overlap: Default::default(),
5553 });
5554
5555 assert!(
5556 handle.removal_tombstone_age_ms(module_id).is_none(),
5557 "a re-added module must not retain a stale removal tombstone"
5558 );
5559 }
5560
5561 #[derive(Debug, PartialEq, Eq)]
5564 struct OwnerInSpawnWindow {
5565 module_id: String,
5566 configured: bool,
5567 on_roster: bool,
5568 admission_refusal: Option<&'static str>,
5569 }
5570
5571 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5582 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5583 use crate::scopes::ScopeTable;
5584 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5585
5586 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5587 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5588 module_id: module_id.to_string(),
5589 program,
5590 args: Vec::new(),
5591 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5592 .into_iter()
5593 .map(|key| (key.to_string(), dir.path().display().to_string()))
5594 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5595 .collect(),
5596 reserved: false,
5597 reserved_prefixes: Vec::new(),
5598 protocol: ModuleProtocol::Subc,
5599 overlap: Default::default(),
5600 };
5601 let live = super::terminal_history_tests::fake_aft_stub_path();
5602 let missing = dir.path().join("definitely-missing-module");
5603
5604 let handle = SupervisorHandle::new();
5605 let mut supervisor =
5606 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5607 .with_handle(handle.clone());
5608 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5609 let hook_handle = handle.clone();
5610 let hook_observed = Arc::clone(&observed);
5611 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5612 let configured = hook_handle.is_configured(module_id);
5615 let selector = ScopeSelector::new(
5616 Principal::Reserved {
5617 module_id: module_id.to_string(),
5618 },
5619 "s".to_string(),
5620 Some(1),
5621 );
5622 let carrier = Principal::Reserved {
5623 module_id: "carrier".to_string(),
5624 };
5625 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5626 .admit(&carrier, module_id, &selector, configured)
5627 {
5628 Ok(_) => None,
5629 Err(refusal) => Some(refusal.code),
5630 };
5631 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5632 module_id: module_id.to_string(),
5633 configured,
5634 on_roster: hook_handle.get(module_id).is_some(),
5635 admission_refusal,
5636 });
5637 })));
5638
5639 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5640 let configured = supervisor
5641 .supervise_configured(stub("configured", live.clone()), true)
5642 .unwrap();
5643 let with_health = supervisor
5644 .supervise_configured_with_health(
5645 stub("with-health", live.clone()),
5646 true,
5647 HealthConfig::default(),
5648 None,
5649 RestartPolicy::default(),
5650 )
5651 .unwrap();
5652 let failed = supervisor
5655 .supervise_configured_with_health(
5656 stub("failed-spawn", missing.clone()),
5657 true,
5658 HealthConfig::default(),
5659 None,
5660 RestartPolicy::default(),
5661 )
5662 .unwrap();
5663 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5666
5667 let expected = [
5668 "plain",
5669 "configured",
5670 "with-health",
5671 "failed-spawn",
5672 "spawn-error",
5673 ]
5674 .into_iter()
5675 .map(|module_id| OwnerInSpawnWindow {
5676 module_id: module_id.to_string(),
5677 configured: true,
5678 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5679 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5680 })
5681 .collect::<Vec<_>>();
5682 assert_eq!(*observed.lock().unwrap(), expected);
5683
5684 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5685 assert!(
5686 handle.get(module_id).is_some(),
5687 "{module_id} is on the roster"
5688 );
5689 assert!(
5690 handle.is_configured(module_id),
5691 "{module_id} stays configured"
5692 );
5693 }
5694 assert!(handle.get("spawn-error").is_none());
5695 assert!(
5696 !handle.is_configured("spawn-error"),
5697 "a plain spawn that failed must not leave its module marked configured"
5698 );
5699
5700 handle.retire("failed-spawn");
5702 assert!(!handle.is_configured("failed-spawn"));
5703
5704 for module in [plain, configured, with_health] {
5705 module.stop().await.unwrap();
5706 }
5707 drop(failed);
5708 }
5709
5710 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5711 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5712 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5713 snapshot.process_alive = true;
5714 snapshot.pid = Some(41);
5715 snapshot.spawned_at_ms = Some(42);
5716 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5717 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5718 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5719 device: 43,
5720 inode: 44,
5721 #[cfg(windows)]
5722 windows_image: None,
5723 });
5724 })
5725 .unwrap();
5726 snapshot
5727 }
5728
5729 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5730 let snapshot = lock_snapshot(snapshot).unwrap();
5731 assert!(!snapshot.process_alive);
5732 assert_eq!(snapshot.pid, None);
5733 assert_eq!(snapshot.spawned_at_ms, None);
5734 assert_eq!(snapshot.spawned_from, None);
5735 assert_eq!(snapshot.launch_spec_digest, None);
5736 assert_eq!(snapshot.spawned_file_identity, None);
5737 }
5738
5739 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5740 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5741 let supervisor =
5742 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5743 let mut runtime = supervisor.runtime_config();
5744 runtime.test_seed_stale_facts_before_enable_spawn = true;
5745 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5746 let mut child = None;
5747 let spec = ModuleSpec {
5748 module_id: "failed-enable-clears-facts".to_string(),
5749 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5750 args: Vec::new(),
5751 env: Vec::new(),
5752 reserved: false,
5753 reserved_prefixes: Vec::new(),
5754 protocol: ModuleProtocol::Subc,
5755 overlap: Default::default(),
5756 };
5757
5758 let result = set_child_enabled(
5759 &spec,
5760 &runtime,
5761 &supervisor.registry,
5762 &supervisor.process_liveness,
5763 &snapshot,
5764 &mut child,
5765 true,
5766 )
5767 .await;
5768
5769 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5770 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5771 assert_snapshot_process_facts_cleared(&snapshot);
5772 }
5773
5774 #[tokio::test]
5775 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5776 let supervisor =
5777 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5778 let runtime = supervisor.runtime_config();
5779 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5780 ModuleState::Restarting,
5781 true,
5782 )));
5783 let spec = ModuleSpec {
5784 module_id: "start-stranded-restarting".to_string(),
5785 program: super::terminal_history_tests::fake_aft_stub_path(),
5786 args: Vec::new(),
5787 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5788 reserved: false,
5789 reserved_prefixes: Vec::new(),
5790 protocol: ModuleProtocol::None,
5791 overlap: Default::default(),
5792 };
5793 let mut child = None;
5794 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5795 assert!(!super::set_child_enabled(
5796 &spec,
5797 &runtime,
5798 &Registry::default(),
5799 &supervisor.process_liveness,
5800 &snapshot,
5801 &mut child,
5802 true
5803 )
5804 .await
5805 .unwrap());
5806 assert!(child.is_none());
5807 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5808 assert!(super::set_child_enabled(
5809 &spec,
5810 &runtime,
5811 &Registry::default(),
5812 &supervisor.process_liveness,
5813 &snapshot,
5814 &mut child,
5815 true
5816 )
5817 .await
5818 .unwrap());
5819 assert_eq!(
5820 lock_snapshot(&snapshot).unwrap().state,
5821 ModuleState::Running
5822 );
5823 let mut child = child.unwrap();
5824 child.start_kill().unwrap();
5825 child.wait().await.unwrap();
5826 }
5827
5828 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5829 async fn failed_reload_spawn_clears_current_process_facts() {
5830 let supervisor =
5831 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5832 let mut runtime = supervisor.runtime_config();
5833 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5834 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5835 let mut child = None;
5836 let spec = ModuleSpec {
5837 module_id: "failed-reload-clears-facts".to_string(),
5838 program: PathBuf::from("/unused/failed-reload-module"),
5839 args: Vec::new(),
5840 env: Vec::new(),
5841 reserved: false,
5842 reserved_prefixes: Vec::new(),
5843 protocol: ModuleProtocol::Subc,
5844 overlap: Default::default(),
5845 };
5846
5847 let result = handle_reload_spawn_failure(
5848 &spec,
5849 &runtime,
5850 &supervisor.process_liveness,
5851 &snapshot,
5852 &mut child,
5853 "forced reload spawn failure".to_string(),
5854 )
5855 .await;
5856
5857 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5858 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5859 assert_snapshot_process_facts_cleared(&snapshot);
5860 }
5861
5862 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5863 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5864 let supervisor =
5865 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5866 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5867 let module = supervisor.supervised_module(
5868 ModuleSpec {
5869 module_id: "drop-clears-facts".to_string(),
5870 program: PathBuf::from("/unused/drop-module"),
5871 args: Vec::new(),
5872 env: Vec::new(),
5873 reserved: false,
5874 reserved_prefixes: Vec::new(),
5875 protocol: ModuleProtocol::Subc,
5876 overlap: Default::default(),
5877 },
5878 supervisor.runtime_config(),
5879 Arc::clone(&snapshot),
5880 None,
5881 );
5882 assert!(!module
5883 .inner
5884 .monitor
5885 .lock()
5886 .unwrap()
5887 .as_ref()
5888 .unwrap()
5889 .is_finished());
5890
5891 drop(module);
5892
5893 assert_eq!(
5894 lock_snapshot(&snapshot).unwrap().state,
5895 ModuleState::Stopped
5896 );
5897 assert_snapshot_process_facts_cleared(&snapshot);
5898 }
5899
5900 #[cfg(unix)]
5901 #[tokio::test]
5902 async fn rescan_preserves_running_protocol_until_respawn() {
5903 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5904 let initial = ModuleSpec {
5905 module_id: "rescan-protocol".into(),
5906 program: PathBuf::from("/bin/sleep"),
5907 args: vec!["60".into()],
5908 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5909 .into_iter()
5910 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5911 .collect(),
5912 reserved: false,
5913 reserved_prefixes: vec![],
5914 protocol: ModuleProtocol::None,
5915 overlap: Default::default(),
5916 };
5917 let supervisor =
5918 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5919 let module = supervisor.spawn(initial.clone()).unwrap();
5920 assert!(module.status().unwrap().live);
5921 let mut next = initial;
5922 next.protocol = ModuleProtocol::Subc;
5923 module
5924 .update_configuration(next.clone(), HealthConfig::default(), None)
5925 .await
5926 .unwrap();
5927 assert!(
5928 module.status().unwrap().live,
5929 "rescan must not require HELLO from the old non-wire process"
5930 );
5931 let runtime = supervisor.runtime_config();
5932 let action = on_child_exit(
5933 &next,
5934 RestartPolicy::default(),
5935 &supervisor.registry,
5936 &module.inner.snapshot,
5937 &runtime.terminal_ring,
5938 &runtime.spawn_events,
5939 &runtime.child_roster,
5940 ExitReport {
5941 kind: ExitKind::Clean,
5942 code: Some(0),
5943 signal: None,
5944 at_ms: unix_ms_now(),
5945 },
5946 )
5947 .await;
5948 assert!(
5949 matches!(action, NextAction::Restart { .. }),
5950 "the old non-wire process's clean exit must restart"
5951 );
5952 module.drain().await.unwrap();
5953 }
5954
5955 #[cfg(unix)]
5956 #[tokio::test]
5957 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
5958 use std::os::unix::fs::PermissionsExt;
5959 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
5960 let script = dir.join("module.sh");
5961 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
5962 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
5963 let record_path = dir.join("live-children.json");
5964 let supervisor =
5965 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5966 .with_live_children_record(&record_path);
5967 for (program, args) in [
5968 (PathBuf::from("sleep"), vec!["60".into()]),
5969 (script, vec![]),
5970 ] {
5971 let spec = ModuleSpec {
5972 module_id: "image-identity".into(),
5973 program: program.clone(),
5974 args,
5975 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5976 .into_iter()
5977 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
5978 .collect(),
5979 reserved: false,
5980 reserved_prefixes: vec![],
5981 protocol: ModuleProtocol::None,
5982 overlap: Default::default(),
5983 };
5984 let module = supervisor.spawn(spec).unwrap();
5985 #[cfg(target_os = "macos")]
5986 {
5987 let deadline = Instant::now() + Duration::from_secs(5);
5990 while crate::live_children::read_record(&record_path)
5991 .unwrap()
5992 .iter()
5993 .all(|entry| entry.executable.is_none())
5994 {
5995 assert!(Instant::now() < deadline, "module image was not confirmed");
5996 tokio::time::sleep(Duration::from_millis(5)).await;
5997 }
5998 }
5999 let entry = crate::live_children::read_record(&record_path)
6000 .unwrap()
6001 .pop()
6002 .unwrap();
6003 let observed = subc_os::Process::open(entry.pid)
6004 .unwrap()
6005 .unwrap()
6006 .observe()
6007 .unwrap();
6008 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6009 module.drain().await.unwrap();
6010 assert_eq!(
6011 verdict,
6012 crate::live_children::IdentityVerdict::Matches,
6013 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6014 );
6015 }
6016 }
6017
6018 #[cfg(unix)]
6019 fn http_fixture(
6020 dir: &std::path::Path,
6021 url: &str,
6022 threshold: u32,
6023 ) -> crate::daemon_config::ConfiguredModule {
6024 let path = dir.join("subc.jsonc");
6025 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6026 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6027 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6028 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6029 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6030 }}}).to_string()).unwrap();
6031 crate::daemon_config::load(&path)
6032 .unwrap()
6033 .unwrap()
6034 .modules
6035 .pop()
6036 .unwrap()
6037 }
6038
6039 #[cfg(unix)]
6040 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6041 timeout(Duration::from_secs(5), async {
6042 loop {
6043 if module.status().unwrap().health.status == status {
6044 break;
6045 }
6046 sleep(Duration::from_millis(5)).await;
6047 }
6048 })
6049 .await
6050 .unwrap_or_else(|_| {
6051 panic!(
6052 "expected {status:?}, got {:?}",
6053 module.status().unwrap().health
6054 )
6055 });
6056 }
6057
6058 #[cfg(unix)]
6059 #[tokio::test]
6060 async fn http_health_status_flips_ok_failing_ok() {
6061 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6062 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6063 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6064 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6065 let serving_status = status.clone();
6066 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6067 let server = tokio::spawn(async move {
6068 loop {
6069 let (mut stream, _) = listener.accept().await.unwrap();
6070 let mut request = [0u8; 2048];
6071 let count = stream.read(&mut request).await.unwrap();
6072 assert!(count > 0, "a probe must send an HTTP request");
6073 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6074 let body = if code == 200 {
6075 "ready"
6076 } else {
6077 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6078 };
6079 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6080 let _ = stream.write_all(response.as_bytes()).await;
6081 }
6082 });
6083 let configured = http_fixture(&dir, &url, 1000);
6084 let module =
6085 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6086 .supervise_configured_with_health(
6087 configured.module_spec(),
6088 true,
6089 configured.health,
6090 configured.drain_timeout_ms,
6091 configured.restart,
6092 )
6093 .unwrap();
6094 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6095 status.store(503, std::sync::atomic::Ordering::SeqCst);
6096 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6097 assert!(module
6098 .status()
6099 .unwrap()
6100 .health
6101 .detail
6102 .unwrap()
6103 .contains("scratch failure"));
6104 status.store(200, std::sync::atomic::Ordering::SeqCst);
6105 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6106 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6107 let before = module.status().unwrap();
6108 let (spec, mut health) = module.configuration().unwrap();
6109 health.http = None;
6110 module
6111 .update_configuration(spec.clone(), health.clone(), Some(10))
6112 .await
6113 .unwrap();
6114 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6115 health.http = Some(url);
6116 module
6117 .update_configuration(spec, health, Some(10))
6118 .await
6119 .unwrap();
6120 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6121 assert_eq!(
6122 module.status().unwrap().pid,
6123 before.pid,
6124 "changing a probe must apply live, not restart its process"
6125 );
6126 let (spec, mut health) = module.configuration().unwrap();
6127 health.failure_threshold = 2;
6128 module
6129 .update_configuration(spec, health, Some(10))
6130 .await
6131 .unwrap();
6132 status.store(503, std::sync::atomic::Ordering::SeqCst);
6133 timeout(Duration::from_secs(5), async {
6134 while module.status().unwrap().spawn_generation == before.spawn_generation {
6135 sleep(Duration::from_millis(5)).await;
6136 }
6137 })
6138 .await
6139 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6140 module.drain().await.unwrap();
6141 server.abort();
6142 }
6143
6144 #[cfg(unix)]
6145 #[tokio::test]
6146 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6147 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6148 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6149 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6150 drop(unused);
6151 let configured = http_fixture(&dir, &url, 2);
6152 let module =
6153 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6154 .supervise_configured_with_health(
6155 configured.module_spec(),
6156 true,
6157 configured.health,
6158 configured.drain_timeout_ms,
6159 configured.restart,
6160 )
6161 .unwrap();
6162 let before = module.status().unwrap().spawn_generation;
6163 timeout(Duration::from_secs(5), async {
6164 loop {
6165 let status = module.status().unwrap();
6166 if status.spawn_generation > before {
6167 assert!(status.lifetime_restarts > 0);
6168 break;
6169 }
6170 sleep(Duration::from_millis(5)).await;
6171 }
6172 })
6173 .await
6174 .expect("sustained HTTP refusal must trigger the health restart policy");
6175 module.drain().await.unwrap();
6176 }
6177
6178 #[cfg(unix)]
6179 #[tokio::test]
6180 async fn http_health_timeout_honours_deadline() {
6181 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6182 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6183 let server = tokio::spawn(async move {
6184 let _held = listener.accept().await.unwrap();
6185 std::future::pending::<()>().await;
6186 });
6187 let error = timeout(
6188 Duration::from_secs(1),
6189 probe_http_health(&url, Duration::from_millis(10)),
6190 )
6191 .await
6192 .expect("the probe must enforce its own deadline")
6193 .unwrap_err();
6194 server.abort();
6195 assert!(error.to_string().contains("timed out"));
6196 }
6197
6198 #[cfg(unix)]
6199 #[tokio::test]
6200 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6201 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6202 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6203 let url = format!(
6204 "http://localhost:{}/healthz",
6205 listener.local_addr().unwrap().port()
6206 );
6207 let server = tokio::spawn(async move {
6208 let (mut stream, _) = listener.accept().await.unwrap();
6209 let mut request = [0u8; 2048];
6210 assert!(stream.read(&mut request).await.unwrap() > 0);
6211 stream
6212 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6213 .await
6214 .unwrap();
6215 });
6216 assert_eq!(
6221 probe_http_health(&url, Duration::from_secs(10))
6222 .await
6223 .unwrap()
6224 .status,
6225 HealthStatus::Ok
6226 );
6227 server.await.unwrap();
6228 }
6229
6230 #[cfg(unix)]
6231 #[tokio::test]
6232 async fn http_health_timeout_keeps_partial_status_and_body() {
6233 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6234 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6235 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6236 let server = tokio::spawn(async move {
6237 let (mut stream, _) = listener.accept().await.unwrap();
6238 let mut request = [0u8; 2048];
6239 assert!(stream.read(&mut request).await.unwrap() > 0);
6240 stream
6241 .write_all(
6242 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6243 )
6244 .await
6245 .unwrap();
6246 std::future::pending::<()>().await;
6247 });
6248 let error = probe_http_health(&url, Duration::from_secs(1))
6249 .await
6250 .unwrap_err()
6251 .to_string();
6252 server.abort();
6253 assert!(
6254 error.contains("timed out")
6255 && error.contains("503 Unavailable")
6256 && error.contains("partial diagnostic"),
6257 "{error}"
6258 );
6259 }
6260
6261 #[cfg(unix)]
6262 #[tokio::test]
6263 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6264 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6265 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6266 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6267 let server = tokio::spawn(async move {
6268 let (mut stream, _) = listener.accept().await.unwrap();
6269 let mut request = [0u8; 2048];
6270 assert!(stream.read(&mut request).await.unwrap() > 0);
6271 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6272 let response = format!(
6273 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6274 body.len()
6275 );
6276 stream.write_all(response.as_bytes()).await.unwrap();
6277 });
6278 let error = probe_http_health(&url, Duration::from_secs(1))
6279 .await
6280 .unwrap_err()
6281 .to_string();
6282 server.await.unwrap();
6283 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6284 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6285 assert!(!error.contains("not-in-diagnostic"));
6286 }
6287
6288 #[cfg(unix)]
6289 #[tokio::test]
6290 async fn http_health_real_nats_server_monitoring() {
6291 if std::process::Command::new("nats-server")
6292 .arg("--version")
6293 .env("XDG_DATA_HOME", std::env::temp_dir())
6294 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6295 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6296 .output()
6297 .is_err()
6298 {
6299 eprintln!(
6300 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6301 );
6302 return;
6303 }
6304 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6305 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6306 let port = monitor.local_addr().unwrap().port();
6307 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6308 let client_port = client.local_addr().unwrap().port();
6309 let config = dir.join("server.conf");
6310 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6311 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6312 configured.program = PathBuf::from("nats-server");
6313 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6314 drop(monitor);
6315 drop(client);
6316 let module =
6317 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6318 .supervise_configured_with_health(
6319 configured.module_spec(),
6320 true,
6321 configured.health,
6322 configured.drain_timeout_ms,
6323 configured.restart,
6324 )
6325 .unwrap();
6326 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6327 module.drain().await.unwrap();
6328 }
6329
6330 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6331 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6332 let supervisor =
6333 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6334 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6335 let initial = ModuleSpec {
6336 module_id: "rescan-preserves-spawn-facts".to_string(),
6337 program: PathBuf::from("/spawned/module"),
6338 args: Vec::new(),
6339 env: Vec::new(),
6340 reserved: false,
6341 reserved_prefixes: Vec::new(),
6342 protocol: ModuleProtocol::Subc,
6343 overlap: Default::default(),
6344 };
6345 let module = supervisor.supervised_module(
6346 initial.clone(),
6347 supervisor.runtime_config(),
6348 snapshot,
6349 None,
6350 );
6351 let before = module.status().unwrap();
6352 let mut replacement = initial;
6353 replacement.program = PathBuf::from("/rescanned/replacement-module");
6354
6355 module
6356 .update_configuration(replacement, HealthConfig::default(), None)
6357 .await
6358 .unwrap();
6359
6360 let after = module.status().unwrap();
6361 assert_eq!(after.pid, before.pid);
6362 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6363 assert_eq!(after.spawned_from, before.spawned_from);
6364 drop(module);
6365 }
6366}
6367
6368fn unix_ms_now() -> u64 {
6369 SystemTime::now()
6370 .duration_since(UNIX_EPOCH)
6371 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6372 .unwrap_or(0)
6373}
6374
6375async fn supervise_loop(
6376 mut spec: ModuleSpec,
6377 mut runtime: SupervisorRuntimeConfig,
6378 registry: Arc<Registry>,
6379 process_liveness: Arc<SupervisorProcessLiveness>,
6380 snapshot: SharedSnapshot,
6381 mut child: Option<SupervisedChild>,
6382 mut commands: mpsc::Receiver<SupervisorCommand>,
6383) {
6384 let mut health_probe = HealthProbeRuntime::default();
6385 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6389 Ok(changes) => Some(changes),
6390 Err(err) => {
6391 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6394 None
6395 }
6396 };
6397 let mut pending_respawn: Option<PendingRespawn> = None;
6401 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6404 let mut previous_instance_recheck = tokio::time::interval_at(
6405 Instant::now() + Duration::from_secs(5),
6406 Duration::from_secs(5),
6407 );
6408 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6409 loop {
6410 #[cfg(test)]
6411 {
6412 lock_snapshot(&snapshot).unwrap().actor_select = None;
6413 }
6414 #[cfg(target_os = "macos")]
6415 if let Some(active) = child.as_mut() {
6416 active.confirm_privacy_exec().await;
6417 }
6418 if let Some(scheduled) = runtime
6419 .scheduled_respawn
6420 .lock()
6421 .unwrap_or_else(|p| p.into_inner())
6422 .take()
6423 {
6424 pending_respawn = Some(scheduled);
6425 }
6426 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6427 pending_respawn = None;
6428 cancel_deferred_reload(
6429 &runtime,
6430 &spec.module_id,
6431 "respawn cancelled by a supervisor command",
6432 );
6433 }
6434 if child.is_none()
6435 && pending_respawn.is_none()
6436 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6437 {
6438 cancel_deferred_reload(
6439 &runtime,
6440 &spec.module_id,
6441 "respawn cancelled before a replacement was spawned",
6442 );
6443 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6444 state.respawn_pending = false;
6445 state.coalesced_restart_pending = false;
6446 if matches!(
6447 state.state,
6448 ModuleState::Restarting
6449 | ModuleState::Starting
6450 | ModuleState::Draining
6451 | ModuleState::Unresponsive
6452 ) {
6453 error!(module_id = %spec.module_id, state = ?state.state,
6454 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6455 state.state = ModuleState::Failed;
6456 clear_current_process_facts(state);
6457 }
6458 });
6459 }
6460 if let Some(command) = requeued.pop_front() {
6461 if !handle_supervisor_command(
6462 command,
6463 &mut spec,
6464 &mut runtime,
6465 ®istry,
6466 &process_liveness,
6467 &snapshot,
6468 &mut child,
6469 &mut commands,
6470 &mut requeued,
6471 )
6472 .await
6473 {
6474 return;
6475 }
6476 if child.is_some() || !respawn_still_pending(&snapshot) {
6477 pending_respawn = None;
6478 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6479 state.respawn_pending = false
6480 });
6481 }
6482 continue;
6483 }
6484 if child.is_some() {
6485 #[cfg(test)]
6486 {
6487 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6488 }
6489 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6490 let wake_after = health_probe.wake_after();
6491 let probe_sleep = async move {
6492 match wake_after {
6493 Some(delay) => sleep(delay).await,
6494 None => std::future::pending().await,
6495 }
6496 };
6497 tokio::pin!(probe_sleep);
6498 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6503 #[cfg(test)]
6504 {
6505 let registration_pending = wire_child
6506 && registration_changes
6507 .as_ref()
6508 .is_some_and(|changes| changes.has_changed().unwrap());
6509 let mut state = lock_snapshot(&snapshot).unwrap();
6510 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6511 generation: state.spawn_generation,
6512 turn: state.actor_turns,
6513 registered_connection: health_probe.registered_connection,
6514 next_probe_at: health_probe.next_probe_at,
6515 wake_after,
6516 });
6517 }
6518 let active_child = child.as_mut().expect("child checked above");
6519 tokio::select! {
6520 wait_result = active_child.wait() => {
6521 #[cfg(test)]
6522 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6523 let exit_report = match wait_result {
6532 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6533 Err(err) => {
6534 active_child.drain_stderr(&spec.module_id).await;
6535 fail_after_wait_error(
6536 &spec.module_id,
6537 &snapshot,
6538 &runtime.terminal_ring,
6539 &runtime.spawn_events,
6540 format!("failed to wait for supervised module: {err}"),
6541 );
6542 untrack_if_registration_released(
6543 &process_liveness,
6544 ®istry,
6545 &spec.module_id,
6546 &snapshot,
6547 );
6548 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6549 child = None;
6550 continue;
6551 }
6552 };
6553 active_child.drain_stderr(&spec.module_id).await;
6554
6555 let next = on_child_exit(
6556 &spec,
6557 runtime.restart_policy,
6558 ®istry,
6559 &snapshot,
6560 &runtime.terminal_ring,
6561 &runtime.spawn_events,
6562 &runtime.child_roster,
6563 exit_report,
6564 ).await;
6565 runtime.start_requester = None;
6566 runtime.pending_activity_restart = false;
6567 active_child.release_roster();
6570 match next {
6571 NextAction::Stop { registration_released } => {
6572 if registration_released {
6573 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6574 }
6575 child = None;
6576 }
6577 NextAction::Restart { schedule } => {
6578 let delay = schedule.map_or(
6579 runtime.restart_policy.delay_for_restart(0),
6580 |schedule| schedule.delay,
6581 );
6582 if let Some(schedule) = schedule {
6583 log_crash_respawn(&spec.module_id, schedule);
6584 }
6585 child = None;
6593 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6594 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6595 }
6596 }
6597 }
6598 command = commands.recv() => {
6599 #[cfg(test)]
6600 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6601 let Some(command) = command else {
6602 return;
6603 };
6604 if !handle_supervisor_command(
6605 command,
6606 &mut spec,
6607 &mut runtime,
6608 ®istry,
6609 &process_liveness,
6610 &snapshot,
6611 &mut child,
6612 &mut commands,
6613 &mut requeued,
6614 ).await {
6615 return;
6616 }
6617 }
6618 _ = async {
6619 match registration_changes.as_mut() {
6620 Some(changes) => { let _ = changes.changed().await; }
6621 None => std::future::pending().await,
6622 }
6623 }, if wire_child => {
6624 #[cfg(test)]
6625 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6626 }
6627 _ = &mut probe_sleep => {
6628 #[cfg(test)]
6629 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6630 if health_probe.due() {
6631 run_health_probe_cycle(
6632 &spec,
6633 &runtime,
6634 ®istry,
6635 &process_liveness,
6636 &snapshot,
6637 &mut child,
6638 ).await;
6639 if child.is_none() {
6640 runtime.start_requester = None;
6641 runtime.pending_activity_restart = false;
6642 }
6643 if child.is_some() {
6644 health_probe.schedule_next(&spec, runtime.health.cadence);
6645 }
6646 }
6647 }
6648 }
6649 } else if let Some(pending) = pending_respawn {
6650 tokio::select! {
6651 _ = sleep_until(pending.deadline) => {
6652 pending_respawn = None;
6653 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6654 if !respawn_still_pending(&snapshot) {
6658 continue;
6659 }
6660 if runtime.child_roster.is_closed() {
6665 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6666 state.state = ModuleState::Stopped;
6667 });
6668 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6669 continue;
6670 }
6671 if let Err(err) = release_dead_registration(
6672 ®istry,
6673 runtime.forwarding.as_deref(),
6674 &snapshot,
6675 &spec.module_id,
6676 ).await {
6677 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6678 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6679 continue;
6680 }
6681
6682 if matches!(pending.kind, RespawnKind::Reload) {
6683 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6684 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
6685 if result.is_ok() && runtime.pending_activity_restart {
6686 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6687 }
6688 runtime.pending_activity_restart = false;
6689 if let Some(reply) = reply { let _ = reply.send(result); }
6690 continue;
6691 }
6692 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6693 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6694 Ok(next_child) => {
6695 child = Some(next_child);
6696 if runtime.pending_activity_restart {
6697 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6698 runtime.pending_activity_restart = false;
6699 }
6700 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6701 }
6702 Err(err) => {
6703 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6704 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6705 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6706 }
6707 }
6708 }
6709 command = commands.recv() => {
6710 let Some(command) = command else {
6711 return;
6712 };
6713 if !handle_supervisor_command(
6714 command,
6715 &mut spec,
6716 &mut runtime,
6717 ®istry,
6718 &process_liveness,
6719 &snapshot,
6720 &mut child,
6721 &mut commands,
6722 &mut requeued,
6723 ).await {
6724 return;
6725 }
6726 if child.is_some() || !respawn_still_pending(&snapshot) {
6731 pending_respawn = None;
6732 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6733 }
6734 }
6735 }
6736 } else {
6737 let command = if lock_snapshot(&snapshot)
6738 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6739 {
6740 tokio::select! {
6741 command = commands.recv() => command,
6742 _ = previous_instance_recheck.tick() => {
6743 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6744 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6745 if enabled && !runtime.child_roster.is_closed() {
6746 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6747 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6748 Ok(next_child) => child = Some(next_child),
6749 Err(error) => {
6750 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6751 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6752 }
6753 }
6754 } else {
6755 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6756 state.waiting_for_previous_pid = None;
6757 state.waiting_for_previous_detail = None;
6758 });
6759 }
6760 }
6761 continue;
6762 }
6763 }
6764 } else {
6765 commands.recv().await
6766 };
6767 let Some(command) = command else {
6768 return;
6769 };
6770 if !handle_supervisor_command(
6771 command,
6772 &mut spec,
6773 &mut runtime,
6774 ®istry,
6775 &process_liveness,
6776 &snapshot,
6777 &mut child,
6778 &mut commands,
6779 &mut requeued,
6780 )
6781 .await
6782 {
6783 return;
6784 }
6785 }
6786 }
6787}
6788
6789fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6790 info!(
6791 module_id,
6792 restart_in_window = schedule.restart_in_window,
6793 delay_ms = schedule.delay.as_millis() as u64,
6794 "respawning after crash"
6795 );
6796}
6797
6798fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6804 matches!(
6805 lock_snapshot(snapshot),
6806 Ok(state) if state.enabled && state.state == ModuleState::Restarting
6807 )
6808}
6809
6810enum NextAction {
6811 Stop {
6812 registration_released: bool,
6813 },
6814 Restart {
6815 schedule: Option<CrashRestartSchedule>,
6816 },
6817}
6818
6819#[allow(clippy::too_many_arguments)]
6820async fn handle_supervisor_command(
6821 command: SupervisorCommand,
6822 spec: &mut ModuleSpec,
6823 runtime: &mut SupervisorRuntimeConfig,
6824 registry: &Arc<Registry>,
6825 process_liveness: &SupervisorProcessLiveness,
6826 snapshot: &SharedSnapshot,
6827 child: &mut Option<SupervisedChild>,
6828 commands: &mut mpsc::Receiver<SupervisorCommand>,
6829 requeued: &mut VecDeque<SupervisorCommand>,
6830) -> bool {
6831 let command =
6832 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6833 match command {
6834 SupervisorCommand::Restart { reply, .. }
6835 | SupervisorCommand::Reload { reply, .. }
6836 | SupervisorCommand::Swap { reply, .. } => {
6837 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6838 return true;
6839 }
6840 SupervisorCommand::SetEnabled {
6841 enabled: true,
6842 reply,
6843 ..
6844 } => {
6845 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6846 return true;
6847 }
6848 other => other,
6849 }
6850 } else {
6851 command
6852 };
6853 match command {
6854 SupervisorCommand::Drain { requester, reply } => {
6855 let result = drain_optional_child(
6858 &spec.module_id,
6859 spec.protocol,
6860 StopNotice::NotSent,
6861 registry,
6862 runtime.forwarding.as_deref(),
6863 snapshot,
6864 &runtime.terminal_ring,
6865 &runtime.spawn_events,
6866 child,
6867 runtime.drain_timeout,
6868 ModuleState::Stopped,
6869 None,
6870 )
6871 .await;
6872 let registration_released = result.is_ok();
6873 if registration_released {
6874 runtime.record_activity_transition(
6875 "subc.module_stopped",
6876 &spec.module_id,
6877 requester,
6878 );
6879 }
6880 let _ = reply.send(result);
6881 if registration_released {
6882 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6883 }
6884 false
6885 }
6886 SupervisorCommand::Retire { requester, reply } => {
6887 let result = async {
6888 let stop_notice = begin_forwarding_drain_if_configured(
6889 spec,
6890 runtime,
6891 registry,
6892 snapshot,
6893 None,
6894 RouteCloseReason::Disable,
6895 )
6896 .await?;
6897 drain_optional_child(
6898 &spec.module_id,
6899 spec.protocol,
6900 stop_notice,
6901 registry,
6902 runtime.forwarding.as_deref(),
6903 snapshot,
6904 &runtime.terminal_ring,
6905 &runtime.spawn_events,
6906 child,
6907 runtime.drain_timeout,
6908 ModuleState::Stopped,
6909 None,
6910 )
6911 .await
6912 }
6913 .await;
6914 let registration_released = result.is_ok();
6915 if registration_released {
6916 runtime.record_activity_transition(
6917 "subc.module_stopped",
6918 &spec.module_id,
6919 requester,
6920 );
6921 }
6922 let _ = reply.send(result);
6923 if registration_released {
6924 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6925 }
6926 false
6927 }
6928 SupervisorCommand::Restart {
6929 requester,
6930 drain_timeout_ms,
6931 received_at_generation,
6932 queued_at,
6933 reply,
6934 } => {
6935 info!(
6939 module_id = %spec.module_id,
6940 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6941 "restart command dequeued"
6942 );
6943 let validation = match lock_snapshot(snapshot) {
6955 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
6956 module_id: spec.module_id.clone(),
6957 }),
6958 Ok(_) => Ok(()),
6959 Err(err) => Err(err),
6960 };
6961 let initiated = validation.is_ok();
6962 let _ = reply.send(validation);
6963 let satisfied_by_generation = if initiated && child.is_some() {
6974 lock_snapshot(snapshot).ok().and_then(|state| {
6975 (state.spawn_generation > received_at_generation
6976 && !state.configuration_updated_since_spawn)
6977 .then_some(state.spawn_generation)
6978 })
6979 } else {
6980 None
6981 };
6982 let satisfied_by_pending = initiated
6983 && child.is_none()
6984 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
6985 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
6986 if pending {
6987 state.coalesced_restart_pending = true;
6988 }
6989 pending
6990 });
6991 if satisfied_by_pending {
6992 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
6993 } else if let Some(generation) = satisfied_by_generation {
6994 info!(
6995 module_id = %spec.module_id,
6996 received_at_generation,
6997 "restart already satisfied by generation {generation}; not restarting again"
6998 );
6999 } else if initiated {
7000 let drain_timeout = drain_timeout_ms
7003 .map(Duration::from_millis)
7004 .unwrap_or(runtime.drain_timeout);
7005 if let Err(err) = restart_child(
7006 spec,
7007 runtime,
7008 registry,
7009 process_liveness,
7010 snapshot,
7011 child,
7012 drain_timeout,
7013 )
7014 .await
7015 {
7016 warn!(
7017 module_id = %spec.module_id,
7018 error = %err,
7019 "operator restart failed after initiation ack; module state carries the outcome"
7020 );
7021 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7022 state.state = ModuleState::Failed;
7023 clear_current_process_facts(state);
7024 });
7025 } else {
7026 runtime.start_requester = requester;
7027 runtime.pending_activity_restart = true;
7028 }
7029 }
7030 true
7031 }
7032 SupervisorCommand::Reload { requester, reply } => {
7033 let result =
7034 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7035 if result.is_ok() {
7036 runtime.start_requester = requester;
7037 runtime.pending_activity_restart = true;
7038 }
7039 if result.is_ok()
7040 && runtime
7041 .scheduled_respawn
7042 .lock()
7043 .unwrap_or_else(|p| p.into_inner())
7044 .as_ref()
7045 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7046 {
7047 *runtime
7048 .deferred_reload_reply
7049 .lock()
7050 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7051 } else {
7052 let _ = reply.send(result);
7053 }
7054 true
7055 }
7056 SupervisorCommand::SetEnabled {
7057 enabled,
7058 requester,
7059 reply,
7060 } => {
7061 let previous_requester = runtime.start_requester.clone();
7062 if enabled {
7063 runtime.start_requester = requester.clone();
7064 }
7065 let result = set_child_enabled(
7066 spec,
7067 runtime,
7068 registry,
7069 process_liveness,
7070 snapshot,
7071 child,
7072 enabled,
7073 )
7074 .await;
7075 if matches!(result, Ok(true)) {
7076 runtime.pending_activity_restart = false;
7077 if enabled {
7078 runtime.start_requester = requester;
7079 } else {
7080 runtime.start_requester = None;
7081 runtime.record_activity_transition(
7082 "subc.module_disabled",
7083 &spec.module_id,
7084 requester,
7085 );
7086 }
7087 } else {
7088 runtime.start_requester = previous_requester;
7089 }
7090 let _ = reply.send(result);
7091 true
7092 }
7093 SupervisorCommand::UpdateConfiguration {
7094 spec: next_spec,
7095 health,
7096 drain_timeout_ms,
7097 reply,
7098 } => {
7099 if let Some(handle) = &runtime.supervisor_handle {
7100 handle.apply_identity_configuration(&next_spec);
7101 }
7102 *spec = next_spec;
7103 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7104 state.configuration_updated_since_spawn = true;
7105 });
7106 let health_changed = runtime.health != health;
7107 runtime.health = health;
7108 if health_changed {
7111 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7112 state.health = ModuleHealthStatus::default();
7113 });
7114 }
7115 runtime.drain_timeout = drain_timeout_ms
7116 .map(Duration::from_millis)
7117 .unwrap_or(runtime.default_drain_timeout);
7118 *runtime
7119 .effective_drain_timeout
7120 .lock()
7121 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7122 let _ = reply.send(());
7123 true
7124 }
7125 SupervisorCommand::Swap {
7126 requester,
7127 ready_timeout,
7128 reply,
7129 } => {
7130 let mut swap_runtime = runtime.clone();
7131 swap_runtime.start_requester = requester;
7132 let before = lock_snapshot(snapshot)
7133 .ok()
7134 .map(|state| state.spawn_generation);
7135 let end = swap::run_swap(
7136 spec,
7137 &swap_runtime,
7138 registry,
7139 process_liveness,
7140 snapshot,
7141 child,
7142 commands,
7143 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7144 reply,
7145 )
7146 .await;
7147 if before
7148 != lock_snapshot(snapshot)
7149 .ok()
7150 .map(|state| state.spawn_generation)
7151 {
7152 runtime.start_requester = swap_runtime.start_requester;
7153 }
7154 requeued.extend(end.requeue);
7155 true
7156 }
7157 }
7158}
7159
7160async fn restart_child(
7161 spec: &ModuleSpec,
7162 runtime: &SupervisorRuntimeConfig,
7163 registry: &Registry,
7164 process_liveness: &SupervisorProcessLiveness,
7165 snapshot: &SharedSnapshot,
7166 child: &mut Option<SupervisedChild>,
7167 drain_timeout: Duration,
7168) -> Result<(), SuperviseError> {
7169 if !lock_snapshot(snapshot)?.enabled {
7171 return Err(SuperviseError::Disabled {
7172 module_id: spec.module_id.clone(),
7173 });
7174 }
7175 let stop_notice = begin_forwarding_drain_with_timeout(
7176 spec,
7177 runtime,
7178 registry,
7179 snapshot,
7180 None,
7181 RouteCloseReason::Restart,
7182 drain_timeout,
7183 )
7184 .await?;
7185
7186 if child.is_some() {
7187 drain_optional_child(
7188 &spec.module_id,
7189 spec.protocol,
7190 stop_notice,
7191 registry,
7192 runtime.forwarding.as_deref(),
7193 snapshot,
7194 &runtime.terminal_ring,
7195 &runtime.spawn_events,
7196 child,
7197 drain_timeout,
7198 ModuleState::Restarting,
7199 Some(true),
7200 )
7201 .await?;
7202 } else {
7203 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7204 state.enabled = true;
7205 state.state = ModuleState::Restarting;
7206 clear_current_process_facts(state);
7207 })?;
7208 release_dead_registration(
7209 registry,
7210 runtime.forwarding.as_deref(),
7211 snapshot,
7212 &spec.module_id,
7213 )
7214 .await?;
7215 }
7216
7217 reset_restart_count(snapshot, &spec.module_id)?;
7218 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7219 schedule_respawn(
7220 runtime,
7221 snapshot,
7222 &spec.module_id,
7223 runtime.restart_policy.backoff,
7224 RespawnKind::Spawn,
7225 )
7226}
7227
7228async fn reload_child(
7229 spec: &ModuleSpec,
7230 runtime: &SupervisorRuntimeConfig,
7231 registry: &Registry,
7232 process_liveness: &SupervisorProcessLiveness,
7233 snapshot: &SharedSnapshot,
7234 child: &mut Option<SupervisedChild>,
7235) -> Result<(), SuperviseError> {
7236 if !lock_snapshot(snapshot)?.enabled {
7238 return Err(SuperviseError::Disabled {
7239 module_id: spec.module_id.clone(),
7240 });
7241 }
7242 let stop_notice = begin_forwarding_drain(
7243 spec,
7244 runtime,
7245 registry,
7246 snapshot,
7247 Some(true),
7248 RouteCloseReason::Reload,
7249 )
7250 .await?;
7251
7252 if child.is_some() {
7253 drain_optional_child(
7254 &spec.module_id,
7255 spec.protocol,
7256 stop_notice,
7257 registry,
7258 runtime.forwarding.as_deref(),
7259 snapshot,
7260 &runtime.terminal_ring,
7261 &runtime.spawn_events,
7262 child,
7263 runtime.drain_timeout,
7264 ModuleState::Restarting,
7265 Some(true),
7266 )
7267 .await?;
7268 } else {
7269 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7270 state.enabled = true;
7271 state.state = ModuleState::Restarting;
7272 clear_current_process_facts(state);
7273 })?;
7274 release_dead_registration(
7275 registry,
7276 runtime.forwarding.as_deref(),
7277 snapshot,
7278 &spec.module_id,
7279 )
7280 .await?;
7281 }
7282
7283 reset_restart_count(snapshot, &spec.module_id)?;
7284 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7285 schedule_respawn(
7286 runtime,
7287 snapshot,
7288 &spec.module_id,
7289 runtime.restart_policy.backoff,
7290 RespawnKind::Reload,
7291 )
7292}
7293
7294async fn finish_reload_child(
7295 spec: &ModuleSpec,
7296 runtime: &SupervisorRuntimeConfig,
7297 registry: &Registry,
7298 process_liveness: &SupervisorProcessLiveness,
7299 snapshot: &SharedSnapshot,
7300 child: &mut Option<SupervisedChild>,
7301) -> Result<(), SuperviseError> {
7302 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7303 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7304 Ok(next_child) => next_child,
7305 Err(err) => {
7306 return handle_reload_spawn_failure(
7307 spec,
7308 runtime,
7309 process_liveness,
7310 snapshot,
7311 child,
7312 format!("new child failed to spawn: {err}"),
7313 )
7314 .await;
7315 }
7316 };
7317 *child = Some(next_child);
7318
7319 let wait_outcome = {
7320 let active_child = child.as_mut().expect("new reload child was just stored");
7321 wait_for_registration_after_reload(
7322 registry,
7323 &spec.module_id,
7324 snapshot,
7325 active_child,
7326 REGISTRY_RELEASE_TIMEOUT,
7327 )
7328 .await?
7329 };
7330
7331 match wait_outcome {
7332 RegistrationWaitOutcome::Registered => {
7333 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7334 Ok(())
7335 }
7336 RegistrationWaitOutcome::Exited(exit_report) => {
7337 if let Some(active_child) = child.as_mut() {
7338 active_child.drain_stderr(&spec.module_id).await;
7339 }
7340 let mut exited_child = child.take().expect("exited reload child is still stored");
7343 #[cfg(test)]
7344 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7345 gate.reached.notify_one();
7346 gate.resume.notified().await;
7347 }
7348 let result = handle_reload_child_registration_failure(
7349 spec,
7350 runtime,
7351 registry,
7352 process_liveness,
7353 snapshot,
7354 child,
7355 ReloadRegistrationFailure {
7356 exit_report: registration_failure_exit_report(exit_report),
7357 reason: exited_child
7358 .spawn_failure
7359 .clone()
7360 .unwrap_or_else(|| "new child exited before registering".to_string()),
7361 },
7362 )
7363 .await;
7364 exited_child.release_roster();
7365 result
7366 }
7367 RegistrationWaitOutcome::TimedOut => {
7368 let mut timed_out_child = child
7369 .take()
7370 .expect("timed-out reload child is still running");
7371 timed_out_child
7372 .start_kill()
7373 .map_err(|source| SuperviseError::Kill {
7374 module_id: spec.module_id.clone(),
7375 source,
7376 })?;
7377 let status = timed_out_child
7378 .wait()
7379 .await
7380 .map_err(|source| SuperviseError::Wait {
7381 module_id: spec.module_id.clone(),
7382 source,
7383 })?;
7384 timed_out_child.drain_stderr(&spec.module_id).await;
7385 handle_reload_child_registration_failure(
7386 spec,
7387 runtime,
7388 registry,
7389 process_liveness,
7390 snapshot,
7391 child,
7392 ReloadRegistrationFailure {
7393 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7394 snapshot,
7395 &timed_out_child,
7396 &status,
7397 )),
7398 reason: format!(
7399 "new child did not register within {:?}",
7400 REGISTRY_RELEASE_TIMEOUT
7401 ),
7402 },
7403 )
7404 .await
7405 }
7406 }
7407}
7408
7409async fn set_child_enabled(
7410 spec: &ModuleSpec,
7411 runtime: &SupervisorRuntimeConfig,
7412 registry: &Registry,
7413 process_liveness: &SupervisorProcessLiveness,
7414 snapshot: &SharedSnapshot,
7415 child: &mut Option<SupervisedChild>,
7416 enabled: bool,
7417) -> Result<bool, SuperviseError> {
7418 let (current_enabled, current_state, respawn_pending) = {
7419 let state = lock_snapshot(snapshot)?;
7420 (state.enabled, state.state, state.respawn_pending)
7421 };
7422 let revive_terminal = enabled
7430 && current_enabled
7431 && child.is_none()
7432 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7433 || (current_state == ModuleState::Restarting && !respawn_pending));
7434 if current_enabled == enabled && !revive_terminal {
7435 return Ok(false);
7436 }
7437
7438 if enabled {
7439 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7440 state.enabled = true;
7441 state.state = ModuleState::Starting;
7442 clear_current_process_facts(state);
7443 })?;
7444 #[cfg(test)]
7445 if runtime.test_seed_stale_facts_before_enable_spawn {
7446 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7447 state.process_alive = true;
7448 state.pid = Some(41);
7449 state.spawned_at_ms = Some(42);
7450 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7451 state.spawned_file_identity = Some(SpawnedFileIdentity {
7452 device: 43,
7453 inode: 44,
7454 #[cfg(windows)]
7455 windows_image: None,
7456 });
7457 })?;
7458 }
7459 release_dead_registration(
7460 registry,
7461 runtime.forwarding.as_deref(),
7462 snapshot,
7463 &spec.module_id,
7464 )
7465 .await?;
7466 reset_restart_count(snapshot, &spec.module_id)?;
7467 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7468 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7469 Ok(next_child) => next_child,
7470 Err(err) => {
7471 if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7472 state.state = ModuleState::Failed;
7473 clear_current_process_facts(state);
7474 }) {
7475 error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7476 }
7477 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7478 return Err(err);
7479 }
7480 };
7481 *child = Some(next_child);
7482 debug!(module_id = %spec.module_id, "supervised module enabled");
7483 Ok(true)
7484 } else {
7485 let stop_notice = begin_forwarding_drain_if_configured(
7486 spec,
7487 runtime,
7488 registry,
7489 snapshot,
7490 Some(false),
7491 RouteCloseReason::Disable,
7492 )
7493 .await?;
7494 drain_optional_child(
7495 &spec.module_id,
7496 spec.protocol,
7497 stop_notice,
7498 registry,
7499 runtime.forwarding.as_deref(),
7500 snapshot,
7501 &runtime.terminal_ring,
7502 &runtime.spawn_events,
7503 child,
7504 runtime.drain_timeout,
7505 ModuleState::Disabled,
7506 Some(false),
7507 )
7508 .await?;
7509 debug!(module_id = %spec.module_id, "supervised module disabled");
7510 Ok(true)
7511 }
7512}
7513
7514#[allow(clippy::too_many_arguments)]
7515async fn on_child_exit(
7516 spec: &ModuleSpec,
7517 policy: RestartPolicy,
7518 registry: &Registry,
7519 snapshot: &SharedSnapshot,
7520 terminal_ring: &Arc<Mutex<TerminalRing>>,
7521 spawn_events: &SpawnEventFeed,
7522 roster: &ChildRoster,
7523 exit_report: ExitReport,
7524) -> NextAction {
7525 if roster.is_closed() {
7531 return on_child_exit_during_daemon_shutdown(
7532 spec,
7533 registry,
7534 snapshot,
7535 terminal_ring,
7536 spawn_events,
7537 exit_report,
7538 )
7539 .await;
7540 }
7541 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7556 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7557 match exit_report.kind {
7558 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7559 info!(
7560 module_id = %spec.module_id,
7561 exit_code = ?exit_report.code,
7562 exit_signal = ?exit_report.signal,
7563 "supervised module exited cleanly"
7564 );
7565 if let Err(err) = publish_after_terminal_record(
7566 &spec.module_id,
7567 || {
7568 record_terminal(
7569 &spec.module_id,
7570 terminal_ring,
7571 spawn_events,
7572 &exit_report,
7573 TerminalDisposition::Stopped,
7574 )
7575 },
7576 || {
7577 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7578 state.state = ModuleState::Stopped;
7579 clear_current_process_facts(state);
7580 state.last_exit = Some(exit_report.clone());
7581 })
7582 },
7583 ) {
7584 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7585 }
7586 let registration_released = match wait_for_registration_release(
7587 registry,
7588 &spec.module_id,
7589 REGISTRY_RELEASE_TIMEOUT,
7590 )
7591 .await
7592 {
7593 Ok(()) => true,
7594 Err(err) => {
7595 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7596 false
7597 }
7598 };
7599 NextAction::Stop {
7600 registration_released,
7601 }
7602 }
7603 ExitKind::Clean | ExitKind::Crash => {
7604 if unrequested_clean_exit_of_protocol_none {
7605 warn!(
7606 module_id = %spec.module_id,
7607 exit_code = ?exit_report.code,
7608 exit_signal = ?exit_report.signal,
7609 "protocol-none module exited cleanly without a stop request; handling it as a crash"
7610 );
7611 } else {
7612 warn!(
7613 module_id = %spec.module_id,
7614 exit_code = ?exit_report.code,
7615 exit_signal = ?exit_report.signal,
7616 "supervised module exited abnormally (crash)"
7617 );
7618 }
7619 let mut restart_schedule = None;
7620 let mut disposition = TerminalDisposition::Disabled;
7621 let mut final_state = ModuleState::Disabled;
7622 let mut disposition_detail = lock_snapshot(snapshot)
7626 .ok()
7627 .and_then(|mut state| state.spawn_failure.take());
7628 let now = Instant::now();
7629 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7630 if state.enabled {
7631 if let Some(schedule) = state.next_crash_restart(&policy, now) {
7632 state.respawn_pending = true;
7638 final_state = ModuleState::Restarting;
7639 restart_schedule = Some(schedule);
7640 disposition = TerminalDisposition::Restarting;
7641 } else {
7642 final_state = ModuleState::Failed;
7643 disposition = TerminalDisposition::Failed;
7644 let budget = policy.budget_exhausted_detail();
7645 disposition_detail =
7646 Some(disposition_detail.take().map_or_else(
7647 || budget.clone(),
7648 |cause| format!("{cause}; {budget}"),
7649 ));
7650 }
7651 } else {
7652 disposition = TerminalDisposition::Disabled;
7653 }
7654 }) {
7655 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7656 return NextAction::Stop {
7657 registration_released: false,
7658 };
7659 }
7660 if disposition == TerminalDisposition::Failed {
7661 error!(
7666 module_id = %spec.module_id,
7667 max_restarts = policy.max_restarts,
7668 window_secs = policy.window.as_secs(),
7669 "module stopped: {}",
7670 policy.budget_exhausted_detail()
7671 );
7672 }
7673 if let Err(err) = publish_after_terminal_record(
7674 &spec.module_id,
7675 || {
7676 record_terminal_with_detail(
7677 &spec.module_id,
7678 terminal_ring,
7679 spawn_events,
7680 &exit_report,
7681 disposition,
7682 disposition_detail,
7683 )
7684 },
7685 || {
7686 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7687 state.state = final_state;
7688 clear_current_process_facts(state);
7689 state.last_exit = Some(exit_report.clone());
7690 })
7691 },
7692 ) {
7693 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7694 }
7695
7696 if let Some(schedule) = restart_schedule {
7697 NextAction::Restart {
7698 schedule: Some(schedule),
7699 }
7700 } else {
7701 let registration_released = match wait_for_registration_release(
7702 registry,
7703 &spec.module_id,
7704 REGISTRY_RELEASE_TIMEOUT,
7705 )
7706 .await
7707 {
7708 Ok(()) => true,
7709 Err(err) => {
7710 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7711 false
7712 }
7713 };
7714 NextAction::Stop {
7715 registration_released,
7716 }
7717 }
7718 }
7719 ExitKind::DeliberateSeverance => {
7720 warn!(
7721 module_id = %spec.module_id,
7722 exit_code = ?exit_report.code,
7723 exit_signal = ?exit_report.signal,
7724 "supervised module exited after deliberate connection severance"
7725 );
7726 let should_restart = match lock_snapshot(snapshot) {
7727 Ok(state) => state.enabled,
7728 Err(err) => {
7729 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7730 return NextAction::Stop {
7731 registration_released: false,
7732 };
7733 }
7734 };
7735 let final_state = if should_restart {
7736 ModuleState::Restarting
7737 } else {
7738 ModuleState::Disabled
7739 };
7740 if let Err(err) = publish_after_terminal_record(
7741 &spec.module_id,
7742 || {
7743 record_terminal(
7744 &spec.module_id,
7745 terminal_ring,
7746 spawn_events,
7747 &exit_report,
7748 terminal_disposition(final_state),
7749 )
7750 },
7751 || {
7752 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7753 state.state = final_state;
7754 clear_current_process_facts(state);
7755 state.last_exit = Some(exit_report.clone());
7756 state.lifetime_restarts += 1;
7757 })
7758 },
7759 ) {
7760 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7761 return NextAction::Stop {
7762 registration_released: false,
7763 };
7764 }
7765
7766 if should_restart {
7767 NextAction::Restart { schedule: None }
7768 } else {
7769 let registration_released = match wait_for_registration_release(
7770 registry,
7771 &spec.module_id,
7772 REGISTRY_RELEASE_TIMEOUT,
7773 )
7774 .await
7775 {
7776 Ok(()) => true,
7777 Err(err) => {
7778 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7779 false
7780 }
7781 };
7782 NextAction::Stop {
7783 registration_released,
7784 }
7785 }
7786 }
7787 }
7788}
7789
7790async fn on_child_exit_during_daemon_shutdown(
7791 spec: &ModuleSpec,
7792 registry: &Registry,
7793 snapshot: &SharedSnapshot,
7794 terminal_ring: &Arc<Mutex<TerminalRing>>,
7795 spawn_events: &SpawnEventFeed,
7796 exit_report: ExitReport,
7797) -> NextAction {
7798 info!(
7799 module_id = %spec.module_id,
7800 exit_code = ?exit_report.code,
7801 exit_signal = ?exit_report.signal,
7802 exit_kind = ?exit_report.kind,
7803 "supervised module exited during daemon shutdown; not restarting it"
7804 );
7805 if let Err(err) = publish_after_terminal_record(
7806 &spec.module_id,
7807 || {
7808 record_terminal(
7809 &spec.module_id,
7810 terminal_ring,
7811 spawn_events,
7812 &exit_report,
7813 TerminalDisposition::DaemonShutdown,
7814 )
7815 },
7816 || {
7817 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7818 state.state = ModuleState::Stopped;
7819 clear_current_process_facts(state);
7820 state.last_exit = Some(exit_report.clone());
7821 })
7822 },
7823 ) {
7824 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7825 }
7826 let registration_released =
7827 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7828 .await
7829 .is_ok();
7830 NextAction::Stop {
7831 registration_released,
7832 }
7833}
7834
7835fn publish_after_terminal_record<T>(
7840 module_id: &str,
7841 record: impl FnOnce(),
7842 publish: impl FnOnce() -> T,
7843) -> T {
7844 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7845 let published = publish();
7846 if let Err(panic) = recorded {
7847 error!(
7848 module_id,
7849 "terminal recording panicked; published final state before resuming unwind"
7850 );
7851 std::panic::resume_unwind(panic);
7852 }
7853 published
7854}
7855
7856fn fail_after_wait_error(
7857 module_id: &str,
7858 snapshot: &SharedSnapshot,
7859 terminal_ring: &Arc<Mutex<TerminalRing>>,
7860 spawn_events: &SpawnEventFeed,
7861 detail: String,
7862) {
7863 let report = wait_error_exit_report();
7864 publish_after_terminal_record(
7865 module_id,
7866 || {
7867 record_terminal_with_detail(
7868 module_id,
7869 terminal_ring,
7870 spawn_events,
7871 &report,
7872 TerminalDisposition::Failed,
7873 Some(detail),
7874 )
7875 },
7876 || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7877 );
7878}
7879
7880#[cfg(test)]
7881fn record_wait_error_terminal(
7882 module_id: &str,
7883 terminal_ring: &Arc<Mutex<TerminalRing>>,
7884 spawn_events: &SpawnEventFeed,
7885) {
7886 record_terminal(
7887 module_id,
7888 terminal_ring,
7889 spawn_events,
7890 &wait_error_exit_report(),
7891 TerminalDisposition::Failed,
7892 );
7893}
7894
7895fn record_terminal(
7896 module_id: &str,
7897 terminal_ring: &Arc<Mutex<TerminalRing>>,
7898 spawn_events: &SpawnEventFeed,
7899 exit_report: &ExitReport,
7900 disposition: TerminalDisposition,
7901) {
7902 record_terminal_with_detail(
7903 module_id,
7904 terminal_ring,
7905 spawn_events,
7906 exit_report,
7907 disposition,
7908 None,
7909 );
7910}
7911
7912fn durable_terminal_history_of(
7916 terminal_ring: &Mutex<TerminalRing>,
7917 module_id: &str,
7918) -> subc_control::TerminalHistory {
7919 let read = terminal_ring
7920 .lock()
7921 .unwrap_or_else(|p| p.into_inner())
7922 .capture_durable_history();
7923 read.read(module_id)
7924}
7925
7926fn record_terminal_with_detail(
7927 module_id: &str,
7928 terminal_ring: &Arc<Mutex<TerminalRing>>,
7929 spawn_events: &SpawnEventFeed,
7930 exit_report: &ExitReport,
7931 disposition: TerminalDisposition,
7932 disposition_detail: Option<String>,
7933) {
7934 #[cfg(test)]
7935 terminal_history_tests::before_terminal_record(spawn_events);
7936 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7937 let record = TerminalRecord {
7938 exit_code: exit_report.code,
7939 exit_signal: exit_report.signal,
7940 at_ms: exit_report.at_ms,
7941 disposition,
7942 exit_kind: exit_report.kind.into(),
7943 disposition_detail,
7944 };
7945 terminal_ring
7946 .lock()
7947 .unwrap_or_else(|poisoned| poisoned.into_inner())
7948 .record_exit(module_id, record);
7949}
7950
7951fn untrack_if_registration_released(
7952 process_liveness: &SupervisorProcessLiveness,
7953 registry: &Registry,
7954 module_id: &str,
7955 snapshot: &SharedSnapshot,
7956) {
7957 match registry.get_module(module_id) {
7958 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
7959 Ok(Some(_)) => {}
7960 Err(err) => {
7961 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
7962 }
7963 }
7964}
7965
7966#[cfg(test)]
7980fn apply_wire_spawn_args(
7981 command: &mut Command,
7982 spec: &ModuleSpec,
7983 connection_file_path: Option<&std::path::Path>,
7984 handle: Option<&SupervisorHandle>,
7985) -> Result<Option<NonceHandoff>, SuperviseError> {
7986 apply_wire_spawn_args_for_role(
7987 command,
7988 spec,
7989 connection_file_path,
7990 handle,
7991 SpawnRole::Plain,
7992 )
7993}
7994
7995#[cfg(unix)]
8000type NonceHandoff = subc_os::LaunchNonceHandoff;
8001#[cfg(windows)]
8002type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8003#[cfg(not(any(unix, windows)))]
8004type NonceHandoff = std::convert::Infallible;
8005
8006fn apply_wire_spawn_args_for_role(
8027 command: &mut Command,
8028 spec: &ModuleSpec,
8029 connection_file_path: Option<&std::path::Path>,
8030 handle: Option<&SupervisorHandle>,
8031 role: SpawnRole,
8032) -> Result<Option<NonceHandoff>, SuperviseError> {
8033 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8034 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8039 #[cfg(windows)]
8040 {
8041 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8042 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8043 }
8044 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8046 if spec.protocol == ModuleProtocol::None {
8047 return Ok(None);
8048 }
8049 if let Some(connection_file_path) = connection_file_path {
8050 command.arg(SUBC_ARG).arg(connection_file_path);
8051 }
8052
8053 let nonce = generate_launch_nonce()?;
8057 if let Some(handle) = handle {
8058 match role {
8059 SpawnRole::Plain => {
8060 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8061 if spec.reserved {
8062 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8063 }
8064 }
8065 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8066 }
8067 }
8068 #[cfg(unix)]
8069 let handoff = {
8070 let handoff =
8071 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8072 program: spec.program.clone(),
8073 source,
8074 cgroup_path: None,
8075 })?;
8076 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8077 Some(handoff)
8078 };
8079 #[cfg(windows)]
8080 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8081 #[cfg(not(any(unix, windows)))]
8082 let handoff = None;
8083 #[cfg(not(unix))]
8087 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8088 Ok(handoff)
8089}
8090
8091fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8092 command.env_remove(CK_LOG_ENV);
8093 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8100 for (key, value) in &spec.env {
8101 if matches!(
8105 key.as_str(),
8106 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8107 ) || key == SUBC_SPAWN_ROLE_ENV
8108 {
8109 continue;
8110 }
8111 command.env(key, value);
8112 }
8113}
8114
8115#[cfg(windows)]
8116fn prepare_windows_nonce_pipe(
8117 command: &mut Command,
8118 spec: &ModuleSpec,
8119 nonce: &str,
8120) -> Option<NonceHandoff> {
8121 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8122 Ok(handoff) => {
8123 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8124 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8129 Some(handoff)
8130 }
8131 Err(error) => {
8132 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8133 None
8134 }
8135 }
8136}
8137
8138#[cfg(windows)]
8139fn start_windows_nonce_pipe(
8140 handoff: Option<NonceHandoff>,
8141 pid: u32,
8142 spec: &ModuleSpec,
8143) -> Option<subc_os::LaunchNoncePipeDelivery> {
8144 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8148 Ok(delivery) => Some(delivery),
8149 Err(error) => {
8150 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8151 None
8152 }
8153 })
8154}
8155
8156fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8157 #[cfg(windows)]
8158 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8159 return Some(
8160 if snapshot
8161 .pipe_consumed
8162 .as_ref()
8163 .is_some_and(|consumed| consumed.get().is_some())
8164 {
8165 "pipe"
8166 } else {
8167 "env"
8168 }
8169 .to_owned(),
8170 );
8171 }
8172 let _ = snapshot;
8173 None
8174}
8175
8176#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8179enum SpawnRole {
8180 Plain,
8181 SwapCandidate,
8182}
8183
8184fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8187 if role == SpawnRole::SwapCandidate {
8188 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8189 }
8190}
8191
8192fn previous_instance_error(
8193 spec: &ModuleSpec,
8194 pid: u32,
8195 reason: crate::live_children::RetainedReason,
8196) -> SuperviseError {
8197 SuperviseError::Spawn {
8198 program: spec.program.clone(),
8199 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8200 cgroup_path: None,
8201 }
8202}
8203
8204fn spawn_child(
8205 spec: &ModuleSpec,
8206 connection_file_path: Option<&std::path::Path>,
8207 handle: Option<&SupervisorHandle>,
8208 ring: &Arc<Mutex<StderrRing>>,
8209 capture_logs_dir: Option<&std::path::Path>,
8210 roster: &ChildRoster,
8211 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8212) -> Result<SupervisedChild, SuperviseError> {
8213 spawn_child_in_slot(
8214 spec,
8215 connection_file_path,
8216 handle,
8217 ring,
8218 capture_logs_dir,
8219 roster,
8220 #[cfg(target_os = "linux")]
8221 cgroup_placement,
8222 SpawnRole::Plain,
8223 false,
8224 )
8225}
8226
8227#[allow(clippy::too_many_arguments)]
8240fn spawn_child_in_slot(
8241 spec: &ModuleSpec,
8242 connection_file_path: Option<&std::path::Path>,
8243 handle: Option<&SupervisorHandle>,
8244 ring: &Arc<Mutex<StderrRing>>,
8245 capture_logs_dir: Option<&std::path::Path>,
8246 roster: &ChildRoster,
8247 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8248 role: SpawnRole,
8249 alternate_slot: bool,
8250) -> Result<SupervisedChild, SuperviseError> {
8251 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8252 return Err(previous_instance_error(spec, pid, reason));
8253 }
8254 if roster.is_closed() {
8255 return Err(SuperviseError::Spawn {
8256 program: spec.program.clone(),
8257 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8258 cgroup_path: None,
8259 });
8260 }
8261 #[cfg(target_os = "linux")]
8262 let cgroup_name = {
8263 if cgroup_placement.is_none() {
8268 swap::cgroup_name(&spec.module_id, alternate_slot)
8269 } else {
8270 let nonce = generate_launch_nonce()?;
8271 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8272 let mut end = spec.module_id.len().min(64);
8275 while !spec.module_id.is_char_boundary(end) {
8276 end -= 1;
8277 }
8278 format!(
8279 "{}_{suffix}",
8280 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8281 )
8282 }
8283 };
8284 #[cfg(not(target_os = "linux"))]
8285 let _ = alternate_slot;
8286 #[cfg(target_os = "macos")]
8287 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8288 #[cfg(windows)]
8289 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8290 program: spec.program.clone(),
8291 source,
8292 cgroup_path: None,
8293 })?;
8294 #[cfg(windows)]
8295 let mut command = Command::new(image_capture.path());
8296 #[cfg(not(any(target_os = "macos", windows)))]
8297 let mut command = Command::new(&spec.program);
8298 command.args(&spec.args);
8299 apply_child_env(&mut command, spec);
8329 apply_spawn_role(&mut command, role);
8330 let nonce_handoff =
8331 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8332
8333 #[cfg(target_os = "linux")]
8334 let cgroup_path = cgroup_placement
8335 .map(|placement| placement.module_path(&cgroup_name))
8336 .transpose()
8337 .map_err(|source| SuperviseError::Cgroup {
8338 module_id: spec.module_id.clone(),
8339 source,
8340 })?;
8341 #[cfg(not(target_os = "linux"))]
8342 let cgroup_path: Option<PathBuf> = None;
8343 #[cfg(target_os = "linux")]
8344 if let Some(path) = &cgroup_path {
8345 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8346 if let Some(placement) = cgroup_placement {
8347 remove_module_cgroup(placement, &cgroup_name);
8348 }
8349 return Err(error);
8350 }
8351 }
8352
8353 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8354 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8355 match ChildOutputSink::open(&path, capture_retention(spec)) {
8356 Ok(sink) => sink,
8357 Err(error) => {
8358 warn!(
8359 module_id = %spec.module_id,
8360 path = %path.display(),
8361 error = %error,
8362 "could not open child output capture file; forwarding to stderr"
8363 );
8364 ChildOutputSink::Stderr
8365 }
8366 }
8367 } else {
8368 ChildOutputSink::Stderr
8369 };
8370
8371 command.stdout(Stdio::piped());
8372 command.stderr(Stdio::piped());
8373 command.kill_on_drop(true);
8374 #[cfg(unix)]
8391 command.process_group(0);
8392 command.stdin(Stdio::null());
8393 #[cfg(unix)]
8397 if let Some(handoff) = nonce_handoff {
8398 handoff.install_last(command.as_std_mut());
8399 }
8400 #[cfg(not(any(unix, windows)))]
8401 let _ = nonce_handoff;
8402
8403 #[cfg(windows)]
8408 {
8409 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8410 program: spec.program.clone(),
8411 source,
8412 cgroup_path: None,
8413 })?;
8414 subc_jobobject::suspend_on_create_async(&mut command);
8415 }
8416 let mut child = match command.spawn() {
8417 Ok(child) => child,
8418 Err(source) => {
8419 #[cfg(target_os = "linux")]
8420 if let Some(placement) = cgroup_placement {
8421 remove_module_cgroup(placement, &cgroup_name);
8422 }
8423 return Err(SuperviseError::Spawn {
8424 program: spec.program.clone(),
8425 source,
8426 cgroup_path,
8427 });
8428 }
8429 };
8430 #[cfg(test)]
8431 spawn_record_tests::after_create(&child, spec);
8432 #[cfg(target_os = "macos")]
8437 drop(exec_ack);
8438
8439 #[cfg(windows)]
8444 let windows_image = match image_capture.bind_tokio(&child) {
8445 Ok(image) => Arc::new(image),
8446 Err(source) => {
8447 kill_and_reap_rejected_child(&mut child);
8448 return Err(SuperviseError::Spawn {
8449 program: spec.program.clone(),
8450 source,
8451 cgroup_path: None,
8452 });
8453 }
8454 };
8455 #[cfg(windows)]
8458 let job = contain_spawned_child(&mut child, spec)?;
8459 let spawned_at_ms = unix_ms_now();
8460 #[cfg(not(windows))]
8461 let spawned_from = spec.program.clone();
8462 #[cfg(windows)]
8463 let spawned_from = windows_image.path().to_path_buf();
8464 #[cfg(not(windows))]
8465 let spawned_file_identity = spawned_file_identity(&spawned_from);
8466 #[cfg(windows)]
8467 let spawned_file_identity = Some(SpawnedFileIdentity {
8468 device: 0,
8469 inode: 0,
8470 windows_image: Some(Arc::clone(&windows_image)),
8471 });
8472 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8473 program: spec.program.clone(),
8474 source: io::Error::other("spawned child exposed no live pid"),
8475 cgroup_path: cgroup_path.clone(),
8476 })?;
8477 #[cfg(windows)]
8478 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8479 #[cfg(not(windows))]
8480 let process_start_time = crate::provenance::process_start_time(pid);
8481 #[cfg(windows)]
8482 let process_start_time = Some(windows_image.start_time());
8483 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8484 #[cfg(all(test, target_os = "macos"))]
8485 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8486 #[cfg(not(windows))]
8491 let recorded_image = observe_spawned_image(pid);
8492 #[cfg(target_os = "macos")]
8496 let recorded_image = if privacy_exec.is_some() {
8497 None
8498 } else {
8499 recorded_image
8500 };
8501 #[cfg(target_os = "linux")]
8502 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8503 #[cfg(not(target_os = "linux"))]
8504 let recorded_cgroup_name = None;
8505 let roster_guard = roster.admit(
8506 spec.module_id.clone(),
8507 pid,
8508 spec.protocol,
8509 process_start_time,
8510 crate::child_roster::RecordedIdentity {
8511 #[cfg(not(windows))]
8512 start_time: recorded_image.map(|image| image.start_time),
8513 #[cfg(windows)]
8514 start_time: process_start_time,
8515 #[cfg(not(windows))]
8516 executable: recorded_image
8517 .and_then(|image| image.executable)
8518 .map(crate::live_children::ExecutableIdentity::from),
8519 #[cfg(windows)]
8520 executable: Some(crate::live_children::ExecutableIdentity::from(
8521 windows_image.identity(),
8522 )),
8523 cgroup_name: recorded_cgroup_name,
8524 #[cfg(target_os = "linux")]
8525 cgroup_placement: cgroup_placement.cloned(),
8526 },
8527 );
8528 let roster_guard = match roster_guard {
8529 Ok(guard) => guard,
8530 Err(source) => {
8531 #[cfg(windows)]
8532 if let Some(job) = &job {
8533 let _ = job.terminate();
8534 }
8535 #[cfg(target_os = "linux")]
8536 kill_module_cgroup(cgroup_placement, &cgroup_name);
8537 kill_and_reap_rejected_child(&mut child);
8538 #[cfg(target_os = "linux")]
8539 if let Some(placement) = cgroup_placement {
8540 remove_module_cgroup(placement, &cgroup_name);
8541 }
8542 return Err(SuperviseError::Spawn {
8543 program: spec.program.clone(),
8544 source: io::Error::other(format!(
8545 "could not durably record module child: {source}"
8546 )),
8547 cgroup_path,
8548 });
8549 }
8550 };
8551 if roster.is_closed() {
8560 #[cfg(target_os = "linux")]
8562 kill_module_cgroup(cgroup_placement, &cgroup_name);
8563 if let Err(error) = child.start_kill() {
8564 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8565 }
8566 #[cfg(target_os = "linux")]
8567 if let Some(placement) = cgroup_placement {
8568 while matches!(child.try_wait(), Ok(None)) {
8572 std::thread::yield_now();
8573 }
8574 if matches!(
8575 subc_cgroup::kill_module(Some(placement), &cgroup_name),
8576 subc_cgroup::KillOutcome::Killed
8577 ) {
8578 if let Ok(path) = placement.module_path(&cgroup_name) {
8579 while std::fs::read_to_string(path.join("cgroup.events"))
8580 .ok()
8581 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8582 {
8583 std::thread::yield_now();
8584 }
8585 }
8586 }
8587 remove_module_cgroup(placement, &cgroup_name);
8588 }
8589 drop(roster_guard);
8590 return Err(SuperviseError::Spawn {
8591 program: spec.program.clone(),
8592 source: io::Error::other(
8593 "the daemon began shutting down while this process was starting; ended it",
8594 ),
8595 cgroup_path,
8596 });
8597 }
8598
8599 #[cfg(windows)]
8600 if let Err(error) = resume_suspended_child(pid, spec) {
8601 kill_and_reap_rejected_child(&mut child);
8602 return Err(error);
8603 }
8604
8605 let stdout_pump = match child.stdout.take() {
8606 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8607 None => {
8608 warn!(
8609 module_id = %spec.module_id,
8610 "spawned child exposed no stdout pipe; file capture will be incomplete"
8611 );
8612 None
8613 }
8614 };
8615 let stderr_pump = match child.stderr.take() {
8616 Some(stderr) => {
8617 let generation = ring
8618 .lock()
8619 .unwrap_or_else(|poisoned| poisoned.into_inner())
8620 .begin_process();
8621 Some(StderrPump {
8622 task: tokio::spawn(pump_stderr_to(
8623 stderr,
8624 Arc::clone(ring),
8625 generation,
8626 output_sink,
8627 )),
8628 generation,
8629 })
8630 }
8631 None => {
8632 ring.lock()
8636 .unwrap_or_else(|poisoned| poisoned.into_inner())
8637 .mark_not_captured("stderr pipe was not available on spawn");
8638 warn!(
8639 module_id = %spec.module_id,
8640 "spawned child exposed no stderr pipe; tail will be unavailable"
8641 );
8642 None
8643 }
8644 };
8645
8646 let exec = Arc::new(crate::activity::ExecOutcome::new());
8647 #[cfg(target_os = "macos")]
8648 let pending_exec = privacy_exec.is_some();
8649 #[cfg(not(target_os = "macos"))]
8650 let pending_exec = false;
8651 if !pending_exec {
8652 exec.finish(crate::activity::ExecState::CommittedExec);
8653 }
8654 Ok(SupervisedChild {
8655 child,
8656 exec: crate::activity::ExecOwner(exec),
8657 #[cfg(windows)]
8658 nonce_delivery,
8659 protocol: spec.protocol,
8660 #[cfg(target_os = "linux")]
8661 module_id: cgroup_name,
8662 #[cfg(target_os = "linux")]
8663 cgroup_placement: cgroup_placement.cloned(),
8664 #[cfg(windows)]
8665 job,
8666 stdout_pump,
8667 stderr_pump,
8668 stderr_ring: Arc::clone(ring),
8669 spawned_at_ms,
8670 spawned_from,
8671 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8672 spawned_file_identity,
8673 process_start_time,
8674 process_identity,
8675 pid,
8676 roster_guard: Some(roster_guard),
8677 #[cfg(target_os = "macos")]
8678 privacy_exec,
8679 #[cfg(target_os = "macos")]
8680 report_ready: Arc::new(OnceLock::new()),
8681 spawn_failure: None,
8682 })
8683}
8684
8685#[cfg(test)]
8686mod spawn_record_tests {
8687 use super::*;
8688
8689 thread_local! {
8690 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8691 }
8692
8693 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8694 LAST_PID.set(child.id());
8695 #[cfg(windows)]
8696 if let Some((_, address)) = _spec
8697 .env
8698 .iter()
8699 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8700 {
8701 use std::io::{Read, Write};
8702 let mut stream = std::net::TcpStream::connect(address).unwrap();
8703 stream
8704 .write_all(&child.id().unwrap().to_ne_bytes())
8705 .unwrap();
8706 let mut release = [0];
8707 stream.read_exact(&mut release).unwrap();
8708 }
8709 }
8710
8711 #[cfg(windows)]
8712 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8713 if spec
8714 .env
8715 .iter()
8716 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8717 {
8718 Err(io::Error::other("injected module job assignment failure"))
8719 } else {
8720 Ok(())
8721 }
8722 }
8723
8724 fn spec() -> ModuleSpec {
8725 ModuleSpec {
8726 module_id: "spawn-record-worker".into(),
8727 #[cfg(unix)]
8728 program: "/bin/sleep".into(),
8729 #[cfg(windows)]
8730 program: std::env::current_exe().unwrap(),
8731 #[cfg(unix)]
8732 args: vec!["60".into()],
8733 #[cfg(windows)]
8734 args: vec![
8735 "--exact".into(),
8736 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8737 "--nocapture".into(),
8738 ],
8739 env: vec![],
8740 reserved: false,
8741 reserved_prefixes: vec![],
8742 protocol: ModuleProtocol::None,
8743 overlap: Default::default(),
8744 }
8745 }
8746
8747 #[tokio::test]
8748 async fn record_sync_failure_fails_spawn_and_reaps_child() {
8749 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8750 let supervisor =
8751 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8752 .with_live_children_record(home.join("live-children.json"));
8753 crate::live_children::fail_next_file_sync();
8754 let result = supervisor.spawn(spec());
8755 let failed = result.is_err();
8756 if let Ok(module) = result {
8757 module.stop().await.unwrap();
8758 }
8759 assert!(failed, "record failure must fail spawn");
8760 let pid = LAST_PID.get().expect("the real OS child was created");
8761 assert!(
8762 subc_os::Process::open(pid).unwrap().is_none(),
8763 "child {pid} survived rejected admission or was not reaped"
8764 );
8765 }
8766
8767 #[cfg(target_os = "macos")]
8768 #[test]
8769 #[ignore = "manual on-device latency measurement"]
8770 fn durable_record_write_latency_50() {
8771 let home = cortexkit_test_support::ScratchDir::new("record-latency");
8772 let path = home.join("live-children.json");
8773 let children = vec![crate::live_children::LiveChild {
8774 module_id: "latency-worker".into(),
8775 pid: std::process::id(),
8776 protocol: ModuleProtocol::None,
8777 start_time: Some(1),
8778 executable: None,
8779 cgroup_name: None,
8780 }];
8781 let mut samples = Vec::new();
8782 for _ in 0..50 {
8783 let start = std::time::Instant::now();
8784 crate::live_children::write_record(&path, &children).unwrap();
8785 samples.push(start.elapsed());
8786 }
8787 samples.sort();
8788 eprintln!(
8789 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8790 (samples[24] + samples[25]) / 2,
8791 samples[47]
8792 );
8793 }
8794
8795 #[cfg(windows)]
8796 #[test]
8797 fn windows_store_touch_fixture() {
8798 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8799 std::fs::write(path, b"module ran").unwrap();
8800 }
8801 }
8802
8803 #[cfg(windows)]
8804 #[tokio::test]
8805 async fn windows_daemon_fixture() {
8806 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8807 return;
8808 };
8809 let mut spec = spec();
8810 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8811 let supervisor =
8812 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8813 let _module = supervisor.spawn(spec).unwrap();
8814 }
8815
8816 #[cfg(windows)]
8817 #[test]
8818 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8819 use std::io::Read;
8820 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8821 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8824 command
8825 .args([
8826 "--exact",
8827 "supervise::spawn_record_tests::windows_daemon_fixture",
8828 "--nocapture",
8829 ])
8830 .env(
8831 "SUBC_TEST_DAEMON_BARRIER",
8832 listener.local_addr().unwrap().to_string(),
8833 );
8834 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8835 listener.set_nonblocking(true).unwrap();
8836 let deadline = std::time::Instant::now() + Duration::from_secs(30);
8837 let mut stream = loop {
8838 if let Ok((stream, _)) = listener.accept() {
8839 break stream;
8840 }
8841 assert!(
8842 daemon.child.try_wait().unwrap().is_none(),
8843 "daemon failed before rendezvous"
8844 );
8845 assert!(
8846 std::time::Instant::now() < deadline,
8847 "daemon did not reach CreateProcess failpoint"
8848 );
8849 std::thread::yield_now();
8850 };
8851 stream
8852 .set_read_timeout(Some(Duration::from_secs(30)))
8853 .unwrap();
8854 let mut pid = [0; 4];
8855 stream.read_exact(&mut pid).unwrap();
8856 let pid = u32::from_ne_bytes(pid);
8857 assert!(subc_jobobject::process_exists(pid));
8858 daemon.child.kill().unwrap();
8861 daemon.child.wait().unwrap();
8862 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8863 if !exited {
8864 daemon.job.terminate().unwrap();
8865 }
8866 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8867 }
8868
8869 #[cfg(windows)]
8870 #[tokio::test]
8871 async fn windows_containment_failure_never_resumes() {
8872 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8873 let sentinel = home.join("store-touched");
8874 let mut spec = spec();
8875 spec.env
8876 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8877 spec.env.push((
8878 "SUBC_TEST_STORE_TOUCH".into(),
8879 sentinel.display().to_string(),
8880 ));
8881 let supervisor =
8882 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8883 let result = supervisor.spawn(spec);
8884 if let Ok(module) = result {
8887 let _ = tokio::time::timeout(Duration::from_secs(10), async {
8888 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8889 tokio::task::yield_now().await;
8890 }
8891 })
8892 .await;
8893 let _ = module.stop().await;
8894 panic!(
8895 "assignment failure resumed a module; store touched={}",
8896 sentinel.exists()
8897 );
8898 }
8899 assert!(
8900 !sentinel.exists(),
8901 "refused module executed store-touch sentinel"
8902 );
8903 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8904 }
8905
8906 #[cfg(windows)]
8907 #[tokio::test]
8908 async fn windows_nested_daemon_fixture() {
8909 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8910 return;
8911 };
8912 let mut spec = spec();
8913 spec.env.push((
8914 "SUBC_TEST_STORE_TOUCH".into(),
8915 sentinel.to_str().unwrap().into(),
8916 ));
8917 let supervisor =
8918 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8919 let module = supervisor
8920 .spawn(spec)
8921 .expect("per-module job must nest below daemon and host jobs");
8922 tokio::time::timeout(Duration::from_secs(10), async {
8923 while !PathBuf::from(&sentinel).exists() {
8924 tokio::task::yield_now().await;
8925 }
8926 })
8927 .await
8928 .unwrap();
8929 module.stop().await.unwrap();
8930 }
8931
8932 #[cfg(windows)]
8933 #[test]
8934 fn windows_module_job_nests_under_host_job() {
8935 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
8936 let sentinel = home.join("nested-store-touched");
8937 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8938 command
8939 .args([
8940 "--exact",
8941 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
8942 "--nocapture",
8943 ])
8944 .env("SUBC_TEST_NESTED_STORE", &sentinel);
8945 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8946 assert!(daemon.child.wait().unwrap().success());
8947 assert!(sentinel.exists(), "nested protocol:none module never ran");
8948 }
8949}
8950
8951#[cfg(target_os = "linux")]
8952pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
8953 use subc_cgroup::KillOutcome;
8954 match subc_cgroup::kill_module(placement, module_id) {
8955 KillOutcome::Killed => {}
8956 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
8957 debug!(
8958 module_id,
8959 "cgroup tree kill unavailable; using direct-child kill"
8960 );
8961 }
8962 KillOutcome::IoError { path, error } => {
8963 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
8964 }
8965 }
8966}
8967
8968#[cfg(windows)]
8976fn contain_spawned_child(
8977 child: &mut Child,
8978 spec: &ModuleSpec,
8979) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
8980 let assignment = || {
8981 let job = subc_jobobject::JobObject::new()?;
8982 #[cfg(test)]
8983 spawn_record_tests::assignment_failpoint(spec)?;
8984 job.assign(child)?;
8985 Ok::<_, io::Error>(job)
8986 };
8987 match assignment() {
8988 Ok(job) => Ok(Some(job)),
8989 Err(source) => {
8990 kill_and_reap_rejected_child(child);
8991 Err(SuperviseError::Spawn {
8992 program: spec.program.clone(),
8993 source: io::Error::other(format!(
8994 "Windows module job containment failed: {source}"
8995 )),
8996 cgroup_path: None,
8997 })
8998 }
8999 }
9000}
9001
9002fn kill_and_reap_rejected_child(child: &mut Child) {
9005 #[cfg(unix)]
9006 if let Some(pid) = child
9007 .id()
9008 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9009 {
9010 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9011 }
9012 let _ = child.start_kill();
9013 while matches!(child.try_wait(), Ok(None)) {
9014 std::thread::yield_now();
9015 }
9016}
9017
9018#[cfg(windows)]
9023fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9024 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9025 let _ = std::process::Command::new("taskkill.exe")
9029 .args(["/PID", &pid.to_string(), "/T", "/F"])
9030 .stdin(Stdio::null())
9031 .stdout(Stdio::null())
9032 .stderr(Stdio::null())
9033 .status();
9034 return Err(SuperviseError::Spawn {
9035 program: spec.program.clone(),
9036 source,
9037 cgroup_path: None,
9038 });
9039 }
9040 Ok(())
9041}
9042
9043#[cfg(target_os = "linux")]
9044fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9045 match placement.remove_module(module_id) {
9046 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9047 Err(error) => warn!(
9048 module_id,
9049 error = %error,
9050 "could not remove module cgroup after process exit; continuing teardown"
9051 ),
9052 }
9053}
9054
9055#[cfg(target_os = "linux")]
9056async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9057 if matches!(
9061 subc_cgroup::kill_module(Some(placement), module_id),
9062 subc_cgroup::KillOutcome::Killed
9063 ) {
9064 if let Ok(path) = placement.module_path(module_id) {
9065 while std::fs::read_to_string(path.join("cgroup.events"))
9066 .ok()
9067 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9068 {
9069 sleep(Duration::from_millis(1)).await;
9070 }
9071 }
9072 }
9073 remove_module_cgroup(placement, module_id);
9074}
9075
9076#[cfg(target_os = "linux")]
9077fn apply_cgroup_placement(
9078 command: &mut Command,
9079 spec: &ModuleSpec,
9080 path: &std::path::Path,
9081) -> Result<(), SuperviseError> {
9082 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9083 module_id: spec.module_id.clone(),
9084 source,
9085 })
9086}
9087
9088fn capture_retention(spec: &ModuleSpec) -> Retention {
9089 let defaults = Retention::default();
9090 let value = |name: &str| {
9091 spec.env
9092 .iter()
9093 .rev()
9094 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9095 };
9096 Retention {
9097 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9098 .and_then(|value| value.parse().ok())
9099 .unwrap_or(defaults.max_file_mb),
9100 keep: value(CAPTURE_KEEP_ENV)
9101 .and_then(|value| value.parse().ok())
9102 .unwrap_or(defaults.keep),
9103 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9104 .and_then(|value| value.parse().ok())
9105 .unwrap_or(defaults.max_age_days),
9106 }
9107}
9108
9109fn generate_launch_nonce() -> Result<String, SuperviseError> {
9112 let mut bytes = [0u8; 32];
9113 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9114 reason: source.to_string(),
9115 })?;
9116 let mut hex = String::with_capacity(64);
9117 for b in bytes {
9118 use std::fmt::Write;
9119 let _ = write!(hex, "{b:02x}");
9120 }
9121 Ok(hex)
9122}
9123
9124fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9127 if a.len() != b.len() {
9128 return false;
9129 }
9130 let mut diff = 0u8;
9131 for (x, y) in a.iter().zip(b.iter()) {
9132 diff |= x ^ y;
9133 }
9134 diff == 0
9135}
9136
9137#[cfg(not(windows))]
9141fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9142 let process = subc_os::Process::open(pid).ok().flatten()?;
9143 #[cfg(target_os = "linux")]
9144 return sample_after_linux_exec(
9145 || {
9146 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9147 let committed = linux_exec_committed(&stat);
9148 #[cfg(test)]
9149 spawn_image_tests::report_exec_state(committed);
9150 committed
9151 },
9152 || process.observe(),
9153 );
9154 #[cfg(not(target_os = "linux"))]
9155 process.observe()
9156}
9157
9158#[cfg(target_os = "linux")]
9162fn linux_exec_committed(stat: &str) -> Option<bool> {
9163 const PF_FORKNOEXEC: u64 = 0x40;
9164 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9165 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9166}
9167
9168#[cfg(target_os = "linux")]
9169fn sample_after_linux_exec(
9170 mut exec_committed: impl FnMut() -> Option<bool>,
9171 observe: impl FnOnce() -> Option<subc_os::Observation>,
9172) -> Option<subc_os::Observation> {
9173 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9174 while !exec_committed()? {
9175 if std::time::Instant::now() >= deadline {
9179 return None;
9180 }
9181 std::thread::sleep(Duration::from_micros(50));
9182 }
9183 observe()
9184}
9185
9186#[cfg(all(test, target_os = "linux"))]
9187mod spawn_image_tests {
9188 use super::*;
9189 use std::cell::{Cell, RefCell};
9190
9191 enum ExecEvent {
9192 Uncommitted,
9193 Image(Option<subc_os::Observation>),
9194 }
9195
9196 thread_local! {
9197 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9198 }
9199
9200 pub(super) fn report_exec_state(committed: Option<bool>) {
9201 if committed == Some(false) {
9202 EXEC_EVENTS.with(|events| {
9203 if let Some(events) = events.borrow_mut().take() {
9204 let _ = events.send(ExecEvent::Uncommitted);
9205 }
9206 });
9207 }
9208 }
9209
9210 #[test]
9211 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9212 use std::io::{Read, Write};
9213 use std::os::fd::AsRawFd;
9214
9215 let descriptor = std::fs::File::open("/dev/null").unwrap();
9216 let mut command = std::process::Command::new("/bin/sleep");
9217 command.arg("30");
9218 let (mut ready, mut release) =
9219 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9220 .unwrap();
9221 let spawn = std::thread::spawn(move || command.spawn());
9222 let mut message = [0u8; 8];
9223 ready.read_exact(&mut message).unwrap();
9224 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9225 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9226 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9227 let (events, received) = std::sync::mpsc::channel();
9228 let observer = std::thread::spawn(move || {
9229 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9230 let image = observe_spawned_image(pid);
9231 let _ = events.send(ExecEvent::Image(image));
9232 });
9233 let first = received.recv_timeout(Duration::from_secs(5));
9237 release.write_all(&[1]).unwrap();
9238 let mut child = spawn.join().unwrap().unwrap();
9239 let after = match first {
9240 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9241 Ok(ExecEvent::Image(image)) => image,
9242 _ => None,
9243 },
9244 Ok(ExecEvent::Image(image)) => image,
9245 Err(_) => None,
9246 };
9247 observer.join().unwrap();
9248 child.kill().unwrap();
9249 child.wait().unwrap();
9250
9251 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9252 assert_eq!(
9253 inherited.unwrap().executable,
9254 subc_os::file_identity(&std::env::current_exe().unwrap())
9255 );
9256 assert_eq!(
9257 after.unwrap().executable,
9258 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9259 );
9260 }
9261
9262 #[test]
9263 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9264 let committed = Cell::new(false);
9265 let inherited = subc_os::Observation {
9266 start_time: 23,
9267 executable: Some(subc_os::FileIdentity {
9268 device: 29,
9269 inode: 31,
9270 }),
9271 };
9272 let module = subc_os::Observation {
9273 start_time: 23,
9274 executable: Some(subc_os::FileIdentity {
9275 device: 29,
9276 inode: 37,
9277 }),
9278 };
9279 let polls = Cell::new(0);
9280 let image = sample_after_linux_exec(
9281 || {
9282 polls.set(polls.get() + 1);
9283 committed.set(polls.get() == 2);
9286 Some(committed.get())
9287 },
9288 || Some(if committed.get() { module } else { inherited }),
9289 );
9290 assert_eq!(image, Some(module));
9291 assert_eq!(polls.get(), 2);
9292 }
9293
9294 #[test]
9295 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9296 assert_eq!(
9297 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9298 Some(false)
9299 );
9300 assert_eq!(
9301 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9302 Some(true)
9303 );
9304 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9305 assert_eq!(linux_exec_committed("17 (a) R"), None);
9306 }
9307
9308 #[test]
9309 fn unreadable_exec_state_never_samples_an_inherited_image() {
9310 assert_eq!(
9311 sample_after_linux_exec(
9312 || None,
9313 || panic!("must not sample before exec confirmation")
9314 ),
9315 None
9316 );
9317 }
9318
9319 #[test]
9320 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9321 assert_eq!(
9322 sample_after_linux_exec(
9323 || Some(false),
9324 || panic!("must not sample after an exec timeout")
9325 ),
9326 None
9327 );
9328 }
9329}
9330
9331#[cfg(windows)]
9334fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9335 let mut program = spec.program.clone();
9336 if program.is_absolute() || program.components().count() > 1 {
9337 match subc_os::ExecutableCapture::open(&program) {
9340 Ok(capture) => return Ok(capture),
9341 Err(error)
9342 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9343 Err(error) => return Err(error),
9344 }
9345 program.set_extension("exe");
9346 return subc_os::ExecutableCapture::open(&program);
9347 }
9348 if program.extension().is_none() {
9349 program.set_extension("exe");
9350 }
9351 let custom_path = spec
9352 .env
9353 .iter()
9354 .rev()
9355 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9356 let mut directories = Vec::new();
9357 if let Some(parent) = std::env::current_exe()?.parent() {
9360 directories.push(parent.to_path_buf());
9361 }
9362 directories.push(std::env::current_dir()?);
9363 if let Some(windows) = std::env::var_os("SystemRoot") {
9364 let windows = PathBuf::from(windows);
9365 directories.push(windows.join("System32"));
9366 directories.push(windows.clone());
9367 }
9368 let path = custom_path
9369 .map(|(_, value)| std::ffi::OsString::from(value))
9370 .or_else(|| std::env::var_os("PATH"));
9371 if let Some(path) = path {
9372 directories.extend(std::env::split_paths(&path));
9373 }
9374 for directory in directories {
9375 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9376 Ok(capture) => return Ok(capture),
9377 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9378 Err(error) => return Err(error),
9379 }
9380 }
9381 Err(io::Error::new(
9382 io::ErrorKind::NotFound,
9383 "resolved executable not found",
9384 ))
9385}
9386
9387fn spawn_and_mark_running(
9388 spec: &ModuleSpec,
9389 runtime: &SupervisorRuntimeConfig,
9390 snapshot: &SharedSnapshot,
9391) -> Result<SupervisedChild, SuperviseError> {
9392 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9393 let child = spawn_child(
9394 spec,
9395 runtime.connection_file_path.as_deref(),
9396 runtime.supervisor_handle.as_ref(),
9397 &runtime.stderr_ring,
9398 runtime.capture_logs_dir.as_deref(),
9399 &runtime.child_roster,
9400 #[cfg(target_os = "linux")]
9401 runtime.cgroup_placement.as_ref(),
9402 )?;
9403 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9404 if let Some(spawn) = activity_spawn {
9405 spawn.complete(snapshot);
9406 }
9407 Ok(child)
9408}
9409
9410enum RegistrationWaitOutcome {
9411 Registered,
9412 Exited(ExitReport),
9413 TimedOut,
9414}
9415
9416struct ReloadRegistrationFailure {
9417 exit_report: ExitReport,
9418 reason: String,
9419}
9420
9421#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9422enum BusyGaugeObservation {
9423 Quiescent,
9424 Busy,
9425 Omitted,
9426}
9427
9428fn busy_gauge_observation(metrics: Option<&Value>, gauges: &[String]) -> BusyGaugeObservation {
9429 let Some(metrics) = metrics.and_then(Value::as_object) else {
9430 return BusyGaugeObservation::Omitted;
9431 };
9432 let mut sum = 0u128;
9433 for gauge in gauges {
9434 let Some(value) = metrics.get(gauge) else {
9435 return BusyGaugeObservation::Omitted;
9436 };
9437 let Some(value) = value.as_u64() else {
9438 return BusyGaugeObservation::Busy;
9439 };
9440 sum = sum.saturating_add(u128::from(value));
9441 }
9442 if sum == 0 {
9443 BusyGaugeObservation::Quiescent
9444 } else {
9445 BusyGaugeObservation::Busy
9446 }
9447}
9448
9449fn declared_busy_gauges(
9450 registry: &Registry,
9451 module_id: &str,
9452) -> Result<Vec<String>, SuperviseError> {
9453 busy_gauges_of(
9454 registry
9455 .get_module(module_id)
9456 .map_err(SuperviseError::Registry)?,
9457 )
9458}
9459
9460fn declared_busy_gauges_for_connection(
9464 registry: &Registry,
9465 connection_id: ConnectionId,
9466) -> Result<Vec<String>, SuperviseError> {
9467 busy_gauges_of(
9468 registry
9469 .get_module_by_connection(connection_id)
9470 .map_err(SuperviseError::Registry)?,
9471 )
9472}
9473
9474fn busy_gauges_of(
9475 registration: Option<crate::registry::ModuleRegistration>,
9476) -> Result<Vec<String>, SuperviseError> {
9477 let Some(registration) = registration else {
9478 return Ok(Vec::new());
9479 };
9480 let Some(self_signals) = registration.manifest.self_signals else {
9481 return Ok(Vec::new());
9482 };
9483
9484 let mut gauges = Vec::new();
9485 for declaration in self_signals {
9486 if declaration.kind != SelfSignalKind::Busy {
9487 continue;
9488 }
9489 match declaration.anchored_to {
9490 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9491 gauges.extend(declared)
9492 }
9493 _ => {
9494 gauges.push(String::new());
9497 }
9498 }
9499 }
9500 Ok(gauges)
9501}
9502
9503async fn wait_for_forwarding_quiescence(
9510 forwarding: &ForwardingTable,
9511 module_id: &str,
9512 runtime: &SupervisorRuntimeConfig,
9513 endpoint: crate::ModuleEndpointId,
9514 deadline: Instant,
9515 busy_gauges: &[String],
9516 scope: DrainScope,
9517) -> Result<bool, SuperviseError> {
9518 let mut gauges_quiescent = busy_gauges.is_empty();
9519 let mut next_probe_at = Instant::now();
9520 let mut omission_counted = false;
9521
9522 loop {
9523 let now = Instant::now();
9524 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9525 let report = match scope {
9526 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9527 DrainScope::Endpoint(endpoint) => {
9528 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9529 }
9530 };
9531 gauges_quiescent = match report {
9532 Ok(report) => match busy_gauge_observation(report.metrics.as_ref(), busy_gauges) {
9533 BusyGaugeObservation::Quiescent => true,
9534 BusyGaugeObservation::Busy => false,
9535 BusyGaugeObservation::Omitted => {
9536 if !omission_counted {
9537 forwarding
9538 .counters()
9539 .increment_drains_with_undeclared_gauge();
9540 omission_counted = true;
9541 }
9542 false
9543 }
9544 },
9545 Err(err) => {
9546 warn!(
9547 module_id,
9548 error = %err,
9549 "drain health.check did not produce declared busy gauges; treating module as busy"
9550 );
9551 false
9552 }
9553 };
9554 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9557 }
9558
9559 let in_flight = forwarding
9560 .endpoint_in_flight_count(endpoint)
9561 .map_err(SuperviseError::Forwarding)?;
9562 if in_flight == 0 && gauges_quiescent {
9563 return Ok(true);
9564 }
9565
9566 let now = Instant::now();
9567 if now >= deadline {
9568 return Ok(false);
9569 }
9570 let mut wait = deadline
9571 .saturating_duration_since(now)
9572 .min(REGISTRY_RELEASE_POLL);
9573 if !busy_gauges.is_empty() {
9574 wait = wait.min(next_probe_at.saturating_duration_since(now));
9575 }
9576 sleep(wait).await;
9577 }
9578}
9579
9580fn drain_busy_reprobe_interval() -> Duration {
9584 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9585}
9586
9587fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9595 match wait_result {
9596 Ok(drained) => *drained,
9597 Err(_) => false,
9598 }
9599}
9600
9601fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9602 for released in released_routes {
9603 let frame = match Frame::build_with_version(
9604 released.negotiated_ver,
9605 FrameType::Goodbye,
9606 control_flags(),
9607 released.channel,
9608 released.epoch,
9609 0,
9610 Vec::new(),
9611 ) {
9612 Ok(frame) => frame,
9613 Err(err) => {
9614 warn!(
9615 route_channel = released.channel,
9616 error = %err,
9617 "failed to build supervisor drain route GOODBYE frame"
9618 );
9619 continue;
9620 }
9621 };
9622 if !released.close_on_delivery_failure() {
9623 crate::forwarding::send_module_route_goodbye(
9624 &forwarding.counters(),
9625 &released.sink,
9626 frame,
9627 released.module_id.as_deref(),
9628 "supervisor drain",
9629 );
9630 continue;
9631 }
9632 if let Err(err) = released.sink.try_send(frame) {
9633 warn!(
9634 target_connection_id = released.connection_id.get(),
9635 route_channel = released.channel,
9636 error = %err,
9637 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9638 );
9639 let _ = forwarding.escalate_client_delivery_failure(
9640 released.connection_id,
9641 released.channel,
9642 released.epoch,
9643 CloseReason::new(
9644 "route_goodbye_delivery_failed",
9645 format!(
9646 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9647 released.channel
9648 ),
9649 ),
9650 crate::forwarding::UndeliveredFrame {
9651 module_id: released.module_id.as_deref(),
9652 sink: &released.sink,
9653 },
9654 );
9655 }
9656 }
9657}
9658
9659fn send_module_draining(
9660 module_id: &str,
9661 reason: RouteCloseReason,
9662 deadline_ms: u64,
9663 target: &ModuleDrainTarget,
9664) {
9665 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9666 reason,
9667 deadline_ms,
9668 }) {
9669 Ok(body) => body,
9670 Err(err) => {
9671 warn!(
9672 module_id,
9673 error = %err,
9674 "failed to encode module draining command"
9675 );
9676 return;
9677 }
9678 };
9679 let frame = match Frame::build_with_version(
9680 target.negotiated_ver,
9681 FrameType::Push,
9682 control_flags(),
9683 0,
9684 0,
9685 0,
9686 body,
9687 ) {
9688 Ok(frame) => frame,
9689 Err(err) => {
9690 warn!(
9691 module_id,
9692 error = %err,
9693 "failed to build module draining command frame"
9694 );
9695 return;
9696 }
9697 };
9698 if let Err(err) = target.sink.try_send(frame) {
9699 warn!(
9700 module_id,
9701 target_connection_id = target.endpoint.connection_id.get(),
9702 error = %err,
9703 "module draining command was not delivered to peer"
9704 );
9705 }
9706}
9707
9708fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9710 match Frame::build_with_version(
9711 negotiated_ver,
9712 FrameType::Goodbye,
9713 control_flags(),
9714 0,
9715 0,
9716 0,
9717 Vec::new(),
9718 ) {
9719 Ok(frame) => Some(frame),
9720 Err(err) => {
9721 warn!(
9722 module_id,
9723 error = %err,
9724 "failed to build module GOODBYE frame"
9725 );
9726 None
9727 }
9728 }
9729}
9730
9731#[cfg(unix)]
9745async fn send_module_goodbyes_for_daemon_shutdown(
9746 forwarding: &Arc<ForwardingTable>,
9747 reason: &CloseReason,
9748 wait_for_flush: bool,
9749) {
9750 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
9751 let targets = match forwarding.module_connections() {
9752 Ok(targets) => targets,
9753 Err(err) => {
9754 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
9755 return;
9756 }
9757 };
9758 let deadline = Instant::now() + GOODBYE_BUDGET;
9759 let mut sends = tokio::task::JoinSet::new();
9760 for target in targets {
9761 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
9762 continue;
9763 };
9764 if !wait_for_flush {
9765 if let Err(err) = target.sink.try_send(frame) {
9766 debug!(
9767 module_id = %target.module_id,
9768 error = %err,
9769 "shutdown module GOODBYE was not queued"
9770 );
9771 }
9772 continue;
9773 }
9774 let forwarding = Arc::clone(forwarding);
9775 let reason = reason.clone();
9776 sends.spawn(async move {
9777 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
9778 Ok(Ok(())) => {}
9779 Ok(Err(err)) => debug!(
9780 module_id = %target.module_id,
9781 error = %err,
9782 "module connection closed before its shutdown GOODBYE was written"
9783 ),
9784 Err(_) => warn!(
9785 module_id = %target.module_id,
9786 budget = ?GOODBYE_BUDGET,
9787 "shutdown module GOODBYE was not written within its budget; closing anyway"
9788 ),
9789 }
9790 forwarding.request_connection_close(target.endpoint.connection_id, reason);
9791 });
9792 }
9793 while sends.join_next().await.is_some() {}
9795}
9796
9797fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
9798 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
9799 return;
9800 };
9801 if let Err(err) = target.sink.try_send(frame) {
9802 warn!(
9803 module_id,
9804 target_connection_id = target.endpoint.connection_id.get(),
9805 error = %err,
9806 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
9807 );
9808 forwarding.request_connection_close(
9809 target.endpoint.connection_id,
9810 CloseReason::new(
9811 "module_goodbye_delivery_failed",
9812 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
9813 ),
9814 );
9815 }
9816}
9817
9818#[derive(Clone, Copy)]
9819struct ForwardingDrainContext<'a> {
9820 spec: &'a ModuleSpec,
9821 runtime: &'a SupervisorRuntimeConfig,
9822 registry: &'a Registry,
9823 scope: DrainScope,
9824}
9825
9826#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9828enum DrainScope {
9829 Active,
9832 Endpoint(crate::ModuleEndpointId),
9837}
9838
9839#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9847enum StopNotice {
9848 SentOverConnection,
9851 NoConnection,
9855 NotSent,
9859}
9860
9861async fn begin_forwarding_drain(
9862 spec: &ModuleSpec,
9863 runtime: &SupervisorRuntimeConfig,
9864 registry: &Registry,
9865 snapshot: &SharedSnapshot,
9866 enabled: Option<bool>,
9867 reason: RouteCloseReason,
9868) -> Result<StopNotice, SuperviseError> {
9869 let Some(forwarding) = runtime.forwarding.as_ref() else {
9870 return Err(SuperviseError::ReloadUnavailable {
9871 module_id: spec.module_id.clone(),
9872 reason: "supervisor was not configured with a forwarding table".to_string(),
9873 });
9874 };
9875
9876 begin_forwarding_drain_with(
9877 forwarding,
9878 ForwardingDrainContext {
9879 spec,
9880 runtime,
9881 registry,
9882 scope: DrainScope::Active,
9883 },
9884 snapshot,
9885 enabled,
9886 reason,
9887 runtime.drain_timeout,
9888 )
9889 .await
9890}
9891
9892async fn begin_forwarding_drain_if_configured(
9893 spec: &ModuleSpec,
9894 runtime: &SupervisorRuntimeConfig,
9895 registry: &Registry,
9896 snapshot: &SharedSnapshot,
9897 enabled: Option<bool>,
9898 reason: RouteCloseReason,
9899) -> Result<StopNotice, SuperviseError> {
9900 begin_forwarding_drain_with_timeout(
9901 spec,
9902 runtime,
9903 registry,
9904 snapshot,
9905 enabled,
9906 reason,
9907 runtime.drain_timeout,
9908 )
9909 .await
9910}
9911
9912async fn begin_forwarding_drain_with_timeout(
9916 spec: &ModuleSpec,
9917 runtime: &SupervisorRuntimeConfig,
9918 registry: &Registry,
9919 snapshot: &SharedSnapshot,
9920 enabled: Option<bool>,
9921 reason: RouteCloseReason,
9922 drain_timeout: Duration,
9923) -> Result<StopNotice, SuperviseError> {
9924 let Some(forwarding) = runtime.forwarding.as_ref() else {
9925 return Ok(StopNotice::NotSent);
9926 };
9927
9928 begin_forwarding_drain_with(
9929 forwarding,
9930 ForwardingDrainContext {
9931 spec,
9932 runtime,
9933 registry,
9934 scope: DrainScope::Active,
9935 },
9936 snapshot,
9937 enabled,
9938 reason,
9939 drain_timeout,
9940 )
9941 .await
9942}
9943
9944async fn begin_forwarding_drain_with(
9945 forwarding: &ForwardingTable,
9946 context: ForwardingDrainContext<'_>,
9947 snapshot: &SharedSnapshot,
9948 enabled: Option<bool>,
9949 reason: RouteCloseReason,
9950 drain_timeout: Duration,
9951) -> Result<StopNotice, SuperviseError> {
9952 let ForwardingDrainContext {
9953 spec,
9954 runtime,
9955 registry,
9956 scope,
9957 } = context;
9958 debug_assert_ne!(reason, RouteCloseReason::Crash);
9959 let terminal = matches!(reason, RouteCloseReason::Disable);
9960 let drain_started_at = Instant::now();
9961 let drain_deadline = drain_started_at + drain_timeout;
9962 let deadline_ms =
9963 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
9964 let busy_gauges = match scope {
9965 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
9966 DrainScope::Endpoint(endpoint) => {
9967 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
9968 }
9969 };
9970
9971 let gate_started = Instant::now();
9974 let drain_target = match scope {
9975 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
9976 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
9977 }
9978 .map_err(SuperviseError::Forwarding)?;
9979 info!(
9984 module_id = %spec.module_id,
9985 ?reason,
9986 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
9987 connected = drain_target.is_some(),
9988 "module drain began; route admission closed"
9989 );
9990 if scope == DrainScope::Active {
9991 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9992 state.state = ModuleState::Draining;
9993 state.draining_to_replace =
9994 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
9995 if let Some(enabled) = enabled {
9996 state.enabled = enabled;
9997 }
9998 })?;
9999 }
10000
10001 let Some(target) = drain_target.as_ref() else {
10002 return Ok(StopNotice::NoConnection);
10006 };
10007 {
10008 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10009 #[cfg(feature = "test-support")]
10010 forwarding.wait_before_route_closing_for_test().await;
10011 let routes = forwarding
10012 .endpoint_routes(target.endpoint)
10013 .map_err(SuperviseError::Forwarding)?;
10014 let routes_notified = routes.len();
10015 crate::control::send_route_control_pushes(
10016 forwarding,
10017 routes.clone(),
10018 ClientControlPush::RouteClosing {
10019 module_id: spec.module_id.clone(),
10020 channels: Vec::new(),
10021 reason,
10022 },
10023 );
10024 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10025
10026 let wait_result = wait_for_forwarding_quiescence(
10032 forwarding,
10033 &spec.module_id,
10034 runtime,
10035 target.endpoint,
10036 drain_deadline,
10037 &busy_gauges,
10038 scope,
10039 )
10040 .await;
10041 let drained = drained_after_quiescence_wait(&wait_result);
10042 if let Err(err) = &wait_result {
10043 error!(
10044 module_id = %spec.module_id,
10045 ?reason,
10046 error = %err,
10047 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10048 );
10049 } else if !drained {
10050 let holdouts = forwarding
10056 .endpoint_drain_holdouts(target.endpoint)
10057 .unwrap_or_default();
10058 warn!(
10059 module_id = %spec.module_id,
10060 waited = ?drain_timeout,
10061 ?reason,
10062 held_requests = holdouts.requests,
10063 held_routes = holdouts.routes,
10064 total_routes = holdouts.total_routes,
10065 top_connections = ?holdouts.top_connections,
10066 held = %holdouts
10069 .held
10070 .iter()
10071 .map(|(channel, corr)| format!("{channel}:{corr}"))
10072 .collect::<Vec<_>>()
10073 .join(","),
10074 "route drain timed out before request quiescence; forcing teardown"
10075 );
10076 }
10077 crate::control::send_route_control_pushes(
10078 forwarding,
10079 routes,
10080 ClientControlPush::RouteClosed {
10081 module_id: spec.module_id.clone(),
10082 channels: Vec::new(),
10083 reason,
10084 drained,
10085 abandoned: target.abandoned_bindings.len() as u32,
10086 excluded_subscriptions: target.excluded_subscriptions,
10087 terminal: Some(terminal),
10088 },
10089 );
10090 wait_result?;
10091
10092 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10098 Ok(routes) => routes,
10099 Err(err) => {
10100 warn!(
10101 module_id = %spec.module_id,
10102 ?reason,
10103 error = %err,
10104 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10105 );
10106 send_module_goodbye(&spec.module_id, forwarding, target);
10107 return Err(SuperviseError::Forwarding(err));
10108 }
10109 };
10110 let route_goodbye_count = released_routes.len();
10111 send_route_goodbyes(forwarding, released_routes);
10112 send_module_goodbye(&spec.module_id, forwarding, target);
10113
10114 info!(
10120 module_id = %spec.module_id,
10121 ?reason,
10122 routes_notified,
10123 route_goodbyes = route_goodbye_count,
10124 abandoned_reservations = target.abandoned_bindings.len(),
10125 excluded_subscriptions = target.excluded_subscriptions,
10126 drained,
10127 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10128 );
10129 }
10130
10131 Ok(StopNotice::SentOverConnection)
10132}
10133
10134async fn wait_for_registration_after_reload(
10137 registry: &Registry,
10138 module_id: &str,
10139 snapshot: &SharedSnapshot,
10140 child: &mut SupervisedChild,
10141 wait: Duration,
10142) -> Result<RegistrationWaitOutcome, SuperviseError> {
10143 wait_for_slot_registration(
10144 registry,
10145 crate::registry::RegistrationSlot::Active(module_id),
10146 module_id,
10147 snapshot,
10148 child,
10149 wait,
10150 )
10151 .await
10152}
10153
10154async fn wait_for_slot_registration(
10162 registry: &Registry,
10163 slot: crate::registry::RegistrationSlot<'_>,
10164 module_id: &str,
10165 snapshot: &SharedSnapshot,
10166 child: &mut SupervisedChild,
10167 wait: Duration,
10168) -> Result<RegistrationWaitOutcome, SuperviseError> {
10169 let deadline = Instant::now() + wait;
10170 loop {
10171 if registry
10172 .registration(slot)
10173 .map_err(SuperviseError::Registry)?
10174 .is_some()
10175 {
10176 return Ok(RegistrationWaitOutcome::Registered);
10177 }
10178
10179 let now = Instant::now();
10180 if now >= deadline {
10181 return Ok(RegistrationWaitOutcome::TimedOut);
10182 }
10183 let remaining = deadline.saturating_duration_since(now);
10184 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10185
10186 tokio::select! {
10187 wait_result = child.wait() => {
10188 let status = wait_result.map_err(|source| SuperviseError::Wait {
10189 module_id: module_id.to_string(),
10190 source,
10191 })?;
10192 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10193 snapshot,
10194 child,
10195 &status,
10196 )));
10197 }
10198 _ = sleep(poll) => {}
10199 }
10200 }
10201}
10202
10203fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10204 if exit_report.kind != ExitKind::DeliberateSeverance {
10207 exit_report.kind = ExitKind::Crash;
10208 }
10209 exit_report
10210}
10211
10212async fn handle_reload_child_registration_failure(
10213 spec: &ModuleSpec,
10214 runtime: &SupervisorRuntimeConfig,
10215 registry: &Registry,
10216 process_liveness: &SupervisorProcessLiveness,
10217 snapshot: &SharedSnapshot,
10218 _child: &mut Option<SupervisedChild>,
10219 failure: ReloadRegistrationFailure,
10220) -> Result<(), SuperviseError> {
10221 let ReloadRegistrationFailure {
10222 exit_report,
10223 reason,
10224 } = failure;
10225 match on_child_exit(
10226 spec,
10227 runtime.restart_policy,
10228 registry,
10229 snapshot,
10230 &runtime.terminal_ring,
10231 &runtime.spawn_events,
10232 &runtime.child_roster,
10233 exit_report,
10234 )
10235 .await
10236 {
10237 NextAction::Stop {
10238 registration_released,
10239 } => {
10240 if registration_released {
10241 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10242 }
10243 }
10244 NextAction::Restart { schedule } => {
10245 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10246 schedule.delay
10247 });
10248 if let Some(schedule) = schedule {
10249 log_crash_respawn(&spec.module_id, schedule);
10250 }
10251 schedule_respawn(
10252 runtime,
10253 snapshot,
10254 &spec.module_id,
10255 delay,
10256 RespawnKind::Spawn,
10257 )?;
10258 }
10259 }
10260 Err(SuperviseError::ReloadFailed {
10261 module_id: spec.module_id.clone(),
10262 reason,
10263 })
10264}
10265
10266async fn handle_reload_spawn_failure(
10267 spec: &ModuleSpec,
10268 runtime: &SupervisorRuntimeConfig,
10269 process_liveness: &SupervisorProcessLiveness,
10270 snapshot: &SharedSnapshot,
10271 _child: &mut Option<SupervisedChild>,
10272 reason: String,
10273) -> Result<(), SuperviseError> {
10274 let now = Instant::now();
10275 let mut schedule = None;
10276 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10277 clear_current_process_facts(state);
10278 if state.enabled {
10279 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10280 state.state = if schedule.is_some() {
10281 ModuleState::Restarting
10282 } else {
10283 ModuleState::Failed
10284 };
10285 } else {
10286 state.state = ModuleState::Disabled;
10287 }
10288 })?;
10289 if let Some(schedule) = schedule {
10290 schedule_respawn(
10291 runtime,
10292 snapshot,
10293 &spec.module_id,
10294 schedule.delay,
10295 RespawnKind::Spawn,
10296 )?;
10297 } else {
10298 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10299 }
10300 Err(SuperviseError::ReloadFailed {
10301 module_id: spec.module_id.clone(),
10302 reason,
10303 })
10304}
10305
10306fn control_flags() -> Flags {
10307 Flags::new(false, Priority::Passive, false)
10308}
10309
10310#[allow(clippy::too_many_arguments)]
10311async fn drain_optional_child(
10312 module_id: &str,
10313 protocol: ModuleProtocol,
10314 stop_notice: StopNotice,
10315 registry: &Registry,
10316 forwarding: Option<&ForwardingTable>,
10317 snapshot: &SharedSnapshot,
10318 terminal_ring: &Arc<Mutex<TerminalRing>>,
10319 spawn_events: &SpawnEventFeed,
10320 child: &mut Option<SupervisedChild>,
10321 drain_timeout: Duration,
10322 final_state: ModuleState,
10323 enabled: Option<bool>,
10324) -> Result<(), SuperviseError> {
10325 if let Some(child) = child.take() {
10326 drain_child_to_state(
10327 module_id,
10328 protocol,
10329 stop_notice,
10330 registry,
10331 forwarding,
10332 snapshot,
10333 terminal_ring,
10334 spawn_events,
10335 child,
10336 drain_timeout,
10337 final_state,
10338 enabled,
10339 )
10340 .await
10341 } else {
10342 update_snapshot(snapshot, Some(module_id), |state| {
10343 state.state = final_state;
10344 if let Some(enabled) = enabled {
10345 state.enabled = enabled;
10346 }
10347 clear_current_process_facts(state);
10348 })?;
10349 release_dead_registration(registry, forwarding, snapshot, module_id).await
10350 }
10351}
10352
10353#[allow(clippy::too_many_arguments)]
10354async fn drain_child_to_state(
10355 module_id: &str,
10356 _protocol: ModuleProtocol,
10357 stop_notice: StopNotice,
10358 registry: &Registry,
10359 forwarding: Option<&ForwardingTable>,
10360 snapshot: &SharedSnapshot,
10361 terminal_ring: &Arc<Mutex<TerminalRing>>,
10362 spawn_events: &SpawnEventFeed,
10363 mut child: SupervisedChild,
10364 drain_timeout: Duration,
10365 final_state: ModuleState,
10366 enabled: Option<bool>,
10367) -> Result<(), SuperviseError> {
10368 let protocol = child.protocol;
10369 update_snapshot(snapshot, Some(module_id), |state| {
10370 state.state = ModuleState::Draining;
10371 state.draining_to_replace = final_state == ModuleState::Restarting;
10372 if let Some(enabled) = enabled {
10373 state.enabled = enabled;
10374 }
10375 })?;
10376
10377 if stop_notice != StopNotice::SentOverConnection {
10388 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10389 info!(
10390 module_id,
10391 pid = child.pid,
10392 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10393 "module has no connection yet; requesting stop by signal"
10394 );
10395 }
10396 request_graceful_stop(module_id, &child);
10397 }
10398
10399 let exit_report = match timeout(drain_timeout, child.wait()).await {
10400 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10401 Ok(Err(source)) => {
10402 fail_after_wait_error(
10403 module_id,
10404 snapshot,
10405 terminal_ring,
10406 spawn_events,
10407 format!("failed to wait during drain: {source}"),
10408 );
10409 return Err(SuperviseError::Wait {
10410 module_id: module_id.to_string(),
10411 source,
10412 });
10413 }
10414 Err(_) => {
10415 warn!(
10428 module_id,
10429 pid = child.pid,
10430 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10431 reason = ?final_state,
10432 ?stop_notice,
10433 "drain budget expired before the module exited; killing it"
10434 );
10435 child.start_kill().map_err(|source| {
10436 fail_after_wait_error(
10437 module_id,
10438 snapshot,
10439 terminal_ring,
10440 spawn_events,
10441 format!("failed to kill during drain: {source}"),
10442 );
10443 SuperviseError::Kill {
10444 module_id: module_id.to_string(),
10445 source,
10446 }
10447 })?;
10448 let status = child.wait().await.map_err(|source| {
10449 fail_after_wait_error(
10450 module_id,
10451 snapshot,
10452 terminal_ring,
10453 spawn_events,
10454 format!("failed to wait after drain kill: {source}"),
10455 );
10456 SuperviseError::Wait {
10457 module_id: module_id.to_string(),
10458 source,
10459 }
10460 })?;
10461 classify_reaped_child_exit(snapshot, &child, &status)
10462 }
10463 };
10464
10465 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10466 publish_after_terminal_record(
10467 module_id,
10468 || {
10469 record_terminal_with_detail(
10470 module_id,
10471 terminal_ring,
10472 spawn_events,
10473 &exit_report,
10474 terminal_disposition(final_state),
10475 detail,
10476 )
10477 },
10478 || {
10479 update_snapshot(snapshot, Some(module_id), |state| {
10480 state.state = final_state;
10481 if let Some(enabled) = enabled {
10482 state.enabled = enabled;
10483 }
10484 clear_current_process_facts(state);
10485 state.last_exit = Some(exit_report.clone());
10486 if exit_report.kind == ExitKind::DeliberateSeverance {
10487 state.lifetime_restarts += 1;
10488 }
10489 })
10490 },
10491 )?;
10492 child.drain_stderr(module_id).await;
10493
10494 release_dead_registration(registry, forwarding, snapshot, module_id).await
10495}
10496
10497#[cfg(unix)]
10517fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10518 let Some(pid) = child
10519 .id()
10520 .and_then(|pid| i32::try_from(pid).ok())
10521 .and_then(rustix::process::Pid::from_raw)
10522 else {
10523 debug!(
10524 module_id,
10525 "no pid to signal for teardown; falling through to the drain wait"
10526 );
10527 return;
10528 };
10529 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10530 Ok(()) => debug!(
10531 module_id,
10532 "sent SIGTERM to a module nothing else asked to stop"
10533 ),
10534 Err(err) => debug!(
10535 module_id,
10536 error = %err,
10537 "SIGTERM to module failed; the drain wait and kill still apply"
10538 ),
10539 }
10540}
10541
10542#[cfg(not(unix))]
10550fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10551 debug!(
10552 module_id,
10553 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10554 );
10555}
10556
10557fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10558 match final_state {
10559 ModuleState::Stopped => TerminalDisposition::Stopped,
10560 ModuleState::Disabled => TerminalDisposition::Disabled,
10561 ModuleState::Restarting => TerminalDisposition::Restarting,
10562 ModuleState::Failed => TerminalDisposition::Failed,
10563 ModuleState::Starting
10564 | ModuleState::Running
10565 | ModuleState::Unresponsive
10566 | ModuleState::Draining => {
10567 unreachable!("terminal exits only finish in terminal or restarting states")
10568 }
10569 }
10570}
10571
10572async fn release_dead_registration(
10580 registry: &Registry,
10581 forwarding: Option<&ForwardingTable>,
10582 snapshot: &SharedSnapshot,
10583 module_id: &str,
10584) -> Result<(), SuperviseError> {
10585 let result = async {
10586 let registration = registry
10587 .get_module(module_id)
10588 .map_err(SuperviseError::Registry)?;
10589 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10590 Ok(()) => return Ok(()),
10591 Err(SuperviseError::RegistrationStillActive { .. }) => {}
10592 Err(err) => return Err(err),
10593 }
10594 let pid = lock_snapshot(snapshot)?.reaped_pid;
10595 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10596 warn!(
10597 module_id,
10598 pid,
10599 connection_id = registration.connection_id.get(),
10600 "reaped module registration outlived release grace; closing dead connection"
10601 );
10602 forwarding.request_connection_close(
10603 registration.connection_id,
10604 CloseReason::new(
10605 "supervised_process_reaped",
10606 format!("module '{module_id}' pid {pid} exited"),
10607 ),
10608 );
10609 wait_for_slot_registration_release(
10610 registry,
10611 crate::registry::RegistrationSlot::Connection(registration.connection_id),
10612 REGISTRY_RELEASE_TIMEOUT,
10613 )
10614 .await?;
10615 }
10616 wait_for_registration_release(registry, module_id, Duration::ZERO).await
10617 }
10618 .await;
10619 if let Err(err) = &result {
10620 fail_snapshot(snapshot, Some(module_id), None);
10621 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10622 }
10623 result
10624}
10625
10626async fn wait_for_registration_release(
10629 registry: &Registry,
10630 module_id: &str,
10631 wait: Duration,
10632) -> Result<(), SuperviseError> {
10633 wait_for_slot_registration_release(
10634 registry,
10635 crate::registry::RegistrationSlot::Active(module_id),
10636 wait,
10637 )
10638 .await
10639}
10640
10641async fn wait_for_slot_registration_release(
10649 registry: &Registry,
10650 slot: crate::registry::RegistrationSlot<'_>,
10651 wait: Duration,
10652) -> Result<(), SuperviseError> {
10653 let deadline = Instant::now() + wait;
10654 let mut release_events = registration_release_events().subscribe();
10655 let still_active = |registration: &crate::registry::ModuleRegistration| {
10656 SuperviseError::RegistrationStillActive {
10657 module_id: registration.manifest.module_id.clone(),
10658 waited: wait,
10659 }
10660 };
10661 loop {
10662 let _observed_generation = *release_events.borrow_and_update();
10663 let Some(registration) = registry
10664 .registration(slot)
10665 .map_err(SuperviseError::Registry)?
10666 else {
10667 return Ok(());
10668 };
10669
10670 let now = Instant::now();
10671 if now >= deadline {
10672 return Err(still_active(®istration));
10673 }
10674
10675 let remaining = deadline.saturating_duration_since(now);
10676 match timeout(remaining, release_events.changed()).await {
10677 Ok(Ok(())) | Ok(Err(_)) => {}
10678 Err(_) => return Err(still_active(®istration)),
10679 }
10680 }
10681}
10682
10683#[cfg(test)]
10684mod slot_registration_wait_tests {
10685 use super::*;
10686 use crate::registry::{ConnectionId, RegistrationSlot};
10687 use subc_protocol::manifest::ModuleManifest;
10688
10689 #[tokio::test]
10690 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10691 let registry = Arc::new(Registry::default());
10692 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
10693 let runtime = supervisor.runtime_config();
10694 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10695 let spec = ModuleSpec {
10696 module_id: "enable-stale-registration".to_string(),
10697 program: PathBuf::from("/missing/enable-retry-test"),
10698 args: Vec::new(),
10699 env: Vec::new(),
10700 reserved: false,
10701 reserved_prefixes: Vec::new(),
10702 protocol: ModuleProtocol::Subc,
10703 overlap: Default::default(),
10704 };
10705 let connection = ConnectionId::new(90);
10706 registry
10707 .register_with_control_ops(
10708 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10709 1,
10710 connection,
10711 Vec::new(),
10712 )
10713 .unwrap();
10714 let mut child = None;
10715 let err = set_child_enabled(
10716 &spec,
10717 &runtime,
10718 ®istry,
10719 &supervisor.process_liveness,
10720 &snapshot,
10721 &mut child,
10722 true,
10723 )
10724 .await
10725 .unwrap_err();
10726 assert!(matches!(
10727 err,
10728 SuperviseError::RegistrationStillActive { .. }
10729 ));
10730 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10731 assert!(child.is_none());
10732 registry.deregister_connection(connection).unwrap();
10733 let err = set_child_enabled(
10734 &spec,
10735 &runtime,
10736 ®istry,
10737 &supervisor.process_liveness,
10738 &snapshot,
10739 &mut child,
10740 true,
10741 )
10742 .await
10743 .unwrap_err();
10744 assert!(
10745 matches!(err, SuperviseError::Spawn { .. }),
10746 "second enable must attempt a spawn: {err}"
10747 );
10748 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10749 }
10750
10751 const INCUMBENT: u64 = 1;
10752 const CANDIDATE: u64 = 2;
10753
10754 fn swapped_registry() -> Arc<Registry> {
10755 let registry = Arc::new(Registry::default());
10756 let manifest = ModuleManifest::builder("m", "0.1.0").build();
10757 registry
10758 .register_with_control_ops(
10759 manifest.clone(),
10760 1,
10761 ConnectionId::new(INCUMBENT),
10762 Vec::new(),
10763 )
10764 .unwrap();
10765 registry
10766 .register_candidate_with_control_ops(
10767 manifest,
10768 1,
10769 ConnectionId::new(CANDIDATE),
10770 Vec::new(),
10771 )
10772 .unwrap();
10773 registry
10774 }
10775
10776 #[tokio::test]
10780 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
10781 let registry = swapped_registry();
10782 registry.promote_candidate("m").unwrap().unwrap();
10783
10784 assert!(matches!(
10785 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
10786 Err(SuperviseError::RegistrationStillActive { .. })
10787 ));
10788
10789 assert!(matches!(
10791 wait_for_slot_registration_release(
10792 ®istry,
10793 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10794 Duration::from_millis(50),
10795 )
10796 .await,
10797 Err(SuperviseError::RegistrationStillActive { .. })
10798 ));
10799
10800 let releaser = Arc::clone(®istry);
10801 let release = tokio::spawn(async move {
10802 sleep(Duration::from_millis(20)).await;
10803 releaser
10804 .deregister_connection(ConnectionId::new(INCUMBENT))
10805 .unwrap();
10806 notify_registration_release();
10807 });
10808 wait_for_slot_registration_release(
10809 ®istry,
10810 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10811 Duration::from_secs(5),
10812 )
10813 .await
10814 .expect("the incumbent's own registration is released");
10815 release.await.unwrap();
10816 assert!(registry.get_module("m").unwrap().is_some());
10817 }
10818
10819 #[tokio::test]
10822 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
10823 let registry = swapped_registry();
10824 assert!(matches!(
10825 wait_for_slot_registration_release(
10826 ®istry,
10827 RegistrationSlot::Candidate("m"),
10828 Duration::from_millis(50),
10829 )
10830 .await,
10831 Err(SuperviseError::RegistrationStillActive { .. })
10832 ));
10833 registry
10834 .deregister_connection(ConnectionId::new(CANDIDATE))
10835 .unwrap();
10836 wait_for_slot_registration_release(
10837 ®istry,
10838 RegistrationSlot::Candidate("m"),
10839 Duration::from_millis(50),
10840 )
10841 .await
10842 .expect("a candidate slot with no candidate is released");
10843 assert!(registry
10844 .registration(RegistrationSlot::Active("m"))
10845 .unwrap()
10846 .is_some());
10847 }
10848}
10849
10850fn classify_exit(status: &ExitStatus) -> ExitReport {
10851 ExitReport {
10852 kind: if status.success() {
10853 ExitKind::Clean
10854 } else {
10855 ExitKind::Crash
10856 },
10857 code: status.code(),
10858 signal: exit_signal(status),
10859 at_ms: unix_ms_now(),
10860 }
10861}
10862
10863fn wait_error_exit_report() -> ExitReport {
10869 ExitReport {
10870 kind: ExitKind::Crash,
10871 code: None,
10872 signal: None,
10873 at_ms: unix_ms_now(),
10874 }
10875}
10876
10877#[cfg(unix)]
10878fn exit_signal(status: &ExitStatus) -> Option<i32> {
10879 use std::os::unix::process::ExitStatusExt;
10880
10881 status.signal()
10882}
10883
10884#[cfg(not(unix))]
10885fn exit_signal(_status: &ExitStatus) -> Option<i32> {
10886 None
10887}
10888
10889fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
10895 update_snapshot(snapshot, Some(module_id), |state| {
10896 state.clear_crash_restarts();
10897 })
10898}
10899
10900fn set_running(
10901 snapshot: &SharedSnapshot,
10902 child: &SupervisedChild,
10903 module_id: &str,
10904 spawn_events: &SpawnEventFeed,
10905) -> Result<(), SuperviseError> {
10906 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10907 module_id: Some(module_id.to_string()),
10908 })?;
10909 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
10910 if std::mem::take(&mut state.coalesced_restart_pending) {
10911 let generation = state.spawn_generation;
10912 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
10913 }
10914 state.drain_disposition_detail = None;
10915 state.spawn_failure = None;
10916 state.waiting_for_previous_pid = None;
10917 state.waiting_for_previous_detail = None;
10918 state.in_alternate_slot = false;
10921 state.configuration_updated_since_spawn = false;
10922 state.spawned_protocol = Some(child.protocol);
10923 state.state = ModuleState::Running;
10924 state.enabled = true;
10925 state.process_alive = true;
10926 state.pid = child.id();
10927 #[cfg(windows)]
10928 {
10929 state.pipe_consumed = child
10930 .nonce_delivery
10931 .as_ref()
10932 .map(|delivery| delivery.consumed());
10933 }
10934 #[cfg(target_os = "macos")]
10935 {
10936 state.report_ready = Some(Arc::clone(&child.report_ready));
10937 }
10938 state.spawned_at_ms = Some(child.spawned_at_ms);
10939 state.exec_outcome = Some(Arc::clone(&child.exec.0));
10940 state.spawned_from = Some(child.spawned_from.clone());
10941 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
10942 state.spawned_file_identity = child.spawned_file_identity.clone();
10943 state.process_start_time = child.process_start_time;
10944 Ok(())
10945}
10946
10947fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
10948 state.waiting_for_previous_pid = None;
10949 state.waiting_for_previous_detail = None;
10950 state.process_alive = false;
10951 state.spawned_protocol = None;
10952 state.pid = None;
10953 #[cfg(windows)]
10954 {
10955 state.pipe_consumed = None;
10956 }
10957 #[cfg(target_os = "macos")]
10958 {
10959 state.report_ready = None;
10960 }
10961 state.spawned_at_ms = None;
10962 state.exec_outcome = None;
10963 state.spawned_from = None;
10964 state.launch_spec_digest = None;
10965 state.spawned_file_identity = None;
10966 state.process_start_time = None;
10967 state.deliberate_severance = None;
10968}
10969
10970#[cfg(test)]
10971fn record_deliberate_severance(
10972 snapshot: &SharedSnapshot,
10973 identity: ProcessIdentity,
10974) -> Result<(), SuperviseError> {
10975 update_snapshot(snapshot, None, |state| {
10976 state.deliberate_severance = Some(identity);
10977 })
10978}
10979
10980fn apply_deliberate_severance_marker(
10981 snapshot: &SharedSnapshot,
10982 exited_identity: Option<ProcessIdentity>,
10983 mut exit_report: ExitReport,
10984) -> ExitReport {
10985 let marker = lock_snapshot(snapshot)
10986 .ok()
10987 .and_then(|mut state| state.deliberate_severance.take());
10988 if marker.is_some() && marker == exited_identity {
10989 exit_report.kind = ExitKind::DeliberateSeverance;
10990 }
10991 exit_report
10992}
10993
10994fn classify_reaped_child_exit(
10995 snapshot: &SharedSnapshot,
10996 child: &SupervisedChild,
10997 status: &ExitStatus,
10998) -> ExitReport {
10999 let _ = update_snapshot(snapshot, None, |state| {
11000 state.reaped_pid = Some(child.pid);
11001 state.spawn_failure = child.spawn_failure.clone();
11002 });
11003 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11004}
11005
11006fn fail_snapshot(
11007 snapshot: &SharedSnapshot,
11008 module_id: Option<&str>,
11009 last_exit: Option<ExitReport>,
11010) {
11011 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11012 state.state = ModuleState::Failed;
11013 clear_current_process_facts(state);
11014 if let Some(last_exit) = last_exit {
11015 state.last_exit = Some(last_exit);
11016 }
11017 }) {
11018 error!(error = %err, "failed to mark supervisor state failed");
11019 }
11020}
11021
11022fn update_snapshot(
11023 snapshot: &SharedSnapshot,
11024 module_id: Option<&str>,
11025 update: impl FnOnce(&mut SupervisorSnapshot),
11026) -> Result<(), SuperviseError> {
11027 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11028 module_id: module_id.map(ToOwned::to_owned),
11029 })?;
11030 update(&mut state);
11031 Ok(())
11032}
11033
11034const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11035
11036fn lock_snapshot_for_control<'a>(
11037 snapshot: &'a SharedSnapshot,
11038 module_id: &str,
11039 caller: &'static str,
11040) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11041 let started_at = Instant::now();
11042 let guard = lock_snapshot(snapshot)?;
11043 let waited = started_at.elapsed();
11044 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11045 warn!(
11046 module_id = %module_id,
11047 waited_ms = waited.as_millis() as u64,
11048 caller = %caller,
11049 "slow snapshot lock"
11050 );
11051 }
11052 Ok(guard)
11053}
11054
11055fn lock_snapshot(
11056 snapshot: &SharedSnapshot,
11057) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11058 snapshot
11059 .lock()
11060 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11061}
11062
11063#[cfg(test)]
11064mod terminal_history_tests {
11065 use std::{
11066 path::PathBuf,
11067 sync::Arc,
11068 time::{Duration, Instant},
11069 };
11070
11071 use tokio::{sync::mpsc, time::sleep};
11072
11073 use super::{
11074 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11075 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11076 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11077 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11078 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11079 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11080 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11081 SupervisorHealthStatus, SupervisorSnapshot,
11082 };
11083 use super::Instant as ClockInstant;
11088 use crate::{
11089 registry::Registry,
11090 terminal_ring::{TerminalRing, TerminalRingConfig},
11091 };
11092 use std::sync::Mutex;
11093 use subc_control::TerminalDisposition;
11094
11095 pub(super) fn fake_aft_stub_path() -> PathBuf {
11100 let mut path = std::env::current_exe().expect("current_exe available in tests");
11101 path.pop();
11102 path.pop();
11103 path.push(if cfg!(windows) {
11104 "fake-aft-stub.exe"
11105 } else {
11106 "fake-aft-stub"
11107 });
11108 assert!(
11109 path.exists(),
11110 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11111 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11112 path.display()
11113 );
11114 path
11115 }
11116
11117 #[test]
11118 fn reserved_never_spawned_refuses_every_hello() {
11119 let supervisor = SupervisorHandle::default();
11124 supervisor.apply_identity_configuration(&ModuleSpec {
11125 module_id: "never-spawned".to_string(),
11126 program: PathBuf::from("/usr/bin/false"),
11127 args: Vec::new(),
11128 env: Vec::new(),
11129 reserved: true,
11130 reserved_prefixes: Vec::new(),
11131 protocol: ModuleProtocol::Subc,
11132 overlap: Default::default(),
11133 });
11134 assert!(
11135 supervisor
11136 .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11137 .is_some(),
11138 "forged nonce must refuse on a reserved never-spawned id"
11139 );
11140 assert!(
11141 supervisor
11142 .reserved_hello_rejection("never-spawned", None)
11143 .is_some(),
11144 "absent nonce must refuse on a reserved never-spawned id"
11145 );
11146 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11148 supervisor.apply_identity_configuration(&ModuleSpec {
11149 module_id: "never-spawned".to_string(),
11150 program: PathBuf::from("/usr/bin/false"),
11151 args: Vec::new(),
11152 env: Vec::new(),
11153 reserved: true,
11154 reserved_prefixes: Vec::new(),
11155 protocol: ModuleProtocol::Subc,
11156 overlap: Default::default(),
11157 });
11158 assert!(supervisor
11159 .reserved_hello_rejection("never-spawned", Some("minted"))
11160 .is_none());
11161 assert!(supervisor
11162 .reserved_hello_rejection("never-spawned", Some("forged"))
11163 .is_some());
11164 }
11165
11166 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11169 let now = ClockInstant::now();
11170 for _ in 0..count {
11171 state.crash_restarts.push_back(now);
11172 }
11173 }
11174
11175 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11179 let aged = state
11180 .crash_restarts
11181 .front()
11182 .expect("a crash restart must be recorded before it can be aged")
11183 .checked_sub(window + Duration::from_secs(1))
11184 .expect("the test clock is far enough from its origin to age an instant");
11185 state.crash_restarts[0] = aged;
11186 }
11187
11188 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11189 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11190 seed_crash_restarts(&mut state, count);
11191 state
11192 }
11193
11194 #[test]
11195 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11196 let policy = RestartPolicy::new(3, Duration::ZERO);
11197 let now = ClockInstant::now();
11198 assert!(daemon_will_restart(
11199 &mut snapshot_with_restarts(true, 2),
11200 &policy,
11201 now
11202 ));
11203 assert!(!daemon_will_restart(
11204 &mut snapshot_with_restarts(true, 3),
11205 &policy,
11206 now
11207 ));
11208 assert!(!daemon_will_restart(
11209 &mut snapshot_with_restarts(false, 0),
11210 &policy,
11211 now
11212 ));
11213 }
11214
11215 #[test]
11216 fn crash_restart_backoff_escalates_with_in_window_count() {
11217 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11218 .with_max_backoff(Duration::from_secs(30));
11219 let now = ClockInstant::now();
11220 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11221 let schedules = (0..4)
11222 .map(|_| {
11223 state
11224 .next_crash_restart(&policy, now)
11225 .expect("the test policy allows four crash restarts")
11226 })
11227 .collect::<Vec<_>>();
11228
11229 assert_eq!(
11230 schedules
11231 .iter()
11232 .map(|schedule| schedule.restart_in_window)
11233 .collect::<Vec<_>>(),
11234 vec![0, 1, 2, 3]
11235 );
11236 assert_eq!(
11237 schedules
11238 .iter()
11239 .map(|schedule| schedule.delay)
11240 .collect::<Vec<_>>(),
11241 vec![
11242 Duration::from_millis(100),
11243 Duration::from_secs(1),
11244 Duration::from_secs(10),
11245 Duration::from_secs(30),
11246 ]
11247 );
11248 }
11249
11250 #[test]
11251 fn crash_restart_backoff_resets_after_ring_clear() {
11252 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11253 let now = ClockInstant::now();
11254 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11255 assert_eq!(
11256 state.next_crash_restart(&policy, now).unwrap().delay,
11257 Duration::from_millis(100)
11258 );
11259 assert_eq!(
11260 state.next_crash_restart(&policy, now).unwrap().delay,
11261 Duration::from_secs(1)
11262 );
11263
11264 state.clear_crash_restarts();
11265 let schedule = state
11266 .next_crash_restart(&policy, now)
11267 .expect("a cleared ring must allow another restart");
11268 assert_eq!(schedule.restart_in_window, 0);
11269 assert_eq!(schedule.delay, Duration::from_millis(100));
11270 }
11271
11272 #[test]
11273 fn crash_restart_backoff_ignores_aged_restarts() {
11274 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11275 let now = ClockInstant::now();
11276 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11277 state
11278 .next_crash_restart(&policy, now)
11279 .expect("the first restart is allowed");
11280 state
11281 .next_crash_restart(&policy, now)
11282 .expect("the second restart is allowed");
11283 state.crash_restarts[0] = now
11284 .checked_sub(policy.window + Duration::from_secs(1))
11285 .expect("the fake clock can age a restart past the window");
11286
11287 let schedule = state
11288 .next_crash_restart(&policy, now)
11289 .expect("an aged restart must release its slot");
11290 assert_eq!(schedule.restart_in_window, 1);
11291 assert_eq!(schedule.delay, Duration::from_secs(1));
11292 assert_eq!(state.crash_restarts.len(), 2);
11293 }
11294
11295 #[test]
11299 fn a_budget_spent_before_the_window_no_longer_refuses() {
11300 let policy = RestartPolicy::new(3, Duration::ZERO);
11301 let mut state = snapshot_with_restarts(true, 3);
11302 let now = ClockInstant::now();
11303 assert!(!daemon_will_restart(&mut state, &policy, now));
11304
11305 assert!(daemon_will_restart(
11306 &mut state,
11307 &policy,
11308 now + policy.window + Duration::from_secs(1)
11309 ));
11310 assert!(
11311 state.crash_restarts.is_empty(),
11312 "reading the budget must drop the instants that left the window"
11313 );
11314 }
11315
11316 fn module_with_recovery_snapshot(
11317 state: ModuleState,
11318 enabled: bool,
11319 restart_count: u32,
11320 ) -> SupervisedModule {
11321 let registry = Arc::new(Registry::default());
11322 let supervisor =
11323 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
11324 let runtime = supervisor.runtime_config();
11325 let spec = ModuleSpec {
11326 module_id: "recovery-snapshot".to_string(),
11327 program: fake_aft_stub_path(),
11328 args: Vec::new(),
11329 env: Vec::new(),
11330 reserved: false,
11331 reserved_prefixes: Vec::new(),
11332 protocol: ModuleProtocol::Subc,
11333 overlap: Default::default(),
11334 };
11335 let mut snapshot = SupervisorSnapshot::new(state, enabled);
11336 seed_crash_restarts(&mut snapshot, restart_count);
11337 let (commands, _rx) = mpsc::channel(4);
11340 SupervisedModule {
11341 inner: Arc::new(SupervisedModuleInner {
11342 module_id: spec.module_id.clone(),
11343 registry,
11344 snapshot: Arc::new(Mutex::new(snapshot)),
11345 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11346 spec,
11347 health: runtime.health,
11348 })),
11349 stderr_ring: runtime.stderr_ring,
11350 terminal_ring: runtime.terminal_ring,
11351 commands,
11352 monitor: Mutex::new(None),
11353 restart_policy: runtime.restart_policy,
11354 effective_drain_timeout: runtime.effective_drain_timeout,
11355 provenance_probe: supervisor.provenance_probe.clone(),
11356 }),
11357 }
11358 }
11359
11360 #[cfg(target_os = "linux")]
11361 #[tokio::test]
11362 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11363 let supervisor =
11364 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11365 .with_cgroup_placement(None);
11366 let result = supervisor.spawn(ModuleSpec {
11367 module_id: "no-cgroup-placement".to_string(),
11368 program: fake_aft_stub_path(),
11369 args: Vec::new(),
11370 env: Vec::new(),
11371 reserved: false,
11372 reserved_prefixes: Vec::new(),
11373 protocol: ModuleProtocol::Subc,
11374 overlap: Default::default(),
11375 });
11376
11377 assert!(
11378 result.is_ok(),
11379 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11380 );
11381 }
11382
11383 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11384 async fn undecided_snapshot_uses_shared_restart_predicate() {
11385 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11386 .will_recover_after_connection_loss()
11387 .unwrap());
11388 assert!(
11389 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11390 .will_recover_after_connection_loss()
11391 .unwrap()
11392 );
11393 }
11394
11395 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11396 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11397 assert!(
11398 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11399 .will_recover_after_connection_loss()
11400 .unwrap()
11401 );
11402 }
11403
11404 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11405 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11406 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11407 .will_recover_after_connection_loss()
11408 .unwrap());
11409 assert!(
11410 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11411 .will_recover_after_connection_loss()
11412 .unwrap()
11413 );
11414 }
11415
11416 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11417 async fn warming_snapshot_is_limited_to_startup_phases() {
11418 for state in [
11419 ModuleState::Starting,
11420 ModuleState::Running,
11421 ModuleState::Restarting,
11422 ] {
11423 assert!(
11424 module_with_recovery_snapshot(state, true, 0)
11425 .is_warming()
11426 .unwrap(),
11427 "{state:?} should be warming"
11428 );
11429 }
11430 for state in [
11431 ModuleState::Unresponsive,
11432 ModuleState::Draining,
11433 ModuleState::Stopped,
11434 ModuleState::Failed,
11435 ModuleState::Disabled,
11436 ] {
11437 assert!(
11438 !module_with_recovery_snapshot(state, true, 0)
11439 .is_warming()
11440 .unwrap(),
11441 "{state:?} should not be warming"
11442 );
11443 }
11444 }
11445
11446 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11447 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11448 let registry = Arc::new(Registry::default());
11449 let supervisor =
11450 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
11451 let module = supervisor
11452 .spawn(ModuleSpec {
11453 module_id: "terminal-history".to_string(),
11454 program: fake_aft_stub_path(),
11455 args: Vec::new(),
11456 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11457 reserved: false,
11458 reserved_prefixes: Vec::new(),
11459 protocol: ModuleProtocol::Subc,
11460 overlap: Default::default(),
11461 })
11462 .unwrap();
11463
11464 let deadline = Instant::now() + Duration::from_secs(5);
11465 loop {
11466 let history = module.terminal_history();
11467 if history.entries.len() == 2 {
11468 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11469 assert_eq!(history.dropped, 0);
11470 assert_eq!(
11471 history
11472 .entries
11473 .iter()
11474 .map(|entry| entry.exit_code)
11475 .collect::<Vec<_>>(),
11476 vec![Some(23), Some(23)]
11477 );
11478 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11479 return;
11480 }
11481 assert!(
11482 Instant::now() < deadline,
11483 "module did not retain two terminal exits: {history:?}"
11484 );
11485 sleep(Duration::from_millis(10)).await;
11486 }
11487 }
11488
11489 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11493 async fn disable_during_crash_backoff_cancels_pending_respawn() {
11494 let backoff = Duration::from_secs(2);
11495 let supervisor = Supervisor::new_for_test(
11496 Arc::new(Registry::default()),
11497 RestartPolicy::new(10, backoff),
11498 );
11499 let module = supervisor
11500 .spawn(ModuleSpec {
11501 module_id: "disable-during-backoff".to_string(),
11502 program: fake_aft_stub_path(),
11503 args: Vec::new(),
11504 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11505 reserved: false,
11506 reserved_prefixes: Vec::new(),
11507 protocol: ModuleProtocol::Subc,
11508 overlap: Default::default(),
11509 })
11510 .unwrap();
11511
11512 let deadline = Instant::now() + Duration::from_secs(5);
11514 loop {
11515 if module.status().unwrap().state == ModuleState::Restarting {
11516 break;
11517 }
11518 assert!(
11519 Instant::now() < deadline,
11520 "module never entered the crash backoff"
11521 );
11522 sleep(Duration::from_millis(10)).await;
11523 }
11524
11525 let started = Instant::now();
11526 module.set_enabled(false).await.unwrap();
11527 let waited = started.elapsed();
11528
11529 assert!(
11530 waited < backoff / 2,
11531 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11532 );
11533 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11534
11535 sleep(backoff + Duration::from_millis(500)).await;
11537 let status = module.status().unwrap();
11538 assert_eq!(status.state, ModuleState::Disabled);
11539 assert_eq!(
11540 status.spawn_generation, 1,
11541 "module respawned after the operator disabled it"
11542 );
11543 }
11544
11545 #[cfg(unix)]
11548 fn protocol_none_sigterm_exits_clean_spec(
11549 module_id: &str,
11550 dir: &std::path::Path,
11551 ) -> (ModuleSpec, PathBuf, PathBuf) {
11552 let ready = dir.join("ready");
11553 let marker = dir.join("sigterm");
11554 let spec = ModuleSpec {
11555 module_id: module_id.to_string(),
11556 program: fake_aft_stub_path(),
11557 args: Vec::new(),
11558 env: vec![
11559 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11560 (
11561 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11562 marker.display().to_string(),
11563 ),
11564 (
11565 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11566 ready.display().to_string(),
11567 ),
11568 ],
11569 reserved: false,
11570 reserved_prefixes: Vec::new(),
11571 protocol: ModuleProtocol::None,
11572 overlap: Default::default(),
11573 };
11574 (spec, ready, marker)
11575 }
11576
11577 #[cfg(unix)]
11581 async fn wait_for_file(path: &std::path::Path) {
11582 let deadline = Instant::now() + Duration::from_secs(10);
11583 while !path.exists() {
11584 assert!(
11585 Instant::now() < deadline,
11586 "{} never appeared",
11587 path.display()
11588 );
11589 sleep(Duration::from_millis(10)).await;
11590 }
11591 }
11592
11593 #[cfg(unix)]
11597 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11598 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11599 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11600 let (spec, ready, marker) =
11601 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11602 let supervisor = Supervisor::new_for_test(
11603 Arc::new(Registry::default()),
11604 RestartPolicy::new(3, Duration::ZERO),
11605 );
11606 let module = supervisor.spawn(spec).unwrap();
11607 wait_for_file(&ready).await;
11608 let deadline = Instant::now() + Duration::from_secs(10);
11612 let first_pid = loop {
11613 let status = module.status().unwrap();
11614 if status.state == ModuleState::Running {
11615 if let Some(pid) = status.pid {
11616 break pid;
11617 }
11618 }
11619 assert!(
11620 Instant::now() < deadline,
11621 "a running module must report its pid after exec confirmation: {status:?}"
11622 );
11623 sleep(Duration::from_millis(10)).await;
11624 };
11625
11626 rustix::process::kill_process(
11627 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11628 rustix::process::Signal::TERM,
11629 )
11630 .unwrap();
11631
11632 let deadline = Instant::now() + Duration::from_secs(10);
11633 let respawned = loop {
11634 let status = module.status().unwrap();
11635 if status.state == ModuleState::Running
11636 && status.pid.is_some_and(|pid| pid != first_pid)
11637 {
11638 break status;
11639 }
11640 assert!(
11641 Instant::now() < deadline,
11642 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11643 );
11644 sleep(Duration::from_millis(10)).await;
11645 };
11646 assert_eq!(respawned.spawn_generation, 2);
11647 assert!(
11648 marker.exists(),
11649 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11650 );
11651
11652 let history = module.terminal_history();
11653 assert_eq!(history.entries.len(), 1, "{history:?}");
11654 let entry = &history.entries[0];
11655 assert_eq!(entry.exit_code, Some(0));
11656 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11657 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11658
11659 module.stop().await.unwrap();
11660 }
11661
11662 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11666 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11667 let supervisor = Supervisor::new_for_test(
11668 Arc::new(Registry::default()),
11669 RestartPolicy::new(1, Duration::ZERO),
11670 );
11671 let module = supervisor
11672 .spawn(ModuleSpec {
11673 module_id: "none-clean-exit-budget".to_string(),
11674 program: fake_aft_stub_path(),
11675 args: Vec::new(),
11676 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11677 reserved: false,
11678 reserved_prefixes: Vec::new(),
11679 protocol: ModuleProtocol::None,
11680 overlap: Default::default(),
11681 })
11682 .unwrap();
11683
11684 let deadline = Instant::now() + Duration::from_secs(10);
11687 loop {
11688 let status = module.status().unwrap();
11689 if status.state == ModuleState::Failed {
11690 break;
11691 }
11692 assert!(
11693 Instant::now() < deadline,
11694 "module never exhausted its budget: {status:?} {:?}",
11695 module.terminal_history()
11696 );
11697 sleep(Duration::from_millis(10)).await;
11698 }
11699 let history = module.terminal_history();
11700 assert_eq!(
11701 history
11702 .entries
11703 .iter()
11704 .map(|entry| (entry.exit_code, entry.disposition.clone()))
11705 .collect::<Vec<_>>(),
11706 vec![
11707 (Some(0), TerminalDisposition::Restarting),
11708 (Some(0), TerminalDisposition::Failed),
11709 ]
11710 );
11711 let detail = history.entries[1]
11712 .disposition_detail
11713 .as_deref()
11714 .expect("a budget failure names the budget");
11715 assert!(detail.contains("max_restarts=1"), "{detail}");
11716 assert_eq!(module.status().unwrap().spawn_generation, 2);
11717 }
11718
11719 #[test]
11720 fn restart_budget_failure_is_published_after_its_terminal_record() {
11721 use std::io::{Read, Write};
11722 let supervisor = Supervisor::new_for_test(
11723 Arc::new(Registry::default()),
11724 RestartPolicy::new(0, Duration::ZERO),
11725 );
11726 let runtime = supervisor.runtime_config();
11727 let spec = ModuleSpec {
11728 protocol: ModuleProtocol::None,
11729 ..windowed_crash_spec("budget-publication-order")
11730 };
11731 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11732 ModuleState::Running,
11733 true,
11734 )));
11735 lock_snapshot(&snapshot).unwrap().process_alive = true;
11736 let reaped_snapshot = snapshot.clone();
11737 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11738 let history = runtime.terminal_ring.clone();
11739 let reap = std::thread::spawn(move || {
11740 tokio::runtime::Builder::new_current_thread()
11741 .enable_all()
11742 .build()
11743 .unwrap()
11744 .block_on(on_child_exit(
11745 &spec,
11746 runtime.restart_policy,
11747 &supervisor.registry,
11748 &reaped_snapshot,
11749 &runtime.terminal_ring,
11750 &runtime.spawn_events,
11751 &runtime.child_roster,
11752 ExitReport {
11753 kind: ExitKind::Clean,
11754 code: Some(0),
11755 signal: None,
11756 at_ms: 1,
11757 },
11758 ))
11759 });
11760 let mut reached = [0];
11761 barrier.read_exact(&mut reached).unwrap();
11762 assert_eq!(&reached, b"R");
11763 let (before_state, before_alive) = {
11764 let state = lock_snapshot(&snapshot).unwrap();
11765 (state.state, state.process_alive)
11766 };
11767 let before_history = history.lock().unwrap().snapshot();
11768 barrier.write_all(b"X").unwrap();
11769 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
11770 assert!(before_history.entries.is_empty());
11771 assert_ne!(
11772 before_state,
11773 ModuleState::Failed,
11774 "Failed was visible before its terminal record could be written"
11775 );
11776 assert!(before_alive);
11777 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11778 let history = history.lock().unwrap().snapshot();
11779 assert_eq!(history.entries.len(), 1);
11780 assert_eq!(history.entries[0].exit_code, Some(0));
11781 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11782 }
11783
11784 #[test]
11785 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
11786 use std::io::{Read, Write};
11787
11788 let supervisor = Supervisor::new_for_test(
11789 Arc::new(Registry::default()),
11790 RestartPolicy::new(3, Duration::ZERO),
11791 );
11792 let mut runtime = supervisor.runtime_config();
11793 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
11794 let snapshot = module.inner.snapshot.clone();
11795 lock_snapshot(&snapshot).unwrap().process_alive = true;
11796 runtime.terminal_ring = module.inner.terminal_ring.clone();
11797 let spec = windowed_crash_spec(module.module_id());
11798 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11799 let reaped_snapshot = snapshot.clone();
11800 let reap = std::thread::spawn(move || {
11801 tokio::runtime::Builder::new_current_thread()
11802 .enable_all()
11803 .build()
11804 .unwrap()
11805 .block_on(on_child_exit(
11806 &spec,
11807 runtime.restart_policy,
11808 &supervisor.registry,
11809 &reaped_snapshot,
11810 &runtime.terminal_ring,
11811 &runtime.spawn_events,
11812 &runtime.child_roster,
11813 ExitReport {
11814 kind: ExitKind::Crash,
11815 code: Some(2),
11816 signal: None,
11817 at_ms: 1,
11818 },
11819 ))
11820 });
11821
11822 let mut reached = [0];
11826 barrier.read_exact(&mut reached).unwrap();
11827 assert_eq!(&reached, b"R");
11828 let before_status = module.status().unwrap();
11829 let before_history = module.terminal_history();
11830 let before_recovers = module.will_recover_after_connection_loss().unwrap();
11831 barrier.write_all(b"X").unwrap();
11832 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
11833
11834 assert_eq!(before_status.state, ModuleState::Running);
11835 assert!(before_status.process_alive);
11836 assert_eq!(before_status.restart_count, 3);
11837 assert!(before_history.entries.is_empty());
11838 assert!(
11839 before_recovers,
11840 "connection cleanup would report a terminal close for a scheduled crash restart"
11841 );
11842 let restarting = module.status().unwrap();
11843 assert_eq!(restarting.state, ModuleState::Restarting);
11844 assert_eq!(restarting.restart_count, 3);
11845 assert!(module.will_recover_after_connection_loss().unwrap());
11846 let history = module.terminal_history();
11847 assert_eq!(history.entries.len(), 1);
11848 assert_eq!(
11849 history.entries[0].disposition,
11850 TerminalDisposition::Restarting
11851 );
11852
11853 update_snapshot(&snapshot, Some(module.module_id()), |state| {
11856 state.state = ModuleState::Running;
11857 state.respawn_pending = false;
11858 })
11859 .unwrap();
11860 assert!(!module.will_recover_after_connection_loss().unwrap());
11861 update_snapshot(&snapshot, Some(module.module_id()), |state| {
11862 state.enabled = false;
11863 state.respawn_pending = true;
11864 })
11865 .unwrap();
11866 assert!(!module.will_recover_after_connection_loss().unwrap());
11867 }
11868
11869 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
11870 use std::net::{TcpListener, TcpStream};
11871 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
11872 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
11873 let (barrier, _) = listener.accept().unwrap();
11874 for stream in [&writer, &barrier] {
11875 stream
11876 .set_read_timeout(Some(Duration::from_secs(10)))
11877 .unwrap();
11878 stream
11879 .set_write_timeout(Some(Duration::from_secs(10)))
11880 .unwrap();
11881 }
11882 *events.1.lock().unwrap() = Some(writer);
11883 barrier
11884 }
11885
11886 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
11887 use std::io::{Read, Write};
11888 let barrier = events.1.lock().unwrap().take();
11889 if let Some(mut barrier) = barrier {
11890 barrier.write_all(b"R").unwrap();
11891 let mut release = [0];
11892 barrier.read_exact(&mut release).unwrap();
11893 assert_eq!(&release, b"X");
11894 }
11895 }
11896
11897 #[test]
11898 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
11899 use std::io::{Read, Write};
11900
11901 let supervisor = Supervisor::new_for_test(
11902 Arc::new(Registry::default()),
11903 RestartPolicy::new(0, Duration::ZERO),
11904 );
11905 let mut runtime = supervisor.runtime_config();
11906 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
11907 let snapshot = module.inner.snapshot.clone();
11908 runtime.terminal_ring = module.inner.terminal_ring.clone();
11909 let spec = ModuleSpec {
11910 module_id: module.module_id().into(),
11911 program: fake_aft_stub_path(),
11912 protocol: ModuleProtocol::None,
11913 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11914 ..windowed_crash_spec(module.module_id())
11915 };
11916 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11917 let reap = std::thread::spawn(move || {
11918 tokio::runtime::Builder::new_current_thread()
11919 .enable_all()
11920 .build()
11921 .unwrap()
11922 .block_on(async {
11923 let mut child =
11924 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11925 health_restart_child(
11926 &spec,
11927 &runtime,
11928 &supervisor.registry,
11929 &supervisor.process_liveness,
11930 &snapshot,
11931 &mut child,
11932 SupervisorHealthStatus::Failing,
11933 Some("health probe did not answer"),
11934 1,
11935 )
11936 .await
11937 .unwrap();
11938 assert!(child.is_none());
11939 });
11940 });
11941 let mut reached = [0];
11946 barrier.read_exact(&mut reached).unwrap();
11947 assert_eq!(&reached, b"R");
11948 let before_status = module.status().unwrap();
11949 let before_history = module.terminal_history();
11950 barrier.write_all(b"X").unwrap();
11951 reap.join().unwrap();
11952 assert!(before_history.entries.is_empty());
11953 assert_ne!(
11954 before_status.state,
11955 ModuleState::Failed,
11956 "Failed was visible before its terminal record could be written"
11957 );
11958 assert!(
11959 before_status.process_alive,
11960 "process death was visible before its terminal record could be written"
11961 );
11962 let failed = module.status().unwrap();
11963 assert_eq!(failed.state, ModuleState::Failed);
11964 assert!(!failed.process_alive);
11965 let history = module.terminal_history();
11966 assert_eq!(history.entries.len(), 1);
11967 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11968 assert_eq!(
11969 history.entries[0].disposition_detail.as_deref(),
11970 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
11971 );
11972 }
11973
11974 #[test]
11975 fn observed_exits_publish_process_death_after_terminal_history() {
11976 use std::io::{Read, Write};
11977 for (kind, enabled, shutdown, wait_error, expected_state) in [
11978 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
11979 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
11980 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
11981 (
11982 ExitKind::DeliberateSeverance,
11983 true,
11984 false,
11985 false,
11986 ModuleState::Restarting,
11987 ),
11988 #[cfg(unix)]
11989 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
11990 (ExitKind::Crash, true, false, true, ModuleState::Failed),
11991 ] {
11992 let supervisor = Supervisor::new_for_test(
11993 Arc::new(Registry::default()),
11994 RestartPolicy::new(1, Duration::ZERO),
11995 );
11996 let runtime = supervisor.runtime_config();
11997 #[cfg(unix)]
11998 if shutdown {
11999 runtime.child_roster.close();
12000 }
12001 #[cfg(not(unix))]
12002 let _ = shutdown;
12003 let spec = windowed_crash_spec("exit-publication-order");
12004 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12005 ModuleState::Running,
12006 enabled,
12007 )));
12008 lock_snapshot(&snapshot).unwrap().process_alive = true;
12009 let reaped_snapshot = snapshot.clone();
12010 let history = runtime.terminal_ring.clone();
12011 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12012 let reap = std::thread::spawn(move || {
12013 if wait_error {
12014 super::fail_after_wait_error(
12015 &spec.module_id,
12016 &reaped_snapshot,
12017 &runtime.terminal_ring,
12018 &runtime.spawn_events,
12019 "wait failed".into(),
12020 );
12021 } else {
12022 tokio::runtime::Builder::new_current_thread()
12023 .enable_all()
12024 .build()
12025 .unwrap()
12026 .block_on(on_child_exit(
12027 &spec,
12028 runtime.restart_policy,
12029 &supervisor.registry,
12030 &reaped_snapshot,
12031 &runtime.terminal_ring,
12032 &runtime.spawn_events,
12033 &runtime.child_roster,
12034 ExitReport {
12035 kind,
12036 code: Some(0),
12037 signal: None,
12038 at_ms: 1,
12039 },
12040 ));
12041 }
12042 });
12043 let mut reached = [0];
12044 barrier.read_exact(&mut reached).unwrap();
12045 assert_eq!(&reached, b"R");
12046 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12047 let before_history = history.lock().unwrap().snapshot();
12048 barrier.write_all(b"X").unwrap();
12049 reap.join().unwrap();
12050 assert!(
12051 before_alive,
12052 "{kind:?}: death was published before recording"
12053 );
12054 assert!(before_history.entries.is_empty());
12055 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12056 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12057 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12058 }
12059 }
12060
12061 #[tokio::test]
12062 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12063 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12064 let path = dir.join("terminals.jsonl");
12065 std::fs::create_dir(&path).unwrap();
12066 let supervisor = Supervisor::new_for_test(
12067 Arc::new(Registry::default()),
12068 RestartPolicy::new(0, Duration::ZERO),
12069 )
12070 .with_terminal_journal(path, "health-budget-journal-failure".into());
12071 let runtime = supervisor.runtime_config();
12072 let spec = ModuleSpec {
12073 program: fake_aft_stub_path(),
12074 protocol: ModuleProtocol::None,
12075 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12076 ..windowed_crash_spec("health-budget-journal-failure")
12077 };
12078 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12079 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12080 health_restart_child(
12081 &spec,
12082 &runtime,
12083 &supervisor.registry,
12084 &supervisor.process_liveness,
12085 &snapshot,
12086 &mut child,
12087 SupervisorHealthStatus::Failing,
12088 None,
12089 1,
12090 )
12091 .await
12092 .unwrap();
12093 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12094 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12095 let history = runtime
12096 .terminal_ring
12097 .lock()
12098 .unwrap()
12099 .durable_history(&spec.module_id);
12100 assert!(history.journal_write_failures > 0);
12101 assert_eq!(history.entries.len(), 1);
12102 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12103 }
12104
12105 #[test]
12106 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12107 let supervisor = Supervisor::new_for_test(
12108 Arc::new(Registry::default()),
12109 RestartPolicy::new(0, Duration::ZERO),
12110 );
12111 let runtime = supervisor.runtime_config();
12112 let spec = ModuleSpec {
12113 program: fake_aft_stub_path(),
12114 protocol: ModuleProtocol::None,
12115 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12116 ..windowed_crash_spec("health-budget-recording-panic")
12117 };
12118 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12119 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12120 tokio::runtime::Builder::new_current_thread()
12121 .enable_all()
12122 .build()
12123 .unwrap()
12124 .block_on(async {
12125 let mut child =
12126 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12127 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12128 health_restart_child(
12129 &spec,
12130 &runtime,
12131 &supervisor.registry,
12132 &supervisor.process_liveness,
12133 &snapshot,
12134 &mut child,
12135 SupervisorHealthStatus::Failing,
12136 None,
12137 1,
12138 )
12139 .await
12140 .unwrap();
12141 });
12142 }));
12143 let panic = result.expect_err("recording must still unwind");
12144 assert_eq!(
12145 panic.downcast_ref::<String>().map(String::as_str),
12146 Some("spawn event sequence exhausted")
12147 );
12148 let state = lock_snapshot(&snapshot).unwrap();
12149 assert_eq!(state.state, ModuleState::Failed);
12150 assert!(!state.process_alive);
12151 assert!(state.last_exit.is_some());
12152 }
12153
12154 #[tokio::test]
12155 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12156 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12157 let path = dir.join("terminals.jsonl");
12158 std::fs::create_dir(&path).unwrap();
12159 let supervisor = Supervisor::new_for_test(
12160 Arc::new(Registry::default()),
12161 RestartPolicy::new(0, Duration::ZERO),
12162 )
12163 .with_terminal_journal(path, "budget-journal-failure".into());
12164 let runtime = supervisor.runtime_config();
12165 let spec = windowed_crash_spec("budget-journal-failure");
12166 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12167 assert!(matches!(
12168 on_child_exit(
12169 &spec,
12170 runtime.restart_policy,
12171 &supervisor.registry,
12172 &snapshot,
12173 &runtime.terminal_ring,
12174 &runtime.spawn_events,
12175 &runtime.child_roster,
12176 crash_exit_report(1),
12177 )
12178 .await,
12179 NextAction::Stop { .. }
12180 ));
12181 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12182 let history = runtime
12183 .terminal_ring
12184 .lock()
12185 .unwrap()
12186 .durable_history(&spec.module_id);
12187 assert!(history.journal_write_failures > 0);
12188 assert_eq!(history.entries.len(), 1);
12189 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12190 }
12191
12192 #[test]
12193 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12194 let supervisor = Supervisor::new_for_test(
12195 Arc::new(Registry::default()),
12196 RestartPolicy::new(0, Duration::ZERO),
12197 );
12198 let runtime = supervisor.runtime_config();
12199 let spec = windowed_crash_spec("budget-recording-panic");
12200 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12201 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12202 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12205 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12206 tokio::runtime::Builder::new_current_thread()
12207 .enable_all()
12208 .build()
12209 .unwrap()
12210 .block_on(on_child_exit(
12211 &spec,
12212 runtime.restart_policy,
12213 &supervisor.registry,
12214 &snapshot,
12215 &runtime.terminal_ring,
12216 &runtime.spawn_events,
12217 &runtime.child_roster,
12218 crash_exit_report(1),
12219 ))
12220 }));
12221 let panic = result.err().expect("recording must still unwind");
12222 assert_eq!(
12223 panic.downcast_ref::<String>().map(String::as_str),
12224 Some("spawn event sequence exhausted")
12225 );
12226 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12227 }
12228
12229 #[cfg(unix)]
12232 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12233 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12234 for disable in [false, true] {
12235 let label = if disable {
12236 "none-requested-disable"
12237 } else {
12238 "none-requested-stop"
12239 };
12240 let dir = cortexkit_test_support::ScratchDir::new(label);
12241 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12242 let supervisor = Supervisor::new_for_test(
12243 Arc::new(Registry::default()),
12244 RestartPolicy::new(3, Duration::ZERO),
12245 );
12246 let module = supervisor.spawn(spec).unwrap();
12247 wait_for_file(&ready).await;
12248
12249 if disable {
12250 module.set_enabled(false).await.unwrap();
12251 } else {
12252 module.stop().await.unwrap();
12253 }
12254 assert!(
12255 marker.exists(),
12256 "{label}: the child must have left through its SIGTERM handler with exit 0"
12257 );
12258
12259 sleep(Duration::from_millis(500)).await;
12262 let status = module.status().unwrap();
12263 let expected = if disable {
12264 ModuleState::Disabled
12265 } else {
12266 ModuleState::Stopped
12267 };
12268 assert_eq!(status.state, expected, "{label}");
12269 assert_eq!(
12270 status.spawn_generation, 1,
12271 "{label}: respawned after a requested stop"
12272 );
12273 let history = module.terminal_history();
12274 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12275 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12276 assert_ne!(
12277 history.entries[0].disposition,
12278 TerminalDisposition::Restarting,
12279 "{label}"
12280 );
12281 }
12282 }
12283
12284 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12287 async fn subc_wire_clean_exit_is_still_a_stop() {
12288 let supervisor = Supervisor::new_for_test(
12289 Arc::new(Registry::default()),
12290 RestartPolicy::new(3, Duration::ZERO),
12291 );
12292 let module = supervisor
12293 .spawn(ModuleSpec {
12294 module_id: "wire-clean-exit".to_string(),
12295 program: fake_aft_stub_path(),
12296 args: Vec::new(),
12297 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12298 reserved: false,
12299 reserved_prefixes: Vec::new(),
12300 protocol: ModuleProtocol::Subc,
12301 overlap: Default::default(),
12302 })
12303 .unwrap();
12304
12305 let deadline = Instant::now() + Duration::from_secs(10);
12306 while module.terminal_history().entries.is_empty() {
12307 assert!(Instant::now() < deadline, "module never exited");
12308 sleep(Duration::from_millis(10)).await;
12309 }
12310 sleep(Duration::from_millis(500)).await;
12312 let status = module.status().unwrap();
12313 assert_eq!(status.state, ModuleState::Stopped);
12314 assert_eq!(status.spawn_generation, 1);
12315 let history = module.terminal_history();
12316 assert_eq!(history.entries.len(), 1, "{history:?}");
12317 assert_eq!(history.entries[0].exit_code, Some(0));
12318 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12319 }
12320
12321 #[cfg(unix)]
12322 #[tokio::test]
12323 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12324 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12325 let record = dir.join("live-children.json");
12326 let supervisor = Supervisor::new_for_test(
12327 Arc::new(Registry::default()),
12328 RestartPolicy::new(0, Duration::ZERO),
12329 );
12330 let mut runtime = supervisor.runtime_config();
12331 runtime.child_roster.record_to(record.clone());
12332 let gate = Arc::new(super::ReloadExitRecordGate::default());
12333 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12334 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12335 let spec = ModuleSpec {
12336 module_id: "reload-exit-roster".into(),
12337 program: fake_aft_stub_path(),
12338 args: Vec::new(),
12339 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12340 reserved: false,
12341 reserved_prefixes: Vec::new(),
12342 protocol: ModuleProtocol::Subc,
12343 overlap: Default::default(),
12344 };
12345 let mut child = None;
12346 let reload = super::finish_reload_child(
12347 &spec,
12348 &runtime,
12349 &supervisor.registry,
12350 &supervisor.process_liveness,
12351 &snapshot,
12352 &mut child,
12353 );
12354 tokio::pin!(reload);
12355 tokio::select! {
12356 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12357 _ = gate.reached.notified() => {}
12358 }
12359 assert!(runtime
12360 .terminal_ring
12361 .lock()
12362 .unwrap()
12363 .snapshot()
12364 .entries
12365 .is_empty());
12366 assert_eq!(
12367 crate::live_children::read_record(&record).unwrap().len(),
12368 1,
12369 "shutdown must still wait for the reaped child until its terminal record exists"
12370 );
12371 runtime.child_roster.close();
12372 gate.resume.notify_one();
12373 assert!(reload.await.is_err());
12374 assert!(crate::live_children::read_record(&record)
12375 .unwrap()
12376 .is_empty());
12377 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12378 assert_eq!(history.entries.len(), 1);
12379 assert_eq!(
12380 history.entries[0].disposition,
12381 TerminalDisposition::DaemonShutdown
12382 );
12383 }
12384
12385 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12389 async fn every_restart_increment_path_advances_lifetime_count() {
12390 let supervisor = Supervisor::new_for_test(
12391 Arc::new(Registry::default()),
12392 RestartPolicy::new(1, Duration::ZERO),
12393 );
12394 let runtime = supervisor.runtime_config();
12395 let spec = ModuleSpec {
12396 module_id: "lifetime-increment-path".to_string(),
12397 program: PathBuf::from("/unused/lifetime-increment-path"),
12398 args: Vec::new(),
12399 env: Vec::new(),
12400 reserved: false,
12401 reserved_prefixes: Vec::new(),
12402 protocol: ModuleProtocol::Subc,
12403 overlap: Default::default(),
12404 };
12405
12406 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12407 assert!(matches!(
12408 on_child_exit(
12409 &spec,
12410 runtime.restart_policy,
12411 &supervisor.registry,
12412 &crash_snapshot,
12413 &runtime.terminal_ring,
12414 &runtime.spawn_events,
12415 &runtime.child_roster,
12416 ExitReport {
12417 kind: ExitKind::Crash,
12418 code: Some(1),
12419 signal: None,
12420 at_ms: 1,
12421 },
12422 )
12423 .await,
12424 NextAction::Restart { schedule: _ }
12425 ));
12426 let (crash_restarts, crash_lifetime) = {
12427 let state = lock_snapshot(&crash_snapshot).unwrap();
12428 (state.crash_restarts.len(), state.lifetime_restarts)
12429 };
12430 assert_eq!(crash_restarts, 1);
12431 assert_eq!(crash_lifetime, 1);
12432
12433 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12434 let mut health_child = None;
12435 assert!(matches!(
12436 health_restart_child(
12437 &spec,
12438 &runtime,
12439 &supervisor.registry,
12440 &supervisor.process_liveness,
12441 &health_snapshot,
12442 &mut health_child,
12443 SupervisorHealthStatus::Failing,
12444 None,
12445 2,
12446 )
12447 .await,
12448 Ok(())
12449 ));
12450 assert!(health_child.is_none());
12451 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12452 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12453 let (health_restarts, health_lifetime) = {
12454 let state = lock_snapshot(&health_snapshot).unwrap();
12455 (state.crash_restarts.len(), state.lifetime_restarts)
12456 };
12457 assert_eq!(health_restarts, 1);
12458 assert_eq!(health_lifetime, 1);
12459
12460 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12461 let mut reload_child = None;
12462 assert!(matches!(
12463 handle_reload_spawn_failure(
12464 &spec,
12465 &runtime,
12466 &supervisor.process_liveness,
12467 &reload_snapshot,
12468 &mut reload_child,
12469 "forced reload spawn failure".to_string(),
12470 )
12471 .await,
12472 Err(SuperviseError::ReloadFailed { .. })
12473 ));
12474 let (reload_restarts, reload_lifetime) = {
12475 let state = lock_snapshot(&reload_snapshot).unwrap();
12476 (state.crash_restarts.len(), state.lifetime_restarts)
12477 };
12478 assert_eq!(reload_restarts, 1);
12479 assert_eq!(reload_lifetime, 1);
12480 }
12481
12482 #[tokio::test]
12483 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12484 let supervisor = Supervisor::new_for_test(
12485 Arc::new(Registry::default()),
12486 RestartPolicy::new(3, Duration::ZERO),
12487 );
12488 let runtime = supervisor.runtime_config();
12489 let spec = ModuleSpec {
12490 module_id: "deliberately-severed".to_string(),
12491 program: PathBuf::from("/unused/deliberately-severed"),
12492 args: Vec::new(),
12493 env: Vec::new(),
12494 reserved: false,
12495 reserved_prefixes: Vec::new(),
12496 protocol: ModuleProtocol::Subc,
12497 overlap: Default::default(),
12498 };
12499 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12500 let process = ProcessIdentity {
12501 pid: 41,
12502 start_time: 101,
12503 };
12504 record_deliberate_severance(&snapshot, process).unwrap();
12505 let exit_report = apply_deliberate_severance_marker(
12506 &snapshot,
12507 Some(process),
12508 ExitReport {
12509 kind: ExitKind::Crash,
12510 code: Some(1),
12511 signal: None,
12512 at_ms: 1,
12513 },
12514 );
12515 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12516
12517 assert!(matches!(
12518 on_child_exit(
12519 &spec,
12520 runtime.restart_policy,
12521 &supervisor.registry,
12522 &snapshot,
12523 &runtime.terminal_ring,
12524 &runtime.spawn_events,
12525 &runtime.child_roster,
12526 exit_report,
12527 )
12528 .await,
12529 NextAction::Restart { schedule: _ }
12530 ));
12531 let state = lock_snapshot(&snapshot).unwrap();
12532 assert_eq!(state.lifetime_restarts, 1);
12533 assert_eq!(state.crash_restarts.len(), 0);
12534 }
12535
12536 #[tokio::test]
12537 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12538 let supervisor = Supervisor::new_for_test(
12539 Arc::new(Registry::default()),
12540 RestartPolicy::new(3, Duration::ZERO),
12541 );
12542 let runtime = supervisor.runtime_config();
12543 let spec = ModuleSpec {
12544 module_id: "genuine-crash".to_string(),
12545 program: PathBuf::from("/unused/genuine-crash"),
12546 args: Vec::new(),
12547 env: Vec::new(),
12548 reserved: false,
12549 reserved_prefixes: Vec::new(),
12550 protocol: ModuleProtocol::Subc,
12551 overlap: Default::default(),
12552 };
12553 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12554
12555 assert!(matches!(
12556 on_child_exit(
12557 &spec,
12558 runtime.restart_policy,
12559 &supervisor.registry,
12560 &snapshot,
12561 &runtime.terminal_ring,
12562 &runtime.spawn_events,
12563 &runtime.child_roster,
12564 ExitReport {
12565 kind: ExitKind::Crash,
12566 code: Some(1),
12567 signal: None,
12568 at_ms: 1,
12569 },
12570 )
12571 .await,
12572 NextAction::Restart { schedule: _ }
12573 ));
12574 let state = lock_snapshot(&snapshot).unwrap();
12575 assert_eq!(state.lifetime_restarts, 1);
12576 assert_eq!(state.crash_restarts.len(), 1);
12577 }
12578
12579 fn crash_exit_report(at_ms: u64) -> ExitReport {
12580 ExitReport {
12581 kind: ExitKind::Crash,
12582 code: Some(1),
12583 signal: None,
12584 at_ms,
12585 }
12586 }
12587
12588 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12589 ModuleSpec {
12590 module_id: module_id.to_string(),
12591 program: PathBuf::from("/unused").join(module_id),
12592 args: Vec::new(),
12593 env: Vec::new(),
12594 reserved: false,
12595 reserved_prefixes: Vec::new(),
12596 protocol: ModuleProtocol::Subc,
12597 overlap: Default::default(),
12598 }
12599 }
12600
12601 #[tokio::test]
12607 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12608 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12609 let supervisor = Supervisor::new_for_test(
12610 Arc::new(Registry::default()),
12611 RestartPolicy::new(2, Duration::ZERO),
12612 );
12613 let runtime = supervisor.runtime_config();
12614 let spec = windowed_crash_spec("crash-loop-in-window");
12615 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12616
12617 for attempt in 1..=2 {
12618 assert!(
12619 matches!(
12620 on_child_exit(
12621 &spec,
12622 runtime.restart_policy,
12623 &supervisor.registry,
12624 &snapshot,
12625 &runtime.terminal_ring,
12626 &runtime.spawn_events,
12627 &runtime.child_roster,
12628 crash_exit_report(attempt),
12629 )
12630 .await,
12631 NextAction::Restart { schedule: _ }
12632 ),
12633 "crash {attempt} is inside the budget and must respawn"
12634 );
12635 }
12636
12637 assert!(matches!(
12638 on_child_exit(
12639 &spec,
12640 runtime.restart_policy,
12641 &supervisor.registry,
12642 &snapshot,
12643 &runtime.terminal_ring,
12644 &runtime.spawn_events,
12645 &runtime.child_roster,
12646 crash_exit_report(3),
12647 )
12648 .await,
12649 NextAction::Stop { .. }
12650 ));
12651
12652 {
12653 let state = lock_snapshot(&snapshot).unwrap();
12654 assert_eq!(state.state, ModuleState::Failed);
12655 assert_eq!(state.crash_restarts.len(), 2);
12656 assert_eq!(state.lifetime_restarts, 2);
12657 }
12658
12659 let history = runtime
12660 .terminal_ring
12661 .lock()
12662 .expect("terminal ring is not poisoned")
12663 .snapshot();
12664 let last = history
12665 .entries
12666 .last()
12667 .expect("the refused crash is retained");
12668 assert_eq!(last.disposition, TerminalDisposition::Failed);
12669 assert_eq!(
12670 last.disposition_detail.as_deref(),
12671 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12672 );
12673
12674 let captured = crate::router::test_log::captured_logs(&logs);
12675 assert!(
12676 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12677 "the stop must be logged with its window: {captured}"
12678 );
12679 }
12680
12681 #[tokio::test]
12689 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12690 let supervisor = Supervisor::new_for_test(
12691 Arc::new(Registry::default()),
12692 RestartPolicy::new(2, Duration::ZERO),
12693 );
12694 let runtime = supervisor.runtime_config();
12695 let spec = windowed_crash_spec("crash-across-windows");
12696 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12697
12698 for attempt in 1..=2 {
12699 assert!(matches!(
12700 on_child_exit(
12701 &spec,
12702 runtime.restart_policy,
12703 &supervisor.registry,
12704 &snapshot,
12705 &runtime.terminal_ring,
12706 &runtime.spawn_events,
12707 &runtime.child_roster,
12708 crash_exit_report(attempt),
12709 )
12710 .await,
12711 NextAction::Restart { schedule: _ }
12712 ));
12713 }
12714
12715 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12718 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12719 })
12720 .unwrap();
12721
12722 assert!(
12723 matches!(
12724 on_child_exit(
12725 &spec,
12726 runtime.restart_policy,
12727 &supervisor.registry,
12728 &snapshot,
12729 &runtime.terminal_ring,
12730 &runtime.spawn_events,
12731 &runtime.child_roster,
12732 crash_exit_report(3),
12733 )
12734 .await,
12735 NextAction::Restart { schedule: _ }
12736 ),
12737 "a crash older than the window must not hold a budget slot"
12738 );
12739
12740 let state = lock_snapshot(&snapshot).unwrap();
12741 assert_eq!(state.state, ModuleState::Restarting);
12742 assert_eq!(
12743 state.crash_restarts.len(),
12744 2,
12745 "the aged instant is dropped and the new one takes its place"
12746 );
12747 assert_eq!(
12748 state.lifetime_restarts, 3,
12749 "the ledger counts every restart, including the ones the window forgot"
12750 );
12751 }
12752
12753 #[tokio::test]
12758 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
12759 let supervisor = Supervisor::new_for_test(
12760 Arc::new(Registry::default()),
12761 RestartPolicy::new(2, Duration::ZERO),
12762 );
12763 let runtime = supervisor.runtime_config();
12764 let spec = windowed_crash_spec("operator-cleared-budget");
12765 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12766
12767 for attempt in 1..=2 {
12768 assert!(matches!(
12769 on_child_exit(
12770 &spec,
12771 runtime.restart_policy,
12772 &supervisor.registry,
12773 &snapshot,
12774 &runtime.terminal_ring,
12775 &runtime.spawn_events,
12776 &runtime.child_roster,
12777 crash_exit_report(attempt),
12778 )
12779 .await,
12780 NextAction::Restart { schedule: _ }
12781 ));
12782 }
12783
12784 reset_restart_count(&snapshot, &spec.module_id).unwrap();
12785 {
12786 let state = lock_snapshot(&snapshot).unwrap();
12787 assert!(
12788 state.crash_restarts.is_empty(),
12789 "an operator restart returns the full budget"
12790 );
12791 assert_eq!(
12792 state.lifetime_restarts, 2,
12793 "clearing the budget must not unmake the crashes"
12794 );
12795 }
12796
12797 assert!(
12798 matches!(
12799 on_child_exit(
12800 &spec,
12801 runtime.restart_policy,
12802 &supervisor.registry,
12803 &snapshot,
12804 &runtime.terminal_ring,
12805 &runtime.spawn_events,
12806 &runtime.child_roster,
12807 crash_exit_report(3),
12808 )
12809 .await,
12810 NextAction::Restart { schedule: _ }
12811 ),
12812 "the cleared budget must be spendable again"
12813 );
12814 let state = lock_snapshot(&snapshot).unwrap();
12815 assert_eq!(state.crash_restarts.len(), 1);
12816 assert_eq!(state.lifetime_restarts, 3);
12817 }
12818
12819 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12820 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
12821 let severed = ProcessIdentity {
12822 pid: 41,
12823 start_time: 101,
12824 };
12825 let successor = ProcessIdentity {
12826 pid: 41,
12827 start_time: 202,
12828 };
12829 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12830 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
12831 state.pid = Some(successor.pid);
12832 state.process_start_time = Some(successor.start_time);
12833 })
12834 .unwrap();
12835 assert!(!module.record_deliberate_severance(severed).unwrap());
12836
12837 let exit_report = apply_deliberate_severance_marker(
12838 &module.inner.snapshot,
12839 Some(successor),
12840 ExitReport {
12841 kind: ExitKind::Crash,
12842 code: Some(1),
12843 signal: None,
12844 at_ms: 1,
12845 },
12846 );
12847
12848 assert_eq!(exit_report.kind, ExitKind::Crash);
12849 }
12850
12851 #[tokio::test]
12852 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
12853 let registry = Registry::default();
12854 let supervisor = Supervisor::new_for_test(
12855 Arc::new(Registry::default()),
12856 RestartPolicy::new(3, Duration::ZERO),
12857 );
12858 let runtime = supervisor.runtime_config();
12859 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12860 let spec = ModuleSpec {
12861 module_id: "drain-deliberate-severance".to_string(),
12862 program: fake_aft_stub_path(),
12863 args: Vec::new(),
12864 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12865 reserved: false,
12866 reserved_prefixes: Vec::new(),
12867 protocol: ModuleProtocol::Subc,
12868 overlap: Default::default(),
12869 };
12870 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12871 let process = ProcessIdentity {
12872 pid: 41,
12873 start_time: 101,
12874 };
12875 child.process_identity = Some(process);
12876 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12877 state.pid = Some(process.pid);
12878 state.process_start_time = Some(process.start_time);
12879 })
12880 .unwrap();
12881 record_deliberate_severance(&snapshot, process).unwrap();
12882
12883 drain_child_to_state(
12884 &spec.module_id,
12885 spec.protocol,
12886 StopNotice::SentOverConnection,
12889 ®istry,
12890 None,
12891 &snapshot,
12892 &runtime.terminal_ring,
12893 &runtime.spawn_events,
12894 child,
12895 Duration::from_secs(1),
12896 ModuleState::Stopped,
12897 Some(false),
12898 )
12899 .await
12900 .unwrap();
12901
12902 let state = lock_snapshot(&snapshot).unwrap();
12903 assert_eq!(
12904 state.last_exit.as_ref().map(|exit| exit.kind),
12905 Some(ExitKind::DeliberateSeverance)
12906 );
12907 assert_eq!(state.lifetime_restarts, 1);
12908 assert_eq!(state.crash_restarts.len(), 0);
12909 drop(state);
12910 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12911 assert_eq!(
12912 history.entries[0].exit_kind,
12913 subc_control::TerminalExitKind::DeliberateSeverance
12914 );
12915 }
12916
12917 #[tokio::test]
12918 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
12919 let registry = Registry::default();
12920 let supervisor = Supervisor::new_for_test(
12921 Arc::new(Registry::default()),
12922 RestartPolicy::new(3, Duration::ZERO),
12923 );
12924 let runtime = supervisor.runtime_config();
12925 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12926 let spec = ModuleSpec {
12927 module_id: "ordinary-drain".to_string(),
12928 program: fake_aft_stub_path(),
12929 args: Vec::new(),
12930 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12931 reserved: false,
12932 reserved_prefixes: Vec::new(),
12933 protocol: ModuleProtocol::Subc,
12934 overlap: Default::default(),
12935 };
12936 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12937
12938 drain_child_to_state(
12939 &spec.module_id,
12940 spec.protocol,
12941 StopNotice::SentOverConnection,
12944 ®istry,
12945 None,
12946 &snapshot,
12947 &runtime.terminal_ring,
12948 &runtime.spawn_events,
12949 child,
12950 Duration::from_secs(1),
12951 ModuleState::Stopped,
12952 Some(false),
12953 )
12954 .await
12955 .unwrap();
12956
12957 let state = lock_snapshot(&snapshot).unwrap();
12958 assert_eq!(
12959 state.last_exit.as_ref().map(|exit| exit.kind),
12960 Some(ExitKind::Crash)
12961 );
12962 assert_eq!(state.lifetime_restarts, 0);
12963 assert_eq!(state.crash_restarts.len(), 0);
12964 }
12965
12966 #[test]
12967 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
12968 assert!(!include_str!("server.rs")
12974 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
12975 }
12976
12977 #[test]
12984 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
12985 assert!(drained_after_quiescence_wait(&Ok(true)));
12986 assert!(!drained_after_quiescence_wait(&Ok(false)));
12987 assert!(!drained_after_quiescence_wait(&Err(
12988 SuperviseError::StatePoisoned { module_id: None }
12989 )));
12990 }
12991
12992 #[test]
13001 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13002 let ring = Arc::new(Mutex::new(TerminalRing::new(
13003 TerminalRingConfig::default(),
13004 0,
13005 )));
13006 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13007
13008 let snapshot = ring.lock().unwrap().snapshot();
13009 assert_eq!(snapshot.entries.len(), 1);
13010 let entry = &snapshot.entries[0];
13011 assert_eq!(entry.exit_code, None);
13012 assert_eq!(entry.exit_signal, None);
13013 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13014 }
13015
13016 #[test]
13017 fn wait_error_exit_path_preserves_spawn_event_density() {
13018 let feed = super::SpawnEventFeed::default();
13019 feed.configure_incarnation("wait-error-density".to_string());
13020 feed.emit_spawned("wait-error", 41, 1);
13021 let ring = Arc::new(Mutex::new(TerminalRing::new(
13022 TerminalRingConfig::default(),
13023 0,
13024 )));
13025
13026 record_wait_error_terminal("wait-error", &ring, &feed);
13027 feed.emit_spawned("after-wait-error", 42, 2);
13028
13029 let state = feed.0.lock().unwrap();
13030 let sequences = state
13031 .events
13032 .iter()
13033 .map(|event| event.cursor.seq)
13034 .collect::<Vec<_>>();
13035 assert_eq!(sequences, vec![1, 2, 3]);
13036 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13037 assert_eq!(state.events[1].exit_code, None);
13038 assert_eq!(state.events[1].exit_signal, None);
13039 }
13040
13041 #[test]
13045 fn wait_error_exit_report_is_classified_as_a_crash() {
13046 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13047 }
13048}
13049
13050#[cfg(all(test, unix))]
13051mod health_event_tests {
13052 use super::*;
13053 use cortexkit_test_support::ScratchDir;
13054 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13055
13056 struct Actor {
13057 module: SupervisedModule,
13058 registry: Arc<Registry>,
13059 forwarding: Arc<ForwardingTable>,
13060 spec: ModuleSpec,
13061 health: HealthConfig,
13062 rx: mpsc::Receiver<crate::router::OutboundFrame>,
13063 _home: ScratchDir,
13064 }
13065
13066 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13067 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13071 let tick = Instant::now();
13072 loop {
13073 if let Some(value) = observe() {
13074 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13075 return value;
13076 }
13077 assert!(
13078 std::time::Instant::now() < deadline,
13079 "timed out waiting for {reason}"
13080 );
13081 tokio::task::yield_now().await;
13082 }
13083 }
13084
13085 impl Actor {
13086 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13087 let home = ScratchDir::new("health-events");
13088 let registry = Arc::new(Registry::default());
13089 let forwarding = Arc::new(ForwardingTable::default());
13090 let health = HealthConfig {
13091 cadence,
13092 deadline: Duration::from_secs(3600),
13093 ..HealthConfig::default()
13094 };
13095 let supervisor =
13096 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13097 .with_forwarding(forwarding.clone())
13098 .with_health_config(health.clone());
13099 let spec = ModuleSpec {
13100 module_id: "event-health-child".into(),
13101 program: PathBuf::from("/bin/sh"),
13102 args: vec!["-c".into(), "exec sleep 600".into()],
13105 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13106 .into_iter()
13107 .map(|key| (key.into(), home.path().display().to_string()))
13108 .collect(),
13109 reserved: false,
13110 reserved_prefixes: vec![],
13111 protocol,
13112 overlap: Default::default(),
13113 };
13114 let module = supervisor.spawn(spec.clone()).unwrap();
13115 let (_, rx) = mpsc::channel(8);
13116 let actor = Self {
13117 module,
13118 registry,
13119 forwarding,
13120 spec,
13121 health,
13122 rx,
13123 _home: home,
13124 };
13125 actor
13126 .wait_for_select(|checkpoint| checkpoint.generation > 0)
13127 .await;
13128 actor
13129 }
13130
13131 fn turns(&self) -> u64 {
13132 lock_snapshot(&self.module.inner.snapshot)
13133 .unwrap()
13134 .actor_turns
13135 }
13136
13137 async fn wait_for_select(
13138 &self,
13139 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13140 ) -> ActorSelectCheckpoint {
13141 wait_for("actor select checkpoint", || {
13142 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13143 state.actor_select.filter(|checkpoint| {
13144 checkpoint.generation == state.spawn_generation
13145 && state.state == ModuleState::Running
13146 && state.process_alive
13147 && state.reported_pid().is_some()
13148 && expected(checkpoint)
13149 })
13150 })
13151 .await
13152 }
13153
13154 fn hello(&mut self, connection: u64, health: bool) {
13155 let manifest =
13156 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13157 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13158 .build();
13159 let connection = ConnectionId::new(connection);
13160 let (tx, rx) = mpsc::channel(8);
13161 self.rx = rx;
13162 self.forwarding
13163 .register_module_connection(
13164 connection,
13165 self.spec.module_id.clone(),
13166 subc_protocol::PROTOCOL_VERSION,
13167 Concurrency::ModuleManaged,
13168 FrameSink::new(tx),
13169 )
13170 .unwrap();
13171 self.registry
13172 .register_with_control_ops(
13173 manifest,
13174 subc_protocol::PROTOCOL_VERSION,
13175 connection,
13176 if health {
13177 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13178 } else {
13179 vec![]
13180 },
13181 )
13182 .unwrap();
13183 }
13184
13185 async fn probe(&mut self) -> crate::router::OutboundFrame {
13186 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13187 Ok(frame) => Some(frame),
13188 Err(mpsc::error::TryRecvError::Empty) => None,
13189 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13190 })
13191 .await;
13192 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13193 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13194 frame
13195 }
13196
13197 fn no_probe(&mut self) {
13198 assert!(matches!(
13199 self.rx.try_recv(),
13200 Err(mpsc::error::TryRecvError::Empty)
13201 ));
13202 }
13203
13204 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13205 self.forwarding
13206 .complete_module_control_rpc(
13207 ConnectionId::new(connection),
13208 frame.header.corr,
13209 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13210 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13211 status: HealthStatus::Ok,
13212 detail: None,
13213 metrics: None,
13214 }),
13215 )
13216 .unwrap();
13217 }
13218 }
13219
13220 #[tokio::test(start_paused = true)]
13221 async fn no_health_children_park_for_sixty_seconds() {
13222 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13223 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13224 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13225 unadvertised.hello(1, false);
13226 for _ in 0..6000 {
13231 tokio::time::advance(Duration::from_millis(10)).await;
13232 tokio::task::yield_now().await;
13233 }
13234 for actor in [&none, &unregistered, &unadvertised] {
13235 assert!(
13236 actor.turns() <= 5,
13237 "no-health actor woke {} times",
13238 actor.turns()
13239 );
13240 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13241 }
13242 }
13243
13244 #[tokio::test(start_paused = true)]
13245 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13246 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13250 let tick = Instant::now();
13251 actor.hello(1, true);
13252 actor.probe().await;
13253 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13254 }
13255
13256 #[tokio::test(start_paused = true)]
13257 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13258 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13259 let hello_tick = Instant::now();
13260 actor.hello(1, true);
13261 let armed = actor
13262 .wait_for_select(|checkpoint| {
13263 checkpoint.registered_connection == Some(ConnectionId::new(1))
13264 && checkpoint.next_probe_at.is_some()
13265 })
13266 .await;
13267 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13268 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13269 let first_deadline = hello_tick + delay;
13270 assert_eq!(armed.next_probe_at, Some(first_deadline));
13271 tokio::time::advance(Duration::from_secs(29)).await;
13272 let turns = actor.turns();
13273 actor
13274 .registry
13275 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13276 .unwrap();
13277 let updated = actor
13278 .wait_for_select(|checkpoint| checkpoint.turn > turns)
13279 .await;
13280 assert_eq!(
13281 updated.next_probe_at,
13282 Some(first_deadline),
13283 "catalog update must not reset cadence"
13284 );
13285 actor.no_probe();
13286 tokio::time::advance(first_deadline - Instant::now()).await;
13287 let frame = actor.probe().await;
13288 actor.answer(1, frame);
13289 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13290 let next_deadline = Instant::now() + next;
13291 let rearmed = actor
13292 .wait_for_select(|checkpoint| {
13293 checkpoint
13294 .next_probe_at
13295 .is_some_and(|deadline| deadline > first_deadline)
13296 })
13297 .await;
13298 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13299 tokio::time::advance(next - Duration::from_nanos(1)).await;
13300 actor.no_probe();
13301 tokio::time::advance(Duration::from_nanos(1)).await;
13302 actor.probe().await;
13303 }
13304
13305 #[tokio::test(start_paused = true)]
13306 async fn disconnect_disarms_and_reconnect_rearms_health() {
13307 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13308 actor.hello(1, true);
13309 actor
13310 .wait_for_select(|checkpoint| {
13311 checkpoint.registered_connection == Some(ConnectionId::new(1))
13312 && checkpoint.next_probe_at.is_some()
13313 })
13314 .await;
13315 let turns = actor.turns();
13316 let tick = Instant::now();
13317 actor
13318 .registry
13319 .deregister_connection(ConnectionId::new(1))
13320 .unwrap();
13321 let disarmed = actor
13322 .wait_for_select(|checkpoint| {
13323 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13324 })
13325 .await;
13326 assert_eq!(disarmed.next_probe_at, None);
13327 assert_eq!(disarmed.wake_after, None);
13328 assert_eq!(
13329 Instant::now(),
13330 tick,
13331 "disconnect must disarm in the notification tick"
13332 );
13333 tokio::time::advance(Duration::from_secs(60)).await;
13334 actor.no_probe();
13335 let reconnect_tick = Instant::now();
13336 actor.hello(2, true);
13337 let rearmed = actor
13338 .wait_for_select(|checkpoint| {
13339 checkpoint.registered_connection == Some(ConnectionId::new(2))
13340 && checkpoint.next_probe_at.is_some()
13341 })
13342 .await;
13343 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13344 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13345 tokio::time::advance(delay).await;
13346 actor.probe().await;
13347 }
13348
13349 #[tokio::test(start_paused = true)]
13350 async fn reregistration_without_health_disarms_the_old_schedule() {
13351 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13352 actor.hello(1, true);
13353 actor
13354 .wait_for_select(|checkpoint| {
13355 checkpoint.registered_connection == Some(ConnectionId::new(1))
13356 && checkpoint.next_probe_at.is_some()
13357 })
13358 .await;
13359 actor
13360 .registry
13361 .deregister_connection(ConnectionId::new(1))
13362 .unwrap();
13363 actor.hello(2, false);
13364 let disarmed = actor
13365 .wait_for_select(|checkpoint| {
13366 checkpoint.registered_connection == Some(ConnectionId::new(2))
13367 })
13368 .await;
13369 assert_eq!(disarmed.next_probe_at, None);
13370 assert_eq!(disarmed.wake_after, None);
13371 tokio::time::advance(Duration::from_secs(60)).await;
13372 actor.no_probe();
13373 }
13374
13375 #[tokio::test(start_paused = true)]
13376 async fn restart_waits_for_its_own_hello_before_rearming_health() {
13377 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13378 actor.hello(1, true);
13379 let armed = actor
13380 .wait_for_select(|checkpoint| {
13381 checkpoint.registered_connection == Some(ConnectionId::new(1))
13382 && checkpoint.next_probe_at.is_some()
13383 })
13384 .await;
13385 actor
13389 .registry
13390 .deregister_connection(ConnectionId::new(1))
13391 .unwrap();
13392 actor.module.restart(Some(0)).await.unwrap();
13393 let parked = actor
13394 .wait_for_select(|checkpoint| {
13395 checkpoint.generation > armed.generation
13396 && checkpoint.registered_connection.is_none()
13397 && checkpoint.next_probe_at.is_none()
13398 })
13399 .await;
13400 assert_eq!(parked.wake_after, None);
13401 actor.rx.close();
13402 while actor.rx.try_recv().is_ok() {}
13405 let turns = actor.turns();
13406 tokio::time::advance(Duration::from_secs(60)).await;
13407 let after = actor.turns();
13408 eprintln!(
13409 "restart park turns: before={turns} after={after} delta={}",
13410 after - turns
13411 );
13412 assert_eq!(after, turns, "replacement must park before HELLO");
13413 let hello_tick = Instant::now();
13414 actor.hello(2, true);
13415 let rearmed = actor
13416 .wait_for_select(|checkpoint| {
13417 checkpoint.registered_connection == Some(ConnectionId::new(2))
13418 && checkpoint.next_probe_at.is_some()
13419 })
13420 .await;
13421 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13422 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13423 tokio::time::advance(delay).await;
13424 actor.probe().await;
13425 }
13426
13427 #[tokio::test(start_paused = true)]
13428 async fn rescan_adds_and_removes_http_health_without_polling() {
13429 use tokio::{
13430 io::{AsyncReadExt, AsyncWriteExt},
13431 net::TcpListener,
13432 };
13433 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13434 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13435 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13436 actor.health.cadence = Duration::from_millis(1);
13438 let (started_tx, mut started_rx) = mpsc::channel(8);
13439 let peer = tokio::spawn(async move {
13440 loop {
13441 let (mut socket, _) = listener.accept().await.unwrap();
13442 let mut request = [0; 1024];
13443 assert!(socket.read(&mut request).await.unwrap() > 0);
13444 started_tx.send(()).await.unwrap();
13445 socket
13446 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13447 .await
13448 .unwrap();
13449 }
13450 });
13451 let tick = Instant::now();
13452 let turns = actor.turns();
13453 actor
13454 .module
13455 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13456 .await
13457 .unwrap();
13458 let armed = actor
13459 .wait_for_select(|checkpoint| {
13460 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13461 })
13462 .await;
13463 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13464 assert_eq!(Instant::now(), tick);
13465 tokio::time::advance(Duration::from_millis(1)).await;
13466 let mut started = false;
13467 wait_for("successful HTTP probe", || {
13468 started |= started_rx.try_recv().is_ok();
13469 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13470 .then_some(())
13471 })
13472 .await;
13473 assert!(started, "added HTTP check must start at its first deadline");
13474 assert_eq!(
13475 actor.module.status().unwrap().health.status,
13476 SupervisorHealthStatus::Ok
13477 );
13478 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13479 actor.health.http = None;
13480 let turns = actor.turns();
13481 actor
13482 .module
13483 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13484 .await
13485 .unwrap();
13486 let parked = actor
13487 .wait_for_select(|checkpoint| {
13488 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13489 })
13490 .await;
13491 assert_eq!(parked.wake_after, None);
13492 let turns = actor.turns();
13493 tokio::time::advance(Duration::from_secs(60)).await;
13494 assert_eq!(
13495 actor.turns(),
13496 turns,
13497 "removed HTTP check must park the actor"
13498 );
13499 assert!(
13500 started_rx.try_recv().is_err(),
13501 "removed HTTP check must stay disarmed"
13502 );
13503 assert_eq!(
13504 actor.module.status().unwrap().health,
13505 ModuleHealthStatus::default()
13506 );
13507 peer.abort();
13508 }
13509}
13510
13511#[cfg(test)]
13512mod drain_busy_reprobe_tests {
13513 use super::*;
13517 use std::sync::Mutex as StdMutex;
13518 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13519
13520 const MODULE_ID: &str = "drain-reprobe-module";
13521 const CONNECTION: u64 = 7;
13522 const GAUGE: &str = "runs_in_flight";
13523 const CADENCE: Duration = Duration::from_secs(30);
13526 const CEILING: Duration = Duration::from_secs(30);
13527
13528 #[derive(Clone, Copy)]
13530 enum Answer {
13531 BusyUntil { until: Instant, latency: Duration },
13534 Silent,
13536 }
13537
13538 struct Drain {
13539 forwarding: Arc<ForwardingTable>,
13540 runtime: SupervisorRuntimeConfig,
13541 endpoint: crate::ModuleEndpointId,
13542 probes: Arc<StdMutex<Vec<Instant>>>,
13544 responder: tokio::task::JoinHandle<()>,
13545 }
13546
13547 impl Drain {
13548 fn start(answer: Answer) -> Self {
13549 let registry = Arc::new(Registry::default());
13550 let forwarding = Arc::new(ForwardingTable::default());
13551 let health = HealthConfig {
13552 cadence: CADENCE,
13553 ..HealthConfig::default()
13554 };
13555 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
13556 .with_forwarding(forwarding.clone())
13557 .with_health_config(health)
13558 .runtime_config();
13559 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
13560 let endpoint = forwarding
13561 .register_module_connection(
13562 ConnectionId::new(CONNECTION),
13563 MODULE_ID.to_string(),
13564 subc_protocol::PROTOCOL_VERSION,
13565 Concurrency::ModuleManaged,
13566 FrameSink::new(tx),
13567 )
13568 .unwrap();
13569 let probes = Arc::new(StdMutex::new(Vec::new()));
13570 let responder = tokio::spawn({
13571 let forwarding = forwarding.clone();
13572 let probes = probes.clone();
13573 async move {
13574 while let Some(frame) = rx.recv().await {
13575 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13576 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13577 probes.lock().unwrap().push(Instant::now());
13578 let Answer::BusyUntil { until, latency } = answer else {
13579 continue;
13580 };
13581 sleep(latency).await;
13582 let busy = u64::from(Instant::now() < until);
13583 let _ = forwarding.complete_module_control_rpc(
13586 ConnectionId::new(CONNECTION),
13587 frame.header.corr,
13588 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13589 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13590 status: HealthStatus::Ok,
13591 detail: None,
13592 metrics: Some(serde_json::json!({ GAUGE: busy })),
13593 }),
13594 );
13595 }
13596 }
13597 });
13598 Self {
13599 forwarding,
13600 runtime,
13601 endpoint,
13602 probes,
13603 responder,
13604 }
13605 }
13606
13607 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
13610 let started = Instant::now();
13611 let drained = wait_for_forwarding_quiescence(
13612 &self.forwarding,
13613 MODULE_ID,
13614 &self.runtime,
13615 self.endpoint,
13616 started + ceiling,
13617 &[GAUGE.to_string()],
13618 DrainScope::Active,
13619 )
13620 .await
13621 .unwrap();
13622 (drained, started.elapsed())
13623 }
13624
13625 fn probes(&self) -> Vec<Instant> {
13626 self.probes.lock().unwrap().clone()
13627 }
13628 }
13629
13630 impl Drop for Drain {
13631 fn drop(&mut self) {
13632 self.responder.abort();
13633 }
13634 }
13635
13636 #[tokio::test(start_paused = true)]
13637 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
13638 let drain = Drain::start(Answer::BusyUntil {
13639 until: Instant::now() + Duration::from_secs(1),
13640 latency: Duration::ZERO,
13641 });
13642 let (drained, elapsed) = drain.run(CEILING).await;
13643 assert!(
13644 drained,
13645 "gauges that read zero during the drain must let it finish drained, \
13646 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
13647 );
13648 assert!(
13649 elapsed < Duration::from_secs(3),
13650 "drain took {elapsed:?} after the gauges went quiet at 1s"
13651 );
13652 let probes = drain.probes();
13653 assert!(
13654 probes.len() >= 2,
13655 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
13656 );
13657 }
13658
13659 #[tokio::test(start_paused = true)]
13660 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
13661 let started = Instant::now();
13662 let drain = Drain::start(Answer::BusyUntil {
13663 until: started + Duration::from_secs(3600),
13664 latency: Duration::ZERO,
13665 });
13666 let (drained, elapsed) = drain.run(CEILING).await;
13667 assert!(!drained, "a module busy throughout must not drain");
13668 assert!(
13669 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
13670 "a busy module is held until the ceiling and no longer: {elapsed:?}"
13671 );
13672 let probes = drain.probes();
13675 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
13676 assert!(
13677 probes.len() as u128 <= max_probes,
13678 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
13679 probes.len()
13680 );
13681 for pair in probes.windows(2) {
13682 assert!(
13683 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
13684 "consecutive probes {:?} apart",
13685 pair[1] - pair[0]
13686 );
13687 }
13688 }
13689
13690 #[tokio::test(start_paused = true)]
13691 async fn a_slow_probe_is_never_overlapped_by_the_next() {
13692 let latency = Duration::from_secs(1);
13695 let drain = Drain::start(Answer::BusyUntil {
13696 until: Instant::now() + Duration::from_secs(3600),
13697 latency,
13698 });
13699 let (drained, _) = drain.run(Duration::from_secs(5)).await;
13700 assert!(!drained);
13701 let probes = drain.probes();
13702 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
13703 for pair in probes.windows(2) {
13704 assert!(
13705 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
13706 "a probe started {:?} after the previous one, before it could finish",
13707 pair[1] - pair[0]
13708 );
13709 }
13710 }
13711
13712 #[tokio::test(start_paused = true)]
13713 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
13714 let drain = Drain::start(Answer::Silent);
13715 let ceiling = Duration::from_secs(2);
13716 let (drained, elapsed) = drain.run(ceiling).await;
13717 assert!(!drained, "an unanswered probe counts as busy");
13718 assert!(
13719 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
13720 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
13721 );
13722 assert_eq!(
13723 drain.probes().len(),
13724 1,
13725 "the unanswered probe holds the only slot until the ceiling"
13726 );
13727 }
13728}
13729
13730#[cfg(test)]
13731mod health_evidence_tests {
13732 use super::{HealthProbeError, HealthProbeEvidence};
13733 use std::collections::HashSet;
13734
13735 #[test]
13743 fn only_a_dead_lane_is_proof_of_death() {
13744 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
13745 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
13749 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
13750 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
13751 }
13752
13753 #[test]
13759 fn every_evidence_class_has_a_distinct_label() {
13760 let labels = [
13761 HealthProbeError::lane_dead("").label(),
13762 HealthProbeError::no_answer("").label(),
13763 HealthProbeError::bad_answer("").label(),
13764 HealthProbeError::misconfigured("").label(),
13765 ];
13766 let unique: HashSet<_> = labels.iter().collect();
13767 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
13768 }
13769
13770 #[test]
13776 fn classification_preserves_the_original_message() {
13777 let err = HealthProbeError::no_answer("module did not answer within 5s");
13778 assert_eq!(err.to_string(), "module did not answer within 5s");
13779 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13780 }
13781}
13782
13783#[cfg(test)]
13784mod health_tombstone_tests {
13785 use std::{path::PathBuf, sync::Arc, time::Duration};
13786
13787 use subc_protocol::{
13788 manifest::Concurrency,
13789 session::{HealthStatus, ModuleControlResponse},
13790 };
13791 use tokio::sync::mpsc;
13792
13793 use super::{
13794 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
13795 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
13796 };
13797 use crate::{
13798 control::ControlHandler,
13799 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
13800 registry::{ConnectionId, Registry},
13801 router::FrameSink,
13802 };
13803
13804 struct ProbeHarness {
13805 spec: ModuleSpec,
13806 runtime: SupervisorRuntimeConfig,
13807 forwarding: Arc<ForwardingTable>,
13808 module_connection: ConnectionId,
13809 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13810 handler: ControlHandler,
13811 module: super::SupervisedModule,
13812 }
13813
13814 fn probe_harness() -> ProbeHarness {
13815 let registry = Arc::new(Registry::default());
13816 let forwarding = Arc::new(ForwardingTable::default());
13817 let supervisor_handle = super::SupervisorHandle::new();
13818 let health = HealthConfig {
13819 http: None,
13820 cadence: Duration::from_secs(30),
13821 deadline: Duration::from_secs(5),
13822 failure_threshold: 3,
13823 on_degraded: HealthAction::Report,
13824 on_failing: HealthAction::Report,
13825 critical: false,
13826 };
13827 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13828 .with_forwarding(Arc::clone(&forwarding))
13829 .with_handle(supervisor_handle.clone())
13830 .with_health_config(health);
13831 let spec = ModuleSpec {
13832 module_id: "late-health-module".to_string(),
13833 program: PathBuf::from("disabled-module"),
13834 args: Vec::new(),
13835 env: Vec::new(),
13836 reserved: false,
13837 reserved_prefixes: Vec::new(),
13838 protocol: ModuleProtocol::Subc,
13839 overlap: Default::default(),
13840 };
13841 let module = supervisor
13842 .supervise_configured(spec.clone(), false)
13843 .unwrap();
13844 let runtime = supervisor.runtime_config();
13845 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
13846 .with_supervisor(supervisor_handle);
13847 let module_connection = ConnectionId::new(700);
13848 let (module_tx, module_rx) = mpsc::channel(8);
13849 forwarding
13850 .register_module_connection(
13851 module_connection,
13852 spec.module_id.clone(),
13853 subc_protocol::PROTOCOL_VERSION,
13854 Concurrency::ModuleManaged,
13855 FrameSink::new(module_tx),
13856 )
13857 .unwrap();
13858
13859 ProbeHarness {
13860 spec,
13861 runtime,
13862 forwarding,
13863 module_connection,
13864 module_rx,
13865 handler,
13866 module,
13867 }
13868 }
13869
13870 async fn finish_after(
13871 harness: &mut ProbeHarness,
13872 stall: Duration,
13873 ) -> ModuleControlRpcCompletion {
13874 assert!(stall > harness.runtime.health.deadline);
13875 let deadline = harness.runtime.health.deadline;
13876 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13877 let answer = async {
13878 let frame = harness.module_rx.recv().await.expect("health.check frame");
13879 tokio::time::advance(deadline).await;
13880 tokio::task::yield_now().await;
13881 tokio::time::advance(stall - deadline).await;
13882 harness
13883 .forwarding
13884 .complete_module_control_rpc(
13885 harness.module_connection,
13886 frame.header.corr,
13887 Some("health.check"),
13888 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13889 status: HealthStatus::Ok,
13890 detail: None,
13891 metrics: None,
13892 }),
13893 )
13894 .unwrap()
13895 };
13896 let (probe_result, completion) = tokio::join!(probe, answer);
13897 let err = probe_result.expect_err("probe must miss its deadline");
13898 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13899 completion
13900 }
13901
13902 async fn time_out_without_answer(harness: &mut ProbeHarness) {
13903 let deadline = harness.runtime.health.deadline;
13904 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13905 let exhaust_deadline = async {
13906 let _frame = harness.module_rx.recv().await.expect("health.check frame");
13907 tokio::time::advance(deadline).await;
13908 tokio::task::yield_now().await;
13909 };
13910 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
13911 let err = probe_result.expect_err("probe must miss its deadline");
13912 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13913 }
13914
13915 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
13916 let registry = Arc::clone(&harness.module.inner.registry);
13917 let snapshot = Arc::clone(&harness.module.inner.snapshot);
13918 let process_liveness = super::SupervisorProcessLiveness::default();
13919 let mut child = None;
13920 let cycle = super::run_health_probe_cycle(
13921 &harness.spec,
13922 &harness.runtime,
13923 ®istry,
13924 &process_liveness,
13925 &snapshot,
13926 &mut child,
13927 );
13928 let peer = async {
13929 let frame = harness.module_rx.recv().await.expect("health.check frame");
13930 if answer {
13931 harness
13932 .forwarding
13933 .complete_module_control_rpc(
13934 harness.module_connection,
13935 frame.header.corr,
13936 Some("health.check"),
13937 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13938 status: HealthStatus::Ok,
13939 detail: None,
13940 metrics: Some(serde_json::json!({"ready": true})),
13941 }),
13942 )
13943 .unwrap();
13944 } else {
13945 tokio::time::advance(harness.runtime.health.deadline).await;
13946 tokio::task::yield_now().await;
13947 }
13948 };
13949 tokio::join!(cycle, peer);
13950 }
13951
13952 #[tokio::test(start_paused = true)]
13953 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
13954 let mut harness = probe_harness();
13955 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
13959 monitor.abort();
13960 let _ = monitor.await;
13961 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
13962 state.enabled = true;
13963 state.state = super::ModuleState::Running;
13964 state.process_alive = true;
13965 })
13966 .unwrap();
13967
13968 run_probe_cycle(&mut harness, true).await;
13969 assert_eq!(
13970 harness.module.status().unwrap().health.status,
13971 super::SupervisorHealthStatus::Ok
13972 );
13973
13974 for failures in 1..harness.runtime.health.failure_threshold {
13975 run_probe_cycle(&mut harness, false).await;
13976 let status = harness.module.status().unwrap();
13977 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
13978 assert_eq!(status.health.consecutive_failures, failures);
13979 assert!(status.health.last_probe_ms.is_some());
13980 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
13981 assert!(status.health.metrics.is_none());
13982 assert_eq!(status.state, super::ModuleState::Running);
13983 assert!(status.process_alive);
13984 assert_eq!(status.restart_count, 0);
13985 assert_eq!(status.lifetime_restarts, 0);
13986 assert!(status.health.last_action.is_none());
13987 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
13988 }
13989
13990 run_probe_cycle(&mut harness, true).await;
13991 let recovered = harness.module.status().unwrap();
13992 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
13993 assert_eq!(recovered.health.consecutive_failures, 0);
13994 assert!(recovered.health.detail.is_none());
13995 assert_eq!(
13996 recovered.health.metrics,
13997 Some(serde_json::json!({"ready": true}))
13998 );
13999 assert_eq!(recovered.lifetime_restarts, 0);
14000
14001 for failures in 1..=harness.runtime.health.failure_threshold {
14002 run_probe_cycle(&mut harness, false).await;
14003 let status = harness.module.status().unwrap();
14004 assert_eq!(status.health.consecutive_failures, failures);
14005 if failures < harness.runtime.health.failure_threshold {
14006 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14007 assert_eq!(status.state, super::ModuleState::Running);
14008 assert_eq!(status.lifetime_restarts, 0);
14009 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14010 } else {
14011 assert_eq!(
14012 status.health.status,
14013 super::SupervisorHealthStatus::Unresponsive
14014 );
14015 assert_eq!(status.state, super::ModuleState::Restarting);
14016 assert_eq!(status.restart_count, 1);
14017 assert_eq!(status.lifetime_restarts, 1);
14018 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14019 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14020 }
14021 }
14022 }
14023
14024 #[tokio::test(start_paused = true)]
14025 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14026 let mut harness = probe_harness();
14027
14028 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14029 let first_latency = match &first {
14030 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14031 other => panic!("late answer was not retained: {other:?}"),
14032 };
14033 assert!(harness.handler.observe_module_control_completion(first));
14034
14035 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14036 let second_latency = match &second {
14037 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14038 other => panic!("late answer was not retained: {other:?}"),
14039 };
14040 assert!(harness.handler.observe_module_control_completion(second));
14041
14042 assert_eq!(first_latency, Duration::from_secs(8));
14043 assert_eq!(
14044 second_latency - first_latency,
14045 Duration::from_secs(3),
14046 "latency must grow linearly with the additional stall"
14047 );
14048 let health = harness.module.status().unwrap().health;
14049 assert_eq!(health.late_answer_count, 2);
14050 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14051 }
14052
14053 #[tokio::test(start_paused = true)]
14061 async fn late_answer_clears_the_consecutive_failure_streak() {
14062 let mut harness = probe_harness();
14063
14064 time_out_without_answer(&mut harness).await;
14066 harness
14067 .module
14068 .record_health_probe_failure_for_test("[no-answer] test miss")
14069 .unwrap();
14070 assert_eq!(
14071 harness.module.status().unwrap().health.consecutive_failures,
14072 1,
14073 "precondition: the miss must be on the streak before the late answer"
14074 );
14075
14076 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14078 assert!(matches!(
14079 late,
14080 ModuleControlRpcCompletion::LateHealthAnswer { .. }
14081 ));
14082 assert!(harness.handler.observe_module_control_completion(late));
14083
14084 let health = harness.module.status().unwrap().health;
14085 assert_eq!(
14086 health.consecutive_failures, 0,
14087 "a late answer is an answer: the streak must reset"
14088 );
14089 assert_eq!(health.late_answer_count, 1);
14090 }
14091
14092 #[tokio::test(start_paused = true)]
14093 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14094 let mut harness = probe_harness();
14095
14096 for _ in 0..20 {
14097 time_out_without_answer(&mut harness).await;
14098 assert_eq!(
14099 harness.forwarding.health_probe_tombstone_count().unwrap(),
14100 1
14101 );
14102 }
14103 }
14104}
14105
14106#[cfg(test)]
14107mod child_env_tests {
14108 #[cfg(windows)]
14109 use super::{
14110 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14111 };
14112 #[cfg(windows)]
14113 use std::sync::{Arc, OnceLock};
14114 #[cfg(windows)]
14115 #[test]
14116 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14117 let spec = spec(Vec::new());
14118 let mut command = Command::new("unused");
14119 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14120 .unwrap()
14121 .unwrap();
14122 let name = command
14123 .as_std()
14124 .get_envs()
14125 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14126 .and_then(|(_, value)| value)
14127 .unwrap();
14128 assert_eq!(name, OsStr::new(handoff.name()));
14129 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14130 assert!(command.as_std().get_envs().any(|(key, value)| key
14131 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14132 && value == Some(OsStr::new("env"))));
14133 assert!(command
14134 .as_std()
14135 .get_envs()
14136 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14137 }
14138
14139 #[cfg(windows)]
14140 #[test]
14141 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14142 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14143 snapshot.process_alive = true;
14144 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14145 assert_eq!(
14146 observed_launch_nonce_source(&snapshot).as_deref(),
14147 Some("env")
14148 );
14149 let consumed = Arc::new(OnceLock::new());
14150 snapshot.pipe_consumed = Some(consumed.clone());
14151 assert_eq!(
14152 observed_launch_nonce_source(&snapshot).as_deref(),
14153 Some("env")
14154 );
14155 consumed.set(()).unwrap();
14156 assert_eq!(
14157 observed_launch_nonce_source(&snapshot).as_deref(),
14158 Some("pipe")
14159 );
14160 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14161 assert_eq!(
14162 observed_launch_nonce_source(&snapshot).as_deref(),
14163 Some("env")
14164 );
14165 clear_current_process_facts(&mut snapshot);
14166 assert_eq!(observed_launch_nonce_source(&snapshot), None);
14167 }
14168 use super::{
14169 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
14170 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
14171 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14172 };
14173 use std::{ffi::OsStr, path::PathBuf};
14174 use tokio::process::Command;
14175
14176 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14177 ModuleSpec {
14178 module_id: "env-plan".to_string(),
14179 program: PathBuf::from("/nonexistent"),
14180 args: Vec::new(),
14181 env,
14182 reserved: false,
14183 reserved_prefixes: Vec::new(),
14184 protocol: ModuleProtocol::Subc,
14185 overlap: Default::default(),
14186 }
14187 }
14188
14189 #[test]
14190 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
14191 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
14192 let digest = LaunchSpecDigest::from_spec(&base);
14193 let mut reordered = base.clone();
14194 reordered.env.reverse();
14195 reordered.module_id = "different-id".into();
14196 reordered.reserved = true;
14197 reordered.overlap = Default::default();
14198 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
14199 let mut delimiter_collision = base.clone();
14200 delimiter_collision.env[0] = ("A=B".into(), "C".into());
14201 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
14202 let mut changed = base.clone();
14203 changed.protocol = ModuleProtocol::None;
14204 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14205 changed = base.clone();
14206 changed.args.push("--flag".into());
14207 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14208 changed = base.clone();
14209 changed.program = PathBuf::from("/other");
14210 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14211 changed = base.clone();
14212 changed.env[1].1 = "other".into();
14213 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14214 assert!(!format!("{digest:?}").contains("B=C"));
14215 assert!(!format!("{digest:?}").contains("[u8"));
14216 }
14217
14218 #[cfg(unix)]
14219 #[test]
14220 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
14221 use std::os::unix::ffi::OsStringExt;
14222 let mut left = spec(Vec::new());
14223 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
14224 let mut right = left.clone();
14225 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
14226 assert_eq!(
14227 left.program.to_string_lossy(),
14228 right.program.to_string_lossy()
14229 );
14230 assert_ne!(
14231 LaunchSpecDigest::from_spec(&left),
14232 LaunchSpecDigest::from_spec(&right)
14233 );
14234 }
14235
14236 #[test]
14250 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14251 let mut command = Command::new("/nonexistent");
14252 apply_child_env(&mut command, &spec(Vec::new()));
14253 let removed = command
14254 .as_std()
14255 .get_envs()
14256 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14257 assert!(
14258 removed,
14259 "ambient CK_LOG must be explicitly removed for an unconfigured module"
14260 );
14261
14262 let mut configured = Command::new("/nonexistent");
14263 apply_child_env(
14264 &mut configured,
14265 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14266 );
14267 let effective = configured
14268 .as_std()
14269 .get_envs()
14270 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14271 .last()
14272 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14273 assert_eq!(
14274 effective,
14275 Some(Some("debug".to_string())),
14276 "a module's configured CK_LOG must survive the ambient removal"
14277 );
14278 }
14279
14280 #[test]
14289 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14290 let connection_file = std::path::Path::new("/run/subc-connection.json");
14291 let handle = SupervisorHandle::new();
14292
14293 let mut none_spec = spec(Vec::new());
14294 none_spec.protocol = ModuleProtocol::None;
14295 let mut none = Command::new("/nonexistent");
14296 let none_handoff =
14297 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14298 .expect("protocol-none spawn args apply");
14299 assert!(
14300 none_handoff.is_none(),
14301 "protocol:none spawn must not receive a nonce descriptor"
14302 );
14303 #[cfg(windows)]
14304 assert!(!none.as_std().get_envs().any(|(key, value)| key
14305 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14306 && value.is_some()));
14307 assert!(
14308 !none.as_std().get_envs().any(|(key, value)| key
14309 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14310 && value.is_some()),
14311 "protocol:none spawn must not name a nonce descriptor"
14312 );
14313 let none_args: Vec<String> = none
14314 .as_std()
14315 .get_args()
14316 .map(|a| a.to_string_lossy().into_owned())
14317 .collect();
14318 assert!(
14319 !none_args.iter().any(|a| a == SUBC_ARG),
14320 "protocol:none argv must not carry --subc; got {none_args:?}"
14321 );
14322 let none_has_nonce = none
14323 .as_std()
14324 .get_envs()
14325 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
14326 assert!(
14327 !none_has_nonce,
14328 "protocol:none spawn must not receive a launch nonce"
14329 );
14330 let none_has_module_id = none
14331 .as_std()
14332 .get_envs()
14333 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
14334 assert!(
14335 none_has_module_id,
14336 "SUBC_MODULE_ID is inert and stays on every path"
14337 );
14338 assert!(
14339 handle.spawn_nonce(&none_spec.module_id).is_none(),
14340 "no nonce record for a process that will never present one"
14341 );
14342
14343 let wire_spec = spec(Vec::new());
14345 let mut wire = Command::new("/nonexistent");
14346 let wire_handoff =
14347 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
14348 .expect("subc-wire spawn args apply");
14349 let wire_fd_env = wire
14350 .as_std()
14351 .get_envs()
14352 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
14353 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14354 #[cfg(unix)]
14355 assert_eq!(
14356 wire_fd_env,
14357 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
14358 "a subc-wire spawn names the pipe it will receive at descriptor 3"
14359 );
14360 #[cfg(windows)]
14361 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
14362 #[cfg(not(any(unix, windows)))]
14363 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
14364 let wire_args: Vec<String> = wire
14365 .as_std()
14366 .get_args()
14367 .map(|a| a.to_string_lossy().into_owned())
14368 .collect();
14369 assert_eq!(
14370 wire_args,
14371 vec![
14372 SUBC_ARG.to_string(),
14373 connection_file.to_string_lossy().into_owned()
14374 ],
14375 "a subc-wire spawn still carries --subc <path>"
14376 );
14377 assert_eq!(
14378 wire.as_std()
14379 .get_envs()
14380 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
14381 !cfg!(unix),
14382 "only Windows supplies the environment nonce"
14383 );
14384 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
14385 }
14386
14387 #[test]
14397 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
14398 let role = |command: &Command| {
14399 command
14400 .as_std()
14401 .get_envs()
14402 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
14403 .last()
14404 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
14405 };
14406 let forged = spec(vec![(
14407 SUBC_SPAWN_ROLE_ENV.to_string(),
14408 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
14409 )]);
14410
14411 let mut plain = Command::new("/nonexistent");
14412 apply_child_env(&mut plain, &forged);
14413 apply_spawn_role(&mut plain, SpawnRole::Plain);
14414 assert_eq!(
14415 role(&plain),
14416 Some(None),
14417 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
14418 );
14419
14420 let mut candidate = Command::new("/nonexistent");
14421 apply_child_env(&mut candidate, &spec(Vec::new()));
14422 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
14423 assert_eq!(
14424 role(&candidate),
14425 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
14426 );
14427 }
14428
14429 #[test]
14435 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
14436 let mut command = Command::new("/nonexistent");
14437 apply_child_env(
14438 &mut command,
14439 &spec(vec![
14440 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
14441 ("KEPT".to_string(), "yes".to_string()),
14442 ]),
14443 );
14444 let keys: Vec<String> = command
14445 .as_std()
14446 .get_envs()
14447 .filter(|(_, value)| value.is_some())
14448 .map(|(key, _)| key.to_string_lossy().into_owned())
14449 .collect();
14450 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
14451 assert!(
14452 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
14453 "daemon-private capture key leaked to the child: {keys:?}"
14454 );
14455 }
14456}
14457
14458#[cfg(test)]
14459mod jitter_tests {
14460 use super::jittered_health_delay;
14461 use std::{collections::HashSet, time::Duration};
14462
14463 const FLEET: [&str; 14] = [
14472 "aft",
14473 "alfonso-core",
14474 "magic-context",
14475 "broca",
14476 "thalamus",
14477 "quota",
14478 "engram",
14479 "plexus",
14480 "cerebellum",
14481 "astrocyte",
14482 "synapse",
14483 "subc-mcp",
14484 "cortexkit-credentials",
14485 "subc-federation",
14486 ];
14487
14488 #[test]
14496 fn probe_delays_disperse_across_the_fleet() {
14497 let cadence = Duration::from_secs(30);
14498 let delays: HashSet<Duration> = FLEET
14499 .iter()
14500 .map(|id| jittered_health_delay(id, 0, cadence))
14501 .collect();
14502 assert_eq!(
14503 delays.len(),
14504 FLEET.len(),
14505 "every supervised module must land on its own probe offset"
14506 );
14507 }
14508
14509 #[test]
14515 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
14516 let cadence = Duration::from_secs(30);
14517 let span = cadence / 10;
14518 for id in FLEET {
14519 for probe_index in 0..8 {
14520 let delay = jittered_health_delay(id, probe_index, cadence);
14521 assert!(
14522 delay >= cadence,
14523 "{id}#{probe_index}: jitter must not shorten the cadence"
14524 );
14525 assert!(
14526 delay < cadence + span,
14527 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
14528 );
14529 }
14530 }
14531 }
14532
14533 #[test]
14539 fn a_module_offset_is_stable_across_restarts() {
14540 let cadence = Duration::from_secs(30);
14541 for id in FLEET {
14542 assert_eq!(
14543 jittered_health_delay(id, 0, cadence),
14544 jittered_health_delay(id, 0, cadence),
14545 "{id}: the same module and probe index must produce the same offset"
14546 );
14547 }
14548 }
14549
14550 #[test]
14552 fn zero_cadence_yields_zero_delay() {
14553 assert_eq!(
14554 jittered_health_delay("aft", 0, Duration::ZERO),
14555 Duration::ZERO
14556 );
14557 }
14558}
14559
14560#[cfg(all(test, target_os = "linux"))]
14561mod cgroup_placement_tests {
14562 use super::{
14563 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
14564 SuperviseError, SupervisedChild,
14565 };
14566 use crate::stderr_tail::{StderrRing, StderrTailConfig};
14567 use cortexkit_test_support::ScratchDir;
14568 use std::{
14569 fs, io,
14570 path::{Path, PathBuf},
14571 sync::{Arc, Mutex},
14572 };
14573 use tokio::process::Command;
14574
14575 #[tokio::test]
14576 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
14577 use super::*;
14578 let dir = ScratchDir::new("unique-spawn-cgroups");
14579 let root = PathBuf::from(format!(
14580 "/sys/fs/cgroup/subc-unique-test-{}-{}",
14581 std::process::id(),
14582 unix_ms_now()
14583 ));
14584 if let Err(error) = fs::create_dir(&root) {
14585 assert!(
14586 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
14587 "required cgroup test cannot execute: {error}"
14588 );
14589 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
14590 return;
14591 }
14592 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
14593 let group_count = || {
14594 fs::read_dir(root.join("subc-modules"))
14595 .unwrap()
14596 .map(|entry| entry.unwrap().file_type().unwrap())
14597 .filter(|kind| kind.is_dir())
14598 .count()
14599 };
14600 let supervisor =
14601 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
14602 .with_cgroup_placement(Some(placement.clone()));
14603 let runtime = supervisor.runtime_config();
14604 let mut spec = ModuleSpec {
14605 module_id: "unique-spawn".into(),
14606 program: PathBuf::from("/bin/sleep"),
14607 args: vec!["60".into()],
14608 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14609 .into_iter()
14610 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
14611 .collect(),
14612 reserved: false,
14613 reserved_prefixes: vec![],
14614 protocol: ModuleProtocol::None,
14615 overlap: Default::default(),
14616 };
14617 let spawn = |spec: &ModuleSpec| {
14618 spawn_child(
14619 spec,
14620 None,
14621 None,
14622 &runtime.stderr_ring,
14623 None,
14624 &runtime.child_roster,
14625 Some(&placement),
14626 )
14627 .unwrap()
14628 };
14629 let mut live = spawn(&spec);
14630 for _ in 0..3 {
14631 let next = spawn(&spec);
14633 assert_ne!(live.module_id, next.module_id);
14634 live.start_kill().unwrap();
14635 live.wait().await.unwrap();
14636 live = next;
14637 assert!(
14638 live.child.try_wait().unwrap().is_none(),
14639 "retiring the old slot must not kill the replacement"
14640 );
14641 assert_eq!(
14642 group_count(),
14643 1,
14644 "only the live spawn's cgroup should remain"
14645 );
14646 }
14647 supervisor.begin_daemon_shutdown();
14648 let reap = tokio::spawn(async move {
14649 live.wait().await.unwrap();
14650 });
14651 supervisor
14652 .end_children_for_daemon_shutdown(false, std::future::pending())
14653 .await;
14654 reap.await.unwrap();
14655 assert_eq!(group_count(), 0);
14656 spec.program = PathBuf::from("/bin/true");
14658 spec.args.clear();
14659 let fresh_roster = ChildRoster::default();
14660 let mut short = spawn_child(
14661 &spec,
14662 None,
14663 None,
14664 &runtime.stderr_ring,
14665 None,
14666 &fresh_roster,
14667 Some(&placement),
14668 )
14669 .unwrap();
14670 short.wait().await.unwrap();
14671 assert_eq!(group_count(), 0);
14672 spec.module_id = "_".repeat(255);
14673 let mut long_id = spawn_child(
14674 &spec,
14675 None,
14676 None,
14677 &runtime.stderr_ring,
14678 None,
14679 &fresh_roster,
14680 Some(&placement),
14681 )
14682 .unwrap();
14683 long_id.wait().await.unwrap();
14684 assert_eq!(
14685 group_count(),
14686 0,
14687 "valid long module IDs must not exceed cgroup NAME_MAX"
14688 );
14689 fs::remove_dir(root.join("subc-modules")).unwrap();
14690 fs::remove_dir(root).unwrap();
14691 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
14692 }
14693
14694 #[test]
14695 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
14696 let path = Path::new("/definitely-missing-subc-cgroup");
14697 let mut command = Command::new("true");
14698 let error = apply_cgroup_placement(
14699 &mut command,
14700 &ModuleSpec {
14701 module_id: "broken-cgroup".to_string(),
14702 program: PathBuf::from("true"),
14703 args: Vec::new(),
14704 env: Vec::new(),
14705 reserved: false,
14706 reserved_prefixes: Vec::new(),
14707 protocol: ModuleProtocol::Subc,
14708 overlap: Default::default(),
14709 },
14710 path,
14711 )
14712 .expect_err("a parent cgroup open failure must reject the supervised spawn");
14713 let reason = error.to_string();
14714
14715 assert!(
14716 matches!(error, SuperviseError::Cgroup { .. }),
14717 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
14718 );
14719 assert!(
14720 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
14721 "parent cgroup open failure must name cgroup.procs: {reason}"
14722 );
14723 }
14724
14725 #[tokio::test]
14726 async fn reaping_a_child_removes_its_empty_module_cgroup() {
14727 let root = ScratchDir::new("supervisor-reap-cgroup");
14728 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14729 let placement = subc_cgroup::prepare_at(&root)
14730 .expect("prepare scratch cgroup root")
14731 .expect("scratch root has a cgroup.procs marker");
14732 let module_id = "reaped-module";
14733 let module = placement
14734 .module_path(module_id)
14735 .expect("create scratch module cgroup");
14736 let child = Command::new("true")
14737 .env("XDG_DATA_HOME", root.path())
14738 .env("XDG_RUNTIME_DIR", root.path())
14739 .env("XDG_CONFIG_HOME", root.path())
14740 .spawn()
14741 .expect("spawn short-lived child");
14742 let pid = child.id().expect("spawned child has pid");
14743 let mut child = SupervisedChild {
14744 child,
14745 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
14746 protocol: ModuleProtocol::Subc,
14747 module_id: module_id.to_string(),
14748 cgroup_placement: Some(placement),
14749 stdout_pump: None,
14750 stderr_pump: None,
14751 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
14752 spawned_at_ms: 0,
14753 spawned_from: PathBuf::from("true"),
14754 launch_spec_digest: LaunchSpecDigest([0; 32]),
14755 spawned_file_identity: None,
14756 process_start_time: None,
14757 process_identity: None,
14758 pid,
14759 roster_guard: None,
14760 #[cfg(target_os = "macos")]
14761 privacy_exec: None,
14762 spawn_failure: None,
14763 };
14764
14765 child.wait().await.expect("reap short-lived child");
14766
14767 assert!(
14768 !module.exists(),
14769 "reaping the supervised child must remove its empty cgroup"
14770 );
14771 }
14772
14773 #[test]
14774 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
14775 let root = ScratchDir::new("supervisor-non-empty-cgroup");
14776 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14777 let placement = subc_cgroup::prepare_at(&root)
14778 .expect("prepare scratch cgroup root")
14779 .expect("scratch root has a cgroup.procs marker");
14780 let module = placement
14781 .module_path("surviving-module")
14782 .expect("create scratch module cgroup");
14783 fs::write(module.join("surviving-process"), b"still present")
14784 .expect("make scratch cgroup non-empty");
14785 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
14786
14787 remove_module_cgroup(&placement, "surviving-module");
14788
14789 let logs = crate::router::test_log::captured_logs(&logs);
14790 assert!(
14791 module.exists(),
14792 "failed removal must leave the cgroup intact"
14793 );
14794 assert!(
14795 logs.contains("could not remove module cgroup after process exit; continuing teardown")
14796 && logs.contains("surviving-module"),
14797 "best-effort removal must report the failure without returning it: {logs}"
14798 );
14799 }
14800
14801 #[test]
14802 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
14803 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
14804 let reason = SuperviseError::Spawn {
14805 program: PathBuf::from("/bin/true"),
14806 source: io::Error::from_raw_os_error(13),
14807 cgroup_path: Some(cgroup_path.clone()),
14808 }
14809 .to_string();
14810
14811 assert!(
14812 reason.contains(&cgroup_path.display().to_string()),
14813 "a pre_exec spawn failure must name the cgroup path: {reason}"
14814 );
14815 }
14816}
14817
14818#[cfg(test)]
14819mod spawn_subscriber_lag_tests {
14820 use super::*;
14821
14822 #[tokio::test]
14827 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
14828 let feed = SpawnEventFeed::default();
14829 feed.configure_incarnation("lag-incarnation".to_string());
14830 let (tx, mut rx) = mpsc::channel(1);
14833 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
14834 .expect("subscribe");
14835 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
14836 for index in 0..emitted {
14837 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
14838 tokio::task::yield_now().await;
14841 }
14842 assert_eq!(
14843 feed.subscriber_count(),
14844 0,
14845 "the lagged subscriber must be removed"
14846 );
14847
14848 let mut data = Vec::new();
14849 let mut last = None;
14850 loop {
14851 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14852 .await
14853 .expect("the forwarder must finish once the subscriber is dropped");
14854 let Some(outbound) = next else { break };
14855 let frame = outbound.frame;
14856 if frame.header.ty == FrameType::StreamData {
14857 assert!(last.is_none(), "no data may follow the terminal frame");
14858 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
14859 data.push(event.cursor.seq);
14860 } else {
14861 assert!(last.is_none(), "exactly one terminal frame");
14862 last = Some(frame);
14863 }
14864 }
14865 assert!(!data.is_empty(), "queued frames drain before the terminal");
14866 for pair in data.windows(2) {
14867 assert_eq!(
14868 pair[1],
14869 pair[0] + 1,
14870 "queued frames arrive dense and in order"
14871 );
14872 }
14873 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
14874 assert_eq!(terminal.header.ty, FrameType::Error);
14875 assert_eq!(terminal.header.corr, 7);
14876 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
14877 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
14878 let detail = body.detail.expect("lagged error carries detail");
14879 assert_eq!(
14880 detail["first_undelivered_cursor"]["seq"],
14881 data.last().unwrap() + 1,
14882 "the named cursor is the first event the subscriber did not receive"
14883 );
14884 assert_eq!(
14885 detail["first_undelivered_cursor"]["daemon_incarnation"],
14886 "lag-incarnation"
14887 );
14888 }
14889}
14890
14891#[cfg(test)]
14892mod terminal_history_read_concurrency_tests {
14893 use super::*;
14894 use crate::terminal_journal::read_pause;
14895 use cortexkit_test_support::ScratchDir;
14896 use std::sync::mpsc as std_mpsc;
14897
14898 fn journaled_ring(
14899 journal: &Arc<crate::terminal_journal::TerminalJournal>,
14900 ) -> Arc<Mutex<TerminalRing>> {
14901 Arc::new(Mutex::new(
14902 TerminalRing::new(TerminalRingConfig::default(), 1)
14903 .with_journal(Some(Arc::clone(journal))),
14904 ))
14905 }
14906
14907 fn crash(at_ms: u64) -> ExitReport {
14908 ExitReport {
14909 kind: ExitKind::Crash,
14910 code: Some(1),
14911 signal: None,
14912 at_ms,
14913 }
14914 }
14915
14916 fn record_within(
14919 module_id: &'static str,
14920 ring: &Arc<Mutex<TerminalRing>>,
14921 at_ms: u64,
14922 bound: Duration,
14923 ) -> bool {
14924 let ring = Arc::clone(ring);
14925 let (done, done_rx) = std_mpsc::channel();
14926 std::thread::spawn(move || {
14927 record_terminal(
14928 module_id,
14929 &ring,
14930 &SpawnEventFeed::default(),
14931 &crash(at_ms),
14932 TerminalDisposition::Restarting,
14933 );
14934 let _ = done.send(());
14935 });
14936 done_rx.recv_timeout(bound).is_ok()
14937 }
14938
14939 #[test]
14944 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
14945 let dir = ScratchDir::new("terminal-history-concurrent-read");
14946 let path = dir.join("terminals.jsonl");
14947 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
14948 path.clone(),
14949 "daemon".into(),
14950 ));
14951 let reader_ring = journaled_ring(&journal);
14952 let other_ring = journaled_ring(&journal);
14953 assert!(record_within(
14954 "reader-module",
14955 &reader_ring,
14956 10,
14957 Duration::from_secs(5)
14958 ));
14959
14960 let (started, release) = read_pause::install(&path);
14961 let reading = {
14962 let ring = Arc::clone(&reader_ring);
14963 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
14964 };
14965 started
14966 .recv_timeout(Duration::from_secs(5))
14967 .expect("the history read reached its pause");
14968
14969 let bound = Duration::from_secs(1);
14970 assert!(
14971 record_within("other-module", &other_ring, 20, bound),
14972 "another module's exit waited on a history read (journal writer held)"
14973 );
14974 assert!(
14975 record_within("reader-module", &reader_ring, 30, bound),
14976 "the read module's own exit waited on its history read (ring held)"
14977 );
14978
14979 drop(release);
14980 let paused = reading.join().unwrap();
14981 assert_eq!(
14982 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14983 vec![10],
14984 "an exit recorded after the read began lands in neither half of it"
14985 );
14986 assert_eq!(paused.journal_skipped_lines, 0);
14987 assert_eq!(paused.journal_read_errors, 0);
14988
14989 let after = durable_terminal_history_of(&reader_ring, "reader-module");
14990 assert_eq!(
14991 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14992 vec![10, 30],
14993 "the next read merges ring and journal with no duplicate"
14994 );
14995 assert_eq!(after.journal_skipped_lines, 0);
14996 }
14997}
14998
14999#[cfg(test)]
15004mod stderr_settle_tests {
15005 use std::{
15006 future::Future,
15007 io,
15008 pin::Pin,
15009 sync::{Arc, Mutex},
15010 task::{Context, Poll},
15011 time::Duration,
15012 };
15013
15014 use tokio::{
15015 io::{AsyncRead, ReadBuf},
15016 sync::oneshot,
15017 time::Instant,
15018 };
15019
15020 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15021 use crate::stderr_tail::{
15022 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15023 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15024 };
15025
15026 const BOUND: Duration = Duration::from_millis(250);
15027
15028 struct HeldReader {
15032 before: Option<Vec<u8>>,
15033 gate: Option<oneshot::Receiver<()>>,
15034 after: io::Cursor<Vec<u8>>,
15035 }
15036
15037 impl AsyncRead for HeldReader {
15038 fn poll_read(
15039 mut self: Pin<&mut Self>,
15040 cx: &mut Context<'_>,
15041 buf: &mut ReadBuf<'_>,
15042 ) -> Poll<io::Result<()>> {
15043 if let Some(bytes) = self.before.take() {
15044 buf.put_slice(&bytes);
15045 return Poll::Ready(Ok(()));
15046 }
15047 if let Some(gate) = self.gate.as_mut() {
15048 match Pin::new(gate).poll(cx) {
15049 Poll::Pending => return Poll::Pending,
15050 Poll::Ready(_) => self.gate = None,
15051 }
15052 }
15053 Pin::new(&mut self.after).poll_read(cx, buf)
15054 }
15055 }
15056
15057 struct DiscardSink;
15058
15059 impl OutputSink for DiscardSink {
15060 fn write_line(&mut self, _line: &[u8]) {}
15061 }
15062
15063 fn line(text: &str) -> TailEntry {
15064 TailEntry::Line {
15065 text: text.to_string(),
15066 truncated: false,
15067 at_ms: None,
15068 }
15069 }
15070
15071 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
15072 ring.lock().unwrap()
15073 }
15074
15075 fn held_pump(
15079 ring: &Arc<Mutex<StderrRing>>,
15080 before: &str,
15081 after: &str,
15082 ) -> (StderrPump, oneshot::Sender<()>) {
15083 let generation = lock(ring).begin_process();
15084 let (release, gate) = oneshot::channel();
15085 let reader = HeldReader {
15086 before: Some(before.as_bytes().to_vec()),
15087 gate: Some(gate),
15088 after: io::Cursor::new(after.as_bytes().to_vec()),
15089 };
15090 let task = tokio::spawn(pump_stderr_to(
15091 reader,
15092 Arc::clone(ring),
15093 generation,
15094 DiscardSink,
15095 ));
15096 (StderrPump { task, generation }, release)
15097 }
15098
15099 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
15100 for _ in 0..1000 {
15101 if done(&lock(ring)) {
15102 return;
15103 }
15104 tokio::time::sleep(Duration::from_millis(1)).await;
15105 }
15106 panic!(
15107 "ring never reached the expected state: {:?}",
15108 lock(ring).snapshot(None, None)
15109 );
15110 }
15111
15112 #[tokio::test(start_paused = true)]
15113 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
15114 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15115 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
15116
15117 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
15118 let before_release = lock(&ring).snapshot(None, None);
15119 assert!(
15120 matches!(before_release.capture, CaptureState::Incomplete { .. }),
15121 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
15122 );
15123
15124 let next = lock(&ring).begin_process();
15127 lock(&ring).push_line_from(next, "next process booting");
15128 release.send(()).unwrap();
15129 wait_until(&ring, |ring| {
15130 ring.snapshot(None, None).capture == CaptureState::Captured
15131 })
15132 .await;
15133
15134 assert_eq!(
15135 untimed(lock(&ring).snapshot(None, None).entries),
15136 vec![
15137 line("booting"),
15138 line("config error: missing storage"),
15139 TailEntry::ProcessStart,
15140 line("next process booting"),
15141 ],
15142 "the crash's last line must survive a slow reader and stay in the crashed process's section"
15143 );
15144 }
15145
15146 #[tokio::test(start_paused = true)]
15147 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15148 ) {
15149 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15150 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15153
15154 let started = Instant::now();
15155 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15156 assert_eq!(
15157 started.elapsed(),
15158 BOUND,
15159 "the restart must wait exactly the bound for a pipe that stays open, no longer"
15160 );
15161
15162 let next = lock(&ring).begin_process();
15163 lock(&ring).push_line_from(next, "next process booting");
15164 tokio::time::sleep(Duration::from_secs(60)).await;
15165
15166 let snapshot = lock(&ring).snapshot(None, None);
15167 match &snapshot.capture {
15168 CaptureState::Incomplete { reason } => assert!(
15169 reason.contains("had not reached EOF") && reason.contains("250ms"),
15170 "the reason must say what is missing and after how long: {reason}"
15171 ),
15172 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15173 }
15174 assert_eq!(
15175 untimed(snapshot.entries),
15176 vec![
15177 line("parent exiting"),
15178 TailEntry::ProcessStart,
15179 line("next process booting"),
15180 ]
15181 );
15182 }
15183
15184 #[tokio::test(start_paused = true)]
15185 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15186 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15187 let (pump, release) = held_pump(&ring, "one\n", "two\n");
15188 release.send(()).unwrap();
15189
15190 settle_stderr_pump("clean", &ring, pump, BOUND).await;
15191
15192 let snapshot = lock(&ring).snapshot(None, None);
15193 assert_eq!(snapshot.capture, CaptureState::Captured);
15194 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15195 }
15196
15197 #[tokio::test(start_paused = true)]
15198 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15199 const BURST: usize = 150;
15200 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15201 let path = capture.join("two-pipe.stderr.log");
15202 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15203 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15204 let generation = lock(&ring).begin_process();
15205 let stdout = (0..BURST)
15206 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15207 .collect::<String>();
15208 let stderr = (0..BURST)
15209 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15210 .collect::<String>();
15211 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15216 let (release, gate) = oneshot::channel();
15217 let reader = HeldReader {
15218 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15219 gate: Some(gate),
15220 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15221 };
15222 let (finished, completion) = oneshot::channel();
15223 let stdout_sink = sink.clone();
15224 let pump = tokio::spawn(async move {
15225 pump_stdout_to(reader, stdout_sink).await;
15226 finished.send(()).unwrap();
15227 });
15228 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15229
15230 let started = Instant::now();
15231 settle_stdout_pump("two-pipe", pump, BOUND).await;
15232 assert_eq!(
15233 started.elapsed(),
15234 BOUND,
15235 "a late reader must not hold up restart"
15236 );
15237 assert_eq!(
15238 std::fs::read_to_string(&path).unwrap().lines().count(),
15239 BURST + 1
15240 );
15241
15242 release
15243 .send(())
15244 .expect("stdout reader must survive the exit drain bound");
15245 tokio::time::timeout(Duration::from_secs(1), completion)
15246 .await
15247 .expect("released stdout reader must reach EOF")
15248 .expect("stdout capture must finish at EOF, not be cancelled");
15249 let contents = std::fs::read_to_string(&path).unwrap();
15250 let mut lines = contents
15251 .lines()
15252 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15253 .collect::<Vec<_>>();
15254 let mut expected = (0..BURST)
15255 .flat_map(|i| {
15256 [
15257 format!("out-{i:04}-{}", "o".repeat(64)),
15258 format!("err-{i:04}-{}", "e".repeat(64)),
15259 ]
15260 })
15261 .collect::<Vec<_>>();
15262 lines.sort_unstable();
15263 expected.sort_unstable();
15264 assert_eq!(
15265 lines, expected,
15266 "both pipes must retain every complete line exactly once"
15267 );
15268 }
15269}
15270
15271#[cfg(all(test, windows))]
15285mod job_containment_tests {
15286 use super::*;
15287 use cortexkit_test_support::ScratchDir;
15288 use std::{
15289 path::{Path, PathBuf},
15290 sync::{Arc, Mutex},
15291 time::{Duration, Instant},
15292 };
15293
15294 fn stub_path() -> PathBuf {
15300 let mut path = std::env::current_exe().expect("current_exe available in tests");
15301 path.pop();
15302 path.pop();
15303 path.push("fake-aft-stub.exe");
15304 assert!(
15305 path.exists(),
15306 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15307 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15308 path.display()
15309 );
15310 path
15311 }
15312
15313 fn read_grandchild_pid(path: &Path) -> u32 {
15315 let deadline = Instant::now() + Duration::from_secs(10);
15316 loop {
15317 if let Ok(contents) = std::fs::read_to_string(path) {
15318 if let Ok(pid) = contents.trim().parse() {
15319 return pid;
15320 }
15321 }
15322 assert!(
15323 Instant::now() < deadline,
15324 "the stub never recorded a grandchild pid at {}",
15325 path.display()
15326 );
15327 std::thread::sleep(Duration::from_millis(10));
15328 }
15329 }
15330
15331 struct Fixture {
15334 _dir: ScratchDir,
15335 module_id: String,
15336 grandchild: u32,
15337 child: Option<SupervisedChild>,
15338 registry: Arc<Registry>,
15339 snapshot: Arc<Mutex<SupervisorSnapshot>>,
15340 terminal_ring: Arc<Mutex<TerminalRing>>,
15341 spawn_events: SpawnEventFeed,
15342 }
15343
15344 fn fixture(label: &str, module_id: &str) -> Fixture {
15345 let dir = ScratchDir::new(label);
15346 let pid_file = dir.join("grandchild.pid");
15347 let supervisor = Supervisor::new_for_test(
15348 Arc::new(Registry::default()),
15349 RestartPolicy::new(3, Duration::ZERO),
15350 );
15351 let runtime = supervisor.runtime_config();
15352 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15353 let spec = ModuleSpec {
15354 module_id: module_id.to_string(),
15355 program: stub_path(),
15356 args: Vec::new(),
15360 env: vec![
15361 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
15362 (
15363 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
15364 pid_file.display().to_string(),
15365 ),
15366 ],
15367 reserved: false,
15368 reserved_prefixes: Vec::new(),
15369 protocol: ModuleProtocol::Subc,
15370 overlap: Default::default(),
15371 };
15372 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
15373 .expect("spawn the supervised fixture");
15374 let grandchild = read_grandchild_pid(&pid_file);
15375 Fixture {
15376 _dir: dir,
15377 module_id: module_id.to_string(),
15378 grandchild,
15379 child: Some(child),
15380 registry: Arc::new(Registry::default()),
15381 snapshot,
15382 terminal_ring: Arc::clone(&runtime.terminal_ring),
15383 spawn_events: SpawnEventFeed::default(),
15384 }
15385 }
15386
15387 impl Fixture {
15388 async fn drain(&mut self) {
15390 let child = self
15391 .child
15392 .take()
15393 .expect("the fixture child is still present");
15394 drain_child_to_state(
15395 &self.module_id,
15396 ModuleProtocol::Subc,
15397 StopNotice::NotSent,
15400 &self.registry,
15401 None,
15402 &self.snapshot,
15403 &self.terminal_ring,
15404 &self.spawn_events,
15405 child,
15406 Duration::from_millis(500),
15407 ModuleState::Stopped,
15408 Some(false),
15409 )
15410 .await
15411 .expect("drain the supervised fixture");
15412 }
15413 }
15414
15415 #[tokio::test]
15421 async fn teardown_reaps_the_grandchild() {
15422 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
15423 let grandchild = fixture.grandchild;
15424
15425 assert!(
15426 subc_jobobject::process_exists(grandchild),
15427 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
15428 );
15429
15430 fixture.drain().await;
15431
15432 assert!(
15433 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15434 "grandchild {grandchild} outlived module teardown: the tree was not contained"
15435 );
15436 }
15437
15438 #[test]
15451 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
15452 let dir = ScratchDir::new("teardown-uncontained");
15453 let pid_file = dir.join("grandchild.pid");
15454 let mut child = std::process::Command::new(stub_path())
15455 .env("FAKE_AFT_NEVER_CONNECT", "1")
15456 .env(
15457 "FAKE_AFT_GRANDCHILD_PID_FILE",
15458 pid_file.display().to_string(),
15459 )
15460 .stdin(std::process::Stdio::null())
15461 .stdout(std::process::Stdio::null())
15462 .stderr(std::process::Stdio::null())
15463 .spawn()
15464 .expect("spawn the uncontained fixture");
15465 let grandchild = read_grandchild_pid(&pid_file);
15466
15467 child.kill().expect("kill the direct child");
15469 let _ = child.wait();
15470
15471 assert!(
15472 subc_jobobject::process_exists(grandchild),
15473 "grandchild {grandchild} died with the direct child, so this control no longer \
15474 distinguishes contained from uncontained teardown and the regression test is \
15475 passing vacuously"
15476 );
15477
15478 kill_tree(grandchild);
15481 }
15482
15483 #[tokio::test]
15492 async fn dropping_containment_reaps_the_grandchild() {
15493 let mut fixture = fixture("drop-containment", "tree-drop");
15494 let grandchild = fixture.grandchild;
15495
15496 assert!(subc_jobobject::process_exists(grandchild));
15497
15498 fixture.child.as_mut().expect("child present").job = None;
15500
15501 assert!(
15502 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15503 "grandchild {grandchild} survived the containment handle closing, so a daemon \
15504 crash would leave the tree behind"
15505 );
15506 }
15507
15508 fn kill_tree(pid: u32) {
15510 let _ = std::process::Command::new("taskkill.exe")
15511 .args(["/PID", &pid.to_string(), "/T", "/F"])
15512 .stdin(std::process::Stdio::null())
15513 .stdout(std::process::Stdio::null())
15514 .stderr(std::process::Stdio::null())
15515 .status();
15516 assert!(
15517 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
15518 "could not clean up grandchild {pid}"
15519 );
15520 }
15521}
15522
15523#[cfg(test)]
15524mod privacy_trampoline_configuration_tests {
15525 #[tokio::test]
15526 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15527 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
15528 #[cfg(target_os = "macos")]
15529 {
15530 let supervisor = super::Supervisor::new(
15531 std::sync::Arc::new(crate::Registry::default()),
15532 super::RestartPolicy::default(),
15533 );
15534 let error = supervisor.spawn(spec()).unwrap_err();
15535 assert!(
15536 error
15537 .to_string()
15538 .contains("no privacy trampoline configured"),
15539 "{error}"
15540 );
15541 }
15542 }
15543
15544 #[tokio::test]
15545 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15546 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
15547 #[cfg(target_os = "macos")]
15548 {
15549 let supervisor = super::Supervisor::new(
15550 std::sync::Arc::new(crate::Registry::default()),
15551 super::RestartPolicy::default(),
15552 )
15553 .with_privacy_trampoline(std::env::current_exe().unwrap());
15554 let error = supervisor.spawn(spec()).unwrap_err();
15555 assert!(
15556 error
15557 .to_string()
15558 .contains("binary does not implement the privacy trampoline protocol"),
15559 "{error}"
15560 );
15561 }
15562 }
15563
15564 #[cfg(target_os = "macos")]
15565 fn spec() -> super::ModuleSpec {
15566 super::ModuleSpec {
15567 module_id: "privacy-configuration".into(),
15568 program: "/bin/sleep".into(),
15569 args: vec!["30".into()],
15570 env: vec![],
15571 reserved: false,
15572 reserved_prefixes: vec![],
15573 protocol: subc_control::ModuleProtocol::None,
15574 overlap: super::ModuleOverlap::Exclusive,
15575 }
15576 }
15577}
15578
15579#[cfg(test)]
15580mod privacy_exec_boundary_tests {
15581 #[cfg(target_os = "macos")]
15582 use super::*;
15583 #[cfg(target_os = "macos")]
15584 use std::{
15585 io::{Read, Write},
15586 net::{TcpListener, TcpStream},
15587 };
15588
15589 #[cfg(target_os = "macos")]
15592 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
15593 if let Some((_, path)) = spec
15594 .env
15595 .iter()
15596 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
15597 {
15598 let mut barrier = TcpStream::connect(path).unwrap();
15599 barrier
15600 .set_read_timeout(Some(Duration::from_secs(30)))
15601 .unwrap();
15602 barrier.write_all(&pid.to_ne_bytes()).unwrap();
15603 let mut release = [0];
15604 barrier.read_exact(&mut release).unwrap();
15605 assert_eq!(&release, b"X");
15606 }
15607 }
15608
15609 #[cfg(target_os = "macos")]
15610 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
15611 ModuleSpec {
15612 module_id: "privacy-boundary".into(),
15613 program: program.into(),
15614 args: args.iter().map(|arg| (*arg).into()).collect(),
15615 env: vec![],
15616 reserved: false,
15617 reserved_prefixes: vec![],
15618 protocol: ModuleProtocol::None,
15619 overlap: ModuleOverlap::Exclusive,
15620 }
15621 }
15622
15623 #[cfg(target_os = "macos")]
15624 async fn accept(listener: TcpListener) -> TcpStream {
15625 let listener = tokio::net::TcpListener::from_std({
15627 listener.set_nonblocking(true).unwrap();
15628 listener
15629 })
15630 .unwrap();
15631 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
15632 .await
15633 .unwrap()
15634 .unwrap();
15635 let stream = stream.into_std().unwrap();
15636 stream.set_nonblocking(false).unwrap();
15637 stream
15638 .set_read_timeout(Some(Duration::from_secs(30)))
15639 .unwrap();
15640 stream
15641 }
15642
15643 #[tokio::test]
15644 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15645 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
15646 #[cfg(target_os = "macos")]
15647 {
15648 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
15649 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15652 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15653 let record = root.join("live-children.json");
15654 let supervisor =
15655 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15656 .with_live_children_record(&record);
15657 let runtime = supervisor.runtime_config();
15658 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15659 let mut spec = spec("/bin/sleep", &["30"]);
15660 spec.env = vec![
15661 (
15662 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
15663 exec_listener.local_addr().unwrap().to_string(),
15664 ),
15665 (
15666 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
15667 sample_listener.local_addr().unwrap().to_string(),
15668 ),
15669 ];
15670 let spawn = tokio::task::spawn_blocking(move || {
15671 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
15672 });
15673 let mut sample = accept(sample_listener).await;
15674 let mut pid = [0; 4];
15675 sample.read_exact(&mut pid).unwrap();
15676 let pid = u32::from_ne_bytes(pid);
15677 let mut exec = accept(exec_listener).await;
15678 let mut ready = [0];
15679 exec.read_exact(&mut ready).unwrap();
15680 assert_eq!(&ready, b"R");
15681 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
15684 assert_eq!(
15685 observe_spawned_image(pid).unwrap().executable,
15686 Some(trampoline)
15687 );
15688 sample.write_all(b"X").unwrap();
15689 let mut child = spawn.await.unwrap();
15690 let early = crate::live_children::read_record(&record).unwrap();
15691 assert_eq!(early.len(), 1);
15692 assert_eq!(early[0].pid, pid);
15693 assert_eq!(
15694 early[0].executable, None,
15695 "unconfirmed trampoline image entered the roster"
15696 );
15697 assert!(child.report_ready.get().is_none());
15698 child.privacy_exec.as_mut().unwrap().deadline =
15701 tokio::time::Instant::now() + Duration::from_secs(30);
15702 exec.write_all(b"X").unwrap();
15703 child.confirm_privacy_exec().await;
15704 assert_eq!(child.spawn_failure, None);
15705 assert!(child.report_ready.get().is_some());
15706 let confirmed = crate::live_children::read_record(&record).unwrap();
15707 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
15708 assert_ne!(module, trampoline);
15709 assert_eq!(confirmed[0].executable, Some(module.into()));
15710 child.start_kill().unwrap();
15711 child.wait().await.unwrap();
15712 child.release_roster();
15713 }
15714 }
15715
15716 #[tokio::test]
15717 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15718 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
15719 #[cfg(target_os = "macos")]
15720 {
15721 let registry = Arc::new(Registry::default());
15722 let policy = RestartPolicy::new(0, Duration::ZERO);
15723 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
15724 let runtime = supervisor.runtime_config();
15725 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15726 let spec = spec("/bin/sh", &["-c", "exit 121"]);
15727 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
15731 let pid = child.pid;
15732 tokio::task::spawn_blocking(move || {
15733 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
15734 })
15735 .await
15736 .unwrap()
15737 .unwrap();
15738 child.privacy_exec.as_mut().unwrap().deadline =
15739 tokio::time::Instant::now() + Duration::from_secs(30);
15740 let status = child.wait().await.unwrap();
15741 assert_eq!(status.code(), Some(121));
15742 assert!(child.privacy_exec.is_none());
15743 assert!(
15744 child.report_ready.get().is_none(),
15745 "an exited module must not publish a live pid"
15746 );
15747 let report = classify_reaped_child_exit(&snapshot, &child, &status);
15748 on_child_exit(
15749 &spec,
15750 policy,
15751 ®istry,
15752 &snapshot,
15753 &runtime.terminal_ring,
15754 &runtime.spawn_events,
15755 &runtime.child_roster,
15756 report,
15757 )
15758 .await;
15759 let state = lock_snapshot(&snapshot).unwrap();
15760 assert_eq!(state.state, ModuleState::Failed);
15761 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
15762 assert_eq!(state.reported_pid(), None);
15763 drop(state);
15764 let history = runtime.terminal_ring.lock().unwrap().snapshot();
15765 assert_eq!(history.entries.len(), 1);
15766 let terminal = &history.entries[0];
15767 assert_eq!(terminal.exit_code, Some(121));
15768 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
15769 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
15770 assert_eq!(
15771 terminal.disposition_detail.as_deref(),
15772 Some(policy.budget_exhausted_detail().as_str()),
15773 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
15774 );
15775 assert_eq!(child.spawn_failure, None);
15776 child.release_roster();
15777 }
15778 }
15779}
15780
15781#[cfg(all(test, unix))]
15785mod launch_nonce_descriptor_tests {
15786 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
15787 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15788 use cortexkit_test_support::ScratchDir;
15789 use std::{
15790 path::PathBuf,
15791 sync::{Arc, Mutex},
15792 time::{Duration, Instant},
15793 };
15794
15795 async fn probe(role: super::SpawnRole) {
15796 let scratch = ScratchDir::new("launch-nonce-descriptor");
15797 let fd_copy = scratch.join("from-descriptor");
15798 let env_copy = scratch.join("environment");
15799 let script = format!(
15800 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
15801 fd = fd_copy.display(), env = env_copy.display(),
15802 );
15803 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
15804 let spec = ModuleSpec {
15805 module_id: "nonce-descriptor-probe".to_string(),
15806 program: PathBuf::from("/bin/sh"),
15807 args: vec!["-c".to_string(), script],
15808 env: vec![
15809 xdg("XDG_DATA_HOME"),
15810 xdg("XDG_RUNTIME_DIR"),
15811 xdg("XDG_CONFIG_HOME"),
15812 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
15813 ],
15814 reserved: true,
15815 reserved_prefixes: Vec::new(),
15816 protocol: ModuleProtocol::Subc,
15817 overlap: Default::default(),
15818 };
15819 let handle = SupervisorHandle::new();
15820 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15821 let roster = ChildRoster::default();
15822 #[cfg(target_os = "macos")]
15823 {
15824 let path = super::test_privacy_trampoline();
15825 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
15826 }
15827 let child = super::spawn_child_in_slot(
15828 &spec,
15829 None,
15830 Some(&handle),
15831 &ring,
15832 None,
15833 &roster,
15834 #[cfg(target_os = "linux")]
15835 None,
15836 role,
15837 matches!(role, super::SpawnRole::SwapCandidate),
15838 )
15839 .expect("spawn probe");
15840 let deadline = Instant::now() + Duration::from_secs(10);
15841 while !(fd_copy.exists() && env_copy.exists()) {
15842 assert!(Instant::now() < deadline, "probe never wrote its copies");
15843 tokio::time::sleep(Duration::from_millis(20)).await;
15844 }
15845 let nonce = std::fs::read_to_string(fd_copy).unwrap();
15846 assert!(!nonce.is_empty());
15847 let environment = std::fs::read_to_string(env_copy).unwrap();
15848 assert!(environment
15849 .lines()
15850 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
15851 let copy = environment
15852 .lines()
15853 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
15854 assert_eq!(
15855 copy, None,
15856 "Unix children must never receive the environment nonce"
15857 );
15858 if matches!(role, super::SpawnRole::Plain) {
15859 assert_eq!(
15860 handle.spawn_nonce(&spec.module_id).as_deref(),
15861 Some(nonce.as_str())
15862 );
15863 }
15864 drop(child);
15865 }
15866
15867 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15868 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
15869 probe(super::SpawnRole::Plain).await;
15870 }
15871
15872 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15873 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
15874 probe(super::SpawnRole::SwapCandidate).await;
15875 }
15876}
15877
15878#[cfg(all(test, target_os = "linux"))]
15879mod cgroup_containment_tests {
15880 use super::*;
15881 use cortexkit_test_support::ScratchDir;
15882
15883 fn running(pid: u32) -> bool {
15884 std::fs::read_to_string(format!("/proc/{pid}/stat"))
15886 .ok()
15887 .and_then(|stat| {
15888 stat.rsplit_once(") ")
15889 .map(|(_, rest)| rest.starts_with('Z'))
15890 })
15891 .is_some_and(|zombie| !zombie)
15892 }
15893
15894 #[tokio::test]
15895 async fn linux_teardown_reaps_the_grandchild() {
15896 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
15897 }
15898
15899 #[tokio::test]
15900 async fn linux_shutdown_straggler_reaps_the_grandchild() {
15901 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
15902 }
15903
15904 async fn teardown_tree(test_name: &str, shutdown: bool) {
15905 let dir = ScratchDir::new(test_name);
15906 let root = PathBuf::from(format!(
15907 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
15908 std::process::id(),
15909 unix_ms_now()
15910 ));
15911 if let Err(error) = std::fs::create_dir(&root) {
15912 assert!(
15913 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15914 "required cgroup test cannot execute: {error}"
15915 );
15916 eprintln!(
15917 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
15918 root.display()
15919 );
15920 return;
15921 }
15922 let placement = subc_cgroup::prepare_at(&root)
15923 .expect("prepare isolated kernel cgroup")
15924 .expect("isolated cgroup is delegated");
15925 let module_id = "tree-teardown";
15926 let module = placement
15927 .module_path(module_id)
15928 .expect("create isolated module cgroup");
15929 if !module.join("cgroup.kill").exists() {
15930 std::fs::remove_dir(&module).unwrap();
15931 std::fs::remove_dir(root.join("subc-modules")).unwrap();
15932 std::fs::remove_dir(&root).unwrap();
15933 assert!(
15934 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15935 "required cgroup.kill interface unavailable"
15936 );
15937 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
15938 return;
15939 }
15940 let supervisor = Supervisor::new_for_test(
15941 Arc::new(Registry::default()),
15942 RestartPolicy::new(3, Duration::ZERO),
15943 )
15944 .with_cgroup_placement(Some(placement));
15945 let mut runtime = supervisor.runtime_config();
15946 runtime.child_roster = runtime
15947 .child_roster
15948 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
15949 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15950 let pid_file = dir.join("grandchild.pid");
15951 let spec = ModuleSpec {
15952 module_id: module_id.to_string(),
15953 program: PathBuf::from("/bin/sh"),
15954 args: vec![
15955 "-c".into(),
15956 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
15957 "fixture".into(),
15958 pid_file.display().to_string(),
15959 ],
15960 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15961 .into_iter()
15962 .map(|key| (key.to_string(), dir.display().to_string()))
15963 .collect(),
15964 reserved: false,
15965 reserved_prefixes: Vec::new(),
15966 protocol: ModuleProtocol::None,
15967 overlap: Default::default(),
15968 };
15969 let child =
15970 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
15971 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
15972 let grandchild: u32 = loop {
15973 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
15974 if let Ok(pid) = contents.trim().parse() {
15975 break pid;
15976 }
15977 }
15978 assert!(
15979 tokio::time::Instant::now() < deadline,
15980 "grandchild pid was not recorded"
15981 );
15982 tokio::time::sleep(Duration::from_millis(10)).await;
15983 };
15984 assert!(
15985 running(grandchild),
15986 "grandchild must be alive before teardown"
15987 );
15988 if shutdown {
15989 let mut child = child;
15990 crate::child_roster::end_children_for_daemon_shutdown(
15991 &runtime.child_roster,
15992 false,
15993 std::future::pending(),
15994 )
15995 .await;
15996 child.wait().await.expect("reap shutdown straggler");
15997 } else {
15998 drain_child_to_state(
15999 module_id,
16000 ModuleProtocol::None,
16001 StopNotice::NotSent,
16002 &Registry::default(),
16003 None,
16004 &snapshot,
16005 &runtime.terminal_ring,
16006 &SpawnEventFeed::default(),
16007 child,
16008 Duration::from_millis(100),
16009 ModuleState::Stopped,
16010 Some(false),
16011 )
16012 .await
16013 .expect("real supervisor teardown");
16014 }
16015 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16016 while running(grandchild) && tokio::time::Instant::now() < deadline {
16017 tokio::time::sleep(Duration::from_millis(10)).await;
16018 }
16019 let survived = running(grandchild);
16020 if survived {
16022 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16023 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16024 tokio::time::sleep(Duration::from_millis(100)).await;
16025 }
16026 if module.exists() {
16027 std::fs::remove_dir(&module).expect("remove empty module cgroup");
16028 }
16029 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16030 std::fs::remove_dir(&root).unwrap();
16031 assert!(
16032 !survived,
16033 "grandchild {grandchild} outlived module teardown"
16034 );
16035 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16036 }
16037}