1mod resource_launch;
2pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
3
4use std::{
5 collections::{HashMap, HashSet, VecDeque},
6 error::Error,
7 fmt, io,
8 path::PathBuf,
9 process::{ExitStatus, Stdio},
10 sync::{Arc, Mutex, OnceLock},
11 time::{Duration, SystemTime, UNIX_EPOCH},
12};
13
14use cortexkit_log::Retention;
15use serde_json::Value;
16use sha2::{Digest, Sha256};
17use subc_control::{
18 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
19 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
20};
21use subc_protocol::{
22 manifest::{SelfSignalKind, SignalAnchor},
23 session::{
24 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
25 MODULE_CONTROL_OP_HEALTH_CHECK,
26 },
27 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
28};
29use tokio::{
30 process::{Child, Command},
31 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
32 task::JoinHandle,
33 time::{sleep, sleep_until, timeout, timeout_at, Instant},
34};
35use tracing::{debug, error, info, warn};
36
37use crate::{
38 child_roster::ChildRoster,
39 daemon_config::{
40 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
41 },
42 forwarding::{
43 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
44 ModuleDrainTarget, PendingModuleControlRpc,
45 },
46 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
47 registry::{ConnectionId, RegistrationEndReason, RegistryError},
48 stderr_tail::{
49 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
50 StderrTailSnapshot,
51 },
52 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
53 Frame, FrameSink, Registry,
54};
55
56#[cfg(not(windows))]
57use crate::provenance::spawned_file_identity;
58
59#[path = "supervise_swap.rs"]
60mod swap;
61
62#[path = "supervise_activity.rs"]
63mod activity;
64
65#[cfg(unix)]
66#[path = "supervise/admission.rs"]
67mod admission;
68#[cfg(all(test, windows))]
69mod carried_job_tests;
70pub(crate) mod containment;
71#[cfg(all(test, unix))]
72mod containment_tests;
73mod slow_retry;
74use slow_retry::{FailedRetry, FailureContext};
75mod host_health;
76use host_health::HostHealth;
77
78#[cfg(all(test, unix))]
79mod slow_retry_tests;
80
81pub const SUBC_ARG: &str = "--subc";
87
88const DEFAULT_MAX_RESTARTS: u32 = 3;
89const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
90const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
91const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
95pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
106pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
107const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
108const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
109const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
121const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
144pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
146const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
147pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
152
153#[cfg(target_os = "macos")]
154struct PrivacyExec {
155 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
156 deadline: tokio::time::Instant,
157 expected: Option<subc_os::FileIdentity>,
158 trampoline: Option<subc_os::FileIdentity>,
159 script: bool,
160 module_id: String,
161 #[cfg(test)]
162 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
163}
164
165#[cfg(all(test, target_os = "macos"))]
166pub(crate) fn test_privacy_trampoline() -> PathBuf {
167 let path = cortexkit_test_support::cargo_profile_bin("privacy-trampoline-fixture");
170 assert!(
174 path.exists(),
175 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
176 --bins --features test-support` or `cargo test -p subc-daemon` first",
177 path.display()
178 );
179 path
180}
181
182#[cfg(target_os = "macos")]
183fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
184 use std::io::Read;
185 let mut probe = std::process::Command::new(path)
186 .args(["__disclaim-exec", "--probe"])
187 .stdin(Stdio::null())
188 .stdout(Stdio::piped())
189 .stderr(Stdio::piped())
190 .spawn()
191 .map_err(|error| {
192 format!(
193 "privacy trampoline probe failed for {}: {error}",
194 path.display()
195 )
196 })?;
197 let deadline = std::time::Instant::now() + Duration::from_secs(5);
198 let status = loop {
199 match probe.try_wait() {
200 Ok(Some(status)) => break status,
201 Ok(None) if std::time::Instant::now() < deadline => {
202 std::thread::sleep(Duration::from_millis(5))
203 }
204 result => {
205 let _ = probe.kill();
206 let _ = probe.wait();
207 return Err(format!(
208 "privacy trampoline probe failed or timed out for {}: {result:?}",
209 path.display()
210 ));
211 }
212 }
213 };
214 let mut answer = String::new();
215 if let Some(stdout) = probe.stdout.take() {
216 let _ = stdout.take(256).read_to_string(&mut answer);
217 }
218 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
219 return Ok(());
220 }
221 let mut diagnostic = String::new();
222 if let Some(stderr) = probe.stderr.take() {
223 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
224 }
225 let cause = diagnostic
226 .trim()
227 .strip_prefix("ck-subc: own privacy identity refused: ")
228 .unwrap_or("binary does not implement the privacy trampoline protocol");
229 Err(format!(
230 "{cause}: probe of {} exited {status}",
231 path.display()
232 ))
233}
234
235#[cfg(target_os = "macos")]
236fn privacy_command(
237 spec: &ModuleSpec,
238 roster: &ChildRoster,
239) -> Result<
240 (
241 Command,
242 Option<PrivacyExec>,
243 subc_os::privacy_identity::ExecAcknowledgement,
244 ),
245 SuperviseError,
246> {
247 let failure = |cause: String| {
248 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
249 SuperviseError::Spawn {
250 program: spec.program.clone(),
251 source: io::Error::other(cause),
252 cgroup_path: None,
253 }
254 };
255 let trampoline = roster.privacy_trampoline().map_err(failure)?;
256 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
261 let path = spec
262 .env
263 .iter()
264 .find(|(key, _)| key == "PATH")
265 .map(|(_, value)| std::ffi::OsString::from(value))
266 .or_else(|| std::env::var_os("PATH"))
267 .unwrap_or_else(|| "/usr/bin:/bin".into());
268 std::env::split_paths(&path)
269 .map(|dir| dir.join(&spec.program))
270 .find(|path| path.is_file())
271 .unwrap_or_else(|| spec.program.clone())
272 } else {
273 spec.program.clone()
274 };
275 let expected = subc_os::file_identity(&program);
276 let trampoline_image = subc_os::file_identity(&trampoline);
277 let script = {
278 use std::io::Read;
279 let mut prefix = [0u8; 2];
280 std::fs::File::open(&program)
281 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
282 };
283 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
284 return Err(failure(
285 "privacy identity module executable is missing or is the trampoline itself".to_string(),
286 ));
287 }
288 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
289 .map_err(|error| failure(error.to_string()))?;
290 let reader =
291 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
292 let mut command = Command::new(&trampoline);
293 command
294 .arg("__disclaim-exec")
295 .arg(ack.fd().to_string())
296 .arg(&program);
297 Ok((
298 command,
299 Some(PrivacyExec {
300 reader,
301 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
302 expected,
303 trampoline: trampoline_image,
304 script,
305 module_id: spec.module_id.clone(),
306 #[cfg(test)]
307 confirmation_pause: spec
308 .env
309 .iter()
310 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
311 .map(|(_, address)| (address.clone(), None)),
312 }),
313 ack,
314 ))
315}
316
317struct SupervisedChild {
318 subtree_target: crate::subtree_resources::Target,
319 child: containment::ProcessChild,
320 exec: crate::activity::ExecOwner,
321 #[cfg(windows)]
322 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
323 #[cfg(target_os = "macos")]
324 privacy_exec: Option<PrivacyExec>,
325 #[cfg(target_os = "macos")]
332 report_ready: Arc<OnceLock<()>>,
333 spawn_failure: Option<String>,
336 protocol: ModuleProtocol,
341 #[cfg(target_os = "linux")]
348 module_id: String,
349 #[cfg(target_os = "linux")]
350 cgroup_placement: Option<subc_cgroup::Placement>,
351 #[cfg(windows)]
374 job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
375 stdout_pump: Option<JoinHandle<()>>,
376 stderr_pump: Option<StderrPump>,
377 stderr_ring: Arc<Mutex<StderrRing>>,
378 spawned_at: Instant,
381 registration_deadline: Option<Instant>,
384 registration_watch: Option<RegistrationWatch>,
385 spawned_at_ms: u64,
386 spawned_from: PathBuf,
387 launch_spec_digest: LaunchSpecDigest,
388 spawned_file_identity: Option<SpawnedFileIdentity>,
389 process_start_time: Option<u64>,
390 process_identity: Option<ProcessIdentity>,
391 pid: u32,
392 roster_guard: Option<crate::child_roster::RosterGuard>,
395}
396
397impl SupervisedChild {
398 fn registered_since_spawn(&self) -> bool {
399 self.registration_watch
400 .as_ref()
401 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
402 }
403
404 fn id(&self) -> Option<u32> {
405 Some(self.pid)
406 }
407
408 fn process_identity(&self) -> Option<ProcessIdentity> {
409 self.process_identity
410 }
411
412 async fn wait(&mut self) -> io::Result<ExitStatus> {
413 #[cfg(target_os = "macos")]
414 self.confirm_privacy_exec().await;
415 let result = self.child.wait().await;
423 #[cfg(target_os = "linux")]
424 if result.is_ok() && self.child.container().is_none() {
425 if let Some(placement) = self.cgroup_placement.as_ref() {
426 cleanup_reaped_cgroup(placement, &self.module_id).await;
430 self.cgroup_placement = None;
431 }
432 }
433 result
434 }
435
436 async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
437 if self.child.container().is_some() {
438 if !carry {
439 self.child.terminate_container().await;
440 }
441 } else {
442 #[cfg(target_os = "linux")]
443 if let Some(placement) = self.cgroup_placement.as_ref() {
444 cleanup_reaped_cgroup(placement, &self.module_id).await;
445 self.cgroup_placement = None;
446 }
447 }
448 self.child.reap().await?;
449 self.release_roster();
450 Ok(())
451 }
452
453 #[cfg(target_os = "macos")]
454 async fn confirm_privacy_exec(&mut self) {
455 let Some(pending) = &mut self.privacy_exec else {
456 return;
457 };
458 let result = tokio::time::timeout_at(pending.deadline, async {
459 let mut record = Vec::new();
460 loop {
461 let mut ready = pending.reader.readable().await?;
462 let read = ready.try_io(|reader| {
463 use std::io::Read;
464 let mut reader = reader.get_ref();
465 let mut buffer = [0u8; 256];
466 reader.read(&mut buffer).map(|count| (count, buffer))
467 });
468 match read {
469 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
470 Ok(Ok((count, buffer))) => {
471 if record.len() + count > 1024 {
472 return Err(io::Error::other(
473 "privacy exec refusal record is too long",
474 ));
475 }
476 record.extend_from_slice(&buffer[..count]);
477 }
478 Ok(Err(error)) => return Err(error),
479 Err(_) => continue,
480 }
481 }
482 })
483 .await;
484 #[cfg(test)]
485 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
486 use tokio::io::{AsyncReadExt, AsyncWriteExt};
487 if stream.is_none() {
488 *stream = Some(
489 tokio::net::TcpStream::connect(address.as_str())
490 .await
491 .unwrap(),
492 );
493 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
494 stream
495 .as_mut()
496 .unwrap()
497 .write_all(&self.pid.to_ne_bytes())
498 .await
499 .unwrap();
500 }
501 let mut reply = [0];
502 stream
503 .as_mut()
504 .unwrap()
505 .read_exact(&mut reply)
506 .await
507 .unwrap();
508 pending.confirmation_pause = None;
509 if &reply == b"F" {
510 Ok(Err(io::Error::other("injected late exec refusal")))
511 } else {
512 assert_eq!(&reply, b"X");
513 result
514 }
515 } else {
516 result
517 };
518 let pending = self.privacy_exec.as_ref().expect("pending exec");
521 let cause = match result {
522 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
523 Ok(Err(error)) => Some(format!(
524 "privacy identity exec acknowledgement failed: {error}"
525 )),
526 Ok(Ok(record)) if !record.is_empty() => Some(
527 std::str::from_utf8(&record)
528 .ok()
529 .and_then(|record| {
530 record
531 .trim()
532 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
533 })
534 .filter(|cause| !cause.is_empty())
535 .unwrap_or("invalid privacy exec refusal record")
536 .to_string(),
537 ),
538 Ok(Ok(_)) => match self.child.try_wait() {
539 Ok(Some(_status)) => None,
543 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
544 Ok(None) => {
545 let image = observe_spawned_image(self.pid);
546 if let Some(image) = image.filter(|image| {
547 image.executable.is_some()
548 && image.executable != pending.trampoline
549 && (image.executable == pending.expected || pending.script)
550 }) {
551 if let Err(error) = self
552 .roster_guard
553 .as_ref()
554 .map(|guard| guard.confirm_executable(image))
555 .transpose()
556 {
557 Some(format!(
558 "could not durably record confirmed module image: {error}"
559 ))
560 } else {
561 let _ = self.report_ready.set(());
562 info!(module_id = %pending.module_id, pid = self.pid,
563 "module spawned with own privacy identity (responsibility disclaimed)");
564 None
565 }
566 } else if image.is_none()
567 || image.is_some_and(|image| image.executable.is_none())
568 {
569 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
576 Ok(Ok(_status)) => None,
577 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
578 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
579 }
580 } else {
581 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
582 }
583 }
584 },
585 };
586 let pending = self.privacy_exec.take().expect("pending exec");
587 self.exec.0.finish(if cause.is_some() {
590 crate::activity::ExecState::Refused
591 } else {
592 crate::activity::ExecState::CommittedExec
593 });
594 if let Some(cause) = cause {
595 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
596 self.spawn_failure = Some(cause);
597 if let Some(container) = self.child.container() {
600 container.hard_kill();
601 } else if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
602 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
603 }
604 let _ = self.child.start_kill();
605 }
606 }
607
608 fn release_roster(&mut self) {
612 self.roster_guard = None;
613 }
614
615 fn start_kill(&mut self) -> io::Result<()> {
628 if let Some(container) = self.child.container() {
629 container.hard_kill();
630 return self.kill_leader();
631 }
632 #[cfg(windows)]
633 if let Some(job) = &self.job {
634 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
635 debug!(
636 error = %error,
637 "job termination failed; the direct-child kill still owns the outcome"
638 );
639 }
640 }
641 #[cfg(target_os = "linux")]
642 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
643 self.child.start_kill()
644 }
645
646 fn kill_leader(&mut self) -> io::Result<()> {
647 self.child.start_kill()
648 }
649
650 async fn drain_stderr(&mut self, module_id: &str) {
651 if let Some(pump) = self.stdout_pump.take() {
652 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
653 }
654
655 let Some(pump) = self.stderr_pump.take() else {
656 return;
657 };
658 settle_stderr_pump(
659 module_id,
660 &self.stderr_ring,
661 pump,
662 STDERR_PUMP_DRAIN_TIMEOUT,
663 )
664 .await;
665 }
666}
667
668async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
672 match timeout(bound, &mut pump).await {
673 Ok(Ok(())) => {}
674 Ok(Err(error)) => {
675 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
676 }
677 Err(_) => {
678 drop(pump);
679 warn!(
680 module_id,
681 waited = ?bound,
682 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
683 );
684 }
685 }
686}
687
688struct StderrPump {
691 task: JoinHandle<()>,
692 generation: u64,
693}
694
695async fn settle_stderr_pump(
701 module_id: &str,
702 ring: &Arc<Mutex<StderrRing>>,
703 pump: StderrPump,
704 bound: Duration,
705) {
706 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
707 let StderrPump {
708 mut task,
709 generation,
710 } = pump;
711 lock().retire_pump(generation);
712 match timeout(bound, &mut task).await {
713 Ok(Ok(())) => {}
714 Ok(Err(err)) => {
715 let mut ring = lock();
716 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
717 ring.finish_pump(generation);
718 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
719 }
720 Err(_) => {
721 drop(task);
723 lock().mark_pump_late(
724 generation,
725 format!(
726 "stderr of the exited process had not reached EOF {bound:?} after it was \
727 retired (a descendant may still hold the pipe open); lines it still \
728 writes are kept in that process's section"
729 ),
730 );
731 warn!(
732 module_id,
733 waited = ?bound,
734 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
735 );
736 }
737 }
738}
739
740fn registration_release_events() -> &'static watch::Sender<u64> {
741 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
742 EVENTS.get_or_init(|| {
743 let (sender, _receiver) = watch::channel(0);
744 sender
745 })
746}
747
748pub(crate) fn notify_registration_release() {
749 let events = registration_release_events();
750 let next_generation = (*events.borrow()).wrapping_add(1);
751 events.send_replace(next_generation);
752}
753
754#[derive(Debug, Clone, PartialEq, Eq)]
756pub struct ModuleSpec {
757 pub module_id: String,
758 pub program: PathBuf,
759 pub args: Vec<String>,
760 pub env: Vec<(String, String)>,
761 pub reserved: bool,
766 pub reserved_prefixes: Vec<String>,
771 pub protocol: ModuleProtocol,
790 pub overlap: ModuleOverlap,
795 pub carry_containment: bool,
799}
800
801#[derive(Clone, PartialEq, Eq)]
804pub(crate) struct LaunchSpecDigest([u8; 32]);
805
806impl fmt::Debug for LaunchSpecDigest {
807 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
808 f.write_str("LaunchSpecDigest([redacted])")
809 }
810}
811
812impl LaunchSpecDigest {
813 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
814 fn field(hash: &mut Sha256, bytes: &[u8]) {
815 hash.update((bytes.len() as u64).to_le_bytes());
816 hash.update(bytes);
817 }
818 let mut hash = Sha256::new();
819 if let Some(program) = spec.program.to_str() {
820 field(&mut hash, b"utf8-program");
821 field(&mut hash, program.as_bytes());
822 } else {
823 #[cfg(unix)]
826 {
827 use std::os::unix::ffi::OsStrExt;
828 field(&mut hash, b"unix-program");
829 field(&mut hash, spec.program.as_os_str().as_bytes());
830 }
831 #[cfg(windows)]
832 {
833 use std::os::windows::ffi::OsStrExt;
834 field(&mut hash, b"windows-program");
835 let bytes = spec
836 .program
837 .as_os_str()
838 .encode_wide()
839 .flat_map(u16::to_le_bytes)
840 .collect::<Vec<_>>();
841 field(&mut hash, &bytes);
842 }
843 #[cfg(not(any(unix, windows)))]
844 field(&mut hash, spec.program.to_string_lossy().as_bytes());
845 }
846 hash.update((spec.args.len() as u64).to_le_bytes());
847 for arg in &spec.args {
848 field(&mut hash, arg.as_bytes());
849 }
850 let mut env = spec.env.iter().collect::<Vec<_>>();
851 env.sort_unstable();
852 hash.update((env.len() as u64).to_le_bytes());
853 for (key, value) in env {
854 field(&mut hash, key.as_bytes());
855 field(&mut hash, value.as_bytes());
856 }
857 field(
858 &mut hash,
859 match spec.protocol {
860 ModuleProtocol::Subc => b"subc",
861 ModuleProtocol::None => b"none",
862 },
863 );
864 Self(hash.finalize().into())
865 }
866}
867
868#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
875pub enum ModuleOverlap {
876 #[default]
878 Exclusive,
879 Safe,
891}
892
893impl ModuleOverlap {
894 pub fn as_str(self) -> &'static str {
895 match self {
896 Self::Exclusive => "exclusive",
897 Self::Safe => "safe",
898 }
899 }
900}
901
902pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
912pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
914pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
919
920#[derive(Debug, Clone, Copy, PartialEq, Eq)]
938pub struct RestartPolicy {
939 pub max_restarts: u32,
940 pub backoff: Duration,
943 pub max_backoff: Duration,
945 pub window: Duration,
949}
950
951impl RestartPolicy {
952 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
956 Self {
957 max_restarts,
958 backoff,
959 max_backoff: DEFAULT_MAX_BACKOFF,
960 window: DEFAULT_RESTART_WINDOW,
961 }
962 }
963
964 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
965 self.max_backoff = max_backoff;
966 self
967 }
968
969 pub fn with_window(mut self, window: Duration) -> Self {
970 self.window = window;
971 self
972 }
973
974 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
979 if self.backoff.is_zero() || self.max_backoff.is_zero() {
980 return Duration::ZERO;
981 }
982
983 let mut delay = self.backoff;
984 for _ in 0..restart_in_window {
985 if delay >= self.max_backoff {
986 return self.max_backoff;
987 }
988 delay = delay
989 .checked_mul(10)
990 .unwrap_or(self.max_backoff)
991 .min(self.max_backoff);
992 }
993 delay.min(self.max_backoff)
994 }
995
996 fn budget_exhausted_detail(&self) -> String {
1001 format!(
1002 "crash budget exhausted: max_restarts={} within window_secs={}",
1003 self.max_restarts,
1004 self.window.as_secs()
1005 )
1006 }
1007}
1008
1009impl Default for RestartPolicy {
1010 fn default() -> Self {
1011 Self {
1012 max_restarts: DEFAULT_MAX_RESTARTS,
1013 backoff: DEFAULT_BACKOFF,
1014 max_backoff: DEFAULT_MAX_BACKOFF,
1015 window: DEFAULT_RESTART_WINDOW,
1016 }
1017 }
1018}
1019
1020#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1021struct CrashRestartSchedule {
1022 restart_in_window: u32,
1023 delay: Duration,
1024}
1025
1026fn daemon_will_restart(
1033 state: &mut SupervisorSnapshot,
1034 policy: &RestartPolicy,
1035 now: Instant,
1036) -> bool {
1037 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1038}
1039
1040const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1041const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1042const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1043const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1044
1045#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1046pub enum HealthAction {
1047 Report,
1048 Restart,
1049 Alert,
1050}
1051
1052impl fmt::Display for HealthAction {
1053 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1054 f.write_str(match self {
1055 Self::Report => "report",
1056 Self::Restart => "restart",
1057 Self::Alert => "alert",
1058 })
1059 }
1060}
1061
1062#[derive(Debug, Clone, PartialEq, Eq)]
1063pub struct HealthConfig {
1064 pub http: Option<String>,
1068 pub cadence: Duration,
1069 pub deadline: Duration,
1070 pub failure_threshold: u32,
1071 pub on_degraded: HealthAction,
1072 pub on_failing: HealthAction,
1073 pub critical: bool,
1074}
1075
1076impl Default for HealthConfig {
1077 fn default() -> Self {
1078 Self {
1079 http: None,
1080 cadence: DEFAULT_HEALTH_CADENCE,
1081 deadline: DEFAULT_HEALTH_DEADLINE,
1082 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1083 on_degraded: HealthAction::Report,
1084 on_failing: HealthAction::Report,
1085 critical: false,
1086 }
1087 }
1088}
1089
1090#[derive(Debug, Clone, PartialEq)]
1108pub struct ModuleHealthStatus {
1109 pub status: SupervisorHealthStatus,
1110 pub last_probe_ms: Option<u64>,
1111 pub detail: Option<String>,
1112 pub metrics: Option<Value>,
1113 pub consecutive_failures: u32,
1114 pub late_answer_count: u64,
1117 pub last_late_answer_latency_ms: Option<u64>,
1119 pub last_action: Option<String>,
1120 pub last_action_ms: Option<u64>,
1124}
1125
1126impl Default for ModuleHealthStatus {
1127 fn default() -> Self {
1128 Self {
1129 status: SupervisorHealthStatus::Unknown,
1130 last_probe_ms: None,
1131 detail: None,
1132 metrics: None,
1133 consecutive_failures: 0,
1134 late_answer_count: 0,
1135 last_late_answer_latency_ms: None,
1136 last_action: None,
1137 last_action_ms: None,
1138 }
1139 }
1140}
1141
1142#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1144pub enum ModuleState {
1145 Starting,
1146 Running,
1147 Unresponsive,
1148 Restarting,
1149 Draining,
1150 Stopped,
1151 Failed,
1152 Disabled,
1153}
1154
1155impl fmt::Display for ModuleState {
1156 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1157 f.write_str(match self {
1158 Self::Starting => "starting",
1159 Self::Running => "running",
1160 Self::Unresponsive => "unresponsive",
1161 Self::Restarting => "restarting",
1162 Self::Draining => "draining",
1163 Self::Stopped => "stopped",
1164 Self::Failed => "failed",
1165 Self::Disabled => "disabled",
1166 })
1167 }
1168}
1169
1170#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1172pub enum ExitKind {
1173 Clean,
1174 Crash,
1175 DeliberateSeverance,
1176}
1177
1178impl From<ExitKind> for TerminalExitKind {
1179 fn from(kind: ExitKind) -> Self {
1180 match kind {
1181 ExitKind::Clean => Self::Clean,
1182 ExitKind::Crash => Self::Crash,
1183 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1184 }
1185 }
1186}
1187
1188#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1191pub(crate) struct ProcessIdentity {
1192 pub(crate) pid: u32,
1193 pub(crate) start_time: u64,
1194}
1195
1196#[derive(Debug, Clone, PartialEq, Eq)]
1198pub struct ExitReport {
1199 pub kind: ExitKind,
1200 pub code: Option<i32>,
1201 pub signal: Option<i32>,
1202 pub at_ms: u64,
1203}
1204
1205#[derive(Debug, Clone, PartialEq)]
1208pub struct ModuleStatus {
1209 pub module_id: String,
1210 pub waiting_for_previous_pid: Option<u32>,
1212 pub waiting_for_previous_detail: Option<String>,
1214 pub launch_nonce_source: Option<String>,
1219 pub state: ModuleState,
1220 pub enabled: bool,
1221 pub process_alive: bool,
1222 pub registration_active: bool,
1223 pub protocol: ModuleProtocol,
1228 pub live: bool,
1239 pub restart_count: u32,
1243 pub lifetime_restarts: u32,
1247 pub spawn_generation: u64,
1248 pub max_restarts: u32,
1253 pub restart_window: Duration,
1257 pub drain_timeout: Duration,
1261 pub restart_backoff: Duration,
1262 pub restart_max_backoff: Duration,
1263 pub pid: Option<u32>,
1268 pub spawned_at_ms: Option<u64>,
1269 pub spawned_from: Option<PathBuf>,
1270 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1271 pub process_start_time: Option<u64>,
1272 pub last_exit: Option<ExitReport>,
1273 pub health: ModuleHealthStatus,
1274}
1275
1276#[cfg(test)]
1277#[derive(Debug, Clone, Copy, PartialEq)]
1278struct ActorSelectCheckpoint {
1279 generation: u64,
1280 turn: u64,
1281 registered_connection: Option<ConnectionId>,
1282 next_probe_at: Option<Instant>,
1283 wake_after: Option<Duration>,
1284}
1285
1286#[derive(Debug, Clone, PartialEq)]
1287struct SupervisorSnapshot {
1288 subtree_target: crate::subtree_resources::Target,
1289 failed_retry: FailedRetry,
1290 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1291 #[cfg(windows)]
1292 pipe_consumed: Option<Arc<OnceLock<()>>>,
1293 #[cfg(test)]
1295 actor_turns: u64,
1296 #[cfg(test)]
1299 actor_select: Option<ActorSelectCheckpoint>,
1300 state: ModuleState,
1301 enabled: bool,
1302 process_alive: bool,
1303 spawned_protocol: Option<ModuleProtocol>,
1304 spawn_failure: Option<String>,
1305 waiting_for_previous_pid: Option<u32>,
1306 waiting_for_previous_detail: Option<String>,
1307 crash_restarts: VecDeque<Instant>,
1313 lifetime_restarts: u32,
1314 spawn_generation: u64,
1323 pid: Option<u32>,
1324 #[cfg(target_os = "macos")]
1325 report_ready: Option<Arc<OnceLock<()>>>,
1326 reaped_pid: Option<u32>,
1328 respawn_pending: bool,
1330 coalesced_restart_pending: bool,
1332 spawned_at_ms: Option<u64>,
1333 spawned_from: Option<PathBuf>,
1334 launch_spec_digest: Option<LaunchSpecDigest>,
1335 spawned_file_identity: Option<SpawnedFileIdentity>,
1336 process_start_time: Option<u64>,
1337 deliberate_severance: Option<ProcessIdentity>,
1338 last_exit: Option<ExitReport>,
1339 drain_disposition_detail: Option<String>,
1341 health: ModuleHealthStatus,
1342 last_probe_evidence: Option<HealthProbeEvidence>,
1343 in_alternate_slot: bool,
1348 draining_to_replace: bool,
1355 configuration_updated_since_spawn: bool,
1361}
1362
1363impl SupervisorSnapshot {
1364 fn reported_pid(&self) -> Option<u32> {
1369 #[cfg(target_os = "macos")]
1370 if self
1371 .report_ready
1372 .as_ref()
1373 .is_some_and(|ready| ready.get().is_none())
1374 {
1375 return None;
1376 }
1377 self.pid
1378 }
1379
1380 fn starting() -> Self {
1381 Self::new(ModuleState::Starting, true)
1382 }
1383
1384 fn disabled() -> Self {
1385 Self::new(ModuleState::Disabled, false)
1386 }
1387
1388 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1392 while let Some(oldest) = self.crash_restarts.front() {
1393 if now.duration_since(*oldest) > window {
1394 self.crash_restarts.pop_front();
1395 } else {
1396 break;
1397 }
1398 }
1399 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1400 }
1401
1402 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1408 self.crash_restarts.push_back(now);
1409 while self.crash_restarts.len() > policy.max_restarts as usize {
1410 self.crash_restarts.pop_front();
1411 }
1412 self.lifetime_restarts += 1;
1413 }
1414
1415 fn next_crash_restart(
1419 &mut self,
1420 policy: &RestartPolicy,
1421 now: Instant,
1422 ) -> Option<CrashRestartSchedule> {
1423 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1424 if restart_in_window >= policy.max_restarts {
1425 return None;
1426 }
1427 self.record_crash_restart(policy, now);
1428 Some(CrashRestartSchedule {
1429 restart_in_window,
1430 delay: policy.delay_for_restart(restart_in_window),
1431 })
1432 }
1433
1434 fn clear_crash_restarts(&mut self) {
1439 self.crash_restarts.clear();
1440 }
1441
1442 fn new(state: ModuleState, enabled: bool) -> Self {
1443 Self {
1444 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1445 failed_retry: FailedRetry::default(),
1446 exec_outcome: None,
1447 #[cfg(test)]
1448 actor_turns: 0,
1449 #[cfg(test)]
1450 actor_select: None,
1451 state,
1452 enabled,
1453 process_alive: false,
1454 #[cfg(windows)]
1455 pipe_consumed: None,
1456 spawned_protocol: None,
1457 spawn_failure: None,
1458 waiting_for_previous_pid: None,
1459 waiting_for_previous_detail: None,
1460 crash_restarts: VecDeque::new(),
1461 lifetime_restarts: 0,
1462 spawn_generation: 0,
1463 pid: None,
1464 #[cfg(target_os = "macos")]
1465 report_ready: None,
1466 reaped_pid: None,
1467 respawn_pending: false,
1468 coalesced_restart_pending: false,
1469 spawned_at_ms: None,
1470 spawned_from: None,
1471 launch_spec_digest: None,
1472 spawned_file_identity: None,
1473 process_start_time: None,
1474 deliberate_severance: None,
1475 last_exit: None,
1476 drain_disposition_detail: None,
1477 health: ModuleHealthStatus::default(),
1478 last_probe_evidence: None,
1479 in_alternate_slot: false,
1480 draining_to_replace: false,
1481 configuration_updated_since_spawn: false,
1482 }
1483 }
1484}
1485
1486type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1487
1488type SpawnSubscriberKey = (ConnectionId, u64);
1489
1490#[derive(Debug)]
1491struct SpawnSubscriber {
1492 version: u8,
1493 frames: mpsc::Sender<Frame>,
1494 lagged: Option<oneshot::Sender<SpawnCursor>>,
1498}
1499
1500#[derive(Debug)]
1501struct SpawnEventState {
1502 daemon_incarnation: String,
1503 seq: u64,
1504 capacity: usize,
1505 live: HashMap<String, LiveSpawn>,
1506 generations: HashMap<String, u64>,
1507 events: VecDeque<SpawnEvent>,
1508 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1509}
1510
1511impl Default for SpawnEventState {
1512 fn default() -> Self {
1513 Self {
1514 daemon_incarnation: "unconfigured".to_string(),
1515 seq: 0,
1516 capacity: SPAWN_EVENT_RING_CAPACITY,
1517 live: HashMap::new(),
1518 generations: HashMap::new(),
1519 events: VecDeque::new(),
1520 subscribers: HashMap::new(),
1521 }
1522 }
1523}
1524
1525#[derive(Debug, Clone, Default)]
1526struct SpawnEventFeed(
1527 Arc<Mutex<SpawnEventState>>,
1528 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1529);
1530
1531#[derive(Debug, Clone, PartialEq, Eq)]
1532pub(crate) enum SpawnSubscribeRefusal {
1533 ForeignIncarnation { current: String },
1534 TooOld { oldest: SpawnCursor },
1535 Frame(String),
1536}
1537
1538impl SpawnEventFeed {
1539 fn configure_incarnation(&self, daemon_incarnation: String) {
1540 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1541 state.daemon_incarnation = daemon_incarnation;
1542 state.seq = 0;
1543 state.live.clear();
1544 state.generations.clear();
1545 state.events.clear();
1546 state.subscribers.clear();
1547 }
1548
1549 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1550 SpawnCursor {
1551 daemon_incarnation: state.daemon_incarnation.clone(),
1552 seq: state.seq,
1553 }
1554 }
1555
1556 fn snapshot(&self) -> SpawnSnapshot {
1557 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1558 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1559 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1560 SpawnSnapshot {
1561 cursor: Self::cursor(&state),
1562 ring_bound: state.capacity as u64,
1563 live,
1564 }
1565 }
1566
1567 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1568 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1569 let generation = state
1570 .generations
1571 .get(module_id)
1572 .copied()
1573 .unwrap_or(0)
1574 .checked_add(1)
1575 .expect("spawn generation exhausted");
1576 state.generations.insert(module_id.to_string(), generation);
1577 let live = LiveSpawn {
1578 module_id: module_id.to_string(),
1579 spawn_generation: generation,
1580 pid,
1581 spawned_at_ms,
1582 };
1583 state.live.insert(module_id.to_string(), live);
1584 Self::emit_locked(
1585 &mut state,
1586 SpawnEventKind::Spawned,
1587 module_id.to_string(),
1588 generation,
1589 pid,
1590 None,
1591 None,
1592 );
1593 generation
1594 }
1595
1596 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1597 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1598 let Some(live) = state.live.remove(module_id) else {
1599 warn!(
1600 module_id,
1601 "terminal record had no live spawn event identity"
1602 );
1603 return;
1604 };
1605 Self::emit_locked(
1606 &mut state,
1607 SpawnEventKind::Exited,
1608 module_id.to_string(),
1609 live.spawn_generation,
1610 live.pid,
1611 exit_code,
1612 exit_signal,
1613 );
1614 }
1615
1616 fn emit_superseded_exited(
1623 &self,
1624 module_id: &str,
1625 spawn_generation: u64,
1626 pid: u32,
1627 exit_code: Option<i32>,
1628 exit_signal: Option<i32>,
1629 ) {
1630 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1631 if state
1632 .live
1633 .get(module_id)
1634 .is_some_and(|live| live.spawn_generation == spawn_generation)
1635 {
1636 state.live.remove(module_id);
1637 }
1638 Self::emit_locked(
1639 &mut state,
1640 SpawnEventKind::Exited,
1641 module_id.to_string(),
1642 spawn_generation,
1643 pid,
1644 exit_code,
1645 exit_signal,
1646 );
1647 }
1648
1649 #[allow(clippy::too_many_arguments)]
1650 fn emit_locked(
1651 state: &mut SpawnEventState,
1652 kind: SpawnEventKind,
1653 module_id: String,
1654 spawn_generation: u64,
1655 pid: u32,
1656 exit_code: Option<i32>,
1657 exit_signal: Option<i32>,
1658 ) {
1659 state.seq = state
1660 .seq
1661 .checked_add(1)
1662 .expect("spawn event sequence exhausted");
1663 let event = SpawnEvent {
1664 cursor: Self::cursor(state),
1665 kind,
1666 module_id,
1667 spawn_generation,
1668 pid,
1669 exit_code,
1670 exit_signal,
1671 };
1672 state.events.push_back(event.clone());
1673 while state.events.len() > state.capacity {
1674 state.events.pop_front();
1675 }
1676 let body = match serde_json::to_vec(&event) {
1677 Ok(body) => body,
1678 Err(error) => {
1679 error!(%error, "failed to serialize supervisor spawn event");
1680 return;
1681 }
1682 };
1683 state.subscribers.retain(|(connection_id, corr), subscriber| {
1684 let frame = Frame::build_with_version(
1685 subscriber.version,
1686 FrameType::StreamData,
1687 control_flags(),
1688 0,
1689 0,
1690 *corr,
1691 body.clone(),
1692 );
1693 match frame {
1694 Ok(frame) => {
1695 if subscriber.frames.try_send(frame).is_ok() {
1696 true
1697 } else {
1698 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1699 if let Some(lagged) = subscriber.lagged.take() {
1700 let _ = lagged.send(event.cursor.clone());
1701 }
1702 false
1703 }
1704 }
1705 Err(error) => {
1706 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1707 false
1708 }
1709 }
1710 });
1711 }
1712
1713 fn subscribe(
1714 &self,
1715 connection_id: ConnectionId,
1716 corr: u64,
1717 version: u8,
1718 since: Option<SpawnCursor>,
1719 sink: FrameSink,
1720 ) -> Result<(), SpawnSubscribeRefusal> {
1721 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1722 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1723 {
1724 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1725 let replay = if let Some(since) = since {
1726 if since.daemon_incarnation != state.daemon_incarnation {
1727 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1728 current: state.daemon_incarnation.clone(),
1729 });
1730 }
1731 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1732 if since.seq < oldest.seq.saturating_sub(1) {
1733 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1734 }
1735 }
1736 state
1737 .events
1738 .iter()
1739 .filter(|event| event.cursor.seq > since.seq)
1740 .cloned()
1741 .collect::<Vec<_>>()
1742 } else {
1743 Vec::new()
1744 };
1745 for event in replay {
1746 let body = serde_json::to_vec(&event)
1747 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1748 let frame = Frame::build_with_version(
1749 version,
1750 FrameType::StreamData,
1751 control_flags(),
1752 0,
1753 0,
1754 corr,
1755 body,
1756 )
1757 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1758 frames
1759 .try_send(frame)
1760 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1761 }
1762 state.subscribers.insert(
1763 (connection_id, corr),
1764 SpawnSubscriber {
1765 version,
1766 frames,
1767 lagged: Some(lagged),
1768 },
1769 );
1770 }
1771 tokio::spawn(async move {
1782 while let Some(frame) = receiver.recv().await {
1783 if sink.send(frame).await.is_err() {
1784 return;
1785 }
1786 }
1787 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1788 return;
1789 };
1790 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1791 Ok(frame) => {
1792 let _ = sink.send(frame).await;
1793 }
1794 Err(error) => {
1795 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1796 }
1797 }
1798 });
1799 Ok(())
1800 }
1801
1802 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1803 let Some(subscriber) = self
1804 .0
1805 .lock()
1806 .unwrap_or_else(|p| p.into_inner())
1807 .subscribers
1808 .remove(&(connection_id, corr))
1809 else {
1810 return false;
1811 };
1812 if let Ok(frame) = Frame::build_with_version(
1813 subscriber.version,
1814 FrameType::StreamEnd,
1815 control_flags(),
1816 0,
1817 0,
1818 corr,
1819 Vec::new(),
1820 ) {
1821 tokio::spawn(async move {
1822 let _ = subscriber.frames.send(frame).await;
1823 });
1824 }
1825 true
1826 }
1827
1828 fn remove_connection(&self, connection_id: ConnectionId) {
1829 self.0
1830 .lock()
1831 .unwrap_or_else(|p| p.into_inner())
1832 .subscribers
1833 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1834 }
1835
1836 #[cfg(any(test, feature = "test-support"))]
1837 fn set_capacity(&self, capacity: usize) {
1838 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1839 }
1840
1841 #[cfg(any(test, feature = "test-support"))]
1842 fn subscriber_count(&self) -> usize {
1843 self.0
1844 .lock()
1845 .unwrap_or_else(|p| p.into_inner())
1846 .subscribers
1847 .len()
1848 }
1849}
1850
1851fn spawn_subscriber_lagged_frame(
1854 version: u8,
1855 corr: u64,
1856 first_undelivered: SpawnCursor,
1857) -> Result<Frame, String> {
1858 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1860 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1861 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1862 .to_string(),
1863 detail: Some(serde_json::json!({
1864 "first_undelivered_cursor": first_undelivered
1865 })),
1866 })
1867 .map_err(|error| error.to_string())?;
1868 Frame::build_with_version(
1869 version,
1870 FrameType::Error,
1871 crate::router::daemon_error_flags(),
1872 0,
1873 0,
1874 corr,
1875 body,
1876 )
1877 .map_err(|error| error.to_string())
1878}
1879
1880pub trait ModuleProcessLiveness: Send + Sync {
1881 fn process_live(&self, module_id: &str) -> Option<bool>;
1882
1883 fn process_replacing(&self, _module_id: &str) -> bool {
1889 false
1890 }
1891}
1892
1893#[derive(Debug, Clone, Default)]
1895pub struct SupervisorProcessLiveness {
1896 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1897}
1898
1899impl SupervisorProcessLiveness {
1900 pub fn new() -> Self {
1901 Self::default()
1902 }
1903
1904 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1905 let mut snapshots = self
1906 .snapshots
1907 .lock()
1908 .unwrap_or_else(|poisoned| poisoned.into_inner());
1909 snapshots.insert(module_id, snapshot);
1910 }
1911
1912 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1913 let mut snapshots = self
1914 .snapshots
1915 .lock()
1916 .unwrap_or_else(|poisoned| poisoned.into_inner());
1917 let is_current = snapshots
1918 .get(module_id)
1919 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1920 .unwrap_or(false);
1921 if is_current {
1922 snapshots.remove(module_id);
1923 }
1924 }
1925}
1926
1927impl ModuleProcessLiveness for SupervisorProcessLiveness {
1928 fn process_live(&self, module_id: &str) -> Option<bool> {
1929 let snapshot = {
1930 let snapshots = self
1931 .snapshots
1932 .lock()
1933 .unwrap_or_else(|poisoned| poisoned.into_inner());
1934 snapshots.get(module_id).cloned()
1935 }?;
1936 let snapshot = snapshot
1937 .lock()
1938 .unwrap_or_else(|poisoned| poisoned.into_inner());
1939 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1940 }
1941
1942 fn process_replacing(&self, module_id: &str) -> bool {
1943 let Some(snapshot) = self
1944 .snapshots
1945 .lock()
1946 .unwrap_or_else(|poisoned| poisoned.into_inner())
1947 .get(module_id)
1948 .cloned()
1949 else {
1950 return false;
1951 };
1952 let snapshot = snapshot
1953 .lock()
1954 .unwrap_or_else(|poisoned| poisoned.into_inner());
1955 snapshot.enabled
1956 && match snapshot.state {
1957 ModuleState::Restarting => true,
1958 ModuleState::Draining => snapshot.draining_to_replace,
1959 ModuleState::Starting
1960 | ModuleState::Running
1961 | ModuleState::Unresponsive
1962 | ModuleState::Stopped
1963 | ModuleState::Failed
1964 | ModuleState::Disabled => false,
1965 }
1966 }
1967}
1968
1969#[cfg(test)]
1970#[derive(Debug, Default)]
1971struct ReloadExitRecordGate {
1972 reached: tokio::sync::Notify,
1973 resume: tokio::sync::Notify,
1974}
1975
1976#[derive(Debug, Clone, Copy)]
1977enum RespawnKind {
1978 Spawn,
1979 Reload,
1980}
1981
1982#[derive(Debug, Clone, Copy)]
1983struct PendingRespawn {
1984 deadline: Instant,
1985 kind: RespawnKind,
1986 daemon_initiated: bool,
1987}
1988
1989type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1990
1991#[derive(Debug, Clone)]
1992struct SupervisorRuntimeConfig {
1993 registry: Arc<Registry>,
1994 recovery_is_daemon: bool,
1996 start_requester: Option<Principal>,
1997 pending_activity_restart: bool,
1998 provenance_probe: ExecutableIdentityProbe,
1999 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2001 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2003 restart_policy: RestartPolicy,
2004 register_timeout: Duration,
2005 drain_timeout: Duration,
2008 effective_drain_timeout: Arc<Mutex<Duration>>,
2011 default_drain_timeout: Duration,
2014 health: HealthConfig,
2015 host_health: HostHealth,
2016 connection_file_path: Option<PathBuf>,
2017 capture_logs_dir: Option<PathBuf>,
2018 forwarding: Option<Arc<ForwardingTable>>,
2019 supervisor_handle: Option<SupervisorHandle>,
2022 stderr_ring: Arc<Mutex<StderrRing>>,
2029 terminal_ring: Arc<Mutex<TerminalRing>>,
2030 spawn_events: SpawnEventFeed,
2031 child_roster: ChildRoster,
2032 #[cfg(target_os = "linux")]
2033 cgroup_placement: Option<subc_cgroup::Placement>,
2034 #[cfg(test)]
2035 test_seed_stale_facts_before_enable_spawn: bool,
2036 #[cfg(test)]
2037 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2038}
2039
2040impl SupervisorRuntimeConfig {
2041 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2042 FailureContext {
2043 daemon_initiated,
2044 shutdown: self.child_roster.shutdown_flag(),
2045 }
2046 }
2047}
2048
2049#[derive(Debug, Clone, PartialEq, Eq)]
2050struct SupervisedConfiguration {
2051 spec: ModuleSpec,
2052 health: HealthConfig,
2053}
2054
2055#[derive(Debug, Clone, Default)]
2061pub struct SupervisorHandle {
2062 host_health: HostHealth,
2063 restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2064 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2065 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2066 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2067 configured_ids: Arc<Mutex<HashSet<String>>>,
2079 spawn_events: SpawnEventFeed,
2080 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2091 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2097 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2101 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2109 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2115 promotion_observer: PromotionObserverSlot,
2117 operation_lock: Arc<AsyncMutex<()>>,
2121}
2122
2123pub(crate) trait SwapPromotionObserver: Send + Sync {
2132 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2133}
2134
2135#[derive(Clone, Default)]
2139struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2140
2141impl fmt::Debug for PromotionObserverSlot {
2142 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2143 f.write_str("PromotionObserverSlot")
2144 }
2145}
2146
2147#[derive(Debug, Clone)]
2149struct OpenSwap {
2150 candidate_nonce: String,
2153 incumbent_nonce: Option<String>,
2158 candidate_admitted: bool,
2162}
2163
2164#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2167pub(crate) enum SwapHelloAdmission {
2168 NotSwapping,
2171 Candidate,
2173 Refused,
2176}
2177
2178#[derive(Debug, Clone, PartialEq, Eq)]
2179pub(crate) enum ReservedHelloRejection {
2180 Exact {
2181 module_id: String,
2182 },
2183 Prefix {
2184 prefix: String,
2185 owner_module_id: String,
2186 },
2187}
2188
2189#[derive(Debug, Default)]
2191pub(crate) struct ReservedHelloVerification {
2192 pub(crate) nonce_verified: bool,
2193 pub(crate) prefix_owner: Option<String>,
2194}
2195
2196#[derive(Debug)]
2197struct DaemonRestartIntent {
2198 expires: Option<Duration>,
2199 window: Duration,
2200}
2201
2202impl Default for DaemonRestartIntent {
2203 fn default() -> Self {
2204 Self {
2205 expires: None,
2206 window: Duration::from_secs(120),
2207 }
2208 }
2209}
2210
2211impl SupervisorHandle {
2212 pub(crate) fn announce_daemon_restart(&self) -> u64 {
2213 let mut intent = self
2214 .restart_intent
2215 .lock()
2216 .unwrap_or_else(|p| p.into_inner());
2217 intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2218 unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2219 }
2220
2221 #[cfg(feature = "test-support")]
2222 pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2223 self.restart_intent
2224 .lock()
2225 .unwrap_or_else(|p| p.into_inner())
2226 .window = window;
2227 }
2228
2229 #[cfg(unix)]
2230 fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2231 let expiry = self
2232 .restart_intent
2233 .lock()
2234 .unwrap_or_else(|p| p.into_inner())
2235 .expires
2236 .take();
2237 if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2238 RouteCloseReason::Restart
2239 } else {
2240 RouteCloseReason::Disable
2241 }
2242 }
2243
2244 pub fn new() -> Self {
2245 Self::default()
2246 }
2247
2248 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2249 self.spawn_events.snapshot()
2250 }
2251
2252 pub(crate) fn subscribe_spawns(
2253 &self,
2254 connection_id: ConnectionId,
2255 corr: u64,
2256 version: u8,
2257 since: Option<SpawnCursor>,
2258 sink: FrameSink,
2259 ) -> Result<(), SpawnSubscribeRefusal> {
2260 self.spawn_events
2261 .subscribe(connection_id, corr, version, since, sink)
2262 }
2263
2264 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2265 self.spawn_events.cancel(connection_id, corr)
2266 }
2267
2268 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2269 self.spawn_events.remove_connection(connection_id);
2270 }
2271
2272 #[cfg(any(test, feature = "test-support"))]
2273 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2274 assert!(capacity > 0, "spawn event capacity must be non-zero");
2275 self.spawn_events.set_capacity(capacity);
2276 }
2277
2278 #[cfg(any(test, feature = "test-support"))]
2279 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2280 self.spawn_events.subscriber_count()
2281 }
2282
2283 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2286 self.spawn_nonces
2287 .lock()
2288 .unwrap_or_else(|poisoned| poisoned.into_inner())
2289 .insert(module_id.to_string(), nonce);
2290 }
2291
2292 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2295 self.reserved_nonces
2296 .lock()
2297 .unwrap_or_else(|poisoned| poisoned.into_inner())
2298 .insert(module_id.to_string(), Some(nonce));
2299 }
2300
2301 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2303 let mut owners = self
2304 .reserved_prefix_owners
2305 .lock()
2306 .unwrap_or_else(|poisoned| poisoned.into_inner());
2307 owners.retain(|_, owner| owner != owner_module_id);
2308 for prefix in prefixes {
2309 owners.insert(prefix.clone(), owner_module_id.to_string());
2310 }
2311 }
2312
2313 #[cfg(test)]
2315 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2316 self.spawn_nonces
2317 .lock()
2318 .unwrap_or_else(|poisoned| poisoned.into_inner())
2319 .get(module_id)
2320 .cloned()
2321 }
2322
2323 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2324 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2325 let spawn_nonce = self
2326 .spawn_nonces
2327 .lock()
2328 .unwrap_or_else(|poisoned| poisoned.into_inner())
2329 .get(&spec.module_id)
2330 .cloned();
2331 let mut reserved_nonces = self
2332 .reserved_nonces
2333 .lock()
2334 .unwrap_or_else(|poisoned| poisoned.into_inner());
2335 if spec.reserved {
2336 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2341 }
2342 drop(reserved_nonces);
2343 self.removal_tombstones
2347 .lock()
2348 .unwrap_or_else(|poisoned| poisoned.into_inner())
2349 .remove(&spec.module_id);
2350 }
2351
2352 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2357 self.reserved_hello_verification(module_id, presented)
2358 .is_ok()
2359 }
2360
2361 pub(crate) fn reserved_hello_verification(
2364 &self,
2365 module_id: &str,
2366 presented: Option<&str>,
2367 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2368 let nonces = self
2369 .reserved_nonces
2370 .lock()
2371 .unwrap_or_else(|poisoned| poisoned.into_inner());
2372 if let Some(expected) = nonces.get(module_id) {
2373 let authorized = match expected {
2377 Some(expected) => {
2378 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2379 }
2380 None => false,
2381 };
2382 if authorized {
2383 return Ok(ReservedHelloVerification {
2384 nonce_verified: true,
2385 prefix_owner: None,
2386 });
2387 }
2388 return Err(ReservedHelloRejection::Exact {
2389 module_id: module_id.to_string(),
2390 });
2391 }
2392 drop(nonces);
2393
2394 let matched_prefix = self
2395 .reserved_prefix_owners
2396 .lock()
2397 .unwrap_or_else(|poisoned| poisoned.into_inner())
2398 .iter()
2399 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2400 .max_by_key(|(prefix, _)| prefix.len())
2401 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2402 let Some((prefix, owner_module_id)) = matched_prefix else {
2403 return Ok(ReservedHelloVerification::default());
2404 };
2405
2406 let authorized = presented.is_some_and(|presented| {
2407 self.spawn_nonces
2408 .lock()
2409 .unwrap_or_else(|poisoned| poisoned.into_inner())
2410 .get(&owner_module_id)
2411 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2412 || self.swap_nonce_matches(&owner_module_id, presented)
2415 });
2416 if authorized {
2417 Ok(ReservedHelloVerification {
2418 nonce_verified: true,
2419 prefix_owner: Some(owner_module_id),
2420 })
2421 } else {
2422 Err(ReservedHelloRejection::Prefix {
2423 prefix,
2424 owner_module_id,
2425 })
2426 }
2427 }
2428
2429 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2434 if presented.is_empty() {
2435 return false;
2436 }
2437 let nonces = self
2438 .spawn_nonces
2439 .lock()
2440 .unwrap_or_else(|poisoned| poisoned.into_inner());
2441 let current = nonces
2442 .get(module_id)
2443 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2444 drop(nonces);
2445 current || self.swap_nonce_matches(module_id, presented)
2450 }
2451
2452 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2454 let swaps = self
2455 .swaps
2456 .lock()
2457 .unwrap_or_else(|poisoned| poisoned.into_inner());
2458 swaps.get(module_id).is_some_and(|swap| {
2459 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2460 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2461 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2462 })
2463 })
2464 }
2465
2466 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2469 let incumbent_nonce = self
2470 .spawn_nonces
2471 .lock()
2472 .unwrap_or_else(|poisoned| poisoned.into_inner())
2473 .get(module_id)
2474 .cloned();
2475 self.swaps
2476 .lock()
2477 .unwrap_or_else(|poisoned| poisoned.into_inner())
2478 .insert(
2479 module_id.to_string(),
2480 OpenSwap {
2481 candidate_nonce,
2482 incumbent_nonce,
2483 candidate_admitted: false,
2484 },
2485 );
2486 }
2487
2488 pub(crate) fn close_swap(&self, module_id: &str) {
2491 self.swaps
2492 .lock()
2493 .unwrap_or_else(|poisoned| poisoned.into_inner())
2494 .remove(module_id);
2495 }
2496
2497 pub(crate) fn set_swap_promotion_observer(
2500 &self,
2501 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2502 ) {
2503 *self
2504 .promotion_observer
2505 .0
2506 .lock()
2507 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2508 }
2509
2510 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2513 let observer = self
2514 .promotion_observer
2515 .0
2516 .lock()
2517 .unwrap_or_else(|poisoned| poisoned.into_inner())
2518 .as_ref()
2519 .and_then(std::sync::Weak::upgrade);
2520 if let Some(observer) = observer {
2521 observer.swap_promoted(registration);
2522 }
2523 }
2524
2525 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2527 self.swaps
2528 .lock()
2529 .unwrap_or_else(|poisoned| poisoned.into_inner())
2530 .contains_key(module_id)
2531 }
2532
2533 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2538 let candidate_nonce = self
2539 .swaps
2540 .lock()
2541 .unwrap_or_else(|poisoned| poisoned.into_inner())
2542 .get(module_id)
2543 .map(|swap| swap.candidate_nonce.clone());
2544 let Some(nonce) = candidate_nonce else {
2545 return;
2546 };
2547 self.set_spawn_nonce(module_id, nonce.clone());
2548 if reserved {
2549 self.set_reserved_nonce(module_id, nonce);
2550 }
2551 }
2552
2553 pub(crate) fn swap_hello_admission(
2568 &self,
2569 module_id: &str,
2570 presented: Option<&str>,
2571 ) -> SwapHelloAdmission {
2572 let swaps = self
2573 .swaps
2574 .lock()
2575 .unwrap_or_else(|poisoned| poisoned.into_inner());
2576 let Some(swap) = swaps.get(module_id) else {
2577 return SwapHelloAdmission::NotSwapping;
2578 };
2579 let Some(presented) = presented else {
2580 return SwapHelloAdmission::Refused;
2581 };
2582 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2583 return if swap.candidate_admitted {
2584 SwapHelloAdmission::Refused
2585 } else {
2586 SwapHelloAdmission::Candidate
2587 };
2588 }
2589 if swap
2590 .incumbent_nonce
2591 .as_deref()
2592 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2593 {
2594 return SwapHelloAdmission::NotSwapping;
2595 }
2596 SwapHelloAdmission::Refused
2597 }
2598
2599 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2602 if let Some(swap) = self
2603 .swaps
2604 .lock()
2605 .unwrap_or_else(|poisoned| poisoned.into_inner())
2606 .get_mut(module_id)
2607 {
2608 swap.candidate_admitted = true;
2609 }
2610 }
2611
2612 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2614 self.spawn_nonces
2615 .lock()
2616 .unwrap_or_else(|poisoned| poisoned.into_inner())
2617 .get(module_id)
2618 .cloned()
2619 }
2620
2621 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2623 self.reserved_nonces
2624 .lock()
2625 .unwrap_or_else(|poisoned| poisoned.into_inner())
2626 .get(module_id)
2627 .cloned()
2628 .flatten()
2629 }
2630
2631 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2632 self.mark_configured(module.module_id());
2636 let mut modules = self
2637 .modules
2638 .lock()
2639 .unwrap_or_else(|poisoned| poisoned.into_inner());
2640 modules.insert(module.module_id().to_string(), module)
2641 }
2642
2643 fn mark_configured(&self, module_id: &str) {
2647 self.configured_ids
2648 .lock()
2649 .unwrap_or_else(|poisoned| poisoned.into_inner())
2650 .insert(module_id.to_string());
2651 }
2652
2653 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2657 let modules = self
2658 .modules
2659 .lock()
2660 .unwrap_or_else(|poisoned| poisoned.into_inner());
2661 if !modules.contains_key(module_id) {
2662 self.configured_ids
2663 .lock()
2664 .unwrap_or_else(|poisoned| poisoned.into_inner())
2665 .remove(module_id);
2666 }
2667 }
2668
2669 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2677 self.configured_ids
2678 .lock()
2679 .unwrap_or_else(|poisoned| poisoned.into_inner())
2680 .contains(module_id)
2681 }
2682
2683 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2684 let modules = self
2685 .modules
2686 .lock()
2687 .unwrap_or_else(|poisoned| poisoned.into_inner());
2688 modules.get(module_id).cloned()
2689 }
2690
2691 pub(crate) fn record_late_health_answer(
2692 &self,
2693 module_id: &str,
2694 latency_ms: u64,
2695 ) -> Result<bool, SuperviseError> {
2696 let Some(module) = self.get(module_id) else {
2697 return Ok(false);
2698 };
2699 self.host_health.record_late_module(module_id);
2700 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2701 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2702 state.health.last_late_answer_latency_ms = Some(latency_ms);
2703 state.health.consecutive_failures = 0;
2711 })?;
2712 Ok(true)
2713 }
2714
2715 pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
2716 self.host_health.spawn_monitor()
2717 }
2718
2719 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2725 let Some(module) = self.get(module_id) else {
2726 return Ok(false);
2727 };
2728 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2729 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2730 return Ok(false);
2731 };
2732 drop(snapshot);
2733 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2734 }
2735
2736 pub fn list(&self) -> Vec<SupervisedModule> {
2737 let modules = self
2738 .modules
2739 .lock()
2740 .unwrap_or_else(|poisoned| poisoned.into_inner());
2741 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2742 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2743 modules
2744 }
2745
2746 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2747 self.activity_images
2748 .lock()
2749 .unwrap_or_else(|p| p.into_inner())
2750 .remove(module_id);
2751 self.spawn_nonces
2752 .lock()
2753 .unwrap_or_else(|poisoned| poisoned.into_inner())
2754 .remove(module_id);
2755 self.close_swap(module_id);
2756 let mut reserved_nonces = self
2757 .reserved_nonces
2758 .lock()
2759 .unwrap_or_else(|poisoned| poisoned.into_inner());
2760 if reserved_nonces.contains_key(module_id) {
2761 reserved_nonces.insert(module_id.to_string(), None);
2764 }
2765 drop(reserved_nonces);
2766 self.reserved_prefix_owners
2767 .lock()
2768 .unwrap_or_else(|poisoned| poisoned.into_inner())
2769 .retain(|_, owner| owner != module_id);
2770 let removed = self
2771 .modules
2772 .lock()
2773 .unwrap_or_else(|poisoned| poisoned.into_inner())
2774 .remove(module_id);
2775 self.configured_ids
2776 .lock()
2777 .unwrap_or_else(|poisoned| poisoned.into_inner())
2778 .remove(module_id);
2779 removed
2780 }
2781
2782 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2785 self.removal_tombstones
2786 .lock()
2787 .unwrap_or_else(|poisoned| poisoned.into_inner())
2788 .insert(module_id.to_string(), unix_ms_now());
2789 }
2790
2791 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2793 self.removal_tombstones
2794 .lock()
2795 .unwrap_or_else(|poisoned| poisoned.into_inner())
2796 .get(module_id)
2797 .copied()
2798 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2799 }
2800
2801 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2806 if self.get(module_id).is_some() {
2807 return false;
2808 }
2809 let mut reserved_nonces = self
2810 .reserved_nonces
2811 .lock()
2812 .unwrap_or_else(|poisoned| poisoned.into_inner());
2813 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2814 return false;
2815 }
2816 reserved_nonces.remove(module_id);
2817 true
2818 }
2819
2820 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2821 Arc::clone(&self.operation_lock)
2822 }
2823}
2824
2825#[derive(Debug, Clone)]
2827pub struct Supervisor {
2828 registry: Arc<Registry>,
2829 restart_policy: RestartPolicy,
2830 drain_timeout: Duration,
2831 connection_file_path: Option<PathBuf>,
2832 capture_logs_dir: Option<PathBuf>,
2833 forwarding: Option<Arc<ForwardingTable>>,
2834 process_liveness: Arc<SupervisorProcessLiveness>,
2835 supervisor_handle: Option<SupervisorHandle>,
2836 health: HealthConfig,
2837 host_health: HostHealth,
2838 daemon_start_clock: crate::clock::StartClock,
2839 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2840 spawn_events: SpawnEventFeed,
2841 provenance_probe: ExecutableIdentityProbe,
2842 child_roster: ChildRoster,
2845 #[cfg(target_os = "linux")]
2846 cgroup_placement: Option<subc_cgroup::Placement>,
2847 #[cfg(test)]
2848 test_after_first_spawn: AfterFirstSpawnHook,
2849}
2850
2851pub(crate) struct RegisteredConfiguredModule {
2855 spec: ModuleSpec,
2856 runtime: SupervisorRuntimeConfig,
2857 module: SupervisedModule,
2858 commands: mpsc::Receiver<SupervisorCommand>,
2859}
2860
2861#[cfg(test)]
2867#[derive(Clone, Default)]
2868struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2869
2870#[cfg(test)]
2871type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2872
2873#[cfg(test)]
2874impl fmt::Debug for AfterFirstSpawnHook {
2875 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2876 f.write_str("AfterFirstSpawnHook")
2877 }
2878}
2879
2880#[cfg(test)]
2881impl AfterFirstSpawnHook {
2882 fn run(&self, module_id: &str) {
2883 if let Some(hook) = &self.0 {
2884 hook(module_id);
2885 }
2886 }
2887}
2888
2889impl Supervisor {
2890 #[cfg(test)]
2891 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2892 let supervisor = Self::new(registry, policy);
2893 #[cfg(target_os = "macos")]
2894 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2895 supervisor
2896 }
2897 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2902 let path = path.into();
2903 #[cfg(target_os = "macos")]
2904 {
2905 let result = probe_privacy_trampoline(&path).map(|()| path);
2906 if let Err(cause) = &result {
2907 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2908 }
2909 self.child_roster.set_privacy_trampoline(result);
2910 }
2911 #[cfg(not(target_os = "macos"))]
2912 let _ = path;
2913 self
2914 }
2915 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2918 self.child_roster.shutdown_flag()
2919 }
2920
2921 #[cfg(unix)]
2932 pub(crate) fn begin_daemon_shutdown(&self) {
2933 self.child_roster.close();
2934 if let Some(journal) = &self.terminal_journal {
2935 journal.stamp_shutdown();
2936 }
2937 }
2938
2939 #[cfg(unix)]
2944 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2945 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2946 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2947 let Some(forwarding) = &self.forwarding else {
2948 return Ok(());
2949 };
2950 let notice_started = Instant::now();
2951 let modules = forwarding
2952 .begin_daemon_drain()
2953 .map_err(SuperviseError::Forwarding)?;
2954 let deadline_ms =
2955 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2956 let shutdown_reason = self
2957 .supervisor_handle
2958 .as_ref()
2959 .map(SupervisorHandle::take_daemon_shutdown_reason)
2960 .unwrap_or(RouteCloseReason::Disable);
2961 let mut modules_told = 0;
2962 let mut notices = tokio::task::JoinSet::new();
2963 let mut drains = Vec::new();
2964 for connection in forwarding
2965 .module_connections()
2966 .map_err(SuperviseError::Forwarding)?
2967 {
2968 let target = ModuleDrainTarget {
2969 endpoint: connection.endpoint,
2970 sink: connection.sink,
2971 negotiated_ver: connection.negotiated_ver,
2972 abandoned_bindings: Vec::new(),
2973 excluded_subscriptions: 0,
2974 };
2975 notices.spawn(async move {
2976 (
2977 true,
2978 send_module_draining(
2979 &connection.module_id,
2980 shutdown_reason,
2981 deadline_ms,
2982 &target,
2983 true,
2984 )
2985 .await,
2986 )
2987 });
2988 }
2989 for (module_id, endpoint) in modules {
2990 let Some(target) = forwarding
2991 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2992 .map_err(SuperviseError::Forwarding)?
2993 else {
2994 continue;
2995 };
2996 let routes = forwarding
2997 .endpoint_routes(target.endpoint)
2998 .map_err(SuperviseError::Forwarding)?;
2999 let mut recipients = Vec::new();
3002 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3003 for route in routes {
3004 let client = route.goodbye_target;
3005 if let Some((_, channels)) = clients
3006 .iter_mut()
3007 .find(|(existing, _)| existing.connection_id == client.connection_id)
3008 {
3009 channels.push(client.channel);
3010 } else {
3011 let channel = client.channel;
3012 clients.push((client, vec![channel]));
3013 }
3014 }
3015 for (client, mut channels) in clients {
3016 channels.sort_unstable();
3017 channels.dedup();
3018 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3019 module_id: module_id.clone(),
3020 channels,
3021 reason: RouteCloseReason::Restart,
3022 })
3023 .expect("route closing serializes");
3024 recipients.push((client.sink, client.negotiated_ver, closing));
3025 }
3026 for (sink, version, body) in recipients {
3027 notices.spawn(async move {
3028 let frame = Frame::build_with_version(
3029 version,
3030 FrameType::Push,
3031 control_flags(),
3032 0,
3033 0,
3034 0,
3035 body,
3036 )
3037 .expect("bounded lifecycle notice frame builds");
3038 (false, sink.send_flushed(frame).await.is_ok())
3039 });
3040 }
3041 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3042 drains.push((module_id, target.endpoint, gauges));
3043 }
3044 let notice_deadline = Instant::now() + NOTICE_BUDGET;
3047 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3048 match result {
3049 Ok((true, true)) => modules_told += 1,
3050 Ok((false, true)) => {}
3051 result => warn!(?result, "daemon shutdown notice delivery failed"),
3052 }
3053 }
3054 notices.abort_all();
3055 info!(
3056 ?shutdown_reason,
3057 modules_told, "daemon shutdown module notices delivered"
3058 );
3059 let notice_elapsed = notice_started.elapsed();
3060 let drain_started = Instant::now();
3061 let deadline = drain_started + DRAIN_BUDGET;
3062 let mut waits = tokio::task::JoinSet::new();
3063 for (module_id, endpoint, gauges) in drains {
3064 let forwarding = Arc::clone(forwarding);
3065 let mut runtime = self.runtime_config();
3066 runtime.health.cadence = Duration::from_millis(100);
3067 waits.spawn(async move {
3068 let wait_result = wait_for_forwarding_quiescence(
3069 &forwarding,
3070 &module_id,
3071 &runtime,
3072 endpoint,
3073 deadline,
3074 &gauges,
3075 DrainScope::Endpoint(endpoint),
3076 &mut DrainGaugeTrace::new(&gauges),
3077 )
3078 .await;
3079 forwarding
3083 .record_daemon_drain_result(
3084 endpoint,
3085 drained_after_quiescence_wait(&wait_result),
3086 )
3087 .map_err(SuperviseError::Forwarding)?;
3088 wait_result
3089 });
3090 }
3091 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3092 if !matches!(result, Ok(Ok(true))) {
3093 warn!(?result, "daemon shutdown drain did not reach quiescence");
3094 }
3095 }
3096 info!(
3099 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3100 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3101 "daemon shutdown forwarding drain complete"
3102 );
3103 Ok(())
3104 }
3105
3106 #[cfg(unix)]
3118 pub(crate) async fn end_children_for_daemon_shutdown(
3119 &self,
3120 already_escalated: bool,
3121 escalate: impl std::future::Future<Output = ()>,
3122 ) {
3123 tokio::pin!(escalate);
3124 let mut escalated = already_escalated;
3125 if let Some(forwarding) = &self.forwarding {
3126 let reason = CloseReason::new(
3127 "daemon_shutdown",
3128 "the daemon is exiting after its shutdown notice and drain",
3129 );
3130 if escalated {
3131 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3134 } else {
3135 tokio::select! {
3136 biased;
3137 _ = escalate.as_mut() => {
3138 info!("second SIGTERM: abandoning module GOODBYE delivery");
3139 escalated = true;
3140 }
3141 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3142 }
3143 }
3144 let closed = forwarding.close_all_connections(&reason);
3145 debug!(closed, "closed established connections for daemon shutdown");
3146 }
3147 let escalated_here = escalated && !already_escalated;
3151 let remaining_escalate = async move {
3152 if escalated_here {
3153 std::future::pending::<()>().await;
3154 } else {
3155 escalate.await;
3156 }
3157 };
3158 crate::child_roster::end_children_for_daemon_shutdown(
3159 &self.child_roster,
3160 escalated,
3161 remaining_escalate,
3162 )
3163 .await;
3164 }
3165
3166 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3167 Self {
3168 registry,
3169 restart_policy,
3170 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3171 connection_file_path: None,
3172 capture_logs_dir: None,
3173 forwarding: None,
3174 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3175 supervisor_handle: None,
3176 health: HealthConfig::default(),
3177 host_health: HostHealth::default(),
3178 daemon_start_clock: crate::clock::StartClock::capture(),
3179 terminal_journal: None,
3180 spawn_events: SpawnEventFeed::default(),
3181 provenance_probe: ExecutableIdentityProbe::default(),
3182 child_roster: ChildRoster::default(),
3183 #[cfg(target_os = "linux")]
3184 cgroup_placement: None,
3185 #[cfg(test)]
3186 test_after_first_spawn: AfterFirstSpawnHook::default(),
3187 }
3188 }
3189
3190 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3191 self.drain_timeout = drain_timeout;
3192 self
3193 }
3194
3195 pub(crate) fn default_drain_timeout(&self) -> Duration {
3197 self.drain_timeout
3198 }
3199
3200 pub fn with_process_liveness(
3201 mut self,
3202 process_liveness: Arc<SupervisorProcessLiveness>,
3203 ) -> Self {
3204 self.process_liveness = process_liveness;
3205 self
3206 }
3207
3208 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3209 self.connection_file_path = Some(connection_file_path.into());
3210 self
3211 }
3212
3213 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3215 self.capture_logs_dir = Some(logs_dir.into());
3216 self
3217 }
3218
3219 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3222 self.spawn_events.configure_incarnation(daemon_incarnation);
3226 self
3227 }
3228
3229 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3232 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3233 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3234 path,
3235 daemon_incarnation,
3236 )));
3237 this
3238 }
3239
3240 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3241 self.child_roster.counters = forwarding.counters();
3242 self.forwarding = Some(forwarding);
3243 self
3244 }
3245
3246 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3247 self.host_health = supervisor_handle.host_health.clone();
3248 self.spawn_events = supervisor_handle.spawn_events.clone();
3249 self.supervisor_handle = Some(supervisor_handle);
3250 self
3251 }
3252
3253 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3254 self.health = health;
3255 self
3256 }
3257
3258 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3262 self.child_roster.record_to(path.into());
3263 self
3264 }
3265
3266 #[cfg(target_os = "linux")]
3267 pub fn with_cgroup_placement(
3268 mut self,
3269 cgroup_placement: Option<subc_cgroup::Placement>,
3270 ) -> Self {
3271 self.child_roster
3272 .set_cgroup_placement(cgroup_placement.clone());
3273 self.cgroup_placement = cgroup_placement;
3274 self
3275 }
3276
3277 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3283 validate_spec(&spec)?;
3284 self.establish_identity(&spec);
3285
3286 let runtime = self.runtime_config();
3287 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3288 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3289 let registration_watch = watch_registration(&self.registry, &spec);
3290 let spawned = spawn_child(
3291 &spec,
3292 runtime.connection_file_path.as_deref(),
3293 self.supervisor_handle.as_ref(),
3294 &runtime.stderr_ring,
3295 runtime.capture_logs_dir.as_deref(),
3296 &runtime.child_roster,
3297 #[cfg(target_os = "linux")]
3298 runtime.cgroup_placement.as_ref(),
3299 );
3300 #[cfg(test)]
3301 self.test_after_first_spawn.run(&spec.module_id);
3302 let mut child = match spawned {
3303 Ok(child) => child,
3304 Err(err) => {
3305 self.abandon_unrostered(&spec.module_id);
3308 return Err(err);
3309 }
3310 };
3311 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3312 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3313 if let Some(spawn) = activity_spawn {
3314 spawn.complete(&snapshot);
3315 }
3316
3317 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3318 }
3319
3320 fn establish_identity(&self, spec: &ModuleSpec) {
3333 if let Some(supervisor_handle) = &self.supervisor_handle {
3334 supervisor_handle.apply_identity_configuration(spec);
3335 supervisor_handle.mark_configured(&spec.module_id);
3336 }
3337 }
3338
3339 fn abandon_unrostered(&self, module_id: &str) {
3342 if let Some(supervisor_handle) = &self.supervisor_handle {
3343 supervisor_handle.unmark_configured_unless_rostered(module_id);
3344 }
3345 }
3346
3347 fn mark_first_process_running(
3350 &self,
3351 spec: &ModuleSpec,
3352 runtime: &SupervisorRuntimeConfig,
3353 snapshot: &SharedSnapshot,
3354 child: &mut SupervisedChild,
3355 ) -> Result<(), SuperviseError> {
3356 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3357 containment::terminate_blocking(&runtime.child_roster.containment);
3358 self.abandon_unrostered(&spec.module_id);
3359 return Err(err);
3360 }
3361 self.process_liveness
3362 .track(spec.module_id.clone(), Arc::clone(snapshot));
3363 Ok(())
3364 }
3365
3366 pub fn supervise_configured(
3372 &self,
3373 spec: ModuleSpec,
3374 enabled: bool,
3375 ) -> Result<SupervisedModule, SuperviseError> {
3376 self.supervise_configured_requested(
3377 spec,
3378 enabled,
3379 self.health.clone(),
3380 None,
3381 self.restart_policy,
3382 DEFAULT_REGISTER_TIMEOUT,
3383 None,
3384 )
3385 }
3386
3387 pub fn supervise_configured_with_health(
3393 &self,
3394 spec: ModuleSpec,
3395 enabled: bool,
3396 health: HealthConfig,
3397 drain_timeout_ms: Option<u64>,
3398 restart_policy: RestartPolicy,
3399 ) -> Result<SupervisedModule, SuperviseError> {
3400 self.supervise_configured_requested(
3401 spec,
3402 enabled,
3403 health,
3404 drain_timeout_ms,
3405 restart_policy,
3406 DEFAULT_REGISTER_TIMEOUT,
3407 None,
3408 )
3409 }
3410
3411 #[allow(clippy::too_many_arguments)]
3412 pub(crate) fn supervise_configured_requested(
3413 &self,
3414 spec: ModuleSpec,
3415 enabled: bool,
3416 health: HealthConfig,
3417 drain_timeout_ms: Option<u64>,
3418 restart_policy: RestartPolicy,
3419 register_timeout: Duration,
3420 requester: Option<Principal>,
3421 ) -> Result<SupervisedModule, SuperviseError> {
3422 let registered = self.register_configured_with_health(
3423 spec,
3424 enabled,
3425 health,
3426 drain_timeout_ms,
3427 restart_policy,
3428 register_timeout,
3429 )?;
3430 let daemon_initiated = requester.is_none();
3431 self.start_registered_configured(registered, requester, daemon_initiated)
3432 }
3433
3434 pub(crate) fn register_configured_with_health(
3439 &self,
3440 spec: ModuleSpec,
3441 enabled: bool,
3442 health: HealthConfig,
3443 drain_timeout_ms: Option<u64>,
3444 restart_policy: RestartPolicy,
3445 register_timeout: Duration,
3446 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3447 validate_spec(&spec)?;
3448 self.establish_identity(&spec);
3449
3450 let mut runtime = self.runtime_config();
3451 runtime.health = health;
3452 runtime.restart_policy = restart_policy;
3453 runtime.register_timeout = register_timeout;
3454 if let Some(ms) = drain_timeout_ms {
3455 runtime.drain_timeout = Duration::from_millis(ms);
3456 *runtime
3457 .effective_drain_timeout
3458 .lock()
3459 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3460 }
3461 let snapshot = if enabled {
3462 SupervisorSnapshot::starting()
3463 } else {
3464 SupervisorSnapshot::disabled()
3465 };
3466 let snapshot = {
3467 let mut snapshot = snapshot;
3468 if enabled {
3469 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3470 {
3471 snapshot.waiting_for_previous_pid = Some(pid);
3472 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3473 }
3474 }
3475 snapshot
3476 };
3477 let (module, commands) =
3478 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3479 Ok(RegisteredConfiguredModule {
3480 spec,
3481 runtime,
3482 module,
3483 commands,
3484 })
3485 }
3486
3487 pub(crate) fn start_registered_configured(
3490 &self,
3491 registered: RegisteredConfiguredModule,
3492 requester: Option<Principal>,
3493 daemon_initiated: bool,
3494 ) -> Result<SupervisedModule, SuperviseError> {
3495 let RegisteredConfiguredModule {
3496 spec,
3497 mut runtime,
3498 module,
3499 commands,
3500 } = registered;
3501 runtime.start_requester = requester;
3502 runtime.recovery_is_daemon = daemon_initiated;
3503 let snapshot = &module.inner.snapshot;
3504 let should_spawn = {
3505 let state = lock_snapshot(snapshot)?;
3506 state.enabled && state.waiting_for_previous_pid.is_none()
3507 };
3508 let child = if should_spawn {
3509 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3510 let registration_watch = watch_registration(&self.registry, &spec);
3511 let spawned = spawn_child(
3512 &spec,
3513 runtime.connection_file_path.as_deref(),
3514 self.supervisor_handle.as_ref(),
3515 &runtime.stderr_ring,
3516 runtime.capture_logs_dir.as_deref(),
3517 &runtime.child_roster,
3518 #[cfg(target_os = "linux")]
3519 runtime.cgroup_placement.as_ref(),
3520 );
3521 #[cfg(test)]
3522 self.test_after_first_spawn.run(&spec.module_id);
3523 match spawned {
3524 Ok(mut child) => {
3525 arm_registration_deadline(
3526 &mut child,
3527 runtime.register_timeout,
3528 registration_watch,
3529 );
3530 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3531 if let Some(spawn) = activity_spawn {
3532 spawn.complete(snapshot);
3533 }
3534 Some(child)
3535 }
3536 Err(err) => {
3537 if runtime.health.critical {
3538 error!(
3539 module_id = %spec.module_id,
3540 program = %spec.program.display(),
3541 error = %err,
3542 "critical configured module failed to spawn; marking failed and alerting"
3543 );
3544 } else {
3545 error!(
3546 module_id = %spec.module_id,
3547 program = %spec.program.display(),
3548 error = %err,
3549 "configured module failed to spawn; marking failed and continuing"
3550 );
3551 }
3552 fail_snapshot(
3553 snapshot,
3554 Some(&spec.module_id),
3555 None,
3556 &runtime.failure_context(daemon_initiated),
3557 );
3558 None
3559 }
3560 }
3561 } else {
3562 None
3563 };
3564 self.start_monitor(spec, runtime, &module, child, commands);
3565 Ok(module)
3566 }
3567
3568 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3569 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3570 SupervisorRuntimeConfig {
3571 registry: self.registry.clone(),
3572 recovery_is_daemon: true,
3573 start_requester: None,
3574 pending_activity_restart: false,
3575 provenance_probe: self.provenance_probe.clone(),
3576 scheduled_respawn: Arc::default(),
3577 deferred_reload_reply: Arc::default(),
3578 restart_policy: self.restart_policy,
3579 register_timeout: DEFAULT_REGISTER_TIMEOUT,
3580 drain_timeout: self.drain_timeout,
3581 child_roster: self
3584 .child_roster
3585 .for_module(Arc::clone(&effective_drain_timeout)),
3586 effective_drain_timeout,
3587 default_drain_timeout: self.drain_timeout,
3588 health: self.health.clone(),
3589 host_health: self.host_health.clone(),
3590 connection_file_path: self.connection_file_path.clone(),
3591 capture_logs_dir: self.capture_logs_dir.clone(),
3592 forwarding: self.forwarding.clone(),
3593 supervisor_handle: self.supervisor_handle.clone(),
3594 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3595 terminal_ring: Arc::new(Mutex::new(
3596 TerminalRing::new(
3597 TerminalRingConfig::default(),
3598 self.daemon_start_clock.started_at_ms(),
3599 )
3600 .with_start_clock(self.daemon_start_clock)
3601 .with_journal(self.terminal_journal.clone())
3602 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3603 )),
3604 spawn_events: self.spawn_events.clone(),
3605 #[cfg(target_os = "linux")]
3606 cgroup_placement: self.cgroup_placement.clone(),
3607 #[cfg(test)]
3608 test_seed_stale_facts_before_enable_spawn: false,
3609 #[cfg(test)]
3610 test_reload_exit_record_gate: None,
3611 }
3612 }
3613
3614 fn supervised_module(
3615 &self,
3616 spec: ModuleSpec,
3617 runtime: SupervisorRuntimeConfig,
3618 snapshot: SharedSnapshot,
3619 child: Option<SupervisedChild>,
3620 ) -> SupervisedModule {
3621 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3622 self.start_monitor(spec, runtime, &module, child, commands);
3623 module
3624 }
3625
3626 fn register_module(
3627 &self,
3628 spec: &ModuleSpec,
3629 runtime: &SupervisorRuntimeConfig,
3630 snapshot: SharedSnapshot,
3631 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3632 runtime.host_health.track(spec, &snapshot);
3633 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3634 spec: spec.clone(),
3635 health: runtime.health.clone(),
3636 }));
3637 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3638 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3639 let restart_policy = runtime.restart_policy;
3643 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3644 let (tx, rx) = mpsc::channel(4);
3645 let module_id = spec.module_id.clone();
3646 let module = SupervisedModule {
3647 inner: Arc::new(SupervisedModuleInner {
3648 module_id: module_id.clone(),
3649 registry: Arc::clone(&self.registry),
3650 snapshot,
3651 configuration,
3652 stderr_ring,
3653 terminal_ring,
3654 commands: tx,
3655 monitor: Mutex::new(None),
3656 restart_policy,
3657 effective_drain_timeout,
3658 provenance_probe: self.provenance_probe.clone(),
3659 retained: runtime.child_roster.retained_view(),
3660 }),
3661 };
3662 if let Some(supervisor_handle) = &self.supervisor_handle {
3665 supervisor_handle.insert(module.clone());
3666 }
3667 (module, rx)
3668 }
3669
3670 fn start_monitor(
3671 &self,
3672 spec: ModuleSpec,
3673 runtime: SupervisorRuntimeConfig,
3674 module: &SupervisedModule,
3675 child: Option<SupervisedChild>,
3676 commands: mpsc::Receiver<SupervisorCommand>,
3677 ) {
3678 let monitor = tokio::spawn(supervise_loop(
3679 spec,
3680 runtime,
3681 Arc::clone(&self.registry),
3682 Arc::clone(&self.process_liveness),
3683 Arc::clone(&module.inner.snapshot),
3684 child,
3685 commands,
3686 ));
3687 *module
3688 .inner
3689 .monitor
3690 .lock()
3691 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3692 }
3693}
3694
3695impl Default for Supervisor {
3696 fn default() -> Self {
3697 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3698 }
3699}
3700
3701#[derive(Clone)]
3703pub struct SupervisedModule {
3704 inner: Arc<SupervisedModuleInner>,
3705}
3706
3707struct SupervisedModuleInner {
3708 retained: crate::child_roster::RetainedView,
3709 module_id: String,
3710 registry: Arc<Registry>,
3711 snapshot: SharedSnapshot,
3712 configuration: Arc<Mutex<SupervisedConfiguration>>,
3713 stderr_ring: Arc<Mutex<StderrRing>>,
3714 terminal_ring: Arc<Mutex<TerminalRing>>,
3715 commands: mpsc::Sender<SupervisorCommand>,
3716 monitor: Mutex<Option<JoinHandle<()>>>,
3717 restart_policy: RestartPolicy,
3721 effective_drain_timeout: Arc<Mutex<Duration>>,
3722 provenance_probe: ExecutableIdentityProbe,
3723}
3724
3725impl fmt::Debug for SupervisedModule {
3726 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3727 f.debug_struct("SupervisedModule")
3728 .field("module_id", &self.inner.module_id)
3729 .field("status", &self.status())
3730 .finish_non_exhaustive()
3731 }
3732}
3733
3734impl SupervisedModule {
3735 pub fn module_id(&self) -> &str {
3736 &self.inner.module_id
3737 }
3738
3739 #[cfg(test)]
3743 pub(crate) fn record_health_probe_failure_for_test(
3744 &self,
3745 detail: &str,
3746 ) -> Result<(), SuperviseError> {
3747 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3748 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3749 state.health.detail = Some(detail.to_string());
3750 })
3751 }
3752
3753 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3754 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3755 }
3756
3757 pub(crate) fn registration_end_reason(
3758 &self,
3759 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3760 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3761 Ok(match snapshot.state {
3762 ModuleState::Draining if snapshot.draining_to_replace => {
3763 Some(RegistrationEndReason::SupervisorRestart)
3764 }
3765 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3766 ModuleState::Restarting if snapshot.draining_to_replace => {
3767 Some(RegistrationEndReason::SupervisorRestart)
3768 }
3769 _ => None,
3770 })
3771 }
3772
3773 pub fn stderr_tail(
3780 &self,
3781 max_lines: Option<usize>,
3782 max_bytes: Option<usize>,
3783 ) -> StderrTailSnapshot {
3784 self.inner
3785 .stderr_ring
3786 .lock()
3787 .unwrap_or_else(|poisoned| poisoned.into_inner())
3788 .snapshot(max_lines, max_bytes)
3789 }
3790
3791 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3796 self.inner
3797 .terminal_ring
3798 .lock()
3799 .unwrap_or_else(|poisoned| poisoned.into_inner())
3800 .snapshot()
3801 }
3802
3803 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3808 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3809 }
3810
3811 pub(crate) async fn read_durable_terminal_history(
3816 &self,
3817 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3818 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3819 let module_id = self.inner.module_id.clone();
3820 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3821 .await
3822 }
3823
3824 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3825 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3826 .map(|(status, _)| status)
3827 }
3828
3829 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3830 let state = lock_snapshot(&self.inner.snapshot)?;
3831 Ok(state
3832 .failed_retry
3833 .pending
3834 .filter(|_| state.state == ModuleState::Failed && state.enabled)
3835 .map(|retry| {
3836 unix_ms_now().saturating_add(
3837 u64::try_from(
3838 retry
3839 .deadline
3840 .saturating_duration_since(Instant::now())
3841 .as_millis(),
3842 )
3843 .unwrap_or(u64::MAX),
3844 )
3845 }))
3846 }
3847
3848 pub(crate) fn record_deliberate_severance(
3849 &self,
3850 identity: ProcessIdentity,
3851 ) -> Result<bool, SuperviseError> {
3852 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3853 if snapshot.pid != Some(identity.pid)
3854 || snapshot.process_start_time != Some(identity.start_time)
3855 {
3856 return Ok(false);
3857 }
3858 snapshot.deliberate_severance = Some(identity);
3859 Ok(true)
3860 }
3861
3862 pub(crate) fn status_for_control(
3867 &self,
3868 caller: &'static str,
3869 ) -> Result<ModuleStatus, SuperviseError> {
3870 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3871 .map(|(status, _)| status)
3872 }
3873
3874 fn status_with_snapshot_lock(
3875 &self,
3876 snapshot: &SharedSnapshot,
3877 caller: Option<&'static str>,
3878 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3879 let mut guard = match caller {
3880 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3881 None => lock_snapshot(snapshot)?,
3882 };
3883 let restart_count =
3886 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3887 let snapshot = guard.clone();
3888 drop(guard);
3889 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3890 SuperviseError::StatePoisoned {
3891 module_id: Some(self.inner.module_id.clone()),
3892 }
3893 })?;
3894 let registration_active = self
3895 .inner
3896 .registry
3897 .get_module(&self.inner.module_id)
3898 .map_err(SuperviseError::Registry)?
3899 .is_some();
3900 let protocol = snapshot
3901 .spawned_protocol
3902 .unwrap_or(self.declared_protocol()?);
3903 let running_process =
3904 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3905 let live = match protocol {
3911 ModuleProtocol::Subc => running_process && registration_active,
3912 ModuleProtocol::None => running_process,
3913 };
3914 let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
3917 let carry_configured = self
3918 .inner
3919 .configuration
3920 .lock()
3921 .map_err(|_| SuperviseError::StatePoisoned {
3922 module_id: Some(self.inner.module_id.clone()),
3923 })?
3924 .spec
3925 .carry_containment;
3926 let recovery_block = carries_containment(
3927 carry_configured,
3928 self.inner
3929 .retained
3930 .has_carried_container(&self.inner.module_id),
3931 )
3932 .then(|| self.inner.retained.recovery_block())
3933 .flatten();
3934 let launch_nonce_source = observed_launch_nonce_source(&snapshot);
3935 let pid = snapshot.reported_pid();
3936 let mut health = snapshot.health;
3937 if let Some(detail) = recovery_block
3938 .as_ref()
3939 .or_else(|| retained.as_ref().map(|(_, detail)| detail))
3940 {
3941 health.detail = Some(match health.detail {
3942 Some(previous) => format!("{previous}; {detail}"),
3943 None => detail.clone(),
3944 });
3945 }
3946
3947 Ok((
3948 ModuleStatus {
3949 waiting_for_previous_pid: retained
3950 .as_ref()
3951 .map(|(pid, _)| *pid)
3952 .filter(|pid| *pid > 0)
3953 .or(snapshot.waiting_for_previous_pid),
3954 waiting_for_previous_detail: recovery_block
3955 .or_else(|| retained.map(|(_, detail)| detail))
3956 .or(snapshot.waiting_for_previous_detail),
3957 module_id: self.inner.module_id.clone(),
3958 launch_nonce_source,
3959 state: snapshot.state,
3960 enabled: snapshot.enabled,
3961 process_alive: snapshot.process_alive,
3962 registration_active,
3963 protocol,
3964 live,
3965 restart_count,
3966 lifetime_restarts: snapshot.lifetime_restarts,
3967 spawn_generation: snapshot.spawn_generation,
3968 max_restarts: self.inner.restart_policy.max_restarts,
3969 restart_window: self.inner.restart_policy.window,
3970 drain_timeout,
3971 restart_backoff: self.inner.restart_policy.backoff,
3972 restart_max_backoff: self.inner.restart_policy.max_backoff,
3973 pid,
3974 spawned_at_ms: snapshot.spawned_at_ms,
3975 spawned_from: snapshot.spawned_from,
3976 launch_spec_digest: snapshot.launch_spec_digest,
3977 process_start_time: snapshot.process_start_time,
3978 last_exit: snapshot.last_exit,
3979 health,
3980 },
3981 snapshot.spawned_file_identity.clone(),
3982 ))
3983 }
3984
3985 #[cfg(test)]
3986 pub(crate) fn hold_snapshot_for_test(
3987 &self,
3988 acquired: std::sync::mpsc::Sender<()>,
3989 hold: Duration,
3990 ) -> std::thread::JoinHandle<()> {
3991 let snapshot = Arc::clone(&self.inner.snapshot);
3992 std::thread::spawn(move || {
3993 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3994 acquired
3995 .send(())
3996 .expect("test receiver waits for snapshot lock");
3997 std::thread::sleep(hold);
3998 })
3999 }
4000
4001 pub(crate) async fn status_and_running_image_agreement(
4007 &self,
4008 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
4009 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
4010 let image = self
4011 .inner
4012 .provenance_probe
4013 .observe(
4014 status.pid,
4015 status.spawned_from.as_deref(),
4016 identity,
4017 status.process_start_time,
4018 )
4019 .await;
4020 Ok((status, image))
4021 }
4022
4023 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
4024 let snapshot = match lock_snapshot(&self.inner.snapshot) {
4025 Ok(snapshot) => snapshot.clone(),
4026 Err(_) => {
4027 return subc_control::RunningImageAgreement::Unavailable {
4028 reason: subc_control::RunningImageUnavailableReason::NotRunning,
4029 };
4030 }
4031 };
4032 self.inner
4033 .provenance_probe
4034 .observe(
4035 snapshot.reported_pid(),
4036 snapshot.spawned_from.as_deref(),
4037 snapshot.spawned_file_identity.clone(),
4038 snapshot.process_start_time,
4039 )
4040 .await
4041 }
4042
4043 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
4046 #[cfg(windows)]
4047 {
4048 let image = lock_snapshot(&self.inner.snapshot)
4049 .ok()
4050 .and_then(|snapshot| {
4051 snapshot
4052 .spawned_file_identity
4053 .as_ref()
4054 .and_then(|identity| identity.windows_image.clone())
4055 });
4056 match image {
4057 Some(image) => {
4058 crate::child_resources::read_windows_handle(image.process(), image.start_time())
4059 }
4060 None => subc_control::ChildResourceUsage::Unavailable {
4061 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
4062 },
4063 }
4064 }
4065 #[cfg(not(windows))]
4066 {
4067 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
4068 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
4069 Err(_) => {
4070 return subc_control::ChildResourceUsage::Unavailable {
4071 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
4072 }
4073 }
4074 };
4075 crate::child_resources::read(pid, start_time)
4076 }
4077 }
4078
4079 #[cfg(all(test, any(target_os = "linux", windows)))]
4080 pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
4081 self.inner
4082 .snapshot
4083 .try_lock()
4084 .map(|snapshot| snapshot.subtree_target.clone())
4085 .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
4086 }
4087
4088 #[cfg(test)]
4089 pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
4090 self.inner.snapshot.lock().unwrap().subtree_target = target;
4091 }
4092
4093 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
4094 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4095 Ok(match snapshot.state {
4096 ModuleState::Restarting => true,
4097 ModuleState::Failed | ModuleState::Disabled => false,
4098 _ if snapshot.enabled && snapshot.respawn_pending => true,
4099 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
4100 })
4101 }
4102
4103 #[cfg(test)]
4104 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
4105 self.is_warming_with_snapshot_lock(None)
4106 }
4107
4108 pub(crate) fn is_warming_for_control(
4109 &self,
4110 caller: &'static str,
4111 ) -> Result<bool, SuperviseError> {
4112 self.is_warming_with_snapshot_lock(Some(caller))
4113 }
4114
4115 fn is_warming_with_snapshot_lock(
4116 &self,
4117 caller: Option<&'static str>,
4118 ) -> Result<bool, SuperviseError> {
4119 let snapshot = match caller {
4120 Some(caller) => {
4121 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
4122 }
4123 None => lock_snapshot(&self.inner.snapshot)?,
4124 }
4125 .clone();
4126 Ok(matches!(
4127 snapshot.state,
4128 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
4129 ))
4130 }
4131
4132 pub async fn drain(&self) -> Result<(), SuperviseError> {
4134 self.stop().await
4135 }
4136
4137 pub(crate) async fn retire_requested(
4138 &self,
4139 requester: Option<Principal>,
4140 ) -> Result<(), SuperviseError> {
4141 match self.state()? {
4142 ModuleState::Stopped => return Ok(()),
4143 ModuleState::Failed
4144 if lock_snapshot(&self.inner.snapshot)?
4145 .failed_retry
4146 .pending
4147 .is_none() =>
4148 {
4149 return Ok(())
4150 }
4151 ModuleState::Failed
4152 | ModuleState::Starting
4153 | ModuleState::Running
4154 | ModuleState::Unresponsive
4155 | ModuleState::Restarting
4156 | ModuleState::Draining
4157 | ModuleState::Disabled => {}
4158 }
4159
4160 let (reply_tx, reply_rx) = oneshot::channel();
4161 self.inner
4162 .commands
4163 .send(SupervisorCommand::Retire {
4164 requester,
4165 reply: reply_tx,
4166 })
4167 .await
4168 .map_err(|_| SuperviseError::CommandClosed {
4169 module_id: self.inner.module_id.clone(),
4170 })?;
4171 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4172 module_id: self.inner.module_id.clone(),
4173 })?
4174 }
4175
4176 pub async fn stop(&self) -> Result<(), SuperviseError> {
4177 self.stop_requested(None).await
4178 }
4179
4180 pub(crate) async fn stop_requested(
4181 &self,
4182 requester: Option<Principal>,
4183 ) -> Result<(), SuperviseError> {
4184 match self.state()? {
4185 ModuleState::Stopped => return Ok(()),
4186 ModuleState::Failed
4187 if lock_snapshot(&self.inner.snapshot)?
4188 .failed_retry
4189 .pending
4190 .is_none() =>
4191 {
4192 return Ok(())
4193 }
4194 ModuleState::Failed
4195 | ModuleState::Starting
4196 | ModuleState::Running
4197 | ModuleState::Unresponsive
4198 | ModuleState::Restarting
4199 | ModuleState::Draining
4200 | ModuleState::Disabled => {}
4201 }
4202
4203 let (reply_tx, reply_rx) = oneshot::channel();
4204 self.inner
4205 .commands
4206 .send(SupervisorCommand::Drain {
4207 requester,
4208 reply: reply_tx,
4209 })
4210 .await
4211 .map_err(|_| SuperviseError::CommandClosed {
4212 module_id: self.inner.module_id.clone(),
4213 })?;
4214 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4215 module_id: self.inner.module_id.clone(),
4216 })?
4217 }
4218
4219 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4220 self.restart_requested(drain_timeout_ms, None).await
4221 }
4222
4223 pub(crate) async fn restart_requested(
4224 &self,
4225 drain_timeout_ms: Option<u64>,
4226 requester: Option<Principal>,
4227 ) -> Result<(), SuperviseError> {
4228 let queued_at = Instant::now();
4229 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4230 let (reply_tx, reply_rx) = oneshot::channel();
4231 self.inner
4232 .commands
4233 .send(SupervisorCommand::Restart {
4234 requester,
4235 drain_timeout_ms,
4236 received_at_generation,
4237 queued_at,
4238 reply: reply_tx,
4239 })
4240 .await
4241 .map_err(|_| SuperviseError::CommandClosed {
4242 module_id: self.inner.module_id.clone(),
4243 })?;
4244 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4245 module_id: self.inner.module_id.clone(),
4246 })?
4247 }
4248
4249 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4254 self.swap_requested(ready_timeout, None).await
4255 }
4256
4257 pub(crate) async fn swap_requested(
4258 &self,
4259 ready_timeout: Option<Duration>,
4260 requester: Option<Principal>,
4261 ) -> Result<(), SuperviseError> {
4262 let (reply_tx, reply_rx) = oneshot::channel();
4263 self.inner
4264 .commands
4265 .send(SupervisorCommand::Swap {
4266 requester,
4267 ready_timeout,
4268 reply: reply_tx,
4269 })
4270 .await
4271 .map_err(|_| SuperviseError::CommandClosed {
4272 module_id: self.inner.module_id.clone(),
4273 })?;
4274 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4275 module_id: self.inner.module_id.clone(),
4276 })?
4277 }
4278
4279 pub async fn reload(&self) -> Result<(), SuperviseError> {
4280 self.reload_requested(None).await
4281 }
4282
4283 pub(crate) async fn reload_requested(
4284 &self,
4285 requester: Option<Principal>,
4286 ) -> Result<(), SuperviseError> {
4287 let (reply_tx, reply_rx) = oneshot::channel();
4288 self.inner
4289 .commands
4290 .send(SupervisorCommand::Reload {
4291 requester,
4292 reply: reply_tx,
4293 })
4294 .await
4295 .map_err(|_| SuperviseError::CommandClosed {
4296 module_id: self.inner.module_id.clone(),
4297 })?;
4298 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4299 module_id: self.inner.module_id.clone(),
4300 })?
4301 }
4302
4303 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4304 self.set_enabled_requested(enabled, None).await
4305 }
4306
4307 pub(crate) async fn set_enabled_requested(
4308 &self,
4309 enabled: bool,
4310 requester: Option<Principal>,
4311 ) -> Result<bool, SuperviseError> {
4312 let (reply_tx, reply_rx) = oneshot::channel();
4313 self.inner
4314 .commands
4315 .send(SupervisorCommand::SetEnabled {
4316 requester,
4317 enabled,
4318 reply: reply_tx,
4319 })
4320 .await
4321 .map_err(|_| SuperviseError::CommandClosed {
4322 module_id: self.inner.module_id.clone(),
4323 })?;
4324 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4325 module_id: self.inner.module_id.clone(),
4326 })?
4327 }
4328
4329 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4333 let configured = self
4334 .inner
4335 .configuration
4336 .lock()
4337 .map_err(|_| SuperviseError::StatePoisoned {
4338 module_id: Some(self.inner.module_id.clone()),
4339 })?
4340 .spec
4341 .protocol;
4342 let state = lock_snapshot(&self.inner.snapshot)?;
4343 Ok(state.spawned_protocol.unwrap_or(configured))
4344 }
4345
4346 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4347 let configuration =
4348 self.inner
4349 .configuration
4350 .lock()
4351 .map_err(|_| SuperviseError::StatePoisoned {
4352 module_id: Some(self.inner.module_id.clone()),
4353 })?;
4354 Ok((configuration.spec.clone(), configuration.health.clone()))
4355 }
4356
4357 #[cfg(any(test, feature = "test-support"))]
4361 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4362 let (_, health) = self.configuration()?;
4363 let drain_timeout_ms = u64::try_from(
4364 self.inner
4365 .effective_drain_timeout
4366 .lock()
4367 .unwrap_or_else(|poisoned| poisoned.into_inner())
4368 .as_millis(),
4369 )
4370 .ok();
4371 self.update_configuration(spec, health, drain_timeout_ms)
4372 .await
4373 }
4374
4375 pub(crate) async fn update_configuration(
4376 &self,
4377 spec: ModuleSpec,
4378 health: HealthConfig,
4379 drain_timeout_ms: Option<u64>,
4380 ) -> Result<(), SuperviseError> {
4381 if spec.module_id != self.inner.module_id {
4382 return Err(SuperviseError::InvalidSpec {
4383 reason: "a supervised module's module_id cannot be changed".to_string(),
4384 });
4385 }
4386 validate_spec(&spec)?;
4387 let (reply_tx, reply_rx) = oneshot::channel();
4388 self.inner
4389 .commands
4390 .send(SupervisorCommand::UpdateConfiguration {
4391 spec: spec.clone(),
4392 health: health.clone(),
4393 drain_timeout_ms,
4394 reply: reply_tx,
4395 })
4396 .await
4397 .map_err(|_| SuperviseError::CommandClosed {
4398 module_id: self.inner.module_id.clone(),
4399 })?;
4400 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4401 module_id: self.inner.module_id.clone(),
4402 })?;
4403 let mut configuration =
4404 self.inner
4405 .configuration
4406 .lock()
4407 .map_err(|_| SuperviseError::StatePoisoned {
4408 module_id: Some(self.inner.module_id.clone()),
4409 })?;
4410 configuration.spec = spec;
4411 configuration.health = health;
4412 Ok(())
4413 }
4414}
4415
4416impl Drop for SupervisedModuleInner {
4417 fn drop(&mut self) {
4418 let Ok(mut monitor) = self.monitor.lock() else {
4419 return;
4420 };
4421 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4422 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4423 state.state = ModuleState::Stopped;
4424 clear_current_process_facts(state);
4425 });
4426 monitor.abort();
4427 }
4428 let _ = monitor.take();
4429 }
4430}
4431
4432#[derive(Debug)]
4433enum SupervisorCommand {
4434 Drain {
4435 requester: Option<Principal>,
4436 reply: oneshot::Sender<Result<(), SuperviseError>>,
4437 },
4438 Retire {
4439 requester: Option<Principal>,
4440 reply: oneshot::Sender<Result<(), SuperviseError>>,
4441 },
4442 Restart {
4443 requester: Option<Principal>,
4444 drain_timeout_ms: Option<u64>,
4449 received_at_generation: u64,
4453 queued_at: Instant,
4456 reply: oneshot::Sender<Result<(), SuperviseError>>,
4457 },
4458 Reload {
4459 requester: Option<Principal>,
4460 reply: oneshot::Sender<Result<(), SuperviseError>>,
4461 },
4462 SetEnabled {
4463 requester: Option<Principal>,
4464 enabled: bool,
4465 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4466 },
4467 UpdateConfiguration {
4468 spec: ModuleSpec,
4469 health: HealthConfig,
4470 drain_timeout_ms: Option<u64>,
4473 reply: oneshot::Sender<()>,
4474 },
4475 Swap {
4476 requester: Option<Principal>,
4477 ready_timeout: Option<Duration>,
4480 reply: oneshot::Sender<Result<(), SuperviseError>>,
4482 },
4483}
4484
4485#[derive(Debug)]
4486pub enum SuperviseError {
4487 InvalidSpec {
4488 reason: String,
4489 },
4490 Spawn {
4491 program: PathBuf,
4492 source: io::Error,
4493 cgroup_path: Option<PathBuf>,
4494 },
4495 Cgroup {
4496 module_id: String,
4497 source: io::Error,
4498 },
4499 LaunchNonce {
4502 reason: String,
4503 },
4504 Wait {
4505 module_id: String,
4506 source: io::Error,
4507 },
4508 Kill {
4509 module_id: String,
4510 source: io::Error,
4511 },
4512 Forwarding(ForwardingError),
4513 Registry(RegistryError),
4514 ReloadUnavailable {
4515 module_id: String,
4516 reason: String,
4517 },
4518 Disabled {
4523 module_id: String,
4524 },
4525 ReloadFailed {
4526 module_id: String,
4527 reason: String,
4528 },
4529 RegistrationStillActive {
4530 module_id: String,
4531 waited: Duration,
4532 },
4533 StatePoisoned {
4534 module_id: Option<String>,
4535 },
4536 CommandClosed {
4537 module_id: String,
4538 },
4539 SwapInProgress {
4543 module_id: String,
4544 },
4545 SwapRefused {
4547 module_id: String,
4548 reason: SwapRefusal,
4549 },
4550 SwapFailed {
4554 module_id: String,
4555 arm: SwapFailureArm,
4556 detail: String,
4557 candidate_exit: Option<ExitReport>,
4560 },
4561}
4562
4563#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4565pub enum SwapRefusal {
4566 OverlapExclusive,
4568 NotRegistered,
4571 ProtocolNone,
4574 NotConfigured,
4577 AlreadySwapping,
4579}
4580
4581impl SwapRefusal {
4582 pub fn as_str(self) -> &'static str {
4583 match self {
4584 Self::OverlapExclusive => "overlap_exclusive",
4585 Self::NotRegistered => "not_registered",
4586 Self::ProtocolNone => "protocol_none",
4587 Self::NotConfigured => "not_configured",
4588 Self::AlreadySwapping => "already_swapping",
4589 }
4590 }
4591}
4592
4593#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4596pub enum SwapFailureArm {
4597 SpawnFailed,
4599 NeverRegistered,
4601 NeverReady,
4603 CandidateExited,
4605 CandidateUnhealthy,
4607 Interrupted,
4611 CutoverLost,
4616}
4617
4618impl SwapFailureArm {
4619 pub fn as_str(self) -> &'static str {
4620 match self {
4621 Self::SpawnFailed => "spawn_failed",
4622 Self::NeverRegistered => "never_registered",
4623 Self::NeverReady => "never_ready",
4624 Self::CandidateExited => "candidate_exited",
4625 Self::CandidateUnhealthy => "candidate_unhealthy",
4626 Self::Interrupted => "interrupted",
4627 Self::CutoverLost => "cutover_lost",
4628 }
4629 }
4630}
4631
4632impl fmt::Display for SuperviseError {
4633 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4634 match self {
4635 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4636 Self::Spawn {
4637 program,
4638 source,
4639 cgroup_path: Some(cgroup_path),
4640 } => write!(
4641 f,
4642 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4643 cgroup_path.display(),
4644 program.display()
4645 ),
4646 Self::Spawn {
4647 program,
4648 source,
4649 cgroup_path: None,
4650 } => write!(
4651 f,
4652 "failed to spawn module '{}': {source}",
4653 program.display()
4654 ),
4655 Self::Cgroup { module_id, source } => {
4656 write!(
4657 f,
4658 "failed to prepare cgroup for module '{module_id}': {source}"
4659 )
4660 }
4661 Self::LaunchNonce { reason } => {
4662 write!(
4663 f,
4664 "failed to generate reserved-module launch nonce: {reason}"
4665 )
4666 }
4667 Self::Wait { module_id, source } => {
4668 write!(f, "failed to wait for module '{module_id}': {source}")
4669 }
4670 Self::Kill { module_id, source } => {
4671 write!(f, "failed to kill module '{module_id}': {source}")
4672 }
4673 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4674 Self::Registry(err) => write!(f, "registry error: {err}"),
4675 Self::ReloadUnavailable { module_id, reason } => {
4676 write!(f, "reload unavailable for module '{module_id}': {reason}")
4677 }
4678 Self::Disabled { module_id } => {
4679 write!(
4680 f,
4681 "module '{module_id}' is disabled; enable it before restart or reload"
4682 )
4683 }
4684 Self::ReloadFailed { module_id, reason } => {
4685 write!(f, "reload failed for module '{module_id}': {reason}")
4686 }
4687 Self::RegistrationStillActive { module_id, waited } => write!(
4688 f,
4689 "module '{module_id}' registration remained active after waiting {waited:?}"
4690 ),
4691 Self::StatePoisoned { module_id } => match module_id {
4692 Some(module_id) => {
4693 write!(f, "supervisor state for module '{module_id}' was poisoned")
4694 }
4695 None => write!(f, "supervisor state was poisoned"),
4696 },
4697 Self::CommandClosed { module_id } => {
4698 write!(
4699 f,
4700 "supervisor command channel for module '{module_id}' is closed"
4701 )
4702 }
4703 Self::SwapInProgress { module_id } => write!(
4704 f,
4705 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4706 ),
4707 Self::SwapRefused { module_id, reason } => match reason {
4708 SwapRefusal::OverlapExclusive => write!(
4709 f,
4710 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4711 ),
4712 SwapRefusal::NotRegistered => write!(
4713 f,
4714 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4715 ),
4716 SwapRefusal::ProtocolNone => write!(
4717 f,
4718 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4719 ),
4720 SwapRefusal::NotConfigured => write!(
4721 f,
4722 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4723 ),
4724 SwapRefusal::AlreadySwapping => {
4725 write!(f, "module '{module_id}' is already being swapped")
4726 }
4727 },
4728 Self::SwapFailed {
4729 module_id,
4730 arm,
4731 detail,
4732 ..
4733 } => write!(
4734 f,
4735 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4736 arm.as_str()
4737 ),
4738 }
4739 }
4740}
4741
4742impl Error for SuperviseError {
4743 fn source(&self) -> Option<&(dyn Error + 'static)> {
4744 match self {
4745 Self::Spawn { source, .. }
4746 | Self::Cgroup { source, .. }
4747 | Self::Wait { source, .. }
4748 | Self::Kill { source, .. } => Some(source),
4749 Self::Forwarding(err) => Some(err),
4750 Self::Registry(err) => Some(err),
4751 Self::LaunchNonce { .. }
4752 | Self::InvalidSpec { .. }
4753 | Self::ReloadUnavailable { .. }
4754 | Self::Disabled { .. }
4755 | Self::ReloadFailed { .. }
4756 | Self::RegistrationStillActive { .. }
4757 | Self::StatePoisoned { .. }
4758 | Self::CommandClosed { .. }
4759 | Self::SwapInProgress { .. }
4760 | Self::SwapRefused { .. }
4761 | Self::SwapFailed { .. } => None,
4762 }
4763 }
4764}
4765
4766pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4767 if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
4768 return Err(SuperviseError::InvalidSpec {
4769 reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
4770 });
4771 }
4772 if spec.module_id.trim().is_empty() {
4773 return Err(SuperviseError::InvalidSpec {
4774 reason: "module_id must not be empty".to_string(),
4775 });
4776 }
4777
4778 Ok(())
4779}
4780
4781#[derive(Debug, Default)]
4782struct HealthProbeRuntime {
4783 spawn_generation: Option<u64>,
4784 configured_health: Option<HealthConfig>,
4785 registered_connection: Option<crate::ConnectionId>,
4786 advertised: bool,
4787 next_probe_at: Option<Instant>,
4788 probe_index: u64,
4789}
4790
4791fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4792 lock_snapshot(snapshot)
4793 .ok()
4794 .and_then(|state| state.spawned_protocol)
4795 .unwrap_or(spec.protocol)
4796}
4797
4798impl HealthProbeRuntime {
4799 fn refresh_registration(
4800 &mut self,
4801 spec: &ModuleSpec,
4802 runtime: &SupervisorRuntimeConfig,
4803 registry: &Registry,
4804 snapshot: &SharedSnapshot,
4805 ) {
4806 let generation = lock_snapshot(snapshot)
4807 .ok()
4808 .map(|state| state.spawn_generation);
4809 if self.spawn_generation != generation {
4810 *self = Self {
4814 spawn_generation: generation,
4815 ..Self::default()
4816 };
4817 }
4818 if self.configured_health.as_ref() != Some(&runtime.health) {
4819 self.configured_health = Some(runtime.health.clone());
4820 self.next_probe_at = None;
4821 self.registered_connection = None;
4822 self.probe_index = 0;
4823 }
4824 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4828 self.registered_connection = None;
4829 self.advertised = runtime.health.http.is_some();
4830 if !self.advertised {
4831 self.next_probe_at = None;
4832 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4833 let unknown = ModuleHealthStatus::default();
4834 if state.health != unknown {
4835 state.health = unknown;
4836 state.last_probe_evidence = None;
4837 }
4838 });
4839 } else if self.next_probe_at.is_none() {
4840 self.next_probe_at = Some(
4841 Instant::now()
4842 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4843 );
4844 }
4845 return;
4846 }
4847
4848 let registration = match registry.get_module(&spec.module_id) {
4849 Ok(registration) => registration,
4850 Err(err) => {
4851 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4852 self.advertised = false;
4853 self.next_probe_at = None;
4854 return;
4855 }
4856 };
4857
4858 let Some(registration) = registration else {
4859 self.registered_connection = None;
4860 self.advertised = false;
4861 self.next_probe_at = None;
4862 return;
4863 };
4864
4865 let advertised = registration
4866 .control_ops
4867 .iter()
4868 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4869 if !advertised {
4870 self.registered_connection = Some(registration.connection_id);
4871 self.advertised = false;
4872 self.next_probe_at = None;
4873 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4874 if state.health.status != SupervisorHealthStatus::Unknown
4875 || state.health.consecutive_failures != 0
4876 || state.health.last_probe_ms.is_some()
4877 || state.health.detail.is_some()
4878 || state.health.metrics.is_some()
4879 {
4880 state.health.status = SupervisorHealthStatus::Unknown;
4881 state.last_probe_evidence = None;
4882 state.health.consecutive_failures = 0;
4883 state.health.last_probe_ms = None;
4884 state.health.detail = None;
4885 state.health.metrics = None;
4886 }
4887 });
4888 return;
4889 }
4890
4891 let reregistered = self.registered_connection != Some(registration.connection_id);
4892 self.registered_connection = Some(registration.connection_id);
4893 self.advertised = true;
4894 if reregistered || self.next_probe_at.is_none() {
4895 self.probe_index = 0;
4896 self.next_probe_at = Some(
4897 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4898 );
4899 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4900 state.health.status = SupervisorHealthStatus::Unknown;
4901 state.last_probe_evidence = None;
4902 state.health.consecutive_failures = 0;
4903 state.health.detail = None;
4904 state.health.metrics = None;
4905 });
4906 }
4907 }
4908
4909 fn wake_at(&self) -> Option<Instant> {
4910 if !self.advertised {
4911 return None;
4912 }
4913 self.next_probe_at
4914 }
4915
4916 fn due(&self) -> bool {
4917 self.advertised
4918 && self
4919 .next_probe_at
4920 .is_some_and(|next| Instant::now() >= next)
4921 }
4922
4923 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4924 self.probe_index = self.probe_index.wrapping_add(1);
4925 self.next_probe_at = Some(
4926 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4927 );
4928 }
4929}
4930
4931#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4935enum HealthProbeEvidence {
4936 LaneDead,
4938 NoAnswer,
4940 BadAnswer,
4942 Misconfigured,
4944}
4945
4946#[derive(Debug)]
4947struct HealthProbeError {
4948 evidence: HealthProbeEvidence,
4949 message: String,
4950}
4951
4952impl HealthProbeError {
4953 fn lane_dead(message: impl Into<String>) -> Self {
4954 Self::with(HealthProbeEvidence::LaneDead, message)
4955 }
4956
4957 fn no_answer(message: impl Into<String>) -> Self {
4958 Self::with(HealthProbeEvidence::NoAnswer, message)
4959 }
4960
4961 fn bad_answer(message: impl Into<String>) -> Self {
4962 Self::with(HealthProbeEvidence::BadAnswer, message)
4963 }
4964
4965 fn misconfigured(message: impl Into<String>) -> Self {
4966 Self::with(HealthProbeEvidence::Misconfigured, message)
4967 }
4968
4969 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4970 Self {
4971 evidence,
4972 message: message.into(),
4973 }
4974 }
4975
4976 #[allow(dead_code)]
4985 fn is_proof_of_death(&self) -> bool {
4986 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4987 }
4988
4989 fn label(&self) -> &'static str {
4997 match self.evidence {
4998 HealthProbeEvidence::LaneDead => "lane-dead",
4999 HealthProbeEvidence::NoAnswer => "no-answer",
5000 HealthProbeEvidence::BadAnswer => "bad-answer",
5001 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
5002 }
5003 }
5004}
5005
5006impl fmt::Display for HealthProbeError {
5007 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5008 f.write_str(&self.message)
5009 }
5010}
5011
5012async fn run_health_probe_cycle(
5013 spec: &ModuleSpec,
5014 runtime: &SupervisorRuntimeConfig,
5015 registry: &Registry,
5016 process_liveness: &SupervisorProcessLiveness,
5017 snapshot: &SharedSnapshot,
5018 child: &mut Option<SupervisedChild>,
5019) {
5020 let now_ms = unix_ms_now();
5021 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
5022 .then_some(runtime.health.http.as_deref())
5023 .flatten();
5024 let result = match http {
5025 Some(url) => probe_http_health(url, runtime.health.deadline).await,
5026 None => probe_module_health(&spec.module_id, runtime, None).await,
5027 };
5028 match result {
5029 Ok(report) => {
5030 handle_health_report(
5031 spec,
5032 runtime,
5033 registry,
5034 process_liveness,
5035 snapshot,
5036 child,
5037 report,
5038 now_ms,
5039 )
5040 .await;
5041 }
5042 Err(err) => {
5043 if http.is_some() {
5044 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5045 state.health.status = SupervisorHealthStatus::Failing;
5046 });
5047 }
5048 handle_health_probe_failure(
5049 spec,
5050 runtime,
5051 registry,
5052 process_liveness,
5053 snapshot,
5054 child,
5055 err,
5056 now_ms,
5057 )
5058 .await;
5059 }
5060 }
5061}
5062
5063pub(crate) struct HttpProbeTarget<'a> {
5064 address: std::net::SocketAddr,
5065 localhost: bool,
5066 authority: &'a str,
5067 path: String,
5068}
5069
5070pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
5073 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
5074 return Err("must not contain whitespace, controls, or a fragment".into());
5075 }
5076 let rest = url
5077 .strip_prefix("http://")
5078 .ok_or("must use plain http://")?;
5079 let split = rest.find(['/', '?']).unwrap_or(rest.len());
5080 let (authority, suffix) = rest.split_at(split);
5081 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
5082 ("::1", rest)
5083 } else {
5084 let split = authority.find(':').unwrap_or(authority.len());
5085 authority.split_at(split)
5086 };
5087 let ip: std::net::IpAddr = match host {
5088 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
5089 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
5090 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
5091 };
5092 let port = if port.is_empty() {
5093 80
5094 } else {
5095 port.strip_prefix(':')
5096 .and_then(|p| p.parse::<u16>().ok())
5097 .filter(|p| *p > 0)
5098 .ok_or("must have a valid nonzero TCP port")?
5099 };
5100 let path = if suffix.is_empty() {
5101 "/".into()
5102 } else if suffix.starts_with('?') {
5103 format!("/{suffix}")
5104 } else {
5105 suffix.into()
5106 };
5107 Ok(HttpProbeTarget {
5108 address: std::net::SocketAddr::new(ip, port),
5109 localhost: host == "localhost",
5110 authority,
5111 path,
5112 })
5113}
5114
5115async fn probe_http_health(
5116 url: &str,
5117 deadline: Duration,
5118) -> Result<HealthReport, HealthProbeError> {
5119 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
5120 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
5121 let mut response_status = String::new();
5124 let mut body = Vec::new();
5125 let probe = async {
5126 let connection = match tokio::net::TcpStream::connect(target.address).await {
5129 Err(_) if target.localhost => {
5130 tokio::net::TcpStream::connect((
5131 std::net::Ipv6Addr::LOCALHOST,
5132 target.address.port(),
5133 ))
5134 .await
5135 }
5136 result => result,
5137 };
5138 let mut stream = connection.map_err(|error| {
5139 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
5140 })?;
5141 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
5142 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
5143 let mut reader = BufReader::new(stream);
5144 let mut budget = 16 * 1024;
5145 let status = http_line(&mut reader, &mut budget).await?;
5146 let mut words = status.split_ascii_whitespace();
5147 let version = words.next();
5148 let code = words
5149 .next()
5150 .filter(|word| word.len() == 3)
5151 .and_then(|word| word.parse::<u16>().ok());
5152 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
5153 || !code.is_some_and(|code| (100..600).contains(&code))
5154 {
5155 return Err(HealthProbeError::bad_answer(format!(
5156 "invalid HTTP status: {status}"
5157 )));
5158 }
5159 let code = code.expect("validated status code");
5160 response_status = status.clone();
5161 let mut length = None;
5162 let mut chunked = false;
5163 loop {
5164 let line = http_line(&mut reader, &mut budget).await?;
5165 if line.is_empty() {
5166 break;
5167 }
5168 if let Some((name, value)) = line.split_once(':') {
5169 if name.eq_ignore_ascii_case("content-length") {
5170 length = Some(value.trim().parse::<u64>().map_err(|_| {
5171 HealthProbeError::bad_answer("invalid HTTP Content-Length")
5172 })?);
5173 } else if name.eq_ignore_ascii_case("transfer-encoding") {
5174 chunked = value.trim().eq_ignore_ascii_case("chunked");
5175 }
5176 }
5177 }
5178 if chunked {
5179 while body.len() < 200 {
5180 let line = http_line(&mut reader, &mut budget).await?;
5181 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5182 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5183 if size == 0 {
5184 break;
5185 }
5186 let count = size.min((200 - body.len()) as u64) as usize;
5187 let start = body.len();
5188 (&mut reader)
5189 .take(count as u64)
5190 .read_to_end(&mut body)
5191 .await
5192 .map_err(|error| {
5193 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5194 })?;
5195 if body.len() - start != count {
5196 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5197 }
5198 if size > count as u64 || body.len() == 200 {
5199 break;
5200 }
5201 if !http_line(&mut reader, &mut budget).await?.is_empty() {
5202 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5203 }
5204 }
5205 } else {
5206 reader
5207 .take(length.unwrap_or(200).min(200))
5208 .read_to_end(&mut body)
5209 .await
5210 .map_err(|error| {
5211 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5212 })?;
5213 }
5214 if (200..300).contains(&code) {
5215 Ok(HealthReport::ok())
5216 } else {
5217 Err(HealthProbeError::bad_answer(
5218 "HTTP health endpoint returned non-2xx",
5219 ))
5220 }
5221 };
5222 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5223 Err(HealthProbeError::no_answer(format!(
5224 "HTTP probe timed out after {deadline:?}"
5225 )))
5226 });
5227 if let Err(error) = &mut result {
5228 if !response_status.is_empty() {
5229 error.message = format!(
5230 "{}; {response_status}: {}",
5231 error.message,
5232 String::from_utf8_lossy(&body)
5233 );
5234 }
5235 }
5236 result
5237}
5238
5239async fn http_line(
5240 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5241 remaining: &mut usize,
5242) -> Result<String, HealthProbeError> {
5243 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5244 let mut line = Vec::new();
5245 (&mut *reader)
5246 .take(*remaining as u64)
5247 .read_until(b'\n', &mut line)
5248 .await
5249 .map_err(|error| {
5250 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5251 })?;
5252 *remaining -= line.len();
5253 if !line.ends_with(b"\r\n") {
5254 return Err(HealthProbeError::bad_answer(
5255 "HTTP headers are incomplete or exceed 16 KiB",
5256 ));
5257 }
5258 line.truncate(line.len() - 2);
5259 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5260}
5261
5262async fn probe_module_health(
5263 module_id: &str,
5264 runtime: &SupervisorRuntimeConfig,
5265 drain_deadline: Option<Instant>,
5266) -> Result<HealthReport, HealthProbeError> {
5267 let Some(forwarding) = runtime.forwarding.as_ref() else {
5268 return Err(HealthProbeError::misconfigured(
5269 "supervisor was not configured with a forwarding table",
5270 ));
5271 };
5272 let probe_started_at = Instant::now();
5273 let mut deadline = probe_started_at + runtime.health.deadline;
5274 if let Some(drain_deadline) = drain_deadline {
5275 deadline = deadline.min(drain_deadline);
5276 }
5277 let pending = if drain_deadline.is_some() {
5278 forwarding.begin_drain_health_probe_rpc_for(
5279 module_id,
5280 MODULE_CONTROL_OP_HEALTH_CHECK,
5281 probe_started_at,
5282 deadline,
5283 )
5284 } else {
5285 forwarding.begin_health_probe_rpc_for(
5286 module_id,
5287 MODULE_CONTROL_OP_HEALTH_CHECK,
5288 probe_started_at,
5289 deadline,
5290 )
5291 }
5292 .map_err(|err| {
5293 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5296 })?;
5297 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5298}
5299
5300async fn probe_endpoint_health(
5307 endpoint: crate::ModuleEndpointId,
5308 runtime: &SupervisorRuntimeConfig,
5309 deadline_cap: Option<Instant>,
5310) -> Result<HealthReport, HealthProbeError> {
5311 let Some(forwarding) = runtime.forwarding.as_ref() else {
5312 return Err(HealthProbeError::misconfigured(
5313 "supervisor was not configured with a forwarding table",
5314 ));
5315 };
5316 let probe_started_at = Instant::now();
5317 let mut deadline = probe_started_at + runtime.health.deadline;
5318 if let Some(cap) = deadline_cap {
5319 deadline = deadline.min(cap);
5320 }
5321 let pending = forwarding
5322 .begin_endpoint_health_probe_rpc_for(
5323 endpoint,
5324 MODULE_CONTROL_OP_HEALTH_CHECK,
5325 probe_started_at,
5326 deadline,
5327 )
5328 .map_err(|err| {
5329 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5330 })?;
5331 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5332}
5333
5334async fn await_health_probe(
5336 forwarding: &ForwardingTable,
5337 pending: PendingModuleControlRpc,
5338 deadline: Instant,
5339 probe_budget: Duration,
5340) -> Result<HealthReport, HealthProbeError> {
5341 let PendingModuleControlRpc {
5342 endpoint,
5343 module_sink,
5344 negotiated_ver,
5345 corr,
5346 receiver,
5347 } = pending;
5348 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5349 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5350 })?;
5351 let frame = Frame::build_with_version(
5352 negotiated_ver,
5353 FrameType::Request,
5354 control_flags(),
5355 0,
5356 0,
5357 corr,
5358 body,
5359 )
5360 .map_err(|err| {
5361 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5362 })?;
5363
5364 match timeout_at(deadline, module_sink.send(frame)).await {
5370 Ok(Ok(())) => {}
5371 Ok(Err(err)) => {
5372 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5373 return Err(HealthProbeError::lane_dead(format!(
5376 "failed to send health.check: {err}"
5377 )));
5378 }
5379 Err(_elapsed) => {
5380 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5381 return Err(HealthProbeError::no_answer(
5385 "health.check send timed out before enqueue (module egress full)",
5386 ));
5387 }
5388 }
5389
5390 match timeout_at(deadline, receiver).await {
5391 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5395 response.health_report().ok_or_else(|| {
5396 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5397 })
5398 }
5399 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5400 format!("health.check rejected: {}", body.message),
5401 )),
5402 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5403 Err(HealthProbeError::lane_dead(message))
5404 }
5405 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5406 Err(HealthProbeError::bad_answer(message))
5407 }
5408 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5409 Err(HealthProbeError::bad_answer(format!(
5410 "expected module-control op '{expected}', got '{actual}'"
5411 )))
5412 }
5413 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5417 "module answered health.check after its daemon deadline",
5418 )),
5419 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5420 "health.check waiter was canceled before the module responded",
5421 )),
5422 Err(_) => {
5423 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5424 Err(HealthProbeError::no_answer(format!(
5425 "module did not answer health.check within {probe_budget:?}"
5426 )))
5427 }
5428 }
5429}
5430
5431#[allow(clippy::too_many_arguments)]
5432async fn handle_health_report(
5433 spec: &ModuleSpec,
5434 runtime: &SupervisorRuntimeConfig,
5435 registry: &Registry,
5436 process_liveness: &SupervisorProcessLiveness,
5437 snapshot: &SharedSnapshot,
5438 child: &mut Option<SupervisedChild>,
5439 report: HealthReport,
5440 now_ms: u64,
5441) {
5442 let status = supervisor_health_status(report.status);
5443 let detail = report.detail.clone();
5444 let metrics = truncate_health_metrics(report.metrics);
5445 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5446 state.health.status = status;
5447 state.last_probe_evidence = None;
5448 state.health.last_probe_ms = Some(now_ms);
5449 state.health.detail = detail.clone();
5450 state.health.metrics = metrics.clone();
5451 state.health.consecutive_failures = 0;
5452 state.failed_retry.observe_health(
5453 report.status == HealthStatus::Ok,
5454 runtime.restart_policy.window,
5455 );
5456 });
5457
5458 let action = match report.status {
5459 HealthStatus::Ok => return,
5460 HealthStatus::Degraded => runtime.health.on_degraded,
5461 HealthStatus::Failing => runtime.health.on_failing,
5462 };
5463 apply_l3_health_action(
5464 spec,
5465 runtime,
5466 registry,
5467 process_liveness,
5468 snapshot,
5469 child,
5470 status,
5471 detail.as_deref(),
5472 action,
5473 now_ms,
5474 )
5475 .await;
5476}
5477
5478#[allow(clippy::too_many_arguments)]
5479async fn handle_health_probe_failure(
5480 spec: &ModuleSpec,
5481 runtime: &SupervisorRuntimeConfig,
5482 registry: &Registry,
5483 process_liveness: &SupervisorProcessLiveness,
5484 snapshot: &SharedSnapshot,
5485 child: &mut Option<SupervisedChild>,
5486 err: HealthProbeError,
5487 now_ms: u64,
5488) {
5489 let threshold = runtime.health.failure_threshold.max(1);
5490 let held = if err.evidence == HealthProbeEvidence::NoAnswer {
5491 runtime.host_health.record_late_module(&spec.module_id)
5492 } else {
5493 false
5494 };
5495 let mut failures = 0;
5496 let detail = if held {
5501 format!("[{}] {err}; host overloaded", err.label())
5502 } else {
5503 format!("[{}] {err}", err.label())
5504 };
5505 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5506 state.last_probe_evidence = Some(err.evidence);
5507 state
5508 .failed_retry
5509 .observe_health(false, runtime.restart_policy.window);
5510 if held {
5513 state.health.status = SupervisorHealthStatus::Degraded;
5514 } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5515 state.health.status = SupervisorHealthStatus::Unknown;
5516 }
5517 state.health.last_probe_ms = Some(now_ms);
5518 state.health.consecutive_failures = if held {
5521 0
5522 } else {
5523 state.health.consecutive_failures.saturating_add(1)
5524 };
5525 state.health.detail = Some(detail.clone());
5526 state.health.metrics = None;
5527 failures = state.health.consecutive_failures;
5528 });
5529
5530 if held || failures < threshold {
5531 warn!(
5532 module_id = %spec.module_id,
5533 consecutive_failures = failures,
5534 threshold,
5535 evidence = err.label(),
5536 detail = %detail,
5537 "health.check probe failed"
5538 );
5539 return;
5540 }
5541
5542 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5543 state.state = ModuleState::Unresponsive;
5544 state.health.status = SupervisorHealthStatus::Unresponsive;
5545 });
5546 if runtime.health.critical {
5550 error!(
5551 module_id = %spec.module_id,
5552 status = "unresponsive",
5553 evidence = err.label(),
5554 detail = %detail,
5555 "critical module health alert"
5556 );
5557 } else {
5558 warn!(
5559 module_id = %spec.module_id,
5560 status = "unresponsive",
5561 evidence = err.label(),
5562 detail = %detail,
5563 "module health threshold breached"
5564 );
5565 }
5566 if let Err(err) = health_restart_child(
5567 spec,
5568 runtime,
5569 registry,
5570 process_liveness,
5571 snapshot,
5572 child,
5573 SupervisorHealthStatus::Unresponsive,
5574 Some(&detail),
5575 now_ms,
5576 )
5577 .await
5578 {
5579 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5580 }
5581}
5582
5583#[allow(clippy::too_many_arguments)]
5584async fn apply_l3_health_action(
5585 spec: &ModuleSpec,
5586 runtime: &SupervisorRuntimeConfig,
5587 registry: &Registry,
5588 process_liveness: &SupervisorProcessLiveness,
5589 snapshot: &SharedSnapshot,
5590 child: &mut Option<SupervisedChild>,
5591 status: SupervisorHealthStatus,
5592 detail: Option<&str>,
5593 action: HealthAction,
5594 now_ms: u64,
5595) {
5596 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5597 match action {
5598 HealthAction::Report => {
5599 info!(
5600 module_id = %spec.module_id,
5601 status = ?status,
5602 detail,
5603 "module reported non-ok health"
5604 );
5605 }
5606 HealthAction::Alert => {
5607 error!(
5608 module_id = %spec.module_id,
5609 status = ?status,
5610 detail,
5611 "module health alert"
5612 );
5613 }
5614 HealthAction::Restart => {
5615 if let Err(err) = health_restart_child(
5616 spec,
5617 runtime,
5618 registry,
5619 process_liveness,
5620 snapshot,
5621 child,
5622 status,
5623 detail,
5624 now_ms,
5625 )
5626 .await
5627 {
5628 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5629 }
5630 }
5631 }
5632}
5633
5634#[allow(clippy::too_many_arguments)]
5635async fn health_restart_child(
5636 spec: &ModuleSpec,
5637 runtime: &SupervisorRuntimeConfig,
5638 registry: &Registry,
5639 process_liveness: &SupervisorProcessLiveness,
5640 snapshot: &SharedSnapshot,
5641 child: &mut Option<SupervisedChild>,
5642 status: SupervisorHealthStatus,
5643 detail: Option<&str>,
5644 now_ms: u64,
5645) -> Result<(), SuperviseError> {
5646 let (enabled, schedule) = {
5647 let mut state = lock_snapshot(snapshot)?;
5648 let enabled = state.enabled;
5649 let schedule = if enabled {
5650 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5651 } else {
5652 None
5653 };
5654 (enabled, schedule)
5655 };
5656
5657 if !enabled {
5658 return Err(SuperviseError::Disabled {
5659 module_id: spec.module_id.clone(),
5660 });
5661 }
5662
5663 if schedule.is_none() {
5664 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5665 error!(
5666 module_id = %spec.module_id,
5667 status = ?status,
5668 detail,
5669 max_restarts = runtime.restart_policy.max_restarts,
5670 window_secs = runtime.restart_policy.window.as_secs(),
5671 reason = %runtime.restart_policy.budget_exhausted_detail(),
5672 "health restart budget exhausted; marking module failed"
5673 );
5674 let stop_notice = begin_forwarding_drain_if_configured(
5675 spec,
5676 runtime,
5677 registry,
5678 snapshot,
5679 Some(true),
5680 RouteCloseReason::Disable,
5681 )
5682 .await?;
5683 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5684 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5685 })?;
5686 let result = drain_optional_child(
5687 &spec.module_id,
5688 spec.protocol,
5689 stop_notice,
5690 registry,
5691 runtime.forwarding.as_deref(),
5692 snapshot,
5693 &runtime.terminal_ring,
5694 &runtime.spawn_events,
5695 child,
5696 runtime.drain_timeout,
5697 ModuleState::Failed,
5698 Some(true),
5699 &runtime.failure_context(true),
5700 )
5701 .await;
5702 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5703 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5704 state.failed_retry.schedule(&spec.module_id);
5705 }
5706 })?;
5707 result?;
5708 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5709 return Ok(());
5710 }
5711
5712 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5713 let mut restart_count = 0;
5714 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5715 restart_count = state.crash_restarts.len();
5716 state.state = ModuleState::Unresponsive;
5717 state.health.status = status;
5718 state.health.last_action = Some(HealthAction::Restart.to_string());
5719 state.health.last_action_ms = Some(now_ms);
5720 })?;
5721 warn!(
5722 module_id = %spec.module_id,
5723 status = ?status,
5724 detail,
5725 restart_count,
5726 restart_in_window = schedule.restart_in_window,
5727 delay_ms = schedule.delay.as_millis() as u64,
5728 "health-triggered module restart"
5729 );
5730
5731 let stop_notice = begin_forwarding_drain_if_configured(
5732 spec,
5733 runtime,
5734 registry,
5735 snapshot,
5736 Some(true),
5737 RouteCloseReason::Restart,
5738 )
5739 .await?;
5740 drain_optional_child(
5741 &spec.module_id,
5742 spec.protocol,
5743 stop_notice,
5744 registry,
5745 runtime.forwarding.as_deref(),
5746 snapshot,
5747 &runtime.terminal_ring,
5748 &runtime.spawn_events,
5749 child,
5750 runtime.drain_timeout,
5751 ModuleState::Restarting,
5752 Some(true),
5753 &runtime.failure_context(true),
5754 )
5755 .await?;
5756 schedule_respawn(
5757 runtime,
5758 snapshot,
5759 &spec.module_id,
5760 schedule.delay,
5761 RespawnKind::Spawn,
5762 true,
5763 )
5764}
5765
5766fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5767 if let Some(reply) = runtime
5768 .deferred_reload_reply
5769 .lock()
5770 .unwrap_or_else(|p| p.into_inner())
5771 .take()
5772 {
5773 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5774 module_id: module_id.to_string(),
5775 reason: reason.to_string(),
5776 }));
5777 }
5778}
5779
5780fn schedule_respawn(
5781 runtime: &SupervisorRuntimeConfig,
5782 snapshot: &SharedSnapshot,
5783 module_id: &str,
5784 delay: Duration,
5785 kind: RespawnKind,
5786 daemon_initiated: bool,
5787) -> Result<(), SuperviseError> {
5788 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5789 update_snapshot(snapshot, Some(module_id), |state| {
5790 state.respawn_pending = true
5791 })?;
5792 *runtime
5793 .scheduled_respawn
5794 .lock()
5795 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5796 deadline: Instant::now() + delay,
5797 kind,
5798 daemon_initiated,
5799 });
5800 Ok(())
5801}
5802
5803fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5804 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5805 state.health.last_action = Some(action);
5806 state.health.last_action_ms = Some(now_ms);
5807 });
5808}
5809
5810fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5811 match status {
5812 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5813 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5814 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5815 }
5816}
5817
5818fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5830 let metrics = metrics?;
5831 match serde_json::to_vec(&metrics) {
5832 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5833 "truncated": true,
5834 "original_bytes": encoded.len(),
5835 })),
5836 Ok(_) | Err(_) => Some(metrics),
5837 }
5838}
5839
5840fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5846 if cadence.is_zero() {
5847 return Duration::ZERO;
5848 }
5849 let cadence_ms = cadence.as_millis() as u64;
5850 if cadence_ms == 0 {
5866 return cadence;
5867 }
5868 let jitter_span = (cadence_ms / 10).max(1);
5883 let hash = module_id.as_bytes().iter().fold(
5884 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5885 |acc, byte| {
5886 acc.wrapping_mul(1099511628211)
5887 .wrapping_add(u64::from(*byte))
5888 },
5889 );
5890 cadence + Duration::from_millis(hash % jitter_span)
5891}
5892
5893#[cfg(test)]
5894mod tests {
5895 use super::*;
5896
5897 #[test]
5898 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5899 let handle = SupervisorHandle::new();
5900 let module_id = "readded-tombstone";
5901 handle.record_rescan_removal(module_id);
5902 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5903
5904 handle.apply_identity_configuration(&ModuleSpec {
5905 module_id: module_id.to_string(),
5906 program: PathBuf::from("/test/module"),
5907 args: Vec::new(),
5908 env: Vec::new(),
5909 reserved: false,
5910 reserved_prefixes: Vec::new(),
5911 protocol: ModuleProtocol::Subc,
5912 overlap: Default::default(),
5913 carry_containment: false,
5914 });
5915
5916 assert!(
5917 handle.removal_tombstone_age_ms(module_id).is_none(),
5918 "a re-added module must not retain a stale removal tombstone"
5919 );
5920 }
5921
5922 #[derive(Debug, PartialEq, Eq)]
5925 struct OwnerInSpawnWindow {
5926 module_id: String,
5927 configured: bool,
5928 on_roster: bool,
5929 admission_refusal: Option<&'static str>,
5930 }
5931
5932 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5943 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5944 use crate::scopes::ScopeTable;
5945 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5946
5947 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5948 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5949 module_id: module_id.to_string(),
5950 program,
5951 args: Vec::new(),
5952 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5953 .into_iter()
5954 .map(|key| (key.to_string(), dir.path().display().to_string()))
5955 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5956 .collect(),
5957 reserved: false,
5958 reserved_prefixes: Vec::new(),
5959 protocol: ModuleProtocol::Subc,
5960 overlap: Default::default(),
5961 carry_containment: false,
5962 };
5963 let live = super::terminal_history_tests::fake_aft_stub_path();
5964 let missing = dir.path().join("definitely-missing-module");
5965
5966 let handle = SupervisorHandle::new();
5967 let mut supervisor =
5968 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5969 .with_handle(handle.clone());
5970 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5971 let hook_handle = handle.clone();
5972 let hook_observed = Arc::clone(&observed);
5973 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5974 let configured = hook_handle.is_configured(module_id);
5977 let selector = ScopeSelector::new(
5978 Principal::Reserved {
5979 module_id: module_id.to_string(),
5980 },
5981 "s".to_string(),
5982 Some(1),
5983 );
5984 let carrier = Principal::Reserved {
5985 module_id: "carrier".to_string(),
5986 };
5987 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5988 .admit(&carrier, module_id, &selector, configured)
5989 {
5990 Ok(_) => None,
5991 Err(refusal) => Some(refusal.code),
5992 };
5993 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5994 module_id: module_id.to_string(),
5995 configured,
5996 on_roster: hook_handle.get(module_id).is_some(),
5997 admission_refusal,
5998 });
5999 })));
6000
6001 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
6002 let configured = supervisor
6003 .supervise_configured(stub("configured", live.clone()), true)
6004 .unwrap();
6005 let with_health = supervisor
6006 .supervise_configured_with_health(
6007 stub("with-health", live.clone()),
6008 true,
6009 HealthConfig::default(),
6010 None,
6011 RestartPolicy::default(),
6012 )
6013 .unwrap();
6014 let failed = supervisor
6017 .supervise_configured_with_health(
6018 stub("failed-spawn", missing.clone()),
6019 true,
6020 HealthConfig::default(),
6021 None,
6022 RestartPolicy::default(),
6023 )
6024 .unwrap();
6025 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
6028
6029 let expected = [
6030 "plain",
6031 "configured",
6032 "with-health",
6033 "failed-spawn",
6034 "spawn-error",
6035 ]
6036 .into_iter()
6037 .map(|module_id| OwnerInSpawnWindow {
6038 module_id: module_id.to_string(),
6039 configured: true,
6040 on_roster: !matches!(module_id, "plain" | "spawn-error"),
6041 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
6042 })
6043 .collect::<Vec<_>>();
6044 assert_eq!(*observed.lock().unwrap(), expected);
6045
6046 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
6047 assert!(
6048 handle.get(module_id).is_some(),
6049 "{module_id} is on the roster"
6050 );
6051 assert!(
6052 handle.is_configured(module_id),
6053 "{module_id} stays configured"
6054 );
6055 }
6056 assert!(handle.get("spawn-error").is_none());
6057 assert!(
6058 !handle.is_configured("spawn-error"),
6059 "a plain spawn that failed must not leave its module marked configured"
6060 );
6061
6062 handle.retire("failed-spawn");
6064 assert!(!handle.is_configured("failed-spawn"));
6065
6066 for module in [plain, configured, with_health] {
6067 module.stop().await.unwrap();
6068 }
6069 drop(failed);
6070 }
6071
6072 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6079 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
6080 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
6081 let spec = ModuleSpec {
6082 module_id: "spawned-not-admitted".to_string(),
6083 program: super::terminal_history_tests::fake_aft_stub_path(),
6084 args: Vec::new(),
6085 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6086 .into_iter()
6087 .map(|key| (key.to_string(), dir.path().display().to_string()))
6088 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
6089 .collect(),
6090 reserved: false,
6091 reserved_prefixes: Vec::new(),
6092 protocol: ModuleProtocol::Subc,
6093 overlap: Default::default(),
6094 carry_containment: false,
6095 };
6096 let handle = SupervisorHandle::new();
6097 let mut supervisor =
6098 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6099 .with_handle(handle.clone());
6100 let observed = Arc::new(Mutex::new(None));
6101 let hook_handle = handle.clone();
6102 let hook_observed = Arc::clone(&observed);
6103 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
6104 let state = hook_handle
6105 .get(module_id)
6106 .expect("a configured module is on the roster before it is spawned")
6107 .state()
6108 .unwrap();
6109 *hook_observed.lock().unwrap() = Some(state);
6110 })));
6111
6112 let module = supervisor.supervise_configured(spec, true).unwrap();
6113 assert_eq!(
6114 *observed.lock().unwrap(),
6115 Some(ModuleState::Starting),
6116 "between the spawn and its admission the supervisor still reports starting"
6117 );
6118 assert_eq!(module.state().unwrap(), ModuleState::Running);
6119 module.stop().await.unwrap();
6120 }
6121
6122 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
6123 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
6124 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
6125 snapshot.process_alive = true;
6126 snapshot.pid = Some(41);
6127 snapshot.spawned_at_ms = Some(42);
6128 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
6129 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
6130 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
6131 device: 43,
6132 inode: 44,
6133 #[cfg(windows)]
6134 windows_image: None,
6135 });
6136 })
6137 .unwrap();
6138 snapshot
6139 }
6140
6141 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
6142 let snapshot = lock_snapshot(snapshot).unwrap();
6143 assert!(!snapshot.process_alive);
6144 assert_eq!(snapshot.pid, None);
6145 assert_eq!(snapshot.spawned_at_ms, None);
6146 assert_eq!(snapshot.spawned_from, None);
6147 assert_eq!(snapshot.launch_spec_digest, None);
6148 assert_eq!(snapshot.spawned_file_identity, None);
6149 }
6150
6151 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6152 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
6153 let supervisor =
6154 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6155 let mut runtime = supervisor.runtime_config();
6156 runtime.test_seed_stale_facts_before_enable_spawn = true;
6157 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
6158 let mut child = None;
6159 let spec = ModuleSpec {
6160 module_id: "failed-enable-clears-facts".to_string(),
6161 program: PathBuf::from("/definitely/missing/failed-enable-module"),
6162 args: Vec::new(),
6163 env: Vec::new(),
6164 reserved: false,
6165 reserved_prefixes: Vec::new(),
6166 protocol: ModuleProtocol::Subc,
6167 overlap: Default::default(),
6168 carry_containment: false,
6169 };
6170
6171 let result = set_child_enabled(
6172 &spec,
6173 &runtime,
6174 &supervisor.registry,
6175 &supervisor.process_liveness,
6176 &snapshot,
6177 &mut child,
6178 true,
6179 &runtime.failure_context(false),
6180 )
6181 .await;
6182
6183 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
6184 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6185 assert_snapshot_process_facts_cleared(&snapshot);
6186 }
6187
6188 #[tokio::test]
6189 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
6190 let supervisor =
6191 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6192 let runtime = supervisor.runtime_config();
6193 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
6194 ModuleState::Restarting,
6195 true,
6196 )));
6197 let spec = ModuleSpec {
6198 module_id: "start-stranded-restarting".to_string(),
6199 program: super::terminal_history_tests::fake_aft_stub_path(),
6200 args: Vec::new(),
6201 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6202 reserved: false,
6203 reserved_prefixes: Vec::new(),
6204 protocol: ModuleProtocol::None,
6205 overlap: Default::default(),
6206 carry_containment: false,
6207 };
6208 let mut child = None;
6209 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6210 assert!(!super::set_child_enabled(
6211 &spec,
6212 &runtime,
6213 &Registry::default(),
6214 &supervisor.process_liveness,
6215 &snapshot,
6216 &mut child,
6217 true,
6218 &runtime.failure_context(false),
6219 )
6220 .await
6221 .unwrap());
6222 assert!(child.is_none());
6223 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6224 assert!(super::set_child_enabled(
6225 &spec,
6226 &runtime,
6227 &Registry::default(),
6228 &supervisor.process_liveness,
6229 &snapshot,
6230 &mut child,
6231 true,
6232 &runtime.failure_context(false),
6233 )
6234 .await
6235 .unwrap());
6236 assert_eq!(
6237 lock_snapshot(&snapshot).unwrap().state,
6238 ModuleState::Running
6239 );
6240 let mut child = child.unwrap();
6241 child.start_kill().unwrap();
6242 child.wait().await.unwrap();
6243 }
6244
6245 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6246 async fn failed_reload_spawn_clears_current_process_facts() {
6247 let supervisor =
6248 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6249 let mut runtime = supervisor.runtime_config();
6250 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6251 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6252 let mut child = None;
6253 let spec = ModuleSpec {
6254 module_id: "failed-reload-clears-facts".to_string(),
6255 program: PathBuf::from("/unused/failed-reload-module"),
6256 args: Vec::new(),
6257 env: Vec::new(),
6258 reserved: false,
6259 reserved_prefixes: Vec::new(),
6260 protocol: ModuleProtocol::Subc,
6261 overlap: Default::default(),
6262 carry_containment: false,
6263 };
6264
6265 let result = handle_reload_spawn_failure(
6266 &spec,
6267 &runtime,
6268 &supervisor.process_liveness,
6269 &snapshot,
6270 &mut child,
6271 "forced reload spawn failure".to_string(),
6272 )
6273 .await;
6274
6275 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6276 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6277 assert_snapshot_process_facts_cleared(&snapshot);
6278 }
6279
6280 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6281 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6282 let supervisor =
6283 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6284 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6285 let module = supervisor.supervised_module(
6286 ModuleSpec {
6287 module_id: "drop-clears-facts".to_string(),
6288 program: PathBuf::from("/unused/drop-module"),
6289 args: Vec::new(),
6290 env: Vec::new(),
6291 reserved: false,
6292 reserved_prefixes: Vec::new(),
6293 protocol: ModuleProtocol::Subc,
6294 overlap: Default::default(),
6295 carry_containment: false,
6296 },
6297 supervisor.runtime_config(),
6298 Arc::clone(&snapshot),
6299 None,
6300 );
6301 assert!(!module
6302 .inner
6303 .monitor
6304 .lock()
6305 .unwrap()
6306 .as_ref()
6307 .unwrap()
6308 .is_finished());
6309
6310 drop(module);
6311
6312 assert_eq!(
6313 lock_snapshot(&snapshot).unwrap().state,
6314 ModuleState::Stopped
6315 );
6316 assert_snapshot_process_facts_cleared(&snapshot);
6317 }
6318
6319 #[cfg(unix)]
6320 #[tokio::test]
6321 async fn rescan_preserves_running_protocol_until_respawn() {
6322 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6323 let initial = ModuleSpec {
6324 module_id: "rescan-protocol".into(),
6325 program: PathBuf::from("/bin/sleep"),
6326 args: vec!["60".into()],
6327 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6328 .into_iter()
6329 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6330 .collect(),
6331 reserved: false,
6332 reserved_prefixes: vec![],
6333 protocol: ModuleProtocol::None,
6334 overlap: Default::default(),
6335 carry_containment: false,
6336 };
6337 let supervisor =
6338 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6339 let module = supervisor.spawn(initial.clone()).unwrap();
6340 assert!(module.status().unwrap().live);
6341 let mut next = initial;
6342 next.protocol = ModuleProtocol::Subc;
6343 module
6344 .update_configuration(next.clone(), HealthConfig::default(), None)
6345 .await
6346 .unwrap();
6347 assert!(
6348 module.status().unwrap().live,
6349 "rescan must not require HELLO from the old non-wire process"
6350 );
6351 let runtime = supervisor.runtime_config();
6352 let action = on_child_exit(
6353 &next,
6354 RestartPolicy::default(),
6355 &supervisor.registry,
6356 &module.inner.snapshot,
6357 &runtime.terminal_ring,
6358 &runtime.spawn_events,
6359 &runtime.child_roster,
6360 ExitReport {
6361 kind: ExitKind::Clean,
6362 code: Some(0),
6363 signal: None,
6364 at_ms: unix_ms_now(),
6365 },
6366 true,
6367 )
6368 .await;
6369 assert!(
6370 matches!(action, NextAction::Restart { .. }),
6371 "the old non-wire process's clean exit must restart"
6372 );
6373 module.drain().await.unwrap();
6374 }
6375
6376 #[cfg(unix)]
6377 #[tokio::test]
6378 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6379 use std::os::unix::fs::PermissionsExt;
6380 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6381 let script = dir.join("module.sh");
6382 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6383 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6384 let record_path = dir.join("live-children.json");
6385 let supervisor =
6386 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6387 .with_live_children_record(&record_path);
6388 for (program, args) in [
6389 (PathBuf::from("sleep"), vec!["60".into()]),
6390 (script, vec![]),
6391 ] {
6392 let spec = ModuleSpec {
6393 module_id: "image-identity".into(),
6394 program: program.clone(),
6395 args,
6396 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6397 .into_iter()
6398 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6399 .collect(),
6400 reserved: false,
6401 reserved_prefixes: vec![],
6402 protocol: ModuleProtocol::None,
6403 overlap: Default::default(),
6404 carry_containment: false,
6405 };
6406 let module = supervisor.spawn(spec).unwrap();
6407 #[cfg(target_os = "macos")]
6408 {
6409 let deadline = Instant::now() + Duration::from_secs(5);
6412 while crate::live_children::read_record(&record_path)
6413 .unwrap()
6414 .iter()
6415 .all(|entry| entry.executable.is_none())
6416 {
6417 assert!(Instant::now() < deadline, "module image was not confirmed");
6418 tokio::time::sleep(Duration::from_millis(5)).await;
6419 }
6420 }
6421 let entry = crate::live_children::read_record(&record_path)
6422 .unwrap()
6423 .pop()
6424 .unwrap();
6425 let observed = subc_os::Process::open(entry.pid)
6426 .unwrap()
6427 .unwrap()
6428 .observe()
6429 .unwrap();
6430 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6431 module.drain().await.unwrap();
6432 assert_eq!(
6433 verdict,
6434 crate::live_children::IdentityVerdict::Matches,
6435 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6436 );
6437 }
6438 }
6439
6440 #[cfg(unix)]
6441 fn http_fixture(
6442 dir: &std::path::Path,
6443 url: &str,
6444 threshold: u32,
6445 ) -> crate::daemon_config::ConfiguredModule {
6446 let path = dir.join("subc.jsonc");
6447 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6448 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6449 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6450 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6451 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6452 }}}).to_string()).unwrap();
6453 crate::daemon_config::load(&path)
6454 .unwrap()
6455 .unwrap()
6456 .modules
6457 .pop()
6458 .unwrap()
6459 }
6460
6461 #[cfg(unix)]
6462 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6463 timeout(Duration::from_secs(5), async {
6464 loop {
6465 if module.status().unwrap().health.status == status {
6466 break;
6467 }
6468 sleep(Duration::from_millis(5)).await;
6469 }
6470 })
6471 .await
6472 .unwrap_or_else(|_| {
6473 panic!(
6474 "expected {status:?}, got {:?}",
6475 module.status().unwrap().health
6476 )
6477 });
6478 }
6479
6480 #[cfg(unix)]
6481 #[tokio::test]
6482 async fn http_health_status_flips_ok_failing_ok() {
6483 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6484 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6485 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6486 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6487 let serving_status = status.clone();
6488 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6489 let server = tokio::spawn(async move {
6490 loop {
6491 let (mut stream, _) = listener.accept().await.unwrap();
6492 let mut request = [0u8; 2048];
6493 let count = stream.read(&mut request).await.unwrap();
6494 assert!(count > 0, "a probe must send an HTTP request");
6495 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6496 let body = if code == 200 {
6497 "ready"
6498 } else {
6499 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6500 };
6501 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6502 let _ = stream.write_all(response.as_bytes()).await;
6503 }
6504 });
6505 let configured = http_fixture(&dir, &url, 1000);
6506 let module =
6507 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6508 .supervise_configured_with_health(
6509 configured.module_spec(),
6510 true,
6511 configured.health,
6512 configured.drain_timeout_ms,
6513 configured.restart,
6514 )
6515 .unwrap();
6516 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6517 status.store(503, std::sync::atomic::Ordering::SeqCst);
6518 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6519 assert!(module
6520 .status()
6521 .unwrap()
6522 .health
6523 .detail
6524 .unwrap()
6525 .contains("scratch failure"));
6526 status.store(200, std::sync::atomic::Ordering::SeqCst);
6527 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6528 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6529 let before = module.status().unwrap();
6530 let (spec, mut health) = module.configuration().unwrap();
6531 health.http = None;
6532 module
6533 .update_configuration(spec.clone(), health.clone(), Some(10))
6534 .await
6535 .unwrap();
6536 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6537 health.http = Some(url);
6538 module
6539 .update_configuration(spec, health, Some(10))
6540 .await
6541 .unwrap();
6542 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6543 assert_eq!(
6544 module.status().unwrap().pid,
6545 before.pid,
6546 "changing a probe must apply live, not restart its process"
6547 );
6548 let (spec, mut health) = module.configuration().unwrap();
6549 health.failure_threshold = 2;
6550 module
6551 .update_configuration(spec, health, Some(10))
6552 .await
6553 .unwrap();
6554 status.store(503, std::sync::atomic::Ordering::SeqCst);
6555 timeout(Duration::from_secs(5), async {
6556 while module.status().unwrap().spawn_generation == before.spawn_generation {
6557 sleep(Duration::from_millis(5)).await;
6558 }
6559 })
6560 .await
6561 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6562 module.drain().await.unwrap();
6563 server.abort();
6564 }
6565
6566 #[cfg(unix)]
6567 #[tokio::test]
6568 async fn http_health_replacement_recovers_after_one_restart() {
6569 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6570 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6571 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6572 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6573 let mut configured = http_fixture(&dir, &url, 2);
6574 configured.restart.max_restarts = 2;
6575 let module =
6576 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6577 .supervise_configured_with_health(
6578 configured.module_spec(),
6579 true,
6580 configured.health,
6581 configured.drain_timeout_ms,
6582 configured.restart,
6583 )
6584 .unwrap();
6585 let first_generation = module.status().unwrap().spawn_generation;
6586 let observed = module.clone();
6587 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6588 let server = tokio::spawn(async move {
6589 let mut generation = first_generation;
6590 let mut probes = 0;
6591 loop {
6592 let (mut stream, _) = listener.accept().await.unwrap();
6593 let mut request = [0; 2048];
6594 assert!(stream.read(&mut request).await.unwrap() > 0);
6595 let current = observed.status().unwrap().spawn_generation;
6596 if current != generation {
6597 generation = current;
6598 probes = 0;
6599 }
6600 probes += 1;
6601 let healthy = generation != first_generation && probes > 1;
6604 let code = if healthy { 200 } else { 503 };
6605 let response = format!(
6606 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6607 );
6608 let _ = stream.write_all(response.as_bytes()).await;
6609 if healthy {
6610 healthy_tx.send(()).await.unwrap();
6611 }
6612 }
6613 });
6614 timeout(Duration::from_secs(5), async {
6615 let mut healthy_probes = 0;
6616 loop {
6617 if module.state().unwrap() == ModuleState::Failed {
6618 break;
6619 }
6620 while healthy_rx.try_recv().is_ok() {
6621 healthy_probes += 1;
6622 }
6623 if healthy_probes >= 3
6624 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6625 {
6626 break;
6627 }
6628 sleep(Duration::from_millis(5)).await;
6629 }
6630 })
6631 .await
6632 .expect("the replacement must settle healthy without exhausting its restart budget");
6633 let status = module.status().unwrap();
6634 module.drain().await.unwrap();
6635 server.abort();
6636 assert_eq!(status.state, ModuleState::Running);
6637 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6638 assert_eq!(status.health.consecutive_failures, 0);
6639 assert_eq!(status.lifetime_restarts, 1);
6640 assert_eq!(status.restart_count, 1);
6641 }
6642
6643 #[cfg(unix)]
6644 #[tokio::test]
6645 async fn http_health_sustained_failure_exhausts_restart_budget() {
6646 use std::collections::BTreeMap;
6647 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6648 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6649 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6650 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6651 let mut configured = http_fixture(&dir, &url, 2);
6652 configured.restart.max_restarts = 2;
6653 let module =
6654 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6655 .supervise_configured_with_health(
6656 configured.module_spec(),
6657 true,
6658 configured.health,
6659 configured.drain_timeout_ms,
6660 configured.restart,
6661 )
6662 .unwrap();
6663 let observed = module.clone();
6664 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6665 let served = probes.clone();
6666 let server = tokio::spawn(async move {
6667 loop {
6668 let (mut stream, _) = listener.accept().await.unwrap();
6669 let mut request = [0; 2048];
6670 assert!(stream.read(&mut request).await.unwrap() > 0);
6671 *served
6672 .lock()
6673 .unwrap()
6674 .entry(observed.status().unwrap().spawn_generation)
6675 .or_default() += 1;
6676 let _ = stream
6677 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6678 .await;
6679 }
6680 });
6681 timeout(Duration::from_secs(5), async {
6682 while module.state().unwrap() != ModuleState::Failed {
6683 sleep(Duration::from_millis(5)).await;
6684 }
6685 })
6686 .await
6687 .expect("sustained failures must still exhaust the health restart budget");
6688 let status = module.status().unwrap();
6689 server.abort();
6690 assert_eq!(status.restart_count, 2);
6691 assert_eq!(status.lifetime_restarts, 2);
6692 assert!(!status.process_alive);
6693 assert_eq!(
6694 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6695 vec![2, 2, 2]
6696 );
6697 }
6698
6699 #[cfg(unix)]
6700 #[tokio::test]
6701 async fn spawn_clears_the_previous_process_health() {
6702 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6703 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6704 configured.health.cadence = Duration::from_secs(3600);
6705 let module =
6706 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6707 .supervise_configured_with_health(
6708 configured.module_spec(),
6709 false,
6710 configured.health,
6711 configured.drain_timeout_ms,
6712 configured.restart,
6713 )
6714 .unwrap();
6715 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6716 state.health.status = SupervisorHealthStatus::Unresponsive;
6717 state.health.consecutive_failures = 3;
6718 state.health.detail = Some("old process refused connections".into());
6719 state.health.metrics = Some(serde_json::json!({"old": true}));
6720 state.health.last_probe_ms = Some(123);
6721 state.health.last_action = Some("restart".into());
6722 state.health.last_action_ms = Some(124);
6723 })
6724 .unwrap();
6725 module.set_enabled(true).await.unwrap();
6726 let health = module.status().unwrap().health;
6727 module.drain().await.unwrap();
6728 assert_eq!(health.consecutive_failures, 0);
6729 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6730 assert_eq!(health.detail, None);
6731 assert_eq!(health.metrics, None);
6732 assert_eq!(health.last_probe_ms, None);
6733 assert_eq!(health.last_action.as_deref(), Some("restart"));
6734 assert_eq!(health.last_action_ms, Some(124));
6735 }
6736
6737 #[cfg(unix)]
6738 #[tokio::test]
6739 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6740 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6741 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6742 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6743 drop(unused);
6744 let configured = http_fixture(&dir, &url, 2);
6745 let module =
6746 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6747 .supervise_configured_with_health(
6748 configured.module_spec(),
6749 true,
6750 configured.health,
6751 configured.drain_timeout_ms,
6752 configured.restart,
6753 )
6754 .unwrap();
6755 let before = module.status().unwrap().spawn_generation;
6756 timeout(Duration::from_secs(5), async {
6757 loop {
6758 let status = module.status().unwrap();
6759 if status.spawn_generation > before {
6760 assert!(status.lifetime_restarts > 0);
6761 break;
6762 }
6763 sleep(Duration::from_millis(5)).await;
6764 }
6765 })
6766 .await
6767 .expect("sustained HTTP refusal must trigger the health restart policy");
6768 module.drain().await.unwrap();
6769 }
6770
6771 #[cfg(unix)]
6772 #[tokio::test]
6773 async fn http_health_timeout_honours_deadline() {
6774 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6775 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6776 let server = tokio::spawn(async move {
6777 let _held = listener.accept().await.unwrap();
6778 std::future::pending::<()>().await;
6779 });
6780 let error = timeout(
6781 Duration::from_secs(1),
6782 probe_http_health(&url, Duration::from_millis(10)),
6783 )
6784 .await
6785 .expect("the probe must enforce its own deadline")
6786 .unwrap_err();
6787 server.abort();
6788 assert!(error.to_string().contains("timed out"));
6789 }
6790
6791 #[cfg(unix)]
6792 #[tokio::test]
6793 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6794 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6795 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6796 let url = format!(
6797 "http://localhost:{}/healthz",
6798 listener.local_addr().unwrap().port()
6799 );
6800 let server = tokio::spawn(async move {
6801 let (mut stream, _) = listener.accept().await.unwrap();
6802 let mut request = [0u8; 2048];
6803 assert!(stream.read(&mut request).await.unwrap() > 0);
6804 stream
6805 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6806 .await
6807 .unwrap();
6808 });
6809 assert_eq!(
6814 probe_http_health(&url, Duration::from_secs(10))
6815 .await
6816 .unwrap()
6817 .status,
6818 HealthStatus::Ok
6819 );
6820 server.await.unwrap();
6821 }
6822
6823 #[cfg(unix)]
6824 #[tokio::test]
6825 async fn http_health_timeout_keeps_partial_status_and_body() {
6826 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6827 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6828 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6829 let server = tokio::spawn(async move {
6830 let (mut stream, _) = listener.accept().await.unwrap();
6831 let mut request = [0u8; 2048];
6832 assert!(stream.read(&mut request).await.unwrap() > 0);
6833 stream
6834 .write_all(
6835 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6836 )
6837 .await
6838 .unwrap();
6839 std::future::pending::<()>().await;
6840 });
6841 let error = probe_http_health(&url, Duration::from_secs(1))
6842 .await
6843 .unwrap_err()
6844 .to_string();
6845 server.abort();
6846 assert!(
6847 error.contains("timed out")
6848 && error.contains("503 Unavailable")
6849 && error.contains("partial diagnostic"),
6850 "{error}"
6851 );
6852 }
6853
6854 #[cfg(unix)]
6855 #[tokio::test]
6856 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6857 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6858 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6859 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6860 let server = tokio::spawn(async move {
6861 let (mut stream, _) = listener.accept().await.unwrap();
6862 let mut request = [0u8; 2048];
6863 assert!(stream.read(&mut request).await.unwrap() > 0);
6864 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6865 let response = format!(
6866 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6867 body.len()
6868 );
6869 stream.write_all(response.as_bytes()).await.unwrap();
6870 });
6871 let error = probe_http_health(&url, Duration::from_secs(1))
6872 .await
6873 .unwrap_err()
6874 .to_string();
6875 server.await.unwrap();
6876 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6877 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6878 assert!(!error.contains("not-in-diagnostic"));
6879 }
6880
6881 #[cfg(unix)]
6882 #[tokio::test]
6883 async fn http_health_real_nats_server_monitoring() {
6884 if std::process::Command::new("nats-server")
6885 .arg("--version")
6886 .env("XDG_DATA_HOME", std::env::temp_dir())
6887 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6888 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6889 .output()
6890 .is_err()
6891 {
6892 eprintln!(
6893 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6894 );
6895 return;
6896 }
6897 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6898 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6899 let port = monitor.local_addr().unwrap().port();
6900 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6901 let client_port = client.local_addr().unwrap().port();
6902 let config = dir.join("server.conf");
6903 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6904 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6905 configured.program = PathBuf::from("nats-server");
6906 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6907 drop(monitor);
6908 drop(client);
6909 let module =
6910 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6911 .supervise_configured_with_health(
6912 configured.module_spec(),
6913 true,
6914 configured.health,
6915 configured.drain_timeout_ms,
6916 configured.restart,
6917 )
6918 .unwrap();
6919 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6920 module.drain().await.unwrap();
6921 }
6922
6923 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6924 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6925 let supervisor =
6926 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6927 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6928 let initial = ModuleSpec {
6929 module_id: "rescan-preserves-spawn-facts".to_string(),
6930 program: PathBuf::from("/spawned/module"),
6931 args: Vec::new(),
6932 env: Vec::new(),
6933 reserved: false,
6934 reserved_prefixes: Vec::new(),
6935 protocol: ModuleProtocol::Subc,
6936 overlap: Default::default(),
6937 carry_containment: false,
6938 };
6939 let module = supervisor.supervised_module(
6940 initial.clone(),
6941 supervisor.runtime_config(),
6942 snapshot,
6943 None,
6944 );
6945 let before = module.status().unwrap();
6946 let mut replacement = initial;
6947 replacement.program = PathBuf::from("/rescanned/replacement-module");
6948
6949 module
6950 .update_configuration(replacement, HealthConfig::default(), None)
6951 .await
6952 .unwrap();
6953
6954 let after = module.status().unwrap();
6955 assert_eq!(after.pid, before.pid);
6956 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6957 assert_eq!(after.spawned_from, before.spawned_from);
6958 drop(module);
6959 }
6960}
6961
6962fn unix_ms_now() -> u64 {
6963 SystemTime::now()
6964 .duration_since(UNIX_EPOCH)
6965 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6966 .unwrap_or(0)
6967}
6968
6969async fn supervise_loop(
6970 mut spec: ModuleSpec,
6971 mut runtime: SupervisorRuntimeConfig,
6972 registry: Arc<Registry>,
6973 process_liveness: Arc<SupervisorProcessLiveness>,
6974 snapshot: SharedSnapshot,
6975 mut child: Option<SupervisedChild>,
6976 mut commands: mpsc::Receiver<SupervisorCommand>,
6977) {
6978 let mut health_probe = HealthProbeRuntime::default();
6979 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6983 Ok(changes) => Some(changes),
6984 Err(err) => {
6985 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6988 None
6989 }
6990 };
6991 let mut pending_respawn: Option<PendingRespawn> = None;
6995 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6998 let mut previous_instance_recheck = tokio::time::interval_at(
6999 Instant::now() + Duration::from_secs(5),
7000 Duration::from_secs(5),
7001 );
7002 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
7003 loop {
7004 #[cfg(test)]
7005 {
7006 lock_snapshot(&snapshot).unwrap().actor_select = None;
7007 }
7008 #[cfg(target_os = "macos")]
7009 if let Some(active) = child.as_mut() {
7010 active.confirm_privacy_exec().await;
7011 }
7012 if let Some(scheduled) = runtime
7013 .scheduled_respawn
7014 .lock()
7015 .unwrap_or_else(|p| p.into_inner())
7016 .take()
7017 {
7018 runtime.recovery_is_daemon = scheduled.daemon_initiated;
7019 pending_respawn = Some(scheduled);
7020 }
7021 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
7022 pending_respawn = None;
7023 cancel_deferred_reload(
7024 &runtime,
7025 &spec.module_id,
7026 "respawn cancelled by a supervisor command",
7027 );
7028 }
7029 if child.is_none()
7030 && pending_respawn.is_none()
7031 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
7032 {
7033 cancel_deferred_reload(
7034 &runtime,
7035 &spec.module_id,
7036 "respawn cancelled before a replacement was spawned",
7037 );
7038 let mut stranded = false;
7039 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7040 state.respawn_pending = false;
7041 state.coalesced_restart_pending = false;
7042 if matches!(
7043 state.state,
7044 ModuleState::Restarting
7045 | ModuleState::Starting
7046 | ModuleState::Draining
7047 | ModuleState::Unresponsive
7048 ) {
7049 error!(module_id = %spec.module_id, state = ?state.state,
7050 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
7051 stranded = true;
7052 }
7053 });
7054 if stranded {
7055 fail_snapshot(
7056 &snapshot,
7057 Some(&spec.module_id),
7058 None,
7059 &runtime.failure_context(runtime.recovery_is_daemon),
7060 );
7061 }
7062 }
7063 if child.is_none()
7064 && (runtime.child_roster.is_closed()
7065 || pending_respawn.is_none()
7066 || !lock_snapshot(&snapshot)
7067 .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
7068 {
7069 containment::terminate(&runtime.child_roster.containment).await;
7070 }
7071 if let Some(command) = requeued.pop_front() {
7072 if !handle_supervisor_command(
7073 command,
7074 &mut spec,
7075 &mut runtime,
7076 ®istry,
7077 &process_liveness,
7078 &snapshot,
7079 &mut child,
7080 &mut commands,
7081 &mut requeued,
7082 )
7083 .await
7084 {
7085 containment::terminate(&runtime.child_roster.containment).await;
7086 return;
7087 }
7088 if child.is_some() || !respawn_still_pending(&snapshot) {
7089 pending_respawn = None;
7090 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7091 state.respawn_pending = false
7092 });
7093 }
7094 continue;
7095 }
7096 if child.is_some() {
7097 runtime.recovery_is_daemon = true;
7098 #[cfg(test)]
7099 {
7100 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
7101 }
7102 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
7103 let active = child.as_mut().expect("child checked above");
7104 if runtime.child_roster.is_closed() || active.registered_since_spawn() {
7105 active.registration_deadline = None;
7106 }
7107 let registration_deadline = active.registration_deadline;
7108 let no_checks = runtime.health.http.is_none()
7109 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
7110 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
7111 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
7112 state
7113 .failed_retry
7114 .uptime_reset_at(no_checks, runtime.restart_policy.window)
7115 });
7116 let wake_at = match (health_probe.wake_at(), uptime_reset) {
7117 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
7118 (probe, None) => probe,
7119 };
7120 let wake_at = match (wake_at, registration_deadline) {
7121 (Some(wake), Some(register)) => Some(wake.min(register)),
7122 (wake, register) => wake.or(register),
7123 };
7124 #[cfg(test)]
7125 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
7126 let probe_sleep = supervisor_sleep(wake_at);
7127 tokio::pin!(probe_sleep);
7128 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
7133 #[cfg(test)]
7134 {
7135 let registration_pending = wire_child
7136 && registration_changes
7137 .as_ref()
7138 .is_some_and(|changes| changes.has_changed().unwrap());
7139 let mut state = lock_snapshot(&snapshot).unwrap();
7140 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
7141 generation: state.spawn_generation,
7142 turn: state.actor_turns,
7143 registered_connection: health_probe.registered_connection,
7144 next_probe_at: health_probe.next_probe_at,
7145 wake_after,
7146 });
7147 }
7148 let active_child = child.as_mut().expect("child checked above");
7149 tokio::select! {
7150 wait_result = active_child.wait() => {
7151 if let Ok(mut state) = lock_snapshot(&snapshot) {
7152 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
7153 }
7154 #[cfg(test)]
7155 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7156 let exit_report = match wait_result {
7165 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
7166 Err(err) => {
7167 active_child.drain_stderr(&spec.module_id).await;
7168 fail_after_wait_error(
7169 &spec.module_id,
7170 &snapshot,
7171 &runtime.terminal_ring,
7172 &runtime.spawn_events,
7173 format!("failed to wait for supervised module: {err}"),
7174 &runtime.failure_context(true),
7175 );
7176 untrack_if_registration_released(
7177 &process_liveness,
7178 ®istry,
7179 &spec.module_id,
7180 &snapshot,
7181 );
7182 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
7183 child = None;
7184 continue;
7185 }
7186 };
7187 active_child.drain_stderr(&spec.module_id).await;
7188
7189 let next = on_child_exit(
7190 &spec,
7191 runtime.restart_policy,
7192 ®istry,
7193 &snapshot,
7194 &runtime.terminal_ring,
7195 &runtime.spawn_events,
7196 &runtime.child_roster,
7197 exit_report,
7198 true,
7199 ).await;
7200 runtime.start_requester = None;
7201 runtime.pending_activity_restart = false;
7202 let carry = matches!(next, NextAction::Restart { .. })
7205 && active_child.spawn_failure.is_none()
7206 && !runtime.child_roster.is_closed();
7207 if let Err(error) = active_child.finish_exit(carry).await {
7208 containment::terminate(&runtime.child_roster.containment).await;
7209 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7210 &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
7211 &runtime.failure_context(true));
7212 child = None;
7213 continue;
7214 }
7215 match next {
7216 NextAction::Stop { registration_released } => {
7217 if registration_released {
7218 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7219 }
7220 child = None;
7221 }
7222 NextAction::Restart { schedule } => {
7223 let delay = schedule.map_or(
7224 runtime.restart_policy.delay_for_restart(0),
7225 |schedule| schedule.delay,
7226 );
7227 if let Some(schedule) = schedule {
7228 log_crash_respawn(&spec.module_id, schedule);
7229 }
7230 child = None;
7238 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7239 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7240 }
7241 }
7242 }
7243 command = commands.recv() => {
7244 #[cfg(test)]
7245 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7246 let Some(command) = command else {
7247 containment::terminate(&runtime.child_roster.containment).await;
7248 return;
7249 };
7250 if !handle_supervisor_command(
7251 command,
7252 &mut spec,
7253 &mut runtime,
7254 ®istry,
7255 &process_liveness,
7256 &snapshot,
7257 &mut child,
7258 &mut commands,
7259 &mut requeued,
7260 ).await {
7261 containment::terminate(&runtime.child_roster.containment).await;
7262 return;
7263 }
7264 }
7265 _ = async {
7266 match registration_changes.as_mut() {
7267 Some(changes) => { let _ = changes.changed().await; }
7268 None => std::future::pending().await,
7269 }
7270 }, if wire_child => {
7271 #[cfg(test)]
7272 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7273 }
7274 _ = &mut probe_sleep => {
7275 #[cfg(test)]
7276 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7277 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7278 active_child.registration_deadline = None;
7281 if !runtime.child_roster.is_closed()
7282 && !active_child.registered_since_spawn()
7283 && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7284 {
7285 warn!(module_id = %spec.module_id, pid = active_child.pid,
7286 wait_secs = runtime.register_timeout.as_secs(),
7287 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7288 active_child.spawn_failure = Some("registration_timeout".into());
7289 if let Err(error) = stop_unregistered_child(active_child).await {
7290 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7291 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7292 &runtime.spawn_events, format!("registration_timeout: {error}"),
7293 &runtime.failure_context(true));
7294 child = None;
7295 }
7296 continue;
7299 }
7300 }
7301 if health_probe.due() {
7302 run_health_probe_cycle(
7303 &spec,
7304 &runtime,
7305 ®istry,
7306 &process_liveness,
7307 &snapshot,
7308 &mut child,
7309 ).await;
7310 if child.is_none() {
7311 runtime.start_requester = None;
7312 runtime.pending_activity_restart = false;
7313 }
7314 if child.is_some() {
7315 health_probe.schedule_next(&spec, runtime.health.cadence);
7316 }
7317 }
7318 }
7319 }
7320 } else if let Some(pending) = pending_respawn {
7321 tokio::select! {
7322 _ = runtime.child_roster.closed() => {
7323 pending_respawn = None;
7324 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7325 state.state = ModuleState::Stopped;
7326 state.respawn_pending = false;
7327 });
7328 containment::terminate(&runtime.child_roster.containment).await;
7329 }
7330 _ = sleep_until(pending.deadline) => {
7331 pending_respawn = None;
7332 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7333 if !respawn_still_pending(&snapshot) {
7337 continue;
7338 }
7339 if runtime.child_roster.is_closed() {
7344 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7345 state.state = ModuleState::Stopped;
7346 });
7347 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7348 continue;
7349 }
7350 if let Err(err) = release_dead_registration(
7351 ®istry,
7352 runtime.forwarding.as_deref(),
7353 &snapshot,
7354 &spec.module_id,
7355 &runtime.failure_context(runtime.recovery_is_daemon),
7356 ).await {
7357 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7358 continue;
7359 }
7360
7361 if matches!(pending.kind, RespawnKind::Reload) {
7362 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7363 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
7364 if result.is_ok() && runtime.pending_activity_restart {
7365 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7366 }
7367 runtime.pending_activity_restart = false;
7368 if let Some(reply) = reply { let _ = reply.send(result); }
7369 continue;
7370 }
7371 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7372 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7373 Ok(next_child) => {
7374 child = Some(next_child);
7375 if runtime.pending_activity_restart {
7376 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7377 runtime.pending_activity_restart = false;
7378 }
7379 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7380 }
7381 Err(err) => {
7382 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7383 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7384 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7385 }
7386 }
7387 }
7388 command = commands.recv() => {
7389 let Some(command) = command else {
7390 containment::terminate(&runtime.child_roster.containment).await;
7391 return;
7392 };
7393 if !handle_supervisor_command(
7394 command,
7395 &mut spec,
7396 &mut runtime,
7397 ®istry,
7398 &process_liveness,
7399 &snapshot,
7400 &mut child,
7401 &mut commands,
7402 &mut requeued,
7403 ).await {
7404 containment::terminate(&runtime.child_roster.containment).await;
7405 return;
7406 }
7407 if child.is_some() || !respawn_still_pending(&snapshot) {
7412 pending_respawn = None;
7413 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7414 }
7415 }
7416 }
7417 } else {
7418 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7419 (state.enabled && state.state == ModuleState::Failed)
7420 .then_some(state.failed_retry.pending)
7421 .flatten()
7422 });
7423 let command = if lock_snapshot(&snapshot)
7424 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7425 {
7426 tokio::select! {
7427 command = commands.recv() => command,
7428 _ = previous_instance_recheck.tick() => {
7429 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7430 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7431 if enabled && !runtime.child_roster.is_closed() {
7432 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7433 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7434 Ok(next_child) => child = Some(next_child),
7435 Err(error) => {
7436 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7437 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7438 }
7439 }
7440 } else {
7441 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7442 state.waiting_for_previous_pid = None;
7443 state.waiting_for_previous_detail = None;
7444 });
7445 }
7446 }
7447 continue;
7448 }
7449 }
7450 } else {
7451 tokio::select! {
7452 biased;
7453 command = commands.recv() => command,
7454 _ = async {
7455 match slow_retry {
7456 Some(retry) => sleep_until(retry.deadline).await,
7457 None => std::future::pending().await,
7458 }
7459 } => {
7460 let retry = slow_retry.expect("only a scheduled slow retry can wake");
7461 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7462 state.failed_retry.pending = None;
7463 });
7464 if runtime.child_roster.is_closed() {
7465 continue;
7466 }
7467 info!(module_id = %spec.module_id, attempt = retry.attempt,
7468 delay_secs = retry.delay.as_secs(), "starting failed module retry");
7469 if let Err(error) = set_child_enabled(
7470 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
7471 &runtime.failure_context(true),
7472 ).await {
7473 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7474 }
7475 continue;
7476 }
7477 }
7478 };
7479 let Some(command) = command else {
7480 containment::terminate(&runtime.child_roster.containment).await;
7481 return;
7482 };
7483 if !handle_supervisor_command(
7484 command,
7485 &mut spec,
7486 &mut runtime,
7487 ®istry,
7488 &process_liveness,
7489 &snapshot,
7490 &mut child,
7491 &mut commands,
7492 &mut requeued,
7493 )
7494 .await
7495 {
7496 containment::terminate(&runtime.child_roster.containment).await;
7497 return;
7498 }
7499 }
7500 }
7501}
7502
7503async fn supervisor_sleep(wake_at: Option<Instant>) {
7506 match wake_at {
7507 Some(deadline) => sleep_until(deadline).await,
7508 None => std::future::pending().await,
7509 }
7510}
7511
7512fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7513 info!(
7514 module_id,
7515 restart_in_window = schedule.restart_in_window,
7516 delay_ms = schedule.delay.as_millis() as u64,
7517 "respawning after crash"
7518 );
7519}
7520
7521fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7527 matches!(
7528 lock_snapshot(snapshot),
7529 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7530 )
7531}
7532
7533enum NextAction {
7534 Stop {
7535 registration_released: bool,
7536 },
7537 Restart {
7538 schedule: Option<CrashRestartSchedule>,
7539 },
7540}
7541
7542#[allow(clippy::too_many_arguments)]
7543async fn handle_supervisor_command(
7544 command: SupervisorCommand,
7545 spec: &mut ModuleSpec,
7546 runtime: &mut SupervisorRuntimeConfig,
7547 registry: &Arc<Registry>,
7548 process_liveness: &SupervisorProcessLiveness,
7549 snapshot: &SharedSnapshot,
7550 child: &mut Option<SupervisedChild>,
7551 commands: &mut mpsc::Receiver<SupervisorCommand>,
7552 requeued: &mut VecDeque<SupervisorCommand>,
7553) -> bool {
7554 if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7555 runtime.recovery_is_daemon = false;
7556 }
7557 if matches!(
7558 command,
7559 SupervisorCommand::Drain { .. }
7560 | SupervisorCommand::Retire { .. }
7561 | SupervisorCommand::SetEnabled { .. }
7562 | SupervisorCommand::Restart { .. }
7563 ) {
7564 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7565 state.failed_retry.cancel()
7566 });
7567 }
7568 let blocked = recovery_block_error(spec, &runtime.child_roster).or_else(|| {
7569 runtime
7570 .child_roster
7571 .previous_instance(&spec.module_id)
7572 .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
7573 });
7574 let command = if let Some(error) = blocked {
7575 match command {
7576 SupervisorCommand::Restart { reply, .. }
7577 | SupervisorCommand::Reload { reply, .. }
7578 | SupervisorCommand::Swap { reply, .. } => {
7579 let _ = reply.send(Err(error));
7580 return true;
7581 }
7582 SupervisorCommand::SetEnabled {
7583 enabled: true,
7584 reply,
7585 ..
7586 } => {
7587 let _ = reply.send(Err(error));
7588 return true;
7589 }
7590 other => other,
7591 }
7592 } else {
7593 command
7594 };
7595 match command {
7596 SupervisorCommand::Drain { requester, reply } => {
7597 let result = drain_optional_child(
7600 &spec.module_id,
7601 spec.protocol,
7602 StopNotice::NotSent,
7603 registry,
7604 runtime.forwarding.as_deref(),
7605 snapshot,
7606 &runtime.terminal_ring,
7607 &runtime.spawn_events,
7608 child,
7609 runtime.drain_timeout,
7610 ModuleState::Stopped,
7611 None,
7612 &FailureContext::operator(),
7613 )
7614 .await;
7615 containment::terminate(&runtime.child_roster.containment).await;
7616 let registration_released = result.is_ok();
7617 if registration_released {
7618 runtime.record_activity_transition(
7619 "subc.module_stopped",
7620 &spec.module_id,
7621 requester,
7622 );
7623 }
7624 let _ = reply.send(result);
7625 if registration_released {
7626 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7627 }
7628 false
7629 }
7630 SupervisorCommand::Retire { requester, reply } => {
7631 let result = async {
7632 let stop_notice = begin_forwarding_drain_if_configured(
7633 spec,
7634 runtime,
7635 registry,
7636 snapshot,
7637 None,
7638 RouteCloseReason::Disable,
7639 )
7640 .await?;
7641 drain_optional_child(
7642 &spec.module_id,
7643 spec.protocol,
7644 stop_notice,
7645 registry,
7646 runtime.forwarding.as_deref(),
7647 snapshot,
7648 &runtime.terminal_ring,
7649 &runtime.spawn_events,
7650 child,
7651 runtime.drain_timeout,
7652 ModuleState::Stopped,
7653 None,
7654 &FailureContext::operator(),
7655 )
7656 .await
7657 }
7658 .await;
7659 containment::terminate(&runtime.child_roster.containment).await;
7660 let registration_released = result.is_ok();
7661 if registration_released {
7662 runtime.record_activity_transition(
7663 "subc.module_stopped",
7664 &spec.module_id,
7665 requester,
7666 );
7667 }
7668 let _ = reply.send(result);
7669 if registration_released {
7670 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7671 }
7672 false
7673 }
7674 SupervisorCommand::Restart {
7675 requester,
7676 drain_timeout_ms,
7677 received_at_generation,
7678 queued_at,
7679 reply,
7680 } => {
7681 info!(
7685 module_id = %spec.module_id,
7686 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7687 "restart command dequeued"
7688 );
7689 let validation = match lock_snapshot(snapshot) {
7701 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7702 module_id: spec.module_id.clone(),
7703 }),
7704 Ok(_) => Ok(()),
7705 Err(err) => Err(err),
7706 };
7707 let initiated = validation.is_ok();
7708 let _ = reply.send(validation);
7709 let satisfied_by_generation = if initiated
7723 && child
7724 .as_ref()
7725 .is_some_and(|child| child.spawned_at > queued_at)
7726 {
7727 lock_snapshot(snapshot).ok().and_then(|state| {
7728 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7729 })
7730 } else {
7731 None
7732 };
7733 let satisfied_by_pending = initiated
7734 && child.is_none()
7735 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7736 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7740 if pending {
7741 state.coalesced_restart_pending = true;
7742 }
7743 pending
7744 });
7745 if satisfied_by_pending {
7746 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7747 } else if let Some(generation) = satisfied_by_generation {
7748 info!(
7749 module_id = %spec.module_id,
7750 received_at_generation,
7751 "restart already satisfied by generation {generation}; not restarting again"
7752 );
7753 } else if initiated {
7754 let drain_timeout = drain_timeout_ms
7757 .map(Duration::from_millis)
7758 .unwrap_or(runtime.drain_timeout);
7759 if let Err(err) = restart_child(
7760 spec,
7761 runtime,
7762 registry,
7763 process_liveness,
7764 snapshot,
7765 child,
7766 drain_timeout,
7767 )
7768 .await
7769 {
7770 warn!(
7771 module_id = %spec.module_id,
7772 error = %err,
7773 "operator restart failed after initiation ack; module state carries the outcome"
7774 );
7775 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7776 state.state = ModuleState::Failed;
7777 clear_current_process_facts(state);
7778 });
7779 } else {
7780 runtime.start_requester = requester;
7781 runtime.pending_activity_restart = true;
7782 }
7783 }
7784 true
7785 }
7786 SupervisorCommand::Reload { requester, reply } => {
7787 let result =
7788 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7789 if result.is_ok() {
7790 runtime.start_requester = requester;
7791 runtime.pending_activity_restart = true;
7792 }
7793 if result.is_ok()
7794 && runtime
7795 .scheduled_respawn
7796 .lock()
7797 .unwrap_or_else(|p| p.into_inner())
7798 .as_ref()
7799 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7800 {
7801 *runtime
7802 .deferred_reload_reply
7803 .lock()
7804 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7805 } else {
7806 let _ = reply.send(result);
7807 }
7808 true
7809 }
7810 SupervisorCommand::SetEnabled {
7811 enabled,
7812 requester,
7813 reply,
7814 } => {
7815 let previous_requester = runtime.start_requester.clone();
7816 if enabled {
7817 runtime.start_requester = requester.clone();
7818 }
7819 let result = set_child_enabled(
7820 spec,
7821 runtime,
7822 registry,
7823 process_liveness,
7824 snapshot,
7825 child,
7826 enabled,
7827 &runtime.failure_context(false),
7828 )
7829 .await;
7830 if !enabled {
7831 containment::terminate(&runtime.child_roster.containment).await;
7832 }
7833 if matches!(result, Ok(true)) {
7834 runtime.pending_activity_restart = false;
7835 if enabled {
7836 runtime.start_requester = requester;
7837 } else {
7838 runtime.start_requester = None;
7839 runtime.record_activity_transition(
7840 "subc.module_disabled",
7841 &spec.module_id,
7842 requester,
7843 );
7844 }
7845 } else {
7846 runtime.start_requester = previous_requester;
7847 }
7848 let _ = reply.send(result);
7849 true
7850 }
7851 SupervisorCommand::UpdateConfiguration {
7852 spec: next_spec,
7853 health,
7854 drain_timeout_ms,
7855 reply,
7856 } => {
7857 if let Some(handle) = &runtime.supervisor_handle {
7858 handle.apply_identity_configuration(&next_spec);
7859 }
7860 *spec = next_spec;
7861 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7862 state.configuration_updated_since_spawn = true;
7863 });
7864 let health_changed = runtime.health != health;
7865 runtime.health = health;
7866 if health_changed {
7869 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7870 state.health = ModuleHealthStatus::default();
7871 state.last_probe_evidence = None;
7872 });
7873 }
7874 runtime.drain_timeout = drain_timeout_ms
7875 .map(Duration::from_millis)
7876 .unwrap_or(runtime.default_drain_timeout);
7877 *runtime
7878 .effective_drain_timeout
7879 .lock()
7880 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7881 let _ = reply.send(());
7882 true
7883 }
7884 SupervisorCommand::Swap {
7885 requester,
7886 ready_timeout,
7887 reply,
7888 } => {
7889 if spec.carry_containment
7890 || containment::current(&runtime.child_roster.containment).is_some()
7891 {
7892 let _ = reply.send(Err(SuperviseError::InvalidSpec {
7893 reason: "a carried container cannot overlap a swap candidate".to_owned(),
7894 }));
7895 return true;
7896 }
7897 let mut swap_runtime = runtime.clone();
7898 swap_runtime.start_requester = requester;
7899 let before = lock_snapshot(snapshot)
7900 .ok()
7901 .map(|state| state.spawn_generation);
7902 let end = swap::run_swap(
7903 spec,
7904 &swap_runtime,
7905 registry,
7906 process_liveness,
7907 snapshot,
7908 child,
7909 commands,
7910 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7911 reply,
7912 )
7913 .await;
7914 if before
7915 != lock_snapshot(snapshot)
7916 .ok()
7917 .map(|state| state.spawn_generation)
7918 {
7919 runtime.start_requester = swap_runtime.start_requester;
7920 }
7921 requeued.extend(end.requeue);
7922 true
7923 }
7924 }
7925}
7926
7927async fn restart_child(
7928 spec: &ModuleSpec,
7929 runtime: &SupervisorRuntimeConfig,
7930 registry: &Registry,
7931 process_liveness: &SupervisorProcessLiveness,
7932 snapshot: &SharedSnapshot,
7933 child: &mut Option<SupervisedChild>,
7934 drain_timeout: Duration,
7935) -> Result<(), SuperviseError> {
7936 if !lock_snapshot(snapshot)?.enabled {
7938 return Err(SuperviseError::Disabled {
7939 module_id: spec.module_id.clone(),
7940 });
7941 }
7942 let stop_notice = begin_forwarding_drain_with_timeout(
7943 spec,
7944 runtime,
7945 registry,
7946 snapshot,
7947 None,
7948 RouteCloseReason::Restart,
7949 drain_timeout,
7950 )
7951 .await?;
7952
7953 if child.is_some() {
7954 drain_optional_child(
7955 &spec.module_id,
7956 spec.protocol,
7957 stop_notice,
7958 registry,
7959 runtime.forwarding.as_deref(),
7960 snapshot,
7961 &runtime.terminal_ring,
7962 &runtime.spawn_events,
7963 child,
7964 drain_timeout,
7965 ModuleState::Restarting,
7966 Some(true),
7967 &FailureContext::operator(),
7968 )
7969 .await?;
7970 } else {
7971 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7972 state.enabled = true;
7973 state.state = ModuleState::Restarting;
7974 clear_current_process_facts(state);
7975 })?;
7976 release_dead_registration(
7977 registry,
7978 runtime.forwarding.as_deref(),
7979 snapshot,
7980 &spec.module_id,
7981 &runtime.failure_context(false),
7982 )
7983 .await?;
7984 }
7985
7986 reset_restart_count(snapshot, &spec.module_id)?;
7987 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7988 schedule_respawn(
7989 runtime,
7990 snapshot,
7991 &spec.module_id,
7992 runtime.restart_policy.backoff,
7993 RespawnKind::Spawn,
7994 false,
7995 )
7996}
7997
7998async fn reload_child(
7999 spec: &ModuleSpec,
8000 runtime: &SupervisorRuntimeConfig,
8001 registry: &Registry,
8002 process_liveness: &SupervisorProcessLiveness,
8003 snapshot: &SharedSnapshot,
8004 child: &mut Option<SupervisedChild>,
8005) -> Result<(), SuperviseError> {
8006 if !lock_snapshot(snapshot)?.enabled {
8008 return Err(SuperviseError::Disabled {
8009 module_id: spec.module_id.clone(),
8010 });
8011 }
8012 let stop_notice = begin_forwarding_drain(
8013 spec,
8014 runtime,
8015 registry,
8016 snapshot,
8017 Some(true),
8018 RouteCloseReason::Reload,
8019 )
8020 .await?;
8021
8022 if child.is_some() {
8023 drain_optional_child(
8024 &spec.module_id,
8025 spec.protocol,
8026 stop_notice,
8027 registry,
8028 runtime.forwarding.as_deref(),
8029 snapshot,
8030 &runtime.terminal_ring,
8031 &runtime.spawn_events,
8032 child,
8033 runtime.drain_timeout,
8034 ModuleState::Restarting,
8035 Some(true),
8036 &FailureContext::operator(),
8037 )
8038 .await?;
8039 } else {
8040 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8041 state.enabled = true;
8042 state.state = ModuleState::Restarting;
8043 clear_current_process_facts(state);
8044 })?;
8045 release_dead_registration(
8046 registry,
8047 runtime.forwarding.as_deref(),
8048 snapshot,
8049 &spec.module_id,
8050 &runtime.failure_context(false),
8051 )
8052 .await?;
8053 }
8054
8055 reset_restart_count(snapshot, &spec.module_id)?;
8056 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8057 schedule_respawn(
8058 runtime,
8059 snapshot,
8060 &spec.module_id,
8061 runtime.restart_policy.backoff,
8062 RespawnKind::Reload,
8063 false,
8064 )
8065}
8066
8067async fn finish_reload_child(
8068 spec: &ModuleSpec,
8069 runtime: &SupervisorRuntimeConfig,
8070 registry: &Registry,
8071 process_liveness: &SupervisorProcessLiveness,
8072 snapshot: &SharedSnapshot,
8073 child: &mut Option<SupervisedChild>,
8074) -> Result<(), SuperviseError> {
8075 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8076 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8077 Ok(next_child) => next_child,
8078 Err(err) => {
8079 return handle_reload_spawn_failure(
8080 spec,
8081 runtime,
8082 process_liveness,
8083 snapshot,
8084 child,
8085 format!("new child failed to spawn: {err}"),
8086 )
8087 .await;
8088 }
8089 };
8090 *child = Some(next_child);
8091
8092 let wait_outcome = {
8093 let active_child = child.as_mut().expect("new reload child was just stored");
8094 wait_for_registration_after_reload(
8095 registry,
8096 &spec.module_id,
8097 snapshot,
8098 active_child,
8099 REGISTRY_RELEASE_TIMEOUT,
8100 )
8101 .await?
8102 };
8103
8104 match wait_outcome {
8105 RegistrationWaitOutcome::Registered => {
8106 child
8107 .as_mut()
8108 .expect("registered reload child")
8109 .registration_deadline = None;
8110 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
8111 Ok(())
8112 }
8113 RegistrationWaitOutcome::Exited(exit_report) => {
8114 if let Some(active_child) = child.as_mut() {
8115 active_child.drain_stderr(&spec.module_id).await;
8116 }
8117 let mut exited_child = child.take().expect("exited reload child is still stored");
8120 #[cfg(test)]
8121 if let Some(gate) = &runtime.test_reload_exit_record_gate {
8122 gate.reached.notify_one();
8123 gate.resume.notified().await;
8124 }
8125 let result = handle_reload_child_registration_failure(
8126 spec,
8127 runtime,
8128 registry,
8129 process_liveness,
8130 snapshot,
8131 child,
8132 ReloadRegistrationFailure {
8133 exit_report: registration_failure_exit_report(exit_report),
8134 reason: exited_child
8135 .spawn_failure
8136 .clone()
8137 .unwrap_or_else(|| "new child exited before registering".to_string()),
8138 },
8139 )
8140 .await;
8141 exited_child
8142 .finish_exit(false)
8143 .await
8144 .map_err(|source| SuperviseError::Wait {
8145 module_id: spec.module_id.clone(),
8146 source,
8147 })?;
8148 result
8149 }
8150 RegistrationWaitOutcome::TimedOut => {
8151 let mut timed_out_child = child
8152 .take()
8153 .expect("timed-out reload child is still running");
8154 timed_out_child
8155 .start_kill()
8156 .map_err(|source| SuperviseError::Kill {
8157 module_id: spec.module_id.clone(),
8158 source,
8159 })?;
8160 let status = timed_out_child
8161 .wait()
8162 .await
8163 .map_err(|source| SuperviseError::Wait {
8164 module_id: spec.module_id.clone(),
8165 source,
8166 })?;
8167 timed_out_child.drain_stderr(&spec.module_id).await;
8168 let result = handle_reload_child_registration_failure(
8169 spec,
8170 runtime,
8171 registry,
8172 process_liveness,
8173 snapshot,
8174 child,
8175 ReloadRegistrationFailure {
8176 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
8177 snapshot,
8178 &timed_out_child,
8179 &status,
8180 )),
8181 reason: format!(
8182 "new child did not register within {:?}",
8183 REGISTRY_RELEASE_TIMEOUT
8184 ),
8185 },
8186 )
8187 .await;
8188 timed_out_child
8189 .finish_exit(false)
8190 .await
8191 .map_err(|source| SuperviseError::Wait {
8192 module_id: spec.module_id.clone(),
8193 source,
8194 })?;
8195 result
8196 }
8197 }
8198}
8199
8200#[allow(clippy::too_many_arguments)]
8201async fn set_child_enabled(
8202 spec: &ModuleSpec,
8203 runtime: &SupervisorRuntimeConfig,
8204 registry: &Registry,
8205 process_liveness: &SupervisorProcessLiveness,
8206 snapshot: &SharedSnapshot,
8207 child: &mut Option<SupervisedChild>,
8208 enabled: bool,
8209 failure: &FailureContext,
8210) -> Result<bool, SuperviseError> {
8211 let (current_enabled, current_state, respawn_pending) = {
8212 let state = lock_snapshot(snapshot)?;
8213 (state.enabled, state.state, state.respawn_pending)
8214 };
8215 let revive_terminal = enabled
8223 && current_enabled
8224 && child.is_none()
8225 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
8226 || (current_state == ModuleState::Restarting && !respawn_pending));
8227 if current_enabled == enabled && !revive_terminal {
8228 return Ok(false);
8229 }
8230
8231 if enabled {
8232 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8233 state.enabled = true;
8234 state.state = ModuleState::Starting;
8235 clear_current_process_facts(state);
8236 })?;
8237 #[cfg(test)]
8238 if runtime.test_seed_stale_facts_before_enable_spawn {
8239 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8240 state.process_alive = true;
8241 state.pid = Some(41);
8242 state.spawned_at_ms = Some(42);
8243 state.spawned_from = Some(PathBuf::from("/spawned/module"));
8244 state.spawned_file_identity = Some(SpawnedFileIdentity {
8245 device: 43,
8246 inode: 44,
8247 #[cfg(windows)]
8248 windows_image: None,
8249 });
8250 })?;
8251 }
8252 release_dead_registration(
8253 registry,
8254 runtime.forwarding.as_deref(),
8255 snapshot,
8256 &spec.module_id,
8257 failure,
8258 )
8259 .await?;
8260 reset_restart_count(snapshot, &spec.module_id)?;
8261 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8262 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8263 Ok(next_child) => next_child,
8264 Err(err) => {
8265 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
8266 process_liveness.untrack_if_current(&spec.module_id, snapshot);
8267 return Err(err);
8268 }
8269 };
8270 *child = Some(next_child);
8271 debug!(module_id = %spec.module_id, "supervised module enabled");
8272 Ok(true)
8273 } else {
8274 let stop_notice = begin_forwarding_drain_if_configured(
8275 spec,
8276 runtime,
8277 registry,
8278 snapshot,
8279 Some(false),
8280 RouteCloseReason::Disable,
8281 )
8282 .await?;
8283 drain_optional_child(
8284 &spec.module_id,
8285 spec.protocol,
8286 stop_notice,
8287 registry,
8288 runtime.forwarding.as_deref(),
8289 snapshot,
8290 &runtime.terminal_ring,
8291 &runtime.spawn_events,
8292 child,
8293 runtime.drain_timeout,
8294 ModuleState::Disabled,
8295 Some(false),
8296 failure,
8297 )
8298 .await?;
8299 debug!(module_id = %spec.module_id, "supervised module disabled");
8300 Ok(true)
8301 }
8302}
8303
8304#[allow(clippy::too_many_arguments)]
8305async fn on_child_exit(
8306 spec: &ModuleSpec,
8307 policy: RestartPolicy,
8308 registry: &Registry,
8309 snapshot: &SharedSnapshot,
8310 terminal_ring: &Arc<Mutex<TerminalRing>>,
8311 spawn_events: &SpawnEventFeed,
8312 roster: &ChildRoster,
8313 exit_report: ExitReport,
8314 daemon_initiated: bool,
8315) -> NextAction {
8316 if roster.is_closed() {
8322 return on_child_exit_during_daemon_shutdown(
8323 spec,
8324 registry,
8325 snapshot,
8326 terminal_ring,
8327 spawn_events,
8328 exit_report,
8329 )
8330 .await;
8331 }
8332 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8348 && running_protocol(spec, snapshot) == ModuleProtocol::None;
8349 match exit_report.kind {
8350 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8351 info!(
8352 module_id = %spec.module_id,
8353 exit_code = ?exit_report.code,
8354 exit_signal = ?exit_report.signal,
8355 "supervised module exited cleanly"
8356 );
8357 if let Err(err) = publish_after_terminal_record(
8358 &spec.module_id,
8359 || {
8360 record_terminal(
8361 &spec.module_id,
8362 terminal_ring,
8363 spawn_events,
8364 &exit_report,
8365 TerminalDisposition::Stopped,
8366 )
8367 },
8368 || {
8369 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8370 state.state = ModuleState::Stopped;
8371 clear_current_process_facts(state);
8372 state.last_exit = Some(exit_report.clone());
8373 })
8374 },
8375 ) {
8376 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8377 }
8378 let registration_released = match wait_for_registration_release(
8379 registry,
8380 &spec.module_id,
8381 REGISTRY_RELEASE_TIMEOUT,
8382 )
8383 .await
8384 {
8385 Ok(()) => true,
8386 Err(err) => {
8387 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8388 false
8389 }
8390 };
8391 NextAction::Stop {
8392 registration_released,
8393 }
8394 }
8395 ExitKind::Clean | ExitKind::Crash => {
8396 if unrequested_clean_exit_of_protocol_none {
8397 warn!(
8398 module_id = %spec.module_id,
8399 exit_code = ?exit_report.code,
8400 exit_signal = ?exit_report.signal,
8401 "protocol-none module exited cleanly without a stop request; handling it as a crash"
8402 );
8403 } else {
8404 warn!(
8405 module_id = %spec.module_id,
8406 exit_code = ?exit_report.code,
8407 exit_signal = ?exit_report.signal,
8408 "supervised module exited abnormally (crash)"
8409 );
8410 }
8411 let mut restart_schedule = None;
8412 let mut disposition = TerminalDisposition::Disabled;
8413 let mut final_state = ModuleState::Disabled;
8414 let mut disposition_detail = lock_snapshot(snapshot)
8418 .ok()
8419 .and_then(|mut state| state.spawn_failure.take());
8420 let now = Instant::now();
8421 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8422 if state.enabled {
8423 if let Some(schedule) = state.next_crash_restart(&policy, now) {
8424 state.respawn_pending = true;
8430 final_state = ModuleState::Restarting;
8431 restart_schedule = Some(schedule);
8432 disposition = TerminalDisposition::Restarting;
8433 } else {
8434 final_state = ModuleState::Failed;
8435 disposition = TerminalDisposition::Failed;
8436 let budget = policy.budget_exhausted_detail();
8437 disposition_detail =
8438 Some(disposition_detail.take().map_or_else(
8439 || budget.clone(),
8440 |cause| format!("{cause}; {budget}"),
8441 ));
8442 }
8443 } else {
8444 disposition = TerminalDisposition::Disabled;
8445 }
8446 }) {
8447 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8448 return NextAction::Stop {
8449 registration_released: false,
8450 };
8451 }
8452 if disposition == TerminalDisposition::Failed {
8453 error!(
8458 module_id = %spec.module_id,
8459 max_restarts = policy.max_restarts,
8460 window_secs = policy.window.as_secs(),
8461 "module stopped: {}",
8462 policy.budget_exhausted_detail()
8463 );
8464 }
8465 if let Err(err) = publish_after_terminal_record(
8466 &spec.module_id,
8467 || {
8468 record_terminal_with_detail(
8469 &spec.module_id,
8470 terminal_ring,
8471 spawn_events,
8472 &exit_report,
8473 disposition,
8474 disposition_detail,
8475 )
8476 },
8477 || {
8478 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8479 state.state = final_state;
8480 clear_current_process_facts(state);
8481 if final_state == ModuleState::Failed
8482 && daemon_initiated
8483 && !roster.is_closed()
8484 {
8485 state.failed_retry.schedule(&spec.module_id);
8486 }
8487 state.last_exit = Some(exit_report.clone());
8488 })
8489 },
8490 ) {
8491 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8492 }
8493
8494 if let Some(schedule) = restart_schedule {
8495 NextAction::Restart {
8496 schedule: Some(schedule),
8497 }
8498 } else {
8499 let registration_released = match wait_for_registration_release(
8500 registry,
8501 &spec.module_id,
8502 REGISTRY_RELEASE_TIMEOUT,
8503 )
8504 .await
8505 {
8506 Ok(()) => true,
8507 Err(err) => {
8508 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8509 false
8510 }
8511 };
8512 NextAction::Stop {
8513 registration_released,
8514 }
8515 }
8516 }
8517 ExitKind::DeliberateSeverance => {
8518 warn!(
8519 module_id = %spec.module_id,
8520 exit_code = ?exit_report.code,
8521 exit_signal = ?exit_report.signal,
8522 "supervised module exited after deliberate connection severance"
8523 );
8524 let should_restart = match lock_snapshot(snapshot) {
8525 Ok(state) => state.enabled,
8526 Err(err) => {
8527 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8528 return NextAction::Stop {
8529 registration_released: false,
8530 };
8531 }
8532 };
8533 let final_state = if should_restart {
8534 ModuleState::Restarting
8535 } else {
8536 ModuleState::Disabled
8537 };
8538 if let Err(err) = publish_after_terminal_record(
8539 &spec.module_id,
8540 || {
8541 record_terminal(
8542 &spec.module_id,
8543 terminal_ring,
8544 spawn_events,
8545 &exit_report,
8546 terminal_disposition(final_state),
8547 )
8548 },
8549 || {
8550 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8551 state.state = final_state;
8552 clear_current_process_facts(state);
8553 state.last_exit = Some(exit_report.clone());
8554 state.lifetime_restarts += 1;
8555 })
8556 },
8557 ) {
8558 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8559 return NextAction::Stop {
8560 registration_released: false,
8561 };
8562 }
8563
8564 if should_restart {
8565 NextAction::Restart { schedule: None }
8566 } else {
8567 let registration_released = match wait_for_registration_release(
8568 registry,
8569 &spec.module_id,
8570 REGISTRY_RELEASE_TIMEOUT,
8571 )
8572 .await
8573 {
8574 Ok(()) => true,
8575 Err(err) => {
8576 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8577 false
8578 }
8579 };
8580 NextAction::Stop {
8581 registration_released,
8582 }
8583 }
8584 }
8585 }
8586}
8587
8588async fn on_child_exit_during_daemon_shutdown(
8589 spec: &ModuleSpec,
8590 registry: &Registry,
8591 snapshot: &SharedSnapshot,
8592 terminal_ring: &Arc<Mutex<TerminalRing>>,
8593 spawn_events: &SpawnEventFeed,
8594 exit_report: ExitReport,
8595) -> NextAction {
8596 info!(
8597 module_id = %spec.module_id,
8598 exit_code = ?exit_report.code,
8599 exit_signal = ?exit_report.signal,
8600 exit_kind = ?exit_report.kind,
8601 "supervised module exited during daemon shutdown; not restarting it"
8602 );
8603 if let Err(err) = publish_after_terminal_record(
8604 &spec.module_id,
8605 || {
8606 record_terminal(
8607 &spec.module_id,
8608 terminal_ring,
8609 spawn_events,
8610 &exit_report,
8611 TerminalDisposition::DaemonShutdown,
8612 )
8613 },
8614 || {
8615 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8616 state.state = ModuleState::Stopped;
8617 clear_current_process_facts(state);
8618 state.last_exit = Some(exit_report.clone());
8619 })
8620 },
8621 ) {
8622 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8623 }
8624 let registration_released =
8625 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8626 .await
8627 .is_ok();
8628 NextAction::Stop {
8629 registration_released,
8630 }
8631}
8632
8633fn publish_after_terminal_record<T>(
8638 module_id: &str,
8639 record: impl FnOnce(),
8640 publish: impl FnOnce() -> T,
8641) -> T {
8642 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8643 let published = publish();
8644 if let Err(panic) = recorded {
8645 error!(
8646 module_id,
8647 "terminal recording panicked; published final state before resuming unwind"
8648 );
8649 std::panic::resume_unwind(panic);
8650 }
8651 published
8652}
8653
8654fn fail_after_wait_error(
8655 module_id: &str,
8656 snapshot: &SharedSnapshot,
8657 terminal_ring: &Arc<Mutex<TerminalRing>>,
8658 spawn_events: &SpawnEventFeed,
8659 detail: String,
8660 failure: &FailureContext,
8661) {
8662 let report = wait_error_exit_report();
8663 publish_after_terminal_record(
8664 module_id,
8665 || {
8666 record_terminal_with_detail(
8667 module_id,
8668 terminal_ring,
8669 spawn_events,
8670 &report,
8671 TerminalDisposition::Failed,
8672 Some(detail),
8673 )
8674 },
8675 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8676 );
8677}
8678
8679#[cfg(test)]
8680fn record_wait_error_terminal(
8681 module_id: &str,
8682 terminal_ring: &Arc<Mutex<TerminalRing>>,
8683 spawn_events: &SpawnEventFeed,
8684) {
8685 record_terminal(
8686 module_id,
8687 terminal_ring,
8688 spawn_events,
8689 &wait_error_exit_report(),
8690 TerminalDisposition::Failed,
8691 );
8692}
8693
8694fn record_terminal(
8695 module_id: &str,
8696 terminal_ring: &Arc<Mutex<TerminalRing>>,
8697 spawn_events: &SpawnEventFeed,
8698 exit_report: &ExitReport,
8699 disposition: TerminalDisposition,
8700) {
8701 record_terminal_with_detail(
8702 module_id,
8703 terminal_ring,
8704 spawn_events,
8705 exit_report,
8706 disposition,
8707 None,
8708 );
8709}
8710
8711fn durable_terminal_history_of(
8715 terminal_ring: &Mutex<TerminalRing>,
8716 module_id: &str,
8717) -> subc_control::TerminalHistory {
8718 let read = terminal_ring
8719 .lock()
8720 .unwrap_or_else(|p| p.into_inner())
8721 .capture_durable_history();
8722 read.read(module_id)
8723}
8724
8725fn record_terminal_with_detail(
8726 module_id: &str,
8727 terminal_ring: &Arc<Mutex<TerminalRing>>,
8728 spawn_events: &SpawnEventFeed,
8729 exit_report: &ExitReport,
8730 disposition: TerminalDisposition,
8731 disposition_detail: Option<String>,
8732) {
8733 #[cfg(test)]
8734 terminal_history_tests::before_terminal_record(spawn_events);
8735 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8736 let record = TerminalRecord {
8737 exit_code: exit_report.code,
8738 exit_signal: exit_report.signal,
8739 at_ms: exit_report.at_ms,
8740 disposition,
8741 exit_kind: exit_report.kind.into(),
8742 disposition_detail,
8743 };
8744 terminal_ring
8745 .lock()
8746 .unwrap_or_else(|poisoned| poisoned.into_inner())
8747 .record_exit(module_id, record);
8748}
8749
8750fn untrack_if_registration_released(
8751 process_liveness: &SupervisorProcessLiveness,
8752 registry: &Registry,
8753 module_id: &str,
8754 snapshot: &SharedSnapshot,
8755) {
8756 match registry.get_module(module_id) {
8757 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8758 Ok(Some(_)) => {}
8759 Err(err) => {
8760 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8761 }
8762 }
8763}
8764
8765#[cfg(test)]
8779fn apply_wire_spawn_args(
8780 command: &mut Command,
8781 spec: &ModuleSpec,
8782 connection_file_path: Option<&std::path::Path>,
8783 handle: Option<&SupervisorHandle>,
8784) -> Result<Option<NonceHandoff>, SuperviseError> {
8785 apply_wire_spawn_args_for_role(
8786 command,
8787 spec,
8788 connection_file_path,
8789 handle,
8790 SpawnRole::Plain,
8791 )
8792}
8793
8794#[cfg(unix)]
8799type NonceHandoff = subc_os::LaunchNonceHandoff;
8800#[cfg(windows)]
8801type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8802#[cfg(not(any(unix, windows)))]
8803type NonceHandoff = std::convert::Infallible;
8804
8805fn apply_wire_spawn_args_for_role(
8826 command: &mut Command,
8827 spec: &ModuleSpec,
8828 connection_file_path: Option<&std::path::Path>,
8829 handle: Option<&SupervisorHandle>,
8830 role: SpawnRole,
8831) -> Result<Option<NonceHandoff>, SuperviseError> {
8832 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8833 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8838 #[cfg(windows)]
8839 {
8840 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8841 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8842 }
8843 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8845 if spec.protocol == ModuleProtocol::None {
8846 return Ok(None);
8847 }
8848 if let Some(connection_file_path) = connection_file_path {
8849 command.arg(SUBC_ARG).arg(connection_file_path);
8850 }
8851
8852 let nonce = generate_launch_nonce()?;
8856 if let Some(handle) = handle {
8857 match role {
8858 SpawnRole::Plain => {
8859 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8860 if spec.reserved {
8861 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8862 }
8863 }
8864 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8865 }
8866 }
8867 #[cfg(unix)]
8868 let handoff = {
8869 let handoff =
8870 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8871 program: spec.program.clone(),
8872 source,
8873 cgroup_path: None,
8874 })?;
8875 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8876 Some(handoff)
8877 };
8878 #[cfg(windows)]
8879 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8880 #[cfg(not(any(unix, windows)))]
8881 let handoff = None;
8882 #[cfg(not(unix))]
8886 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8887 Ok(handoff)
8888}
8889
8890fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8891 command.env_remove(CK_LOG_ENV);
8892 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8899 for (key, value) in &spec.env {
8900 if matches!(
8904 key.as_str(),
8905 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8906 ) || key == SUBC_SPAWN_ROLE_ENV
8907 {
8908 continue;
8909 }
8910 command.env(key, value);
8911 }
8912}
8913
8914#[cfg(windows)]
8915fn prepare_windows_nonce_pipe(
8916 command: &mut Command,
8917 spec: &ModuleSpec,
8918 nonce: &str,
8919) -> Option<NonceHandoff> {
8920 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8921 Ok(handoff) => {
8922 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8923 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8928 Some(handoff)
8929 }
8930 Err(error) => {
8931 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8932 None
8933 }
8934 }
8935}
8936
8937#[cfg(windows)]
8938fn start_windows_nonce_pipe(
8939 handoff: Option<NonceHandoff>,
8940 pid: u32,
8941 spec: &ModuleSpec,
8942) -> Option<subc_os::LaunchNoncePipeDelivery> {
8943 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8947 Ok(delivery) => Some(delivery),
8948 Err(error) => {
8949 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8950 None
8951 }
8952 })
8953}
8954
8955fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8956 #[cfg(windows)]
8957 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8958 return Some(
8959 if snapshot
8960 .pipe_consumed
8961 .as_ref()
8962 .is_some_and(|consumed| consumed.get().is_some())
8963 {
8964 "pipe"
8965 } else {
8966 "env"
8967 }
8968 .to_owned(),
8969 );
8970 }
8971 let _ = snapshot;
8972 None
8973}
8974
8975#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8978enum SpawnRole {
8979 Plain,
8980 SwapCandidate,
8981}
8982
8983fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8986 if role == SpawnRole::SwapCandidate {
8987 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8988 }
8989}
8990
8991fn previous_instance_error(
8992 spec: &ModuleSpec,
8993 pid: u32,
8994 reason: crate::live_children::RetainedReason,
8995) -> SuperviseError {
8996 SuperviseError::Spawn {
8997 program: spec.program.clone(),
8998 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8999 cgroup_path: None,
9000 }
9001}
9002
9003fn carries_containment(configured: bool, has_container: bool) -> bool {
9004 configured || has_container
9005}
9006
9007fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
9008 if !carries_containment(
9009 spec.carry_containment,
9010 containment::current(&roster.containment).is_some(),
9011 ) {
9012 return None;
9013 }
9014 Some(SuperviseError::Spawn {
9015 program: spec.program.clone(),
9016 source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
9017 cgroup_path: None,
9018 })
9019}
9020
9021fn spawn_child(
9022 spec: &ModuleSpec,
9023 connection_file_path: Option<&std::path::Path>,
9024 handle: Option<&SupervisorHandle>,
9025 ring: &Arc<Mutex<StderrRing>>,
9026 capture_logs_dir: Option<&std::path::Path>,
9027 roster: &ChildRoster,
9028 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9029) -> Result<SupervisedChild, SuperviseError> {
9030 #[cfg(all(test, unix))]
9031 if let Some(source) = containment_tests::join_failure(spec, roster) {
9032 containment::terminate_blocking(&roster.containment);
9033 return Err(SuperviseError::Spawn {
9034 program: spec.program.clone(),
9035 source,
9036 cgroup_path: None,
9037 });
9038 }
9039 let result = spawn_child_in_slot(
9040 spec,
9041 connection_file_path,
9042 handle,
9043 ring,
9044 capture_logs_dir,
9045 roster,
9046 #[cfg(target_os = "linux")]
9047 cgroup_placement,
9048 SpawnRole::Plain,
9049 false,
9050 );
9051 if result.is_err() {
9052 if roster.admission_pending(&spec.module_id) {
9053 containment::terminate_deferred(&roster.containment);
9054 } else {
9055 containment::terminate_blocking(&roster.containment);
9056 }
9057 }
9058 result
9059}
9060
9061#[allow(clippy::too_many_arguments)]
9074fn spawn_child_in_slot(
9075 spec: &ModuleSpec,
9076 connection_file_path: Option<&std::path::Path>,
9077 handle: Option<&SupervisorHandle>,
9078 ring: &Arc<Mutex<StderrRing>>,
9079 capture_logs_dir: Option<&std::path::Path>,
9080 roster: &ChildRoster,
9081 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9082 role: SpawnRole,
9083 alternate_slot: bool,
9084) -> Result<SupervisedChild, SuperviseError> {
9085 if let Some(error) = recovery_block_error(spec, roster) {
9086 return Err(error);
9087 }
9088 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
9089 return Err(previous_instance_error(spec, pid, reason));
9090 }
9091 if roster.is_closed() {
9092 return Err(SuperviseError::Spawn {
9093 program: spec.program.clone(),
9094 source: io::Error::other("the daemon is shutting down; not starting a new process"),
9095 cgroup_path: None,
9096 });
9097 }
9098 let carried = containment::current(&roster.containment);
9099 let carrying = carries_containment(spec.carry_containment, carried.is_some());
9100 if role != SpawnRole::Plain && carrying {
9101 return Err(SuperviseError::InvalidSpec {
9102 reason: "a carried container cannot overlap a swap candidate".to_owned(),
9103 });
9104 }
9105 #[cfg(unix)]
9106 let launch_identity = if carrying && carried.is_none() {
9107 Some(
9108 containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
9109 program: spec.program.clone(),
9110 source,
9111 cgroup_path: None,
9112 })?,
9113 )
9114 } else {
9115 None
9116 };
9117 #[cfg(target_os = "linux")]
9118 let cgroup_placement = match &carried {
9119 Some(container) => container.cgroup().map(|(placement, _)| placement),
9120 None => cgroup_placement,
9121 };
9122 #[cfg(unix)]
9123 let group_target = if let Some(container) = carried
9124 .as_ref()
9125 .filter(|container| container.group().is_some())
9126 {
9127 containment::GroupTarget::Pinned(Arc::clone(container))
9128 } else {
9129 #[cfg(target_os = "linux")]
9130 let cgroup_container = carrying && cgroup_placement.is_some();
9131 #[cfg(not(target_os = "linux"))]
9132 let cgroup_container = false;
9133 if cgroup_container {
9134 containment::GroupTarget::NoGroup
9135 } else {
9136 containment::GroupTarget::OwnLeader
9137 }
9138 };
9139 #[cfg(target_os = "linux")]
9140 let cgroup_name = {
9141 if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
9146 name.to_owned()
9147 } else if cgroup_placement.is_none() {
9148 swap::cgroup_name(&spec.module_id, alternate_slot)
9149 } else {
9150 let nonce = generate_launch_nonce()?;
9151 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
9152 let mut end = spec.module_id.len().min(64);
9155 while !spec.module_id.is_char_boundary(end) {
9156 end -= 1;
9157 }
9158 format!(
9159 "{}_{suffix}",
9160 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
9161 )
9162 }
9163 };
9164 #[cfg(not(target_os = "linux"))]
9165 let _ = alternate_slot;
9166 #[cfg(target_os = "macos")]
9167 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
9168 #[cfg(windows)]
9169 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
9170 program: spec.program.clone(),
9171 source,
9172 cgroup_path: None,
9173 })?;
9174 #[cfg(windows)]
9175 let mut command = Command::new(image_capture.path());
9176 #[cfg(not(any(target_os = "macos", windows)))]
9177 let mut command = Command::new(&spec.program);
9178 command.args(&spec.args);
9179 apply_child_env(&mut command, spec);
9209 apply_spawn_role(&mut command, role);
9210 let nonce_handoff =
9211 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
9212
9213 #[cfg(target_os = "linux")]
9214 let cgroup_path = cgroup_placement
9215 .map(|placement| {
9216 if let Some(container) = &carried {
9217 let path = container
9218 .cgroup_path()
9219 .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
9220 placement.existing_module_path(path)?.ok_or_else(|| {
9221 io::Error::new(
9222 io::ErrorKind::NotFound,
9223 "carried cgroup disappeared before replacement",
9224 )
9225 })
9226 } else {
9227 if carrying {
9228 containment::prepare_cgroup_path(placement, &cgroup_name)
9229 } else {
9230 placement.module_path(&cgroup_name)
9231 }
9232 }
9233 })
9234 .transpose()
9235 .map_err(|source| SuperviseError::Cgroup {
9236 module_id: spec.module_id.clone(),
9237 source,
9238 })?;
9239 #[cfg(not(target_os = "linux"))]
9240 let cgroup_path: Option<PathBuf> = None;
9241 #[cfg(unix)]
9242 let mut admission_gate = if carrying {
9243 Some(
9244 admission::Gate::install(&mut command, spec, roster).map_err(|source| {
9245 SuperviseError::Spawn {
9246 program: spec.program.clone(),
9247 source,
9248 cgroup_path: cgroup_path.clone(),
9249 }
9250 })?,
9251 )
9252 } else {
9253 None
9254 };
9255 #[cfg(target_os = "macos")]
9256 exec_ack.install(command.as_std_mut());
9257 #[cfg(target_os = "linux")]
9258 if let Some(path) = &cgroup_path {
9259 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
9260 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9261 remove_module_cgroup(placement, &cgroup_name);
9262 }
9263 return Err(error);
9264 }
9265 }
9266
9267 let output_sink = if let Some(logs_dir) = capture_logs_dir {
9268 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
9269 match ChildOutputSink::open(&path, capture_retention(spec)) {
9270 Ok(sink) => sink,
9271 Err(error) => {
9272 warn!(
9273 module_id = %spec.module_id,
9274 path = %path.display(),
9275 error = %error,
9276 "could not open child output capture file; forwarding to stderr"
9277 );
9278 ChildOutputSink::Stderr
9279 }
9280 }
9281 } else {
9282 ChildOutputSink::Stderr
9283 };
9284
9285 command.stdout(Stdio::piped());
9286 command.stderr(Stdio::piped());
9287 command.kill_on_drop(!carrying);
9288 #[cfg(unix)]
9305 command.process_group(
9306 carried
9307 .as_ref()
9308 .map(|container| container.verified_join_group())
9309 .transpose()
9310 .map_err(|source| SuperviseError::Spawn {
9311 program: spec.program.clone(),
9312 source,
9313 cgroup_path: cgroup_path.clone(),
9314 })?
9315 .flatten()
9316 .unwrap_or(0) as i32,
9317 );
9318 command.stdin(Stdio::null());
9319 #[cfg(unix)]
9320 if let Some(gate) = &mut admission_gate {
9321 gate.install_placement(&mut command);
9322 }
9323 #[cfg(unix)]
9327 if let Some(handoff) = nonce_handoff {
9328 handoff.install_last(command.as_std_mut());
9329 }
9330 #[cfg(not(any(unix, windows)))]
9331 let _ = nonce_handoff;
9332
9333 #[cfg(windows)]
9338 {
9339 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
9340 program: spec.program.clone(),
9341 source,
9342 cgroup_path: None,
9343 })?;
9344 subc_jobobject::suspend_on_create_async(&mut command);
9345 }
9346 let spawned_at = Instant::now();
9350 #[cfg(all(test, unix))]
9351 containment_tests::record_spawn_attempt(spec, &mut command);
9352 #[cfg(unix)]
9353 let spawned = match admission_gate {
9354 Some(gate) => gate
9355 .spawn(
9356 command,
9357 spec,
9358 roster,
9359 launch_identity,
9360 #[cfg(target_os = "linux")]
9361 cgroup_placement
9362 .zip(cgroup_path.as_ref())
9363 .map(|(placement, path)| {
9364 (placement.clone(), cgroup_name.clone(), path.clone())
9365 }),
9366 )
9367 .map(|(child, admitted)| (child, Some(admitted))),
9368 None => command.spawn().map(|child| (child, None)),
9369 };
9370 #[cfg(not(unix))]
9371 let spawned = command.spawn();
9372 let spawned = match spawned {
9373 Ok(child) => child,
9374 Err(source) => {
9375 if carried
9376 .as_ref()
9377 .and_then(|container| container.group())
9378 .is_some()
9379 {
9380 containment::terminate_blocking(&roster.containment);
9381 }
9382 #[cfg(target_os = "linux")]
9383 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9384 remove_module_cgroup(placement, &cgroup_name);
9385 }
9386 return Err(SuperviseError::Spawn {
9387 program: spec.program.clone(),
9388 source,
9389 cgroup_path,
9390 });
9391 }
9392 };
9393 #[cfg(unix)]
9394 let (mut child, mut admitted) = spawned;
9395 #[cfg(unix)]
9396 let group_target = admitted
9397 .as_ref()
9398 .map(|attempt| attempt.group_target())
9399 .unwrap_or(group_target);
9400 #[cfg(not(unix))]
9401 let mut child = spawned;
9402 #[cfg(test)]
9403 spawn_record_tests::after_create(&child, spec);
9404 #[cfg(target_os = "macos")]
9409 drop(exec_ack);
9410
9411 #[cfg(windows)]
9416 let windows_image = match image_capture.bind_tokio(&child) {
9417 Ok(image) => Arc::new(image),
9418 Err(source) => {
9419 kill_and_reap_rejected_child(&mut child);
9420 return Err(SuperviseError::Spawn {
9421 program: spec.program.clone(),
9422 source,
9423 cgroup_path: None,
9424 });
9425 }
9426 };
9427 #[cfg(windows)]
9430 let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
9431 let spawned_at_ms = unix_ms_now();
9432 #[cfg(not(windows))]
9433 let spawned_from = spec.program.clone();
9434 #[cfg(windows)]
9435 let spawned_from = windows_image.path().to_path_buf();
9436 #[cfg(not(windows))]
9437 let spawned_file_identity = spawned_file_identity(&spawned_from);
9438 #[cfg(windows)]
9439 let spawned_file_identity = Some(SpawnedFileIdentity {
9440 device: 0,
9441 inode: 0,
9442 windows_image: Some(Arc::clone(&windows_image)),
9443 });
9444 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9445 program: spec.program.clone(),
9446 source: io::Error::other("spawned child exposed no live pid"),
9447 cgroup_path: cgroup_path.clone(),
9448 })?;
9449 #[cfg(windows)]
9450 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9451 #[cfg(not(windows))]
9452 let process_start_time = admitted
9453 .as_ref()
9454 .map(|admitted| admitted.birth)
9455 .or_else(|| crate::provenance::process_start_time(pid));
9456 #[cfg(windows)]
9457 let process_start_time = Some(windows_image.start_time());
9458 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9459 #[cfg(all(test, target_os = "macos"))]
9460 privacy_exec_boundary_tests::before_image_sample(spec, pid);
9461 #[cfg(not(windows))]
9466 let recorded_image = observe_spawned_image(pid);
9467 #[cfg(target_os = "macos")]
9471 let recorded_image = if privacy_exec.is_some() {
9472 None
9473 } else {
9474 recorded_image
9475 };
9476 #[cfg(target_os = "linux")]
9477 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9478 #[cfg(not(target_os = "linux"))]
9479 let recorded_cgroup_name = None;
9480 #[cfg(unix)]
9481 let preadmitted = admitted
9482 .as_mut()
9483 .and_then(|admitted| admitted.leader.take());
9484 #[cfg(not(unix))]
9485 let preadmitted: Option<crate::child_roster::RosterGuard> = None;
9486 let roster_guard = if let Some(guard) = preadmitted {
9487 Ok(guard)
9488 } else {
9489 roster.admit(
9490 spec.module_id.clone(),
9491 pid,
9492 spec.protocol,
9493 process_start_time,
9494 crate::child_roster::RecordedIdentity {
9495 container: None,
9496 #[cfg(unix)]
9497 carried: None,
9498 #[cfg(not(windows))]
9499 start_time: recorded_image.map(|image| image.start_time),
9500 #[cfg(windows)]
9501 start_time: process_start_time,
9502 #[cfg(not(windows))]
9503 executable: recorded_image
9504 .and_then(|image| image.executable)
9505 .map(crate::live_children::ExecutableIdentity::from),
9506 #[cfg(windows)]
9507 executable: Some(crate::live_children::ExecutableIdentity::from(
9508 windows_image.identity(),
9509 )),
9510 cgroup_name: recorded_cgroup_name,
9511 #[cfg(target_os = "linux")]
9512 cgroup_placement: cgroup_placement.cloned(),
9513 },
9514 )
9515 };
9516 let roster_guard = match roster_guard {
9517 Ok(guard) => guard,
9518 Err(failure) => {
9519 return failure.with_cleanup(|source| {
9520 #[cfg(windows)]
9521 if let Some(job) = &job {
9522 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9523 }
9524 #[cfg(target_os = "linux")]
9525 kill_module_cgroup(cgroup_placement, &cgroup_name);
9526 #[cfg(unix)]
9527 let rejected_group = group_target.clone();
9528 kill_and_reap_rejected_child(
9529 &mut child,
9530 #[cfg(unix)]
9531 rejected_group,
9532 );
9533 #[cfg(target_os = "linux")]
9534 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9535 remove_module_cgroup(placement, &cgroup_name);
9536 }
9537 Err(SuperviseError::Spawn {
9538 program: spec.program.clone(),
9539 source: io::Error::other(format!(
9540 "could not durably record module child: {source}"
9541 )),
9542 cgroup_path,
9543 })
9544 });
9545 }
9546 };
9547 #[cfg(unix)]
9548 if admitted.is_some() {
9549 if let Some(image) = recorded_image {
9550 if let Err(source) = roster_guard.confirm_executable(image) {
9551 kill_and_reap_rejected_child(&mut child, group_target.clone());
9552 return Err(SuperviseError::Spawn {
9553 program: spec.program.clone(),
9554 source,
9555 cgroup_path,
9556 });
9557 }
9558 }
9559 }
9560 if roster.is_closed() {
9565 if carrying {
9566 #[cfg(unix)]
9567 let rejected_group = admitted
9568 .as_ref()
9569 .map(|admitted| admitted.rejection_target())
9570 .unwrap_or_else(|| group_target.clone());
9571 #[cfg(windows)]
9572 if let Some(job) = &job {
9573 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9574 }
9575 kill_and_reap_rejected_child(
9576 &mut child,
9577 #[cfg(unix)]
9578 rejected_group,
9579 );
9580 #[cfg(all(test, unix))]
9581 let _ =
9582 roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9583 drop(roster_guard);
9584 return Err(SuperviseError::Spawn {
9585 program: spec.program.clone(),
9586 source: io::Error::other(
9587 "the daemon began shutting down during carried admission; ended the container",
9588 ),
9589 cgroup_path,
9590 });
9591 }
9592 #[cfg(target_os = "linux")]
9594 kill_module_cgroup(cgroup_placement, &cgroup_name);
9595 if let Err(error) = child.start_kill() {
9596 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9597 }
9598 #[cfg(target_os = "linux")]
9599 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9600 while matches!(child.try_wait(), Ok(None)) {
9604 std::thread::yield_now();
9605 }
9606 if matches!(
9607 subc_cgroup::kill_module(Some(placement), &cgroup_name),
9608 subc_cgroup::KillOutcome::Killed
9609 ) {
9610 if let Ok(path) = placement.module_path(&cgroup_name) {
9611 while std::fs::read_to_string(path.join("cgroup.events"))
9612 .ok()
9613 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9614 {
9615 std::thread::yield_now();
9616 }
9617 }
9618 }
9619 remove_module_cgroup(placement, &cgroup_name);
9620 }
9621 drop(roster_guard);
9622 return Err(SuperviseError::Spawn {
9623 program: spec.program.clone(),
9624 source: io::Error::other(
9625 "the daemon began shutting down while this process was starting; ended it",
9626 ),
9627 cgroup_path,
9628 });
9629 }
9630
9631 let stdout_pump = match child.stdout.take() {
9632 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9633 None => {
9634 warn!(
9635 module_id = %spec.module_id,
9636 "spawned child exposed no stdout pipe; file capture will be incomplete"
9637 );
9638 None
9639 }
9640 };
9641 let stderr_pump = match child.stderr.take() {
9642 Some(stderr) => {
9643 let generation = ring
9644 .lock()
9645 .unwrap_or_else(|poisoned| poisoned.into_inner())
9646 .begin_process();
9647 Some(StderrPump {
9648 task: tokio::spawn(pump_stderr_to(
9649 stderr,
9650 Arc::clone(ring),
9651 generation,
9652 output_sink,
9653 )),
9654 generation,
9655 })
9656 }
9657 None => {
9658 ring.lock()
9662 .unwrap_or_else(|poisoned| poisoned.into_inner())
9663 .mark_not_captured("stderr pipe was not available on spawn");
9664 warn!(
9665 module_id = %spec.module_id,
9666 "spawned child exposed no stderr pipe; tail will be unavailable"
9667 );
9668 None
9669 }
9670 };
9671
9672 let exec = Arc::new(crate::activity::ExecOutcome::new());
9673 #[cfg(target_os = "macos")]
9674 let pending_exec = privacy_exec.is_some();
9675 #[cfg(not(target_os = "macos"))]
9676 let pending_exec = false;
9677 if !pending_exec {
9678 exec.finish(crate::activity::ExecState::CommittedExec);
9679 }
9680 #[cfg(windows)]
9681 let subtree_target = job
9682 .as_ref()
9683 .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
9684 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
9685 #[cfg(target_os = "linux")]
9686 let subtree_target = cgroup_path
9687 .map(crate::subtree_resources::Target::Cgroup)
9688 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
9689 #[cfg(not(any(target_os = "linux", windows)))]
9690 let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
9691 let mut supervised = SupervisedChild {
9692 subtree_target,
9693 child: containment::ProcessChild::plain(child, pid),
9694 exec: crate::activity::ExecOwner(exec),
9695 #[cfg(windows)]
9696 nonce_delivery,
9697 protocol: spec.protocol,
9698 #[cfg(target_os = "linux")]
9699 module_id: cgroup_name,
9700 #[cfg(target_os = "linux")]
9701 cgroup_placement: cgroup_placement.cloned(),
9702 #[cfg(windows)]
9703 job,
9704 stdout_pump,
9705 stderr_pump,
9706 stderr_ring: Arc::clone(ring),
9707 spawned_at,
9708 registration_deadline: None,
9709 registration_watch: None,
9710 spawned_at_ms,
9711 spawned_from,
9712 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9713 spawned_file_identity,
9714 process_start_time,
9715 process_identity,
9716 pid,
9717 roster_guard: Some(roster_guard),
9718 #[cfg(target_os = "macos")]
9719 privacy_exec,
9720 #[cfg(target_os = "macos")]
9721 report_ready: Arc::new(OnceLock::new()),
9722 spawn_failure: None,
9723 };
9724 if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
9725 supervised.child.reject(
9726 #[cfg(unix)]
9727 group_target,
9728 );
9729 #[cfg(all(test, unix))]
9730 let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9731 return Err(error);
9732 }
9733 #[cfg(unix)]
9734 if let Some(admitted) = admitted.take() {
9735 admitted.disarm();
9736 }
9737 #[cfg(windows)]
9738 if let Err(error) = resume_suspended_child(pid, spec) {
9739 containment::terminate_blocking(&roster.containment);
9740 let _ = supervised.child.start_kill();
9741 return Err(error);
9742 }
9743 Ok(supervised)
9744}
9745
9746#[cfg(test)]
9747mod spawn_record_tests {
9748 use super::*;
9749
9750 thread_local! {
9751 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9752 }
9753
9754 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9755 LAST_PID.set(child.id());
9756 #[cfg(windows)]
9757 if let Some((_, address)) = _spec
9758 .env
9759 .iter()
9760 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9761 {
9762 use std::io::{Read, Write};
9763 let mut stream = std::net::TcpStream::connect(address).unwrap();
9764 stream
9765 .write_all(&child.id().unwrap().to_ne_bytes())
9766 .unwrap();
9767 let mut release = [0];
9768 stream.read_exact(&mut release).unwrap();
9769 }
9770 }
9771
9772 #[cfg(windows)]
9773 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9774 if spec
9775 .env
9776 .iter()
9777 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9778 {
9779 Err(io::Error::other("injected module job assignment failure"))
9780 } else {
9781 Ok(())
9782 }
9783 }
9784
9785 fn spec() -> ModuleSpec {
9786 ModuleSpec {
9787 module_id: "spawn-record-worker".into(),
9788 #[cfg(unix)]
9789 program: "/bin/sleep".into(),
9790 #[cfg(windows)]
9791 program: std::env::current_exe().unwrap(),
9792 #[cfg(unix)]
9793 args: vec!["60".into()],
9794 #[cfg(windows)]
9795 args: vec![
9796 "--exact".into(),
9797 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9798 "--nocapture".into(),
9799 ],
9800 env: vec![],
9801 reserved: false,
9802 reserved_prefixes: vec![],
9803 protocol: ModuleProtocol::None,
9804 overlap: Default::default(),
9805 carry_containment: false,
9806 }
9807 }
9808
9809 #[tokio::test]
9810 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9811 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9812 let supervisor =
9813 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9814 .with_live_children_record(home.join("live-children.json"));
9815 crate::live_children::fail_next_file_sync();
9816 let result = supervisor.spawn(spec());
9817 let failed = result.is_err();
9818 if let Ok(module) = result {
9819 module.stop().await.unwrap();
9820 }
9821 assert!(failed, "record failure must fail spawn");
9822 let pid = LAST_PID.get().expect("the real OS child was created");
9823 assert!(
9824 subc_os::Process::open(pid).unwrap().is_none(),
9825 "child {pid} survived rejected admission or was not reaped"
9826 );
9827 }
9828
9829 #[cfg(target_os = "macos")]
9830 #[test]
9831 #[ignore = "manual on-device latency measurement"]
9832 fn durable_record_write_latency_50() {
9833 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9834 let path = home.join("live-children.json");
9835 let children = vec![crate::live_children::LiveChild {
9836 module_id: "latency-worker".into(),
9837 pid: std::process::id(),
9838 protocol: ModuleProtocol::None,
9839 start_time: Some(1),
9840 executable: None,
9841 cgroup_name: None,
9842 container: None,
9843 }];
9844 let mut samples = Vec::new();
9845 for _ in 0..50 {
9846 let start = std::time::Instant::now();
9847 crate::live_children::write_record(&path, &children).unwrap();
9848 samples.push(start.elapsed());
9849 }
9850 samples.sort();
9851 eprintln!(
9852 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9853 (samples[24] + samples[25]) / 2,
9854 samples[47]
9855 );
9856 }
9857
9858 #[cfg(windows)]
9859 #[test]
9860 fn windows_store_touch_fixture() {
9861 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9862 std::fs::write(path, b"module ran").unwrap();
9863 }
9864 }
9865
9866 #[cfg(windows)]
9867 #[tokio::test]
9868 async fn windows_daemon_fixture() {
9869 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9870 return;
9871 };
9872 let mut spec = spec();
9873 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9874 let supervisor =
9875 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9876 let _module = supervisor.spawn(spec).unwrap();
9877 }
9878
9879 #[cfg(windows)]
9880 #[test]
9881 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9882 use std::io::Read;
9883 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9884 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9887 command
9888 .args([
9889 "--exact",
9890 "supervise::spawn_record_tests::windows_daemon_fixture",
9891 "--nocapture",
9892 ])
9893 .env(
9894 "SUBC_TEST_DAEMON_BARRIER",
9895 listener.local_addr().unwrap().to_string(),
9896 );
9897 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9898 listener.set_nonblocking(true).unwrap();
9899 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9900 let mut stream = loop {
9901 if let Ok((stream, _)) = listener.accept() {
9902 break stream;
9903 }
9904 assert!(
9905 daemon.child.try_wait().unwrap().is_none(),
9906 "daemon failed before rendezvous"
9907 );
9908 assert!(
9909 std::time::Instant::now() < deadline,
9910 "daemon did not reach CreateProcess failpoint"
9911 );
9912 std::thread::yield_now();
9913 };
9914 stream
9915 .set_read_timeout(Some(Duration::from_secs(30)))
9916 .unwrap();
9917 let mut pid = [0; 4];
9918 stream.read_exact(&mut pid).unwrap();
9919 let pid = u32::from_ne_bytes(pid);
9920 assert!(subc_jobobject::process_exists(pid));
9921 daemon.child.kill().unwrap();
9924 daemon.child.wait().unwrap();
9925 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9926 if !exited {
9927 daemon.job.terminate().unwrap();
9928 }
9929 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9930 }
9931
9932 #[cfg(windows)]
9933 #[tokio::test]
9934 async fn windows_containment_failure_never_resumes() {
9935 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9936 let sentinel = home.join("store-touched");
9937 let mut spec = spec();
9938 spec.env
9939 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9940 spec.env.push((
9941 "SUBC_TEST_STORE_TOUCH".into(),
9942 sentinel.display().to_string(),
9943 ));
9944 let supervisor =
9945 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9946 let result = supervisor.spawn(spec);
9947 if let Ok(module) = result {
9950 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9951 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9952 tokio::task::yield_now().await;
9953 }
9954 })
9955 .await;
9956 let _ = module.stop().await;
9957 panic!(
9958 "assignment failure resumed a module; store touched={}",
9959 sentinel.exists()
9960 );
9961 }
9962 assert!(
9963 !sentinel.exists(),
9964 "refused module executed store-touch sentinel"
9965 );
9966 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9967 }
9968
9969 #[cfg(windows)]
9970 #[tokio::test]
9971 async fn windows_nested_daemon_fixture() {
9972 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9973 return;
9974 };
9975 let mut spec = spec();
9976 spec.env.push((
9977 "SUBC_TEST_STORE_TOUCH".into(),
9978 sentinel.to_str().unwrap().into(),
9979 ));
9980 let supervisor =
9981 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9982 let module = supervisor
9983 .spawn(spec)
9984 .expect("per-module job must nest below daemon and host jobs");
9985 tokio::time::timeout(Duration::from_secs(10), async {
9986 while !PathBuf::from(&sentinel).exists() {
9987 tokio::task::yield_now().await;
9988 }
9989 })
9990 .await
9991 .unwrap();
9992 module.stop().await.unwrap();
9993 }
9994
9995 #[cfg(windows)]
9996 #[test]
9997 fn windows_module_job_nests_under_host_job() {
9998 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9999 let sentinel = home.join("nested-store-touched");
10000 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
10001 command
10002 .args([
10003 "--exact",
10004 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
10005 "--nocapture",
10006 ])
10007 .env("SUBC_TEST_NESTED_STORE", &sentinel);
10008 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
10009 assert!(daemon.child.wait().unwrap().success());
10010 assert!(sentinel.exists(), "nested protocol:none module never ran");
10011 }
10012}
10013
10014#[cfg(target_os = "linux")]
10015pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
10016 use subc_cgroup::KillOutcome;
10017 match subc_cgroup::kill_module(placement, module_id) {
10018 KillOutcome::Killed => {}
10019 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
10020 debug!(
10021 module_id,
10022 "cgroup tree kill unavailable; using direct-child kill"
10023 );
10024 }
10025 KillOutcome::IoError { path, error } => {
10026 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
10027 }
10028 }
10029}
10030
10031#[cfg(windows)]
10039fn contain_spawned_child(
10040 child: &mut Child,
10041 spec: &ModuleSpec,
10042 carried: Option<&Arc<containment::CarriedContainment>>,
10043) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
10044 let assignment = || {
10045 let job = match carried {
10046 Some(container) => container.job(),
10047 None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
10048 };
10049 #[cfg(test)]
10050 spawn_record_tests::assignment_failpoint(spec)?;
10051 job.lock()
10052 .unwrap_or_else(|p| p.into_inner())
10053 .assign(child)?;
10054 Ok::<_, io::Error>(job)
10055 };
10056 match assignment() {
10057 Ok(job) => Ok(Some(job)),
10058 Err(source) => {
10059 kill_and_reap_rejected_child(child);
10060 Err(SuperviseError::Spawn {
10061 program: spec.program.clone(),
10062 source: io::Error::other(format!(
10063 "Windows module job containment failed: {source}"
10064 )),
10065 cgroup_path: None,
10066 })
10067 }
10068 }
10069}
10070
10071fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
10074 #[cfg(unix)]
10075 group.signal_rejected(child);
10076 let _ = child.start_kill();
10077 while matches!(child.try_wait(), Ok(None)) {
10078 std::thread::yield_now();
10079 }
10080}
10081
10082#[cfg(windows)]
10087fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
10088 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
10089 let _ = std::process::Command::new("taskkill.exe")
10093 .args(["/PID", &pid.to_string(), "/T", "/F"])
10094 .stdin(Stdio::null())
10095 .stdout(Stdio::null())
10096 .stderr(Stdio::null())
10097 .status();
10098 return Err(SuperviseError::Spawn {
10099 program: spec.program.clone(),
10100 source,
10101 cgroup_path: None,
10102 });
10103 }
10104 Ok(())
10105}
10106
10107#[cfg(target_os = "linux")]
10108fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10109 match placement.remove_module(module_id) {
10110 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
10111 Err(error) => warn!(
10112 module_id,
10113 error = %error,
10114 "could not remove module cgroup after process exit; continuing teardown"
10115 ),
10116 }
10117}
10118
10119#[cfg(target_os = "linux")]
10120const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
10121#[cfg(target_os = "linux")]
10122const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
10123
10124#[cfg(target_os = "linux")]
10125async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10126 let path = placement.module_path(module_id).ok();
10127 if path.as_ref().is_some_and(|path| {
10128 std::fs::read_to_string(path.join("cgroup.events"))
10129 .ok()
10130 .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
10131 }) {
10132 remove_module_cgroup(placement, module_id);
10133 return;
10134 }
10135
10136 if terminate_reaped_cgroup(placement, module_id).await {
10139 if let Some(path) = &path {
10140 let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
10141 while cgroup_is_populated(path) && Instant::now() < deadline {
10142 sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
10143 }
10144 }
10145 }
10146
10147 if matches!(
10151 subc_cgroup::kill_module(Some(placement), module_id),
10152 subc_cgroup::KillOutcome::Killed
10153 ) {
10154 if let Some(path) = &path {
10155 while cgroup_is_populated(path) {
10156 sleep(Duration::from_millis(1)).await;
10157 }
10158 }
10159 }
10160 remove_module_cgroup(placement, module_id);
10161}
10162
10163#[cfg(target_os = "linux")]
10164fn cgroup_is_populated(path: &std::path::Path) -> bool {
10165 std::fs::read_to_string(path.join("cgroup.events"))
10166 .ok()
10167 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
10168}
10169
10170#[cfg(target_os = "linux")]
10171async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
10172 let frozen =
10173 match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
10174 .await
10175 {
10176 subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
10177 outcome => {
10178 tracing::debug!(
10179 module_id,
10180 ?outcome,
10181 "cgroup freeze failed; skipping SIGTERM grace"
10182 );
10183 return false;
10184 }
10185 };
10186 let listed = match frozen.processes() {
10187 subc_cgroup::ProcsOutcome::Listed(pids) => {
10188 for pid in pids.into_iter().filter(|pid| *pid > 1) {
10189 if let Some(pid) = rustix::process::Pid::from_raw(pid) {
10190 if let Err(error) =
10191 rustix::process::kill_process(pid, rustix::process::Signal::TERM)
10192 {
10193 tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
10194 }
10195 }
10196 }
10197 true
10198 }
10199 outcome => {
10200 tracing::debug!(
10201 module_id,
10202 ?outcome,
10203 "cgroup process listing failed; skipping SIGTERM grace"
10204 );
10205 false
10206 }
10207 };
10208 match frozen.thaw() {
10209 subc_cgroup::ThawOutcome::Thawed => listed,
10210 outcome => {
10211 tracing::debug!(
10212 module_id,
10213 ?outcome,
10214 "cgroup thaw failed; proceeding to cgroup.kill"
10215 );
10216 false
10217 }
10218 }
10219}
10220
10221#[cfg(target_os = "linux")]
10222fn apply_cgroup_placement(
10223 command: &mut Command,
10224 spec: &ModuleSpec,
10225 path: &std::path::Path,
10226) -> Result<(), SuperviseError> {
10227 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
10228 module_id: spec.module_id.clone(),
10229 source,
10230 })
10231}
10232
10233fn capture_retention(spec: &ModuleSpec) -> Retention {
10234 let defaults = Retention::default();
10235 let value = |name: &str| {
10236 spec.env
10237 .iter()
10238 .rev()
10239 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
10240 };
10241 Retention {
10242 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
10243 .and_then(|value| value.parse().ok())
10244 .unwrap_or(defaults.max_file_mb),
10245 keep: value(CAPTURE_KEEP_ENV)
10246 .and_then(|value| value.parse().ok())
10247 .unwrap_or(defaults.keep),
10248 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
10249 .and_then(|value| value.parse().ok())
10250 .unwrap_or(defaults.max_age_days),
10251 }
10252}
10253
10254fn generate_launch_nonce() -> Result<String, SuperviseError> {
10257 let mut bytes = [0u8; 32];
10258 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
10259 reason: source.to_string(),
10260 })?;
10261 let mut hex = String::with_capacity(64);
10262 for b in bytes {
10263 use std::fmt::Write;
10264 let _ = write!(hex, "{b:02x}");
10265 }
10266 Ok(hex)
10267}
10268
10269fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
10272 if a.len() != b.len() {
10273 return false;
10274 }
10275 let mut diff = 0u8;
10276 for (x, y) in a.iter().zip(b.iter()) {
10277 diff |= x ^ y;
10278 }
10279 diff == 0
10280}
10281
10282#[cfg(not(windows))]
10286fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
10287 let process = subc_os::Process::open(pid).ok().flatten()?;
10288 #[cfg(target_os = "linux")]
10289 return sample_after_linux_exec(
10290 || {
10291 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
10292 let committed = linux_exec_committed(&stat);
10293 #[cfg(test)]
10294 spawn_image_tests::report_exec_state(committed);
10295 committed
10296 },
10297 || process.observe(),
10298 );
10299 #[cfg(not(target_os = "linux"))]
10300 process.observe()
10301}
10302
10303#[cfg(target_os = "linux")]
10307fn linux_exec_committed(stat: &str) -> Option<bool> {
10308 const PF_FORKNOEXEC: u64 = 0x40;
10309 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
10310 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
10311}
10312
10313#[cfg(target_os = "linux")]
10314fn sample_after_linux_exec(
10315 mut exec_committed: impl FnMut() -> Option<bool>,
10316 observe: impl FnOnce() -> Option<subc_os::Observation>,
10317) -> Option<subc_os::Observation> {
10318 let deadline = std::time::Instant::now() + Duration::from_secs(1);
10319 while !exec_committed()? {
10320 if std::time::Instant::now() >= deadline {
10324 return None;
10325 }
10326 std::thread::sleep(Duration::from_micros(50));
10327 }
10328 observe()
10329}
10330
10331#[cfg(all(test, target_os = "linux"))]
10332mod spawn_image_tests {
10333 use super::*;
10334 use std::cell::{Cell, RefCell};
10335
10336 enum ExecEvent {
10337 Uncommitted,
10338 Image(Option<subc_os::Observation>),
10339 }
10340
10341 thread_local! {
10342 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
10343 }
10344
10345 pub(super) fn report_exec_state(committed: Option<bool>) {
10346 if committed == Some(false) {
10347 EXEC_EVENTS.with(|events| {
10348 if let Some(events) = events.borrow_mut().take() {
10349 let _ = events.send(ExecEvent::Uncommitted);
10350 }
10351 });
10352 }
10353 }
10354
10355 #[test]
10356 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
10357 use std::io::{Read, Write};
10358 use std::os::fd::AsRawFd;
10359
10360 let descriptor = std::fs::File::open("/dev/null").unwrap();
10361 let mut command = std::process::Command::new("/bin/sleep");
10362 command.arg("30");
10363 let (mut ready, mut release) =
10364 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
10365 .unwrap();
10366 let spawn = std::thread::spawn(move || command.spawn());
10367 let mut message = [0u8; 8];
10368 ready.read_exact(&mut message).unwrap();
10369 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
10370 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
10371 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
10372 let (events, received) = std::sync::mpsc::channel();
10373 let observer = std::thread::spawn(move || {
10374 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
10375 let image = observe_spawned_image(pid);
10376 let _ = events.send(ExecEvent::Image(image));
10377 });
10378 let first = received.recv_timeout(Duration::from_secs(5));
10382 release.write_all(&[1]).unwrap();
10383 let mut child = spawn.join().unwrap().unwrap();
10384 let after = match first {
10385 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
10386 Ok(ExecEvent::Image(image)) => image,
10387 _ => None,
10388 },
10389 Ok(ExecEvent::Image(image)) => image,
10390 Err(_) => None,
10391 };
10392 observer.join().unwrap();
10393 child.kill().unwrap();
10394 child.wait().unwrap();
10395
10396 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
10397 assert_eq!(
10398 inherited.unwrap().executable,
10399 subc_os::file_identity(&std::env::current_exe().unwrap())
10400 );
10401 assert_eq!(
10402 after.unwrap().executable,
10403 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
10404 );
10405 }
10406
10407 #[test]
10408 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
10409 let committed = Cell::new(false);
10410 let inherited = subc_os::Observation {
10411 start_time: 23,
10412 executable: Some(subc_os::FileIdentity {
10413 device: 29,
10414 inode: 31,
10415 }),
10416 };
10417 let module = subc_os::Observation {
10418 start_time: 23,
10419 executable: Some(subc_os::FileIdentity {
10420 device: 29,
10421 inode: 37,
10422 }),
10423 };
10424 let polls = Cell::new(0);
10425 let image = sample_after_linux_exec(
10426 || {
10427 polls.set(polls.get() + 1);
10428 committed.set(polls.get() == 2);
10431 Some(committed.get())
10432 },
10433 || Some(if committed.get() { module } else { inherited }),
10434 );
10435 assert_eq!(image, Some(module));
10436 assert_eq!(polls.get(), 2);
10437 }
10438
10439 #[test]
10440 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
10441 assert_eq!(
10442 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
10443 Some(false)
10444 );
10445 assert_eq!(
10446 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
10447 Some(true)
10448 );
10449 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
10450 assert_eq!(linux_exec_committed("17 (a) R"), None);
10451 }
10452
10453 #[test]
10454 fn unreadable_exec_state_never_samples_an_inherited_image() {
10455 assert_eq!(
10456 sample_after_linux_exec(
10457 || None,
10458 || panic!("must not sample before exec confirmation")
10459 ),
10460 None
10461 );
10462 }
10463
10464 #[test]
10465 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
10466 assert_eq!(
10467 sample_after_linux_exec(
10468 || Some(false),
10469 || panic!("must not sample after an exec timeout")
10470 ),
10471 None
10472 );
10473 }
10474}
10475
10476#[cfg(windows)]
10479fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
10480 let mut program = spec.program.clone();
10481 if program.is_absolute() || program.components().count() > 1 {
10482 match subc_os::ExecutableCapture::open(&program) {
10485 Ok(capture) => return Ok(capture),
10486 Err(error)
10487 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
10488 Err(error) => return Err(error),
10489 }
10490 program.set_extension("exe");
10491 return subc_os::ExecutableCapture::open(&program);
10492 }
10493 if program.extension().is_none() {
10494 program.set_extension("exe");
10495 }
10496 let custom_path = spec
10497 .env
10498 .iter()
10499 .rev()
10500 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
10501 let mut directories = Vec::new();
10502 if let Some(parent) = std::env::current_exe()?.parent() {
10505 directories.push(parent.to_path_buf());
10506 }
10507 directories.push(std::env::current_dir()?);
10508 if let Some(windows) = std::env::var_os("SystemRoot") {
10509 let windows = PathBuf::from(windows);
10510 directories.push(windows.join("System32"));
10511 directories.push(windows.clone());
10512 }
10513 let path = custom_path
10514 .map(|(_, value)| std::ffi::OsString::from(value))
10515 .or_else(|| std::env::var_os("PATH"));
10516 if let Some(path) = path {
10517 directories.extend(std::env::split_paths(&path));
10518 }
10519 for directory in directories {
10520 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
10521 Ok(capture) => return Ok(capture),
10522 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
10523 Err(error) => return Err(error),
10524 }
10525 }
10526 Err(io::Error::new(
10527 io::ErrorKind::NotFound,
10528 "resolved executable not found",
10529 ))
10530}
10531
10532fn spawn_and_mark_running(
10533 spec: &ModuleSpec,
10534 runtime: &SupervisorRuntimeConfig,
10535 snapshot: &SharedSnapshot,
10536) -> Result<SupervisedChild, SuperviseError> {
10537 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10538 let registration_watch = watch_registration(&runtime.registry, spec);
10539 #[cfg(unix)]
10540 let joining_group = containment::current(&runtime.child_roster.containment)
10541 .is_some_and(|container| container.group().is_some());
10542 let spawn = || {
10543 spawn_child(
10544 spec,
10545 runtime.connection_file_path.as_deref(),
10546 runtime.supervisor_handle.as_ref(),
10547 &runtime.stderr_ring,
10548 runtime.capture_logs_dir.as_deref(),
10549 &runtime.child_roster,
10550 #[cfg(target_os = "linux")]
10551 runtime.cgroup_placement.as_ref(),
10552 )
10553 };
10554 let result = spawn();
10555 #[cfg(unix)]
10556 let result = if result.is_err()
10557 && joining_group
10558 && !runtime.child_roster.is_closed()
10559 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
10560 && runtime
10561 .child_roster
10562 .previous_instance(&spec.module_id)
10563 .is_none()
10564 {
10565 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
10569 spawn()
10570 } else {
10571 result
10572 };
10573 let child = result?;
10574 let mut child = child;
10575 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10576 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
10577 containment::terminate_blocking(&runtime.child_roster.containment);
10578 return Err(error);
10579 }
10580 if let Some(spawn) = activity_spawn {
10581 spawn.complete(snapshot);
10582 }
10583 Ok(child)
10584}
10585
10586struct RegistrationWatch {
10587 changes: watch::Receiver<u64>,
10588 baseline: u64,
10589}
10590
10591fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10595 if spec.protocol != ModuleProtocol::Subc {
10596 return None;
10597 }
10598 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10599 let baseline = *changes.borrow();
10600 Some(RegistrationWatch { changes, baseline })
10601}
10602
10603fn arm_registration_deadline(
10604 child: &mut SupervisedChild,
10605 budget: Duration,
10606 registration_watch: Option<RegistrationWatch>,
10607) {
10608 child.registration_deadline =
10609 (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10610 child.registration_watch = registration_watch;
10611}
10612
10613async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10614 if let Some(container) = child.child.container() {
10615 container.stop_members().await;
10616 child.kill_leader()?;
10617 child.wait().await?;
10618 return Ok(());
10619 }
10620 #[cfg(unix)]
10621 {
10622 let group = unreaped_registration_group(child)?;
10625 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10626 sleep(crate::child_roster::TERM_TO_KILL).await;
10631 let group = unreaped_registration_group(child)?;
10632 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10633 }
10634 child.start_kill()?;
10637 child.wait().await?;
10638 Ok(())
10639}
10640
10641#[cfg(unix)]
10642fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10643 if let Some(container) = child.child.container() {
10644 return container
10645 .group()
10646 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
10647 .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
10648 }
10649 let group = i32::try_from(child.pid)
10652 .ok()
10653 .filter(|pid| *pid > 1)
10654 .and_then(rustix::process::Pid::from_raw)
10655 .ok_or_else(|| {
10656 io::Error::new(
10657 io::ErrorKind::InvalidInput,
10658 "invalid supervised process group",
10659 )
10660 })?;
10661 if child.child.id() != Some(child.pid) {
10662 return Err(io::Error::new(
10663 io::ErrorKind::InvalidInput,
10664 "refusing process-group signal after supervised child was reaped",
10665 ));
10666 }
10667 Ok(group)
10668}
10669
10670enum RegistrationWaitOutcome {
10671 Registered,
10672 Exited(ExitReport),
10673 TimedOut,
10674}
10675
10676struct ReloadRegistrationFailure {
10677 exit_report: ExitReport,
10678 reason: String,
10679}
10680
10681#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10682enum BusyGaugeObservation {
10683 Quiescent,
10684 Busy,
10685 Omitted,
10686}
10687
10688#[derive(Debug, Clone, PartialEq, Eq)]
10690enum GaugeVerdict {
10691 Zero,
10693 NonZero(u64),
10695 NotACount(String),
10699 Missing,
10701 ProbeFailed(&'static str),
10704}
10705
10706impl GaugeVerdict {
10707 fn reads_quiet(&self) -> bool {
10708 matches!(self, Self::Zero)
10709 }
10710}
10711
10712impl fmt::Display for GaugeVerdict {
10713 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10714 match self {
10715 Self::Zero => f.write_str("zero"),
10716 Self::NonZero(value) => write!(f, "non_zero({value})"),
10717 Self::NotACount(value) => write!(f, "not_a_count({value})"),
10718 Self::Missing => f.write_str("missing_from_health_body"),
10719 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10720 }
10721 }
10722}
10723
10724const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10727
10728fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10730 let metrics = metrics.and_then(Value::as_object);
10731 gauges
10732 .iter()
10733 .map(
10734 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10735 None => GaugeVerdict::Missing,
10736 Some(value) => match value.as_u64() {
10737 Some(0) => GaugeVerdict::Zero,
10738 Some(count) => GaugeVerdict::NonZero(count),
10739 None => {
10740 let mut text = value.to_string();
10741 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10742 let mut end = GAUGE_VALUE_TEXT_LIMIT;
10743 while !text.is_char_boundary(end) {
10744 end -= 1;
10745 }
10746 text.truncate(end);
10747 text.push_str("...");
10748 }
10749 GaugeVerdict::NotACount(text)
10750 }
10751 },
10752 },
10753 )
10754 .collect()
10755}
10756
10757fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10760 if verdicts
10761 .iter()
10762 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10763 {
10764 BusyGaugeObservation::Omitted
10765 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10766 BusyGaugeObservation::Quiescent
10767 } else {
10768 BusyGaugeObservation::Busy
10769 }
10770}
10771
10772#[derive(Debug, Clone, Default, PartialEq, Eq)]
10777struct GaugeTrace {
10778 latest: Option<(GaugeVerdict, Duration)>,
10779 last_value: Option<(GaugeVerdict, Duration)>,
10780}
10781
10782#[derive(Debug, Clone, Default, PartialEq, Eq)]
10788struct DrainGaugeTrace {
10789 probes: u32,
10791 gauges: Vec<(String, GaugeTrace)>,
10793}
10794
10795impl DrainGaugeTrace {
10796 fn new(gauges: &[String]) -> Self {
10797 Self {
10798 probes: 0,
10799 gauges: gauges
10800 .iter()
10801 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10802 .collect(),
10803 }
10804 }
10805
10806 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10808 self.probes = self.probes.saturating_add(1);
10809 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10810 trace.latest = Some((verdict.clone(), at));
10811 if matches!(
10812 verdict,
10813 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10814 ) {
10815 trace.last_value = Some((verdict.clone(), at));
10816 }
10817 }
10818 }
10819}
10820
10821impl fmt::Display for DrainGaugeTrace {
10826 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10827 if self.gauges.is_empty() {
10828 return f.write_str("none declared");
10829 }
10830 for (index, (name, trace)) in self.gauges.iter().enumerate() {
10831 if index > 0 {
10832 f.write_str(" | ")?;
10833 }
10834 write!(f, "{name}: ")?;
10835 match &trace.latest {
10836 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10837 None => f.write_str("never probed")?,
10838 }
10839 match &trace.last_value {
10840 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10841 None => f.write_str("; no value ever read")?,
10842 }
10843 }
10844 Ok(())
10845 }
10846}
10847
10848pub(crate) fn declared_busy_gauges(
10849 registry: &Registry,
10850 module_id: &str,
10851) -> Result<Vec<String>, SuperviseError> {
10852 busy_gauges_of(
10853 registry
10854 .get_module(module_id)
10855 .map_err(SuperviseError::Registry)?,
10856 )
10857}
10858
10859fn declared_busy_gauges_for_connection(
10863 registry: &Registry,
10864 connection_id: ConnectionId,
10865) -> Result<Vec<String>, SuperviseError> {
10866 busy_gauges_of(
10867 registry
10868 .get_module_by_connection(connection_id)
10869 .map_err(SuperviseError::Registry)?,
10870 )
10871}
10872
10873fn busy_gauges_of(
10874 registration: Option<crate::registry::ModuleRegistration>,
10875) -> Result<Vec<String>, SuperviseError> {
10876 let Some(registration) = registration else {
10877 return Ok(Vec::new());
10878 };
10879 let Some(self_signals) = registration.manifest.self_signals else {
10880 return Ok(Vec::new());
10881 };
10882
10883 let mut gauges = Vec::new();
10884 for declaration in self_signals {
10885 if declaration.kind != SelfSignalKind::Busy {
10886 continue;
10887 }
10888 match declaration.anchored_to {
10889 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10890 gauges.extend(declared)
10891 }
10892 _ => {
10893 gauges.push(String::new());
10896 }
10897 }
10898 }
10899 Ok(gauges)
10900}
10901
10902#[allow(clippy::too_many_arguments)]
10911async fn wait_for_forwarding_quiescence(
10912 forwarding: &ForwardingTable,
10913 module_id: &str,
10914 runtime: &SupervisorRuntimeConfig,
10915 endpoint: crate::ModuleEndpointId,
10916 deadline: Instant,
10917 busy_gauges: &[String],
10918 scope: DrainScope,
10919 trace: &mut DrainGaugeTrace,
10920) -> Result<bool, SuperviseError> {
10921 let wait_started = Instant::now();
10922 let mut gauges_quiescent = busy_gauges.is_empty();
10923 let mut next_probe_at = wait_started;
10924 let mut omission_reported = false;
10925
10926 loop {
10927 let now = Instant::now();
10928 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10929 let report = match scope {
10930 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10931 DrainScope::Endpoint(endpoint) => {
10932 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10933 }
10934 };
10935 let verdicts = match &report {
10936 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10937 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10938 };
10939 trace.record(&verdicts, wait_started.elapsed());
10940 gauges_quiescent = match report {
10941 Ok(_) => match busy_gauge_observation(&verdicts) {
10942 BusyGaugeObservation::Quiescent => true,
10943 BusyGaugeObservation::Busy => false,
10944 BusyGaugeObservation::Omitted => {
10945 if !omission_reported {
10948 forwarding
10949 .counters()
10950 .increment_drains_with_undeclared_gauge();
10951 let missing = busy_gauges
10952 .iter()
10953 .zip(&verdicts)
10954 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10955 .map(|(gauge, _)| gauge.as_str())
10956 .collect::<Vec<_>>()
10957 .join(",");
10958 warn!(
10959 module_id,
10960 missing_gauges = %missing,
10961 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10962 );
10963 omission_reported = true;
10964 }
10965 false
10966 }
10967 },
10968 Err(err) => {
10969 warn!(
10970 module_id,
10971 error = %err,
10972 "drain health.check did not produce declared busy gauges; treating module as busy"
10973 );
10974 false
10975 }
10976 };
10977 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10980 }
10981
10982 let in_flight = forwarding
10983 .endpoint_in_flight_count(endpoint)
10984 .map_err(SuperviseError::Forwarding)?;
10985 if in_flight == 0 && gauges_quiescent {
10986 return Ok(true);
10987 }
10988
10989 let now = Instant::now();
10990 if now >= deadline {
10991 return Ok(false);
10992 }
10993 let mut wait = deadline
10994 .saturating_duration_since(now)
10995 .min(REGISTRY_RELEASE_POLL);
10996 if !busy_gauges.is_empty() {
10997 wait = wait.min(next_probe_at.saturating_duration_since(now));
10998 }
10999 sleep(wait).await;
11000 }
11001}
11002
11003fn drain_busy_reprobe_interval() -> Duration {
11007 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
11008}
11009
11010fn warn_drain_timed_out(
11025 module_id: &str,
11026 drain_timeout: Duration,
11027 reason: RouteCloseReason,
11028 holdouts: &crate::forwarding::DrainHoldouts,
11029 gauges: &DrainGaugeTrace,
11030) {
11031 warn!(
11032 module_id = %module_id,
11033 waited = ?drain_timeout,
11034 ?reason,
11035 held_requests = holdouts.requests,
11036 held_routes = holdouts.routes,
11037 total_routes = holdouts.total_routes,
11038 top_connections = ?holdouts.top_connections,
11039 held = %holdouts
11042 .held
11043 .iter()
11044 .map(|(channel, corr)| format!("{channel}:{corr}"))
11045 .collect::<Vec<_>>()
11046 .join(","),
11047 gauge_probes = gauges.probes,
11048 busy_gauges = %gauges,
11049 "route drain timed out before request quiescence; forcing teardown"
11050 );
11051}
11052
11053fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
11061 match wait_result {
11062 Ok(drained) => *drained,
11063 Err(_) => false,
11064 }
11065}
11066
11067fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
11068 for released in released_routes {
11069 let frame = match Frame::build_with_version(
11070 released.negotiated_ver,
11071 FrameType::Goodbye,
11072 control_flags(),
11073 released.channel,
11074 released.epoch,
11075 0,
11076 Vec::new(),
11077 ) {
11078 Ok(frame) => frame,
11079 Err(err) => {
11080 warn!(
11081 route_channel = released.channel,
11082 error = %err,
11083 "failed to build supervisor drain route GOODBYE frame"
11084 );
11085 continue;
11086 }
11087 };
11088 if !released.close_on_delivery_failure() {
11089 crate::forwarding::send_module_route_goodbye(
11090 &forwarding.counters(),
11091 &released.sink,
11092 frame,
11093 released.module_id.as_deref(),
11094 "supervisor drain",
11095 );
11096 continue;
11097 }
11098 if let Err(err) = released.sink.try_send(frame) {
11099 warn!(
11100 target_connection_id = released.connection_id.get(),
11101 route_channel = released.channel,
11102 error = %err,
11103 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
11104 );
11105 let _ = forwarding.escalate_client_delivery_failure(
11106 released.connection_id,
11107 released.channel,
11108 released.epoch,
11109 CloseReason::new(
11110 "route_goodbye_delivery_failed",
11111 format!(
11112 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
11113 released.channel
11114 ),
11115 ),
11116 crate::forwarding::UndeliveredFrame {
11117 module_id: released.module_id.as_deref(),
11118 sink: &released.sink,
11119 },
11120 );
11121 }
11122 }
11123}
11124
11125async fn send_module_draining(
11126 module_id: &str,
11127 reason: RouteCloseReason,
11128 deadline_ms: u64,
11129 target: &ModuleDrainTarget,
11130 _wait_for_flush: bool,
11131) -> bool {
11132 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
11133 reason,
11134 deadline_ms,
11135 }) {
11136 Ok(body) => body,
11137 Err(err) => {
11138 warn!(
11139 module_id,
11140 error = %err,
11141 "failed to encode module draining command"
11142 );
11143 return false;
11144 }
11145 };
11146 let frame = match Frame::build_with_version(
11147 target.negotiated_ver,
11148 FrameType::Push,
11149 control_flags(),
11150 0,
11151 0,
11152 0,
11153 body,
11154 ) {
11155 Ok(frame) => frame,
11156 Err(err) => {
11157 warn!(
11158 module_id,
11159 error = %err,
11160 "failed to build module draining command frame"
11161 );
11162 return false;
11163 }
11164 };
11165 #[cfg(unix)]
11166 if _wait_for_flush {
11167 return target.sink.send_flushed(frame).await.is_ok();
11168 }
11169 if let Err(err) = target.sink.try_send(frame) {
11170 warn!(
11171 module_id,
11172 target_connection_id = target.endpoint.connection_id.get(),
11173 error = %err,
11174 "module draining command was not delivered to peer"
11175 );
11176 return false;
11177 }
11178 true
11179}
11180
11181fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
11183 match Frame::build_with_version(
11184 negotiated_ver,
11185 FrameType::Goodbye,
11186 control_flags(),
11187 0,
11188 0,
11189 0,
11190 Vec::new(),
11191 ) {
11192 Ok(frame) => Some(frame),
11193 Err(err) => {
11194 warn!(
11195 module_id,
11196 error = %err,
11197 "failed to build module GOODBYE frame"
11198 );
11199 None
11200 }
11201 }
11202}
11203
11204#[cfg(unix)]
11218async fn send_module_goodbyes_for_daemon_shutdown(
11219 forwarding: &Arc<ForwardingTable>,
11220 reason: &CloseReason,
11221 wait_for_flush: bool,
11222) {
11223 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
11224 let targets = match forwarding.module_connections() {
11225 Ok(targets) => targets,
11226 Err(err) => {
11227 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
11228 return;
11229 }
11230 };
11231 let deadline = Instant::now() + GOODBYE_BUDGET;
11232 let mut sends = tokio::task::JoinSet::new();
11233 for target in targets {
11234 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
11235 continue;
11236 };
11237 if !wait_for_flush {
11238 if let Err(err) = target.sink.try_send(frame) {
11239 debug!(
11240 module_id = %target.module_id,
11241 error = %err,
11242 "shutdown module GOODBYE was not queued"
11243 );
11244 }
11245 continue;
11246 }
11247 let forwarding = Arc::clone(forwarding);
11248 let reason = reason.clone();
11249 sends.spawn(async move {
11250 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
11251 Ok(Ok(())) => {}
11252 Ok(Err(err)) => debug!(
11253 module_id = %target.module_id,
11254 error = %err,
11255 "module connection closed before its shutdown GOODBYE was written"
11256 ),
11257 Err(_) => warn!(
11258 module_id = %target.module_id,
11259 budget = ?GOODBYE_BUDGET,
11260 "shutdown module GOODBYE was not written within its budget; closing anyway"
11261 ),
11262 }
11263 forwarding.request_connection_close(target.endpoint.connection_id, reason);
11264 });
11265 }
11266 while sends.join_next().await.is_some() {}
11268}
11269
11270fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
11271 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
11272 return;
11273 };
11274 if let Err(err) = target.sink.try_send(frame) {
11275 warn!(
11276 module_id,
11277 target_connection_id = target.endpoint.connection_id.get(),
11278 error = %err,
11279 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
11280 );
11281 forwarding.request_connection_close(
11282 target.endpoint.connection_id,
11283 CloseReason::new(
11284 "module_goodbye_delivery_failed",
11285 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
11286 ),
11287 );
11288 }
11289}
11290
11291#[derive(Clone, Copy)]
11292struct ForwardingDrainContext<'a> {
11293 spec: &'a ModuleSpec,
11294 runtime: &'a SupervisorRuntimeConfig,
11295 registry: &'a Registry,
11296 scope: DrainScope,
11297}
11298
11299#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11301enum DrainScope {
11302 Active,
11305 Endpoint(crate::ModuleEndpointId),
11310}
11311
11312#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11320enum StopNotice {
11321 SentOverConnection,
11324 NoConnection,
11328 NotSent,
11332}
11333
11334async fn begin_forwarding_drain(
11335 spec: &ModuleSpec,
11336 runtime: &SupervisorRuntimeConfig,
11337 registry: &Registry,
11338 snapshot: &SharedSnapshot,
11339 enabled: Option<bool>,
11340 reason: RouteCloseReason,
11341) -> Result<StopNotice, SuperviseError> {
11342 let Some(forwarding) = runtime.forwarding.as_ref() else {
11343 return Err(SuperviseError::ReloadUnavailable {
11344 module_id: spec.module_id.clone(),
11345 reason: "supervisor was not configured with a forwarding table".to_string(),
11346 });
11347 };
11348
11349 begin_forwarding_drain_with(
11350 forwarding,
11351 ForwardingDrainContext {
11352 spec,
11353 runtime,
11354 registry,
11355 scope: DrainScope::Active,
11356 },
11357 snapshot,
11358 enabled,
11359 reason,
11360 runtime.drain_timeout,
11361 )
11362 .await
11363}
11364
11365async fn begin_forwarding_drain_if_configured(
11366 spec: &ModuleSpec,
11367 runtime: &SupervisorRuntimeConfig,
11368 registry: &Registry,
11369 snapshot: &SharedSnapshot,
11370 enabled: Option<bool>,
11371 reason: RouteCloseReason,
11372) -> Result<StopNotice, SuperviseError> {
11373 begin_forwarding_drain_with_timeout(
11374 spec,
11375 runtime,
11376 registry,
11377 snapshot,
11378 enabled,
11379 reason,
11380 runtime.drain_timeout,
11381 )
11382 .await
11383}
11384
11385async fn begin_forwarding_drain_with_timeout(
11389 spec: &ModuleSpec,
11390 runtime: &SupervisorRuntimeConfig,
11391 registry: &Registry,
11392 snapshot: &SharedSnapshot,
11393 enabled: Option<bool>,
11394 reason: RouteCloseReason,
11395 drain_timeout: Duration,
11396) -> Result<StopNotice, SuperviseError> {
11397 let Some(forwarding) = runtime.forwarding.as_ref() else {
11398 return Ok(StopNotice::NotSent);
11399 };
11400
11401 begin_forwarding_drain_with(
11402 forwarding,
11403 ForwardingDrainContext {
11404 spec,
11405 runtime,
11406 registry,
11407 scope: DrainScope::Active,
11408 },
11409 snapshot,
11410 enabled,
11411 reason,
11412 drain_timeout,
11413 )
11414 .await
11415}
11416
11417async fn begin_forwarding_drain_with(
11418 forwarding: &ForwardingTable,
11419 context: ForwardingDrainContext<'_>,
11420 snapshot: &SharedSnapshot,
11421 enabled: Option<bool>,
11422 reason: RouteCloseReason,
11423 drain_timeout: Duration,
11424) -> Result<StopNotice, SuperviseError> {
11425 let ForwardingDrainContext {
11426 spec,
11427 runtime,
11428 registry,
11429 scope,
11430 } = context;
11431 debug_assert_ne!(reason, RouteCloseReason::Crash);
11432 let terminal = matches!(reason, RouteCloseReason::Disable);
11433 let drain_started_at = Instant::now();
11434 let drain_deadline = drain_started_at + drain_timeout;
11435 let deadline_ms =
11436 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
11437 let busy_gauges = match scope {
11438 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
11439 DrainScope::Endpoint(endpoint) => {
11440 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
11441 }
11442 };
11443
11444 let gate_started = Instant::now();
11447 let drain_target = match scope {
11448 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
11449 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
11450 }
11451 .map_err(SuperviseError::Forwarding)?;
11452 info!(
11457 module_id = %spec.module_id,
11458 ?reason,
11459 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
11460 connected = drain_target.is_some(),
11461 "module drain began; route admission closed"
11462 );
11463 if scope == DrainScope::Active {
11464 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11465 state.state = ModuleState::Draining;
11466 state.draining_to_replace =
11467 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
11468 if let Some(enabled) = enabled {
11469 state.enabled = enabled;
11470 }
11471 })?;
11472 }
11473
11474 let Some(target) = drain_target.as_ref() else {
11475 return Ok(StopNotice::NoConnection);
11479 };
11480 {
11481 send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
11482 #[cfg(feature = "test-support")]
11483 forwarding.wait_before_route_closing_for_test().await;
11484 let routes = forwarding
11485 .endpoint_routes(target.endpoint)
11486 .map_err(SuperviseError::Forwarding)?;
11487 let routes_notified = routes.len();
11488 crate::control::send_route_control_pushes(
11489 forwarding,
11490 routes.clone(),
11491 ClientControlPush::RouteClosing {
11492 module_id: spec.module_id.clone(),
11493 channels: Vec::new(),
11494 reason,
11495 },
11496 );
11497 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
11498
11499 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
11505 let wait_result = wait_for_forwarding_quiescence(
11506 forwarding,
11507 &spec.module_id,
11508 runtime,
11509 target.endpoint,
11510 drain_deadline,
11511 &busy_gauges,
11512 scope,
11513 &mut gauge_trace,
11514 )
11515 .await;
11516 let drained = drained_after_quiescence_wait(&wait_result);
11517 if let Err(err) = &wait_result {
11518 error!(
11519 module_id = %spec.module_id,
11520 ?reason,
11521 error = %err,
11522 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
11523 );
11524 } else if !drained {
11525 let holdouts = forwarding
11527 .endpoint_drain_holdouts(target.endpoint)
11528 .unwrap_or_default();
11529 warn_drain_timed_out(
11530 &spec.module_id,
11531 drain_timeout,
11532 reason,
11533 &holdouts,
11534 &gauge_trace,
11535 );
11536 }
11537 crate::control::send_route_control_pushes(
11538 forwarding,
11539 routes,
11540 ClientControlPush::RouteClosed {
11541 module_id: spec.module_id.clone(),
11542 channels: Vec::new(),
11543 reason,
11544 drained,
11545 abandoned: target.abandoned_bindings.len() as u32,
11546 excluded_subscriptions: target.excluded_subscriptions,
11547 terminal: Some(terminal),
11548 },
11549 );
11550 wait_result?;
11551
11552 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
11558 Ok(routes) => routes,
11559 Err(err) => {
11560 warn!(
11561 module_id = %spec.module_id,
11562 ?reason,
11563 error = %err,
11564 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
11565 );
11566 send_module_goodbye(&spec.module_id, forwarding, target);
11567 return Err(SuperviseError::Forwarding(err));
11568 }
11569 };
11570 let route_goodbye_count = released_routes.len();
11571 send_route_goodbyes(forwarding, released_routes);
11572 send_module_goodbye(&spec.module_id, forwarding, target);
11573
11574 info!(
11580 module_id = %spec.module_id,
11581 ?reason,
11582 routes_notified,
11583 route_goodbyes = route_goodbye_count,
11584 abandoned_reservations = target.abandoned_bindings.len(),
11585 excluded_subscriptions = target.excluded_subscriptions,
11586 drained,
11587 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11588 );
11589 }
11590
11591 Ok(StopNotice::SentOverConnection)
11592}
11593
11594async fn wait_for_registration_after_reload(
11597 registry: &Registry,
11598 module_id: &str,
11599 snapshot: &SharedSnapshot,
11600 child: &mut SupervisedChild,
11601 wait: Duration,
11602) -> Result<RegistrationWaitOutcome, SuperviseError> {
11603 wait_for_slot_registration(
11604 registry,
11605 crate::registry::RegistrationSlot::Active(module_id),
11606 module_id,
11607 snapshot,
11608 child,
11609 wait,
11610 )
11611 .await
11612}
11613
11614async fn wait_for_slot_registration(
11622 registry: &Registry,
11623 slot: crate::registry::RegistrationSlot<'_>,
11624 module_id: &str,
11625 snapshot: &SharedSnapshot,
11626 child: &mut SupervisedChild,
11627 wait: Duration,
11628) -> Result<RegistrationWaitOutcome, SuperviseError> {
11629 let deadline = Instant::now() + wait;
11630 loop {
11631 if registry
11632 .registration(slot)
11633 .map_err(SuperviseError::Registry)?
11634 .is_some()
11635 {
11636 return Ok(RegistrationWaitOutcome::Registered);
11637 }
11638
11639 let now = Instant::now();
11640 if now >= deadline {
11641 return Ok(RegistrationWaitOutcome::TimedOut);
11642 }
11643 let remaining = deadline.saturating_duration_since(now);
11644 let poll = remaining.min(REGISTRY_RELEASE_POLL);
11645
11646 tokio::select! {
11647 wait_result = child.wait() => {
11648 let status = wait_result.map_err(|source| SuperviseError::Wait {
11649 module_id: module_id.to_string(),
11650 source,
11651 })?;
11652 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11653 snapshot,
11654 child,
11655 &status,
11656 )));
11657 }
11658 _ = sleep(poll) => {}
11659 }
11660 }
11661}
11662
11663fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11664 if exit_report.kind != ExitKind::DeliberateSeverance {
11668 exit_report.kind = ExitKind::Crash;
11669 }
11670 exit_report
11671}
11672
11673async fn handle_reload_child_registration_failure(
11674 spec: &ModuleSpec,
11675 runtime: &SupervisorRuntimeConfig,
11676 registry: &Registry,
11677 process_liveness: &SupervisorProcessLiveness,
11678 snapshot: &SharedSnapshot,
11679 _child: &mut Option<SupervisedChild>,
11680 failure: ReloadRegistrationFailure,
11681) -> Result<(), SuperviseError> {
11682 let ReloadRegistrationFailure {
11683 exit_report,
11684 reason,
11685 } = failure;
11686 match on_child_exit(
11687 spec,
11688 runtime.restart_policy,
11689 registry,
11690 snapshot,
11691 &runtime.terminal_ring,
11692 &runtime.spawn_events,
11693 &runtime.child_roster,
11694 exit_report,
11695 runtime.recovery_is_daemon,
11696 )
11697 .await
11698 {
11699 NextAction::Stop {
11700 registration_released,
11701 } => {
11702 if registration_released {
11703 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11704 }
11705 }
11706 NextAction::Restart { schedule } => {
11707 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11708 schedule.delay
11709 });
11710 if let Some(schedule) = schedule {
11711 log_crash_respawn(&spec.module_id, schedule);
11712 }
11713 schedule_respawn(
11714 runtime,
11715 snapshot,
11716 &spec.module_id,
11717 delay,
11718 RespawnKind::Spawn,
11719 true,
11720 )?;
11721 }
11722 }
11723 Err(SuperviseError::ReloadFailed {
11724 module_id: spec.module_id.clone(),
11725 reason,
11726 })
11727}
11728
11729async fn handle_reload_spawn_failure(
11730 spec: &ModuleSpec,
11731 runtime: &SupervisorRuntimeConfig,
11732 process_liveness: &SupervisorProcessLiveness,
11733 snapshot: &SharedSnapshot,
11734 _child: &mut Option<SupervisedChild>,
11735 reason: String,
11736) -> Result<(), SuperviseError> {
11737 let now = Instant::now();
11738 let mut schedule = None;
11739 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11740 clear_current_process_facts(state);
11741 if state.enabled {
11742 schedule = state.next_crash_restart(&runtime.restart_policy, now);
11743 state.state = if schedule.is_some() {
11744 ModuleState::Restarting
11745 } else {
11746 if runtime
11747 .failure_context(runtime.recovery_is_daemon)
11748 .may_retry()
11749 {
11750 state.failed_retry.schedule(&spec.module_id);
11751 }
11752 ModuleState::Failed
11753 };
11754 } else {
11755 state.state = ModuleState::Disabled;
11756 }
11757 })?;
11758 if let Some(schedule) = schedule {
11759 schedule_respawn(
11760 runtime,
11761 snapshot,
11762 &spec.module_id,
11763 schedule.delay,
11764 RespawnKind::Spawn,
11765 true,
11766 )?;
11767 } else {
11768 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11769 }
11770 Err(SuperviseError::ReloadFailed {
11771 module_id: spec.module_id.clone(),
11772 reason,
11773 })
11774}
11775
11776fn control_flags() -> Flags {
11777 Flags::new(false, Priority::Passive, false)
11778}
11779
11780#[allow(clippy::too_many_arguments)]
11781async fn drain_optional_child(
11782 module_id: &str,
11783 protocol: ModuleProtocol,
11784 stop_notice: StopNotice,
11785 registry: &Registry,
11786 forwarding: Option<&ForwardingTable>,
11787 snapshot: &SharedSnapshot,
11788 terminal_ring: &Arc<Mutex<TerminalRing>>,
11789 spawn_events: &SpawnEventFeed,
11790 child: &mut Option<SupervisedChild>,
11791 drain_timeout: Duration,
11792 final_state: ModuleState,
11793 enabled: Option<bool>,
11794 failure: &FailureContext,
11795) -> Result<(), SuperviseError> {
11796 if let Some(child) = child.take() {
11797 drain_child_to_state(
11798 module_id,
11799 protocol,
11800 stop_notice,
11801 registry,
11802 forwarding,
11803 snapshot,
11804 terminal_ring,
11805 spawn_events,
11806 child,
11807 drain_timeout,
11808 final_state,
11809 enabled,
11810 failure,
11811 )
11812 .await
11813 } else {
11814 update_snapshot(snapshot, Some(module_id), |state| {
11815 state.state = final_state;
11816 if final_state == ModuleState::Failed {
11817 state.failed_retry.failed_at = Some(Instant::now());
11818 }
11819 if let Some(enabled) = enabled {
11820 state.enabled = enabled;
11821 }
11822 clear_current_process_facts(state);
11823 })?;
11824 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11825 }
11826}
11827
11828#[allow(clippy::too_many_arguments)]
11829async fn drain_child_to_state(
11830 module_id: &str,
11831 _protocol: ModuleProtocol,
11832 stop_notice: StopNotice,
11833 registry: &Registry,
11834 forwarding: Option<&ForwardingTable>,
11835 snapshot: &SharedSnapshot,
11836 terminal_ring: &Arc<Mutex<TerminalRing>>,
11837 spawn_events: &SpawnEventFeed,
11838 mut child: SupervisedChild,
11839 drain_timeout: Duration,
11840 final_state: ModuleState,
11841 enabled: Option<bool>,
11842 failure: &FailureContext,
11843) -> Result<(), SuperviseError> {
11844 let protocol = child.protocol;
11845 update_snapshot(snapshot, Some(module_id), |state| {
11846 state.state = ModuleState::Draining;
11847 state.draining_to_replace = final_state == ModuleState::Restarting;
11848 if let Some(enabled) = enabled {
11849 state.enabled = enabled;
11850 }
11851 })?;
11852
11853 if stop_notice != StopNotice::SentOverConnection {
11864 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11865 info!(
11866 module_id,
11867 pid = child.pid,
11868 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11869 "module has no connection yet; requesting stop by signal"
11870 );
11871 }
11872 request_graceful_stop(module_id, &child);
11873 }
11874
11875 let exit_report = match timeout(drain_timeout, child.wait()).await {
11876 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11877 Ok(Err(source)) => {
11878 fail_after_wait_error(
11879 module_id,
11880 snapshot,
11881 terminal_ring,
11882 spawn_events,
11883 format!("failed to wait during drain: {source}"),
11884 failure,
11885 );
11886 return Err(SuperviseError::Wait {
11887 module_id: module_id.to_string(),
11888 source,
11889 });
11890 }
11891 Err(_) => {
11892 warn!(
11905 module_id,
11906 pid = child.pid,
11907 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11908 reason = ?final_state,
11909 ?stop_notice,
11910 "drain budget expired before the module exited; killing it"
11911 );
11912 let kill =
11913 if final_state == ModuleState::Restarting && child.child.container().is_some() {
11914 child.kill_leader()
11915 } else {
11916 child.start_kill()
11917 };
11918 kill.map_err(|source| {
11919 fail_after_wait_error(
11920 module_id,
11921 snapshot,
11922 terminal_ring,
11923 spawn_events,
11924 format!("failed to kill during drain: {source}"),
11925 failure,
11926 );
11927 SuperviseError::Kill {
11928 module_id: module_id.to_string(),
11929 source,
11930 }
11931 })?;
11932 let status = child.wait().await.map_err(|source| {
11933 fail_after_wait_error(
11934 module_id,
11935 snapshot,
11936 terminal_ring,
11937 spawn_events,
11938 format!("failed to wait after drain kill: {source}"),
11939 failure,
11940 );
11941 SuperviseError::Wait {
11942 module_id: module_id.to_string(),
11943 source,
11944 }
11945 })?;
11946 classify_reaped_child_exit(snapshot, &child, &status)
11947 }
11948 };
11949
11950 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11951 publish_after_terminal_record(
11952 module_id,
11953 || {
11954 record_terminal_with_detail(
11955 module_id,
11956 terminal_ring,
11957 spawn_events,
11958 &exit_report,
11959 terminal_disposition(final_state),
11960 detail,
11961 )
11962 },
11963 || {
11964 update_snapshot(snapshot, Some(module_id), |state| {
11965 state.state = final_state;
11966 if final_state == ModuleState::Failed {
11967 state.failed_retry.failed_at = Some(Instant::now());
11968 }
11969 if let Some(enabled) = enabled {
11970 state.enabled = enabled;
11971 }
11972 clear_current_process_facts(state);
11973 state.last_exit = Some(exit_report.clone());
11974 if exit_report.kind == ExitKind::DeliberateSeverance {
11975 state.lifetime_restarts += 1;
11976 }
11977 })
11978 },
11979 )?;
11980 child.drain_stderr(module_id).await;
11981
11982 let released =
11983 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
11984 child
11985 .finish_exit(
11986 released.is_ok()
11987 && final_state == ModuleState::Restarting
11988 && child.spawn_failure.is_none(),
11989 )
11990 .await
11991 .map_err(|source| SuperviseError::Wait {
11992 module_id: module_id.to_owned(),
11993 source,
11994 })?;
11995 released
11996}
11997
11998#[cfg(unix)]
12018fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
12019 let Some(pid) = child
12020 .id()
12021 .and_then(|pid| i32::try_from(pid).ok())
12022 .and_then(rustix::process::Pid::from_raw)
12023 else {
12024 debug!(
12025 module_id,
12026 "no pid to signal for teardown; falling through to the drain wait"
12027 );
12028 return;
12029 };
12030 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
12031 Ok(()) => debug!(
12032 module_id,
12033 "sent SIGTERM to a module nothing else asked to stop"
12034 ),
12035 Err(err) => debug!(
12036 module_id,
12037 error = %err,
12038 "SIGTERM to module failed; the drain wait and kill still apply"
12039 ),
12040 }
12041}
12042
12043#[cfg(not(unix))]
12051fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
12052 debug!(
12053 module_id,
12054 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
12055 );
12056}
12057
12058fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
12059 match final_state {
12060 ModuleState::Stopped => TerminalDisposition::Stopped,
12061 ModuleState::Disabled => TerminalDisposition::Disabled,
12062 ModuleState::Restarting => TerminalDisposition::Restarting,
12063 ModuleState::Failed => TerminalDisposition::Failed,
12064 ModuleState::Starting
12065 | ModuleState::Running
12066 | ModuleState::Unresponsive
12067 | ModuleState::Draining => {
12068 unreachable!("terminal exits only finish in terminal or restarting states")
12069 }
12070 }
12071}
12072
12073async fn release_dead_registration(
12081 registry: &Registry,
12082 forwarding: Option<&ForwardingTable>,
12083 snapshot: &SharedSnapshot,
12084 module_id: &str,
12085 failure: &FailureContext,
12086) -> Result<(), SuperviseError> {
12087 let result = async {
12088 let registration = registry
12089 .get_module(module_id)
12090 .map_err(SuperviseError::Registry)?;
12091 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
12092 Ok(()) => return Ok(()),
12093 Err(SuperviseError::RegistrationStillActive { .. }) => {}
12094 Err(err) => return Err(err),
12095 }
12096 let pid = lock_snapshot(snapshot)?.reaped_pid;
12097 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
12098 warn!(
12099 module_id,
12100 pid,
12101 connection_id = registration.connection_id.get(),
12102 "reaped module registration outlived release grace; closing dead connection"
12103 );
12104 forwarding.request_connection_close(
12105 registration.connection_id,
12106 CloseReason::new(
12107 "supervised_process_reaped",
12108 format!("module '{module_id}' pid {pid} exited"),
12109 ),
12110 );
12111 wait_for_slot_registration_release(
12112 registry,
12113 crate::registry::RegistrationSlot::Connection(registration.connection_id),
12114 REGISTRY_RELEASE_TIMEOUT,
12115 )
12116 .await?;
12117 }
12118 wait_for_registration_release(registry, module_id, Duration::ZERO).await
12119 }
12120 .await;
12121 if let Err(err) = &result {
12122 fail_snapshot(snapshot, Some(module_id), None, failure);
12123 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
12124 }
12125 result
12126}
12127
12128async fn wait_for_registration_release(
12131 registry: &Registry,
12132 module_id: &str,
12133 wait: Duration,
12134) -> Result<(), SuperviseError> {
12135 wait_for_slot_registration_release(
12136 registry,
12137 crate::registry::RegistrationSlot::Active(module_id),
12138 wait,
12139 )
12140 .await
12141}
12142
12143async fn wait_for_slot_registration_release(
12151 registry: &Registry,
12152 slot: crate::registry::RegistrationSlot<'_>,
12153 wait: Duration,
12154) -> Result<(), SuperviseError> {
12155 let deadline = Instant::now() + wait;
12156 let mut release_events = registration_release_events().subscribe();
12157 let still_active = |registration: &crate::registry::ModuleRegistration| {
12158 SuperviseError::RegistrationStillActive {
12159 module_id: registration.manifest.module_id.clone(),
12160 waited: wait,
12161 }
12162 };
12163 loop {
12164 let _observed_generation = *release_events.borrow_and_update();
12165 let Some(registration) = registry
12166 .registration(slot)
12167 .map_err(SuperviseError::Registry)?
12168 else {
12169 return Ok(());
12170 };
12171
12172 let now = Instant::now();
12173 if now >= deadline {
12174 return Err(still_active(®istration));
12175 }
12176
12177 let remaining = deadline.saturating_duration_since(now);
12178 match timeout(remaining, release_events.changed()).await {
12179 Ok(Ok(())) | Ok(Err(_)) => {}
12180 Err(_) => return Err(still_active(®istration)),
12181 }
12182 }
12183}
12184
12185#[cfg(test)]
12186mod slot_registration_wait_tests {
12187 use super::*;
12188 use crate::registry::{ConnectionId, RegistrationSlot};
12189 use subc_protocol::manifest::ModuleManifest;
12190
12191 #[tokio::test]
12192 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
12193 let registry = Arc::new(Registry::default());
12194 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
12195 let runtime = supervisor.runtime_config();
12196 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
12197 let spec = ModuleSpec {
12198 module_id: "enable-stale-registration".to_string(),
12199 program: PathBuf::from("/missing/enable-retry-test"),
12200 args: Vec::new(),
12201 env: Vec::new(),
12202 reserved: false,
12203 reserved_prefixes: Vec::new(),
12204 protocol: ModuleProtocol::Subc,
12205 overlap: Default::default(),
12206 carry_containment: false,
12207 };
12208 let connection = ConnectionId::new(90);
12209 registry
12210 .register_with_control_ops(
12211 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
12212 1,
12213 connection,
12214 Vec::new(),
12215 )
12216 .unwrap();
12217 let mut child = None;
12218 let err = set_child_enabled(
12219 &spec,
12220 &runtime,
12221 ®istry,
12222 &supervisor.process_liveness,
12223 &snapshot,
12224 &mut child,
12225 true,
12226 &runtime.failure_context(false),
12227 )
12228 .await
12229 .unwrap_err();
12230 assert!(matches!(
12231 err,
12232 SuperviseError::RegistrationStillActive { .. }
12233 ));
12234 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12235 assert!(child.is_none());
12236 registry.deregister_connection(connection).unwrap();
12237 let err = set_child_enabled(
12238 &spec,
12239 &runtime,
12240 ®istry,
12241 &supervisor.process_liveness,
12242 &snapshot,
12243 &mut child,
12244 true,
12245 &runtime.failure_context(false),
12246 )
12247 .await
12248 .unwrap_err();
12249 assert!(
12250 matches!(err, SuperviseError::Spawn { .. }),
12251 "second enable must attempt a spawn: {err}"
12252 );
12253 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12254 }
12255
12256 const INCUMBENT: u64 = 1;
12257 const CANDIDATE: u64 = 2;
12258
12259 fn swapped_registry() -> Arc<Registry> {
12260 let registry = Arc::new(Registry::default());
12261 let manifest = ModuleManifest::builder("m", "0.1.0").build();
12262 registry
12263 .register_with_control_ops(
12264 manifest.clone(),
12265 1,
12266 ConnectionId::new(INCUMBENT),
12267 Vec::new(),
12268 )
12269 .unwrap();
12270 registry
12271 .register_candidate_with_control_ops(
12272 manifest,
12273 1,
12274 ConnectionId::new(CANDIDATE),
12275 Vec::new(),
12276 )
12277 .unwrap();
12278 registry
12279 }
12280
12281 #[tokio::test]
12285 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
12286 let registry = swapped_registry();
12287 registry.promote_candidate("m").unwrap().unwrap();
12288
12289 assert!(matches!(
12290 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
12291 Err(SuperviseError::RegistrationStillActive { .. })
12292 ));
12293
12294 assert!(matches!(
12296 wait_for_slot_registration_release(
12297 ®istry,
12298 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12299 Duration::from_millis(50),
12300 )
12301 .await,
12302 Err(SuperviseError::RegistrationStillActive { .. })
12303 ));
12304
12305 let releaser = Arc::clone(®istry);
12306 let release = tokio::spawn(async move {
12307 sleep(Duration::from_millis(20)).await;
12308 releaser
12309 .deregister_connection(ConnectionId::new(INCUMBENT))
12310 .unwrap();
12311 notify_registration_release();
12312 });
12313 wait_for_slot_registration_release(
12314 ®istry,
12315 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12316 Duration::from_secs(5),
12317 )
12318 .await
12319 .expect("the incumbent's own registration is released");
12320 release.await.unwrap();
12321 assert!(registry.get_module("m").unwrap().is_some());
12322 }
12323
12324 #[tokio::test]
12327 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
12328 let registry = swapped_registry();
12329 assert!(matches!(
12330 wait_for_slot_registration_release(
12331 ®istry,
12332 RegistrationSlot::Candidate("m"),
12333 Duration::from_millis(50),
12334 )
12335 .await,
12336 Err(SuperviseError::RegistrationStillActive { .. })
12337 ));
12338 registry
12339 .deregister_connection(ConnectionId::new(CANDIDATE))
12340 .unwrap();
12341 wait_for_slot_registration_release(
12342 ®istry,
12343 RegistrationSlot::Candidate("m"),
12344 Duration::from_millis(50),
12345 )
12346 .await
12347 .expect("a candidate slot with no candidate is released");
12348 assert!(registry
12349 .registration(RegistrationSlot::Active("m"))
12350 .unwrap()
12351 .is_some());
12352 }
12353}
12354
12355fn classify_exit(status: &ExitStatus) -> ExitReport {
12356 ExitReport {
12357 kind: if status.success() {
12358 ExitKind::Clean
12359 } else {
12360 ExitKind::Crash
12361 },
12362 code: status.code(),
12363 signal: exit_signal(status),
12364 at_ms: unix_ms_now(),
12365 }
12366}
12367
12368fn wait_error_exit_report() -> ExitReport {
12374 ExitReport {
12375 kind: ExitKind::Crash,
12376 code: None,
12377 signal: None,
12378 at_ms: unix_ms_now(),
12379 }
12380}
12381
12382#[cfg(unix)]
12383fn exit_signal(status: &ExitStatus) -> Option<i32> {
12384 use std::os::unix::process::ExitStatusExt;
12385
12386 status.signal()
12387}
12388
12389#[cfg(not(unix))]
12390fn exit_signal(_status: &ExitStatus) -> Option<i32> {
12391 None
12392}
12393
12394fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
12400 update_snapshot(snapshot, Some(module_id), |state| {
12401 state.clear_crash_restarts();
12402 })
12403}
12404
12405fn set_running(
12406 snapshot: &SharedSnapshot,
12407 child: &SupervisedChild,
12408 module_id: &str,
12409 spawn_events: &SpawnEventFeed,
12410) -> Result<(), SuperviseError> {
12411 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12412 module_id: Some(module_id.to_string()),
12413 })?;
12414 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
12415 if std::mem::take(&mut state.coalesced_restart_pending) {
12416 let generation = state.spawn_generation;
12417 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
12418 }
12419 state.drain_disposition_detail = None;
12420 state.spawn_failure = None;
12421 state.waiting_for_previous_pid = None;
12422 state.waiting_for_previous_detail = None;
12423 state.in_alternate_slot = false;
12426 state.configuration_updated_since_spawn = false;
12427 state.spawned_protocol = Some(child.protocol);
12428 state.state = ModuleState::Running;
12429 state.enabled = true;
12430 state.process_alive = true;
12431 state.pid = child.id();
12432 state.subtree_target = child.subtree_target.clone();
12433 state.health.status = SupervisorHealthStatus::Unknown;
12436 state.last_probe_evidence = None;
12437 state.health.consecutive_failures = 0;
12438 state.health.detail = None;
12439 state.health.metrics = None;
12440 state.health.last_probe_ms = None;
12441 state.failed_retry.healthy_since = None;
12442 state.failed_retry.failed_at = None;
12443 state.failed_retry.running_since = Some(child.spawned_at);
12444 #[cfg(windows)]
12445 {
12446 state.pipe_consumed = child
12447 .nonce_delivery
12448 .as_ref()
12449 .map(|delivery| delivery.consumed());
12450 }
12451 #[cfg(target_os = "macos")]
12452 {
12453 state.report_ready = Some(Arc::clone(&child.report_ready));
12454 }
12455 state.spawned_at_ms = Some(child.spawned_at_ms);
12456 state.exec_outcome = Some(Arc::clone(&child.exec.0));
12457 state.spawned_from = Some(child.spawned_from.clone());
12458 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
12459 state.spawned_file_identity = child.spawned_file_identity.clone();
12460 state.process_start_time = child.process_start_time;
12461 Ok(())
12462}
12463
12464fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
12465 state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
12466 state.failed_retry.healthy_since = None;
12467 state.failed_retry.running_since = None;
12468 state.waiting_for_previous_pid = None;
12469 state.waiting_for_previous_detail = None;
12470 state.process_alive = false;
12471 state.spawned_protocol = None;
12472 state.pid = None;
12473 #[cfg(windows)]
12474 {
12475 state.pipe_consumed = None;
12476 }
12477 #[cfg(target_os = "macos")]
12478 {
12479 state.report_ready = None;
12480 }
12481 state.spawned_at_ms = None;
12482 state.exec_outcome = None;
12483 state.spawned_from = None;
12484 state.launch_spec_digest = None;
12485 state.spawned_file_identity = None;
12486 state.process_start_time = None;
12487 state.deliberate_severance = None;
12488}
12489
12490#[cfg(test)]
12491fn record_deliberate_severance(
12492 snapshot: &SharedSnapshot,
12493 identity: ProcessIdentity,
12494) -> Result<(), SuperviseError> {
12495 update_snapshot(snapshot, None, |state| {
12496 state.deliberate_severance = Some(identity);
12497 })
12498}
12499
12500fn apply_deliberate_severance_marker(
12501 snapshot: &SharedSnapshot,
12502 exited_identity: Option<ProcessIdentity>,
12503 mut exit_report: ExitReport,
12504) -> ExitReport {
12505 let marker = lock_snapshot(snapshot)
12506 .ok()
12507 .and_then(|mut state| state.deliberate_severance.take());
12508 if marker.is_some() && marker == exited_identity {
12509 exit_report.kind = ExitKind::DeliberateSeverance;
12510 }
12511 exit_report
12512}
12513
12514fn classify_reaped_child_exit(
12515 snapshot: &SharedSnapshot,
12516 child: &SupervisedChild,
12517 status: &ExitStatus,
12518) -> ExitReport {
12519 let _ = update_snapshot(snapshot, None, |state| {
12520 state.reaped_pid = Some(child.pid);
12521 state.spawn_failure = child.spawn_failure.clone();
12522 });
12523 if child.spawn_failure.as_deref() == Some("registration_timeout") {
12524 registration_failure_exit_report(classify_exit(status))
12525 } else {
12526 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
12527 }
12528}
12529
12530fn fail_snapshot(
12531 snapshot: &SharedSnapshot,
12532 module_id: Option<&str>,
12533 last_exit: Option<ExitReport>,
12534 failure: &FailureContext,
12535) {
12536 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
12537 state.state = ModuleState::Failed;
12538 clear_current_process_facts(state);
12539 if failure.may_retry() && state.enabled {
12540 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
12541 } else if failure.shutdown.is_set() {
12542 state.failed_retry.cancel();
12543 }
12544 if let Some(last_exit) = last_exit {
12545 state.last_exit = Some(last_exit);
12546 }
12547 }) {
12548 error!(error = %err, "failed to mark supervisor state failed");
12549 }
12550}
12551
12552fn update_snapshot(
12553 snapshot: &SharedSnapshot,
12554 module_id: Option<&str>,
12555 update: impl FnOnce(&mut SupervisorSnapshot),
12556) -> Result<(), SuperviseError> {
12557 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12558 module_id: module_id.map(ToOwned::to_owned),
12559 })?;
12560 update(&mut state);
12561 Ok(())
12562}
12563
12564const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
12565
12566fn lock_snapshot_for_control<'a>(
12567 snapshot: &'a SharedSnapshot,
12568 module_id: &str,
12569 caller: &'static str,
12570) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
12571 let started_at = Instant::now();
12572 let guard = lock_snapshot(snapshot)?;
12573 let waited = started_at.elapsed();
12574 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
12575 warn!(
12576 module_id = %module_id,
12577 waited_ms = waited.as_millis() as u64,
12578 caller = %caller,
12579 "slow snapshot lock"
12580 );
12581 }
12582 Ok(guard)
12583}
12584
12585fn lock_snapshot(
12586 snapshot: &SharedSnapshot,
12587) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
12588 snapshot
12589 .lock()
12590 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
12591}
12592
12593#[cfg(test)]
12594mod terminal_history_tests {
12595 include!("supervise_route_open_recovery_tests.rs");
12596 use std::{
12597 path::PathBuf,
12598 sync::Arc,
12599 time::{Duration, Instant},
12600 };
12601
12602 use tokio::{sync::mpsc, time::sleep};
12603
12604 use super::{
12605 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12606 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12607 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12608 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12609 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12610 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12611 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12612 SupervisorHealthStatus, SupervisorSnapshot,
12613 };
12614 use super::Instant as ClockInstant;
12619 use crate::{
12620 registry::Registry,
12621 terminal_ring::{TerminalRing, TerminalRingConfig},
12622 };
12623 use std::sync::Mutex;
12624 use subc_control::TerminalDisposition;
12625
12626 pub(super) fn fake_aft_stub_path() -> PathBuf {
12631 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
12632 assert!(
12633 path.exists(),
12634 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12635 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12636 path.display()
12637 );
12638 path
12639 }
12640
12641 #[test]
12642 fn reserved_never_spawned_refuses_every_hello() {
12643 let supervisor = SupervisorHandle::default();
12648 supervisor.apply_identity_configuration(&ModuleSpec {
12649 module_id: "never-spawned".to_string(),
12650 program: PathBuf::from("/usr/bin/false"),
12651 args: Vec::new(),
12652 env: Vec::new(),
12653 reserved: true,
12654 reserved_prefixes: Vec::new(),
12655 protocol: ModuleProtocol::Subc,
12656 overlap: Default::default(),
12657 carry_containment: false,
12658 });
12659 assert!(
12660 supervisor
12661 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12662 .is_err(),
12663 "forged nonce must refuse on a reserved never-spawned id"
12664 );
12665 assert!(
12666 supervisor
12667 .reserved_hello_verification("never-spawned", None)
12668 .is_err(),
12669 "absent nonce must refuse on a reserved never-spawned id"
12670 );
12671 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12673 supervisor.apply_identity_configuration(&ModuleSpec {
12674 module_id: "never-spawned".to_string(),
12675 program: PathBuf::from("/usr/bin/false"),
12676 args: Vec::new(),
12677 env: Vec::new(),
12678 reserved: true,
12679 reserved_prefixes: Vec::new(),
12680 protocol: ModuleProtocol::Subc,
12681 overlap: Default::default(),
12682 carry_containment: false,
12683 });
12684 assert!(supervisor
12685 .reserved_hello_verification("never-spawned", Some("minted"))
12686 .is_ok());
12687 assert!(supervisor
12688 .reserved_hello_verification("never-spawned", Some("forged"))
12689 .is_err());
12690 }
12691
12692 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12695 let now = ClockInstant::now();
12696 for _ in 0..count {
12697 state.crash_restarts.push_back(now);
12698 }
12699 }
12700
12701 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12705 let aged = state
12706 .crash_restarts
12707 .front()
12708 .expect("a crash restart must be recorded before it can be aged")
12709 .checked_sub(window + Duration::from_secs(1))
12710 .expect("the test clock is far enough from its origin to age an instant");
12711 state.crash_restarts[0] = aged;
12712 }
12713
12714 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12715 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12716 seed_crash_restarts(&mut state, count);
12717 state
12718 }
12719
12720 #[test]
12721 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12722 let policy = RestartPolicy::new(3, Duration::ZERO);
12723 let now = ClockInstant::now();
12724 assert!(daemon_will_restart(
12725 &mut snapshot_with_restarts(true, 2),
12726 &policy,
12727 now
12728 ));
12729 assert!(!daemon_will_restart(
12730 &mut snapshot_with_restarts(true, 3),
12731 &policy,
12732 now
12733 ));
12734 assert!(!daemon_will_restart(
12735 &mut snapshot_with_restarts(false, 0),
12736 &policy,
12737 now
12738 ));
12739 }
12740
12741 #[test]
12742 fn crash_restart_backoff_escalates_with_in_window_count() {
12743 let policy = RestartPolicy::new(4, Duration::from_millis(100))
12744 .with_max_backoff(Duration::from_secs(30));
12745 let now = ClockInstant::now();
12746 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12747 let schedules = (0..4)
12748 .map(|_| {
12749 state
12750 .next_crash_restart(&policy, now)
12751 .expect("the test policy allows four crash restarts")
12752 })
12753 .collect::<Vec<_>>();
12754
12755 assert_eq!(
12756 schedules
12757 .iter()
12758 .map(|schedule| schedule.restart_in_window)
12759 .collect::<Vec<_>>(),
12760 vec![0, 1, 2, 3]
12761 );
12762 assert_eq!(
12763 schedules
12764 .iter()
12765 .map(|schedule| schedule.delay)
12766 .collect::<Vec<_>>(),
12767 vec![
12768 Duration::from_millis(100),
12769 Duration::from_secs(1),
12770 Duration::from_secs(10),
12771 Duration::from_secs(30),
12772 ]
12773 );
12774 }
12775
12776 #[test]
12777 fn crash_restart_backoff_resets_after_ring_clear() {
12778 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12779 let now = ClockInstant::now();
12780 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12781 assert_eq!(
12782 state.next_crash_restart(&policy, now).unwrap().delay,
12783 Duration::from_millis(100)
12784 );
12785 assert_eq!(
12786 state.next_crash_restart(&policy, now).unwrap().delay,
12787 Duration::from_secs(1)
12788 );
12789
12790 state.clear_crash_restarts();
12791 let schedule = state
12792 .next_crash_restart(&policy, now)
12793 .expect("a cleared ring must allow another restart");
12794 assert_eq!(schedule.restart_in_window, 0);
12795 assert_eq!(schedule.delay, Duration::from_millis(100));
12796 }
12797
12798 #[test]
12799 fn crash_restart_backoff_ignores_aged_restarts() {
12800 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12801 let now = ClockInstant::now();
12802 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12803 state
12804 .next_crash_restart(&policy, now)
12805 .expect("the first restart is allowed");
12806 state
12807 .next_crash_restart(&policy, now)
12808 .expect("the second restart is allowed");
12809 state.crash_restarts[0] = now
12810 .checked_sub(policy.window + Duration::from_secs(1))
12811 .expect("the fake clock can age a restart past the window");
12812
12813 let schedule = state
12814 .next_crash_restart(&policy, now)
12815 .expect("an aged restart must release its slot");
12816 assert_eq!(schedule.restart_in_window, 1);
12817 assert_eq!(schedule.delay, Duration::from_secs(1));
12818 assert_eq!(state.crash_restarts.len(), 2);
12819 }
12820
12821 #[test]
12825 fn a_budget_spent_before_the_window_no_longer_refuses() {
12826 let policy = RestartPolicy::new(3, Duration::ZERO);
12827 let mut state = snapshot_with_restarts(true, 3);
12828 let now = ClockInstant::now();
12829 assert!(!daemon_will_restart(&mut state, &policy, now));
12830
12831 assert!(daemon_will_restart(
12832 &mut state,
12833 &policy,
12834 now + policy.window + Duration::from_secs(1)
12835 ));
12836 assert!(
12837 state.crash_restarts.is_empty(),
12838 "reading the budget must drop the instants that left the window"
12839 );
12840 }
12841
12842 fn module_with_recovery_snapshot(
12843 state: ModuleState,
12844 enabled: bool,
12845 restart_count: u32,
12846 ) -> SupervisedModule {
12847 let registry = Arc::new(Registry::default());
12848 let supervisor =
12849 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
12850 let runtime = supervisor.runtime_config();
12851 let spec = ModuleSpec {
12852 module_id: "recovery-snapshot".to_string(),
12853 program: fake_aft_stub_path(),
12854 args: Vec::new(),
12855 env: Vec::new(),
12856 reserved: false,
12857 reserved_prefixes: Vec::new(),
12858 protocol: ModuleProtocol::Subc,
12859 overlap: Default::default(),
12860 carry_containment: false,
12861 };
12862 let mut snapshot = SupervisorSnapshot::new(state, enabled);
12863 seed_crash_restarts(&mut snapshot, restart_count);
12864 let (commands, _rx) = mpsc::channel(4);
12867 SupervisedModule {
12868 inner: Arc::new(SupervisedModuleInner {
12869 module_id: spec.module_id.clone(),
12870 registry,
12871 snapshot: Arc::new(Mutex::new(snapshot)),
12872 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12873 spec,
12874 health: runtime.health,
12875 })),
12876 stderr_ring: runtime.stderr_ring,
12877 terminal_ring: runtime.terminal_ring,
12878 commands,
12879 monitor: Mutex::new(None),
12880 restart_policy: runtime.restart_policy,
12881 effective_drain_timeout: runtime.effective_drain_timeout,
12882 provenance_probe: supervisor.provenance_probe.clone(),
12883 retained: runtime.child_roster.retained_view(),
12884 }),
12885 }
12886 }
12887
12888 #[cfg(target_os = "linux")]
12889 #[tokio::test]
12890 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12891 let supervisor =
12892 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12893 .with_cgroup_placement(None);
12894 let result = supervisor.spawn(ModuleSpec {
12895 module_id: "no-cgroup-placement".to_string(),
12896 program: fake_aft_stub_path(),
12897 args: Vec::new(),
12898 env: Vec::new(),
12899 reserved: false,
12900 reserved_prefixes: Vec::new(),
12901 protocol: ModuleProtocol::Subc,
12902 overlap: Default::default(),
12903 carry_containment: false,
12904 });
12905
12906 assert!(
12907 result.is_ok(),
12908 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12909 );
12910 }
12911
12912 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12913 async fn undecided_snapshot_uses_shared_restart_predicate() {
12914 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12915 .will_recover_after_connection_loss()
12916 .unwrap());
12917 assert!(
12918 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12919 .will_recover_after_connection_loss()
12920 .unwrap()
12921 );
12922 }
12923
12924 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12925 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12926 assert!(
12927 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12928 .will_recover_after_connection_loss()
12929 .unwrap()
12930 );
12931 }
12932
12933 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12934 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12935 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12936 .will_recover_after_connection_loss()
12937 .unwrap());
12938 assert!(
12939 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12940 .will_recover_after_connection_loss()
12941 .unwrap()
12942 );
12943 }
12944
12945 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12946 async fn warming_snapshot_is_limited_to_startup_phases() {
12947 for state in [
12948 ModuleState::Starting,
12949 ModuleState::Running,
12950 ModuleState::Restarting,
12951 ] {
12952 assert!(
12953 module_with_recovery_snapshot(state, true, 0)
12954 .is_warming()
12955 .unwrap(),
12956 "{state:?} should be warming"
12957 );
12958 }
12959 for state in [
12960 ModuleState::Unresponsive,
12961 ModuleState::Draining,
12962 ModuleState::Stopped,
12963 ModuleState::Failed,
12964 ModuleState::Disabled,
12965 ] {
12966 assert!(
12967 !module_with_recovery_snapshot(state, true, 0)
12968 .is_warming()
12969 .unwrap(),
12970 "{state:?} should not be warming"
12971 );
12972 }
12973 }
12974
12975 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12976 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12977 let registry = Arc::new(Registry::default());
12978 let supervisor =
12979 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
12980 let module = supervisor
12981 .spawn(ModuleSpec {
12982 module_id: "terminal-history".to_string(),
12983 program: fake_aft_stub_path(),
12984 args: Vec::new(),
12985 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12986 reserved: false,
12987 reserved_prefixes: Vec::new(),
12988 protocol: ModuleProtocol::Subc,
12989 overlap: Default::default(),
12990 carry_containment: false,
12991 })
12992 .unwrap();
12993
12994 let deadline = Instant::now() + Duration::from_secs(5);
12995 loop {
12996 let history = module.terminal_history();
12997 if history.entries.len() == 2 {
12998 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12999 assert_eq!(history.dropped, 0);
13000 assert_eq!(
13001 history
13002 .entries
13003 .iter()
13004 .map(|entry| entry.exit_code)
13005 .collect::<Vec<_>>(),
13006 vec![Some(23), Some(23)]
13007 );
13008 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
13009 return;
13010 }
13011 assert!(
13012 Instant::now() < deadline,
13013 "module did not retain two terminal exits: {history:?}"
13014 );
13015 sleep(Duration::from_millis(10)).await;
13016 }
13017 }
13018
13019 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13023 async fn disable_during_crash_backoff_cancels_pending_respawn() {
13024 let backoff = Duration::from_secs(2);
13025 let supervisor = Supervisor::new_for_test(
13026 Arc::new(Registry::default()),
13027 RestartPolicy::new(10, backoff),
13028 );
13029 let module = supervisor
13030 .spawn(ModuleSpec {
13031 module_id: "disable-during-backoff".to_string(),
13032 program: fake_aft_stub_path(),
13033 args: Vec::new(),
13034 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13035 reserved: false,
13036 reserved_prefixes: Vec::new(),
13037 protocol: ModuleProtocol::Subc,
13038 overlap: Default::default(),
13039 carry_containment: false,
13040 })
13041 .unwrap();
13042
13043 let deadline = Instant::now() + Duration::from_secs(5);
13045 loop {
13046 if module.status().unwrap().state == ModuleState::Restarting {
13047 break;
13048 }
13049 assert!(
13050 Instant::now() < deadline,
13051 "module never entered the crash backoff"
13052 );
13053 sleep(Duration::from_millis(10)).await;
13054 }
13055
13056 let started = Instant::now();
13057 module.set_enabled(false).await.unwrap();
13058 let waited = started.elapsed();
13059
13060 assert!(
13061 waited < backoff / 2,
13062 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
13063 );
13064 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
13065
13066 sleep(backoff + Duration::from_millis(500)).await;
13068 let status = module.status().unwrap();
13069 assert_eq!(status.state, ModuleState::Disabled);
13070 assert_eq!(
13071 status.spawn_generation, 1,
13072 "module respawned after the operator disabled it"
13073 );
13074 }
13075
13076 #[cfg(unix)]
13079 fn protocol_none_sigterm_exits_clean_spec(
13080 module_id: &str,
13081 dir: &std::path::Path,
13082 ) -> (ModuleSpec, PathBuf, PathBuf) {
13083 let ready = dir.join("ready");
13084 let marker = dir.join("sigterm");
13085 let spec = ModuleSpec {
13086 module_id: module_id.to_string(),
13087 program: fake_aft_stub_path(),
13088 args: Vec::new(),
13089 env: vec![
13090 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
13091 (
13092 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
13093 marker.display().to_string(),
13094 ),
13095 (
13096 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
13097 ready.display().to_string(),
13098 ),
13099 ],
13100 reserved: false,
13101 reserved_prefixes: Vec::new(),
13102 protocol: ModuleProtocol::None,
13103 overlap: Default::default(),
13104 carry_containment: false,
13105 };
13106 (spec, ready, marker)
13107 }
13108
13109 #[cfg(unix)]
13113 async fn wait_for_file(path: &std::path::Path) {
13114 let deadline = Instant::now() + Duration::from_secs(10);
13115 while !path.exists() {
13116 assert!(
13117 Instant::now() < deadline,
13118 "{} never appeared",
13119 path.display()
13120 );
13121 sleep(Duration::from_millis(10)).await;
13122 }
13123 }
13124
13125 #[cfg(unix)]
13129 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13130 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
13131 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
13132 let (spec, ready, marker) =
13133 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
13134 let supervisor = Supervisor::new_for_test(
13135 Arc::new(Registry::default()),
13136 RestartPolicy::new(3, Duration::ZERO),
13137 );
13138 let module = supervisor.spawn(spec).unwrap();
13139 wait_for_file(&ready).await;
13140 let deadline = Instant::now() + Duration::from_secs(10);
13144 let first_pid = loop {
13145 let status = module.status().unwrap();
13146 if status.state == ModuleState::Running {
13147 if let Some(pid) = status.pid {
13148 break pid;
13149 }
13150 }
13151 assert!(
13152 Instant::now() < deadline,
13153 "a running module must report its pid after exec confirmation: {status:?}"
13154 );
13155 sleep(Duration::from_millis(10)).await;
13156 };
13157
13158 rustix::process::kill_process(
13159 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
13160 rustix::process::Signal::TERM,
13161 )
13162 .unwrap();
13163
13164 let deadline = Instant::now() + Duration::from_secs(10);
13165 let respawned = loop {
13166 let status = module.status().unwrap();
13167 if status.state == ModuleState::Running
13168 && status.pid.is_some_and(|pid| pid != first_pid)
13169 {
13170 break status;
13171 }
13172 assert!(
13173 Instant::now() < deadline,
13174 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
13175 );
13176 sleep(Duration::from_millis(10)).await;
13177 };
13178 assert_eq!(respawned.spawn_generation, 2);
13179 assert!(
13180 marker.exists(),
13181 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
13182 );
13183
13184 let history = module.terminal_history();
13185 assert_eq!(history.entries.len(), 1, "{history:?}");
13186 let entry = &history.entries[0];
13187 assert_eq!(entry.exit_code, Some(0));
13188 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
13189 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
13190
13191 module.stop().await.unwrap();
13192 }
13193
13194 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13198 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
13199 let supervisor = Supervisor::new_for_test(
13200 Arc::new(Registry::default()),
13201 RestartPolicy::new(1, Duration::ZERO),
13202 );
13203 let module = supervisor
13204 .spawn(ModuleSpec {
13205 module_id: "none-clean-exit-budget".to_string(),
13206 program: fake_aft_stub_path(),
13207 args: Vec::new(),
13208 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13209 reserved: false,
13210 reserved_prefixes: Vec::new(),
13211 protocol: ModuleProtocol::None,
13212 overlap: Default::default(),
13213 carry_containment: false,
13214 })
13215 .unwrap();
13216
13217 let deadline = Instant::now() + Duration::from_secs(10);
13220 loop {
13221 let status = module.status().unwrap();
13222 if status.state == ModuleState::Failed {
13223 break;
13224 }
13225 assert!(
13226 Instant::now() < deadline,
13227 "module never exhausted its budget: {status:?} {:?}",
13228 module.terminal_history()
13229 );
13230 sleep(Duration::from_millis(10)).await;
13231 }
13232 let history = module.terminal_history();
13233 assert_eq!(
13234 history
13235 .entries
13236 .iter()
13237 .map(|entry| (entry.exit_code, entry.disposition.clone()))
13238 .collect::<Vec<_>>(),
13239 vec![
13240 (Some(0), TerminalDisposition::Restarting),
13241 (Some(0), TerminalDisposition::Failed),
13242 ]
13243 );
13244 let detail = history.entries[1]
13245 .disposition_detail
13246 .as_deref()
13247 .expect("a budget failure names the budget");
13248 assert!(detail.contains("max_restarts=1"), "{detail}");
13249 assert_eq!(module.status().unwrap().spawn_generation, 2);
13250 }
13251
13252 #[test]
13253 fn restart_budget_failure_is_published_after_its_terminal_record() {
13254 use std::io::{Read, Write};
13255 let supervisor = Supervisor::new_for_test(
13256 Arc::new(Registry::default()),
13257 RestartPolicy::new(0, Duration::ZERO),
13258 );
13259 let runtime = supervisor.runtime_config();
13260 let spec = ModuleSpec {
13261 protocol: ModuleProtocol::None,
13262 ..windowed_crash_spec("budget-publication-order")
13263 };
13264 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13265 ModuleState::Running,
13266 true,
13267 )));
13268 lock_snapshot(&snapshot).unwrap().process_alive = true;
13269 let reaped_snapshot = snapshot.clone();
13270 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13271 let history = runtime.terminal_ring.clone();
13272 let reap = std::thread::spawn(move || {
13273 tokio::runtime::Builder::new_current_thread()
13274 .enable_all()
13275 .build()
13276 .unwrap()
13277 .block_on(on_child_exit(
13278 &spec,
13279 runtime.restart_policy,
13280 &supervisor.registry,
13281 &reaped_snapshot,
13282 &runtime.terminal_ring,
13283 &runtime.spawn_events,
13284 &runtime.child_roster,
13285 ExitReport {
13286 kind: ExitKind::Clean,
13287 code: Some(0),
13288 signal: None,
13289 at_ms: 1,
13290 },
13291 true,
13292 ))
13293 });
13294 let mut reached = [0];
13295 barrier.read_exact(&mut reached).unwrap();
13296 assert_eq!(&reached, b"R");
13297 let (before_state, before_alive) = {
13298 let state = lock_snapshot(&snapshot).unwrap();
13299 (state.state, state.process_alive)
13300 };
13301 let before_history = history.lock().unwrap().snapshot();
13302 barrier.write_all(b"X").unwrap();
13303 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
13304 assert!(before_history.entries.is_empty());
13305 assert_ne!(
13306 before_state,
13307 ModuleState::Failed,
13308 "Failed was visible before its terminal record could be written"
13309 );
13310 assert!(before_alive);
13311 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13312 let history = history.lock().unwrap().snapshot();
13313 assert_eq!(history.entries.len(), 1);
13314 assert_eq!(history.entries[0].exit_code, Some(0));
13315 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13316 }
13317
13318 #[test]
13319 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
13320 use std::io::{Read, Write};
13321
13322 let supervisor = Supervisor::new_for_test(
13323 Arc::new(Registry::default()),
13324 RestartPolicy::new(3, Duration::ZERO),
13325 );
13326 let mut runtime = supervisor.runtime_config();
13327 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
13328 let snapshot = module.inner.snapshot.clone();
13329 lock_snapshot(&snapshot).unwrap().process_alive = true;
13330 runtime.terminal_ring = module.inner.terminal_ring.clone();
13331 let spec = windowed_crash_spec(module.module_id());
13332 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13333 let reaped_snapshot = snapshot.clone();
13334 let reap = std::thread::spawn(move || {
13335 tokio::runtime::Builder::new_current_thread()
13336 .enable_all()
13337 .build()
13338 .unwrap()
13339 .block_on(on_child_exit(
13340 &spec,
13341 runtime.restart_policy,
13342 &supervisor.registry,
13343 &reaped_snapshot,
13344 &runtime.terminal_ring,
13345 &runtime.spawn_events,
13346 &runtime.child_roster,
13347 ExitReport {
13348 kind: ExitKind::Crash,
13349 code: Some(2),
13350 signal: None,
13351 at_ms: 1,
13352 },
13353 true,
13354 ))
13355 });
13356
13357 let mut reached = [0];
13361 barrier.read_exact(&mut reached).unwrap();
13362 assert_eq!(&reached, b"R");
13363 let before_status = module.status().unwrap();
13364 let before_history = module.terminal_history();
13365 let before_recovers = module.will_recover_after_connection_loss().unwrap();
13366 barrier.write_all(b"X").unwrap();
13367 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
13368
13369 assert_eq!(before_status.state, ModuleState::Running);
13370 assert!(before_status.process_alive);
13371 assert_eq!(before_status.restart_count, 3);
13372 assert!(before_history.entries.is_empty());
13373 assert!(
13374 before_recovers,
13375 "connection cleanup would report a terminal close for a scheduled crash restart"
13376 );
13377 let restarting = module.status().unwrap();
13378 assert_eq!(restarting.state, ModuleState::Restarting);
13379 assert_eq!(restarting.restart_count, 3);
13380 assert!(module.will_recover_after_connection_loss().unwrap());
13381 let history = module.terminal_history();
13382 assert_eq!(history.entries.len(), 1);
13383 assert_eq!(
13384 history.entries[0].disposition,
13385 TerminalDisposition::Restarting
13386 );
13387
13388 update_snapshot(&snapshot, Some(module.module_id()), |state| {
13391 state.state = ModuleState::Running;
13392 state.respawn_pending = false;
13393 })
13394 .unwrap();
13395 assert!(!module.will_recover_after_connection_loss().unwrap());
13396 update_snapshot(&snapshot, Some(module.module_id()), |state| {
13397 state.enabled = false;
13398 state.respawn_pending = true;
13399 })
13400 .unwrap();
13401 assert!(!module.will_recover_after_connection_loss().unwrap());
13402 }
13403
13404 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
13405 use std::net::{TcpListener, TcpStream};
13406 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
13407 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
13408 let (barrier, _) = listener.accept().unwrap();
13409 for stream in [&writer, &barrier] {
13410 stream
13411 .set_read_timeout(Some(Duration::from_secs(10)))
13412 .unwrap();
13413 stream
13414 .set_write_timeout(Some(Duration::from_secs(10)))
13415 .unwrap();
13416 }
13417 *events.1.lock().unwrap() = Some(writer);
13418 barrier
13419 }
13420
13421 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
13422 use std::io::{Read, Write};
13423 let barrier = events.1.lock().unwrap().take();
13424 if let Some(mut barrier) = barrier {
13425 barrier.write_all(b"R").unwrap();
13426 let mut release = [0];
13427 barrier.read_exact(&mut release).unwrap();
13428 assert_eq!(&release, b"X");
13429 }
13430 }
13431
13432 #[test]
13433 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
13434 use std::io::{Read, Write};
13435
13436 let supervisor = Supervisor::new_for_test(
13437 Arc::new(Registry::default()),
13438 RestartPolicy::new(0, Duration::ZERO),
13439 );
13440 let mut runtime = supervisor.runtime_config();
13441 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13442 let snapshot = module.inner.snapshot.clone();
13443 runtime.terminal_ring = module.inner.terminal_ring.clone();
13444 let spec = ModuleSpec {
13445 module_id: module.module_id().into(),
13446 program: fake_aft_stub_path(),
13447 protocol: ModuleProtocol::None,
13448 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13449 ..windowed_crash_spec(module.module_id())
13450 };
13451 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13452 let reap = std::thread::spawn(move || {
13453 tokio::runtime::Builder::new_current_thread()
13454 .enable_all()
13455 .build()
13456 .unwrap()
13457 .block_on(async {
13458 let mut child =
13459 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13460 health_restart_child(
13461 &spec,
13462 &runtime,
13463 &supervisor.registry,
13464 &supervisor.process_liveness,
13465 &snapshot,
13466 &mut child,
13467 SupervisorHealthStatus::Failing,
13468 Some("health probe did not answer"),
13469 1,
13470 )
13471 .await
13472 .unwrap();
13473 assert!(child.is_none());
13474 });
13475 });
13476 let mut reached = [0];
13481 barrier.read_exact(&mut reached).unwrap();
13482 assert_eq!(&reached, b"R");
13483 let before_status = module.status().unwrap();
13484 let before_history = module.terminal_history();
13485 barrier.write_all(b"X").unwrap();
13486 reap.join().unwrap();
13487 assert!(before_history.entries.is_empty());
13488 assert_ne!(
13489 before_status.state,
13490 ModuleState::Failed,
13491 "Failed was visible before its terminal record could be written"
13492 );
13493 assert!(
13494 before_status.process_alive,
13495 "process death was visible before its terminal record could be written"
13496 );
13497 let failed = module.status().unwrap();
13498 assert_eq!(failed.state, ModuleState::Failed);
13499 assert!(!failed.process_alive);
13500 let history = module.terminal_history();
13501 assert_eq!(history.entries.len(), 1);
13502 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13503 assert_eq!(
13504 history.entries[0].disposition_detail.as_deref(),
13505 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
13506 );
13507 }
13508
13509 #[test]
13510 fn observed_exits_publish_process_death_after_terminal_history() {
13511 use std::io::{Read, Write};
13512 for (kind, enabled, shutdown, wait_error, expected_state) in [
13513 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
13514 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
13515 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
13516 (
13517 ExitKind::DeliberateSeverance,
13518 true,
13519 false,
13520 false,
13521 ModuleState::Restarting,
13522 ),
13523 #[cfg(unix)]
13524 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
13525 (ExitKind::Crash, true, false, true, ModuleState::Failed),
13526 ] {
13527 let supervisor = Supervisor::new_for_test(
13528 Arc::new(Registry::default()),
13529 RestartPolicy::new(1, Duration::ZERO),
13530 );
13531 let runtime = supervisor.runtime_config();
13532 #[cfg(unix)]
13533 if shutdown {
13534 runtime.child_roster.close();
13535 }
13536 #[cfg(not(unix))]
13537 let _ = shutdown;
13538 let spec = windowed_crash_spec("exit-publication-order");
13539 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13540 ModuleState::Running,
13541 enabled,
13542 )));
13543 lock_snapshot(&snapshot).unwrap().process_alive = true;
13544 let reaped_snapshot = snapshot.clone();
13545 let history = runtime.terminal_ring.clone();
13546 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13547 let reap = std::thread::spawn(move || {
13548 if wait_error {
13549 super::fail_after_wait_error(
13550 &spec.module_id,
13551 &reaped_snapshot,
13552 &runtime.terminal_ring,
13553 &runtime.spawn_events,
13554 "wait failed".into(),
13555 &runtime.failure_context(true),
13556 );
13557 } else {
13558 tokio::runtime::Builder::new_current_thread()
13559 .enable_all()
13560 .build()
13561 .unwrap()
13562 .block_on(on_child_exit(
13563 &spec,
13564 runtime.restart_policy,
13565 &supervisor.registry,
13566 &reaped_snapshot,
13567 &runtime.terminal_ring,
13568 &runtime.spawn_events,
13569 &runtime.child_roster,
13570 ExitReport {
13571 kind,
13572 code: Some(0),
13573 signal: None,
13574 at_ms: 1,
13575 },
13576 true,
13577 ));
13578 }
13579 });
13580 let mut reached = [0];
13581 barrier.read_exact(&mut reached).unwrap();
13582 assert_eq!(&reached, b"R");
13583 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
13584 let before_history = history.lock().unwrap().snapshot();
13585 barrier.write_all(b"X").unwrap();
13586 reap.join().unwrap();
13587 assert!(
13588 before_alive,
13589 "{kind:?}: death was published before recording"
13590 );
13591 assert!(before_history.entries.is_empty());
13592 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
13593 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13594 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
13595 }
13596 }
13597
13598 #[tokio::test]
13599 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
13600 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
13601 let path = dir.join("terminals.jsonl");
13602 std::fs::create_dir(&path).unwrap();
13603 let supervisor = Supervisor::new_for_test(
13604 Arc::new(Registry::default()),
13605 RestartPolicy::new(0, Duration::ZERO),
13606 )
13607 .with_terminal_journal(path, "health-budget-journal-failure".into());
13608 let runtime = supervisor.runtime_config();
13609 let spec = ModuleSpec {
13610 program: fake_aft_stub_path(),
13611 protocol: ModuleProtocol::None,
13612 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13613 ..windowed_crash_spec("health-budget-journal-failure")
13614 };
13615 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13616 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13617 health_restart_child(
13618 &spec,
13619 &runtime,
13620 &supervisor.registry,
13621 &supervisor.process_liveness,
13622 &snapshot,
13623 &mut child,
13624 SupervisorHealthStatus::Failing,
13625 None,
13626 1,
13627 )
13628 .await
13629 .unwrap();
13630 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13631 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13632 let history = runtime
13633 .terminal_ring
13634 .lock()
13635 .unwrap()
13636 .durable_history(&spec.module_id);
13637 assert!(history.journal_write_failures > 0);
13638 assert_eq!(history.entries.len(), 1);
13639 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13640 }
13641
13642 #[test]
13643 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13644 let supervisor = Supervisor::new_for_test(
13645 Arc::new(Registry::default()),
13646 RestartPolicy::new(0, Duration::ZERO),
13647 );
13648 let runtime = supervisor.runtime_config();
13649 let spec = ModuleSpec {
13650 program: fake_aft_stub_path(),
13651 protocol: ModuleProtocol::None,
13652 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13653 ..windowed_crash_spec("health-budget-recording-panic")
13654 };
13655 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13656 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13657 tokio::runtime::Builder::new_current_thread()
13658 .enable_all()
13659 .build()
13660 .unwrap()
13661 .block_on(async {
13662 let mut child =
13663 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13664 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13665 health_restart_child(
13666 &spec,
13667 &runtime,
13668 &supervisor.registry,
13669 &supervisor.process_liveness,
13670 &snapshot,
13671 &mut child,
13672 SupervisorHealthStatus::Failing,
13673 None,
13674 1,
13675 )
13676 .await
13677 .unwrap();
13678 });
13679 }));
13680 let panic = result.expect_err("recording must still unwind");
13681 assert_eq!(
13682 panic.downcast_ref::<String>().map(String::as_str),
13683 Some("spawn event sequence exhausted")
13684 );
13685 let state = lock_snapshot(&snapshot).unwrap();
13686 assert_eq!(state.state, ModuleState::Failed);
13687 assert!(!state.process_alive);
13688 assert!(state.last_exit.is_some());
13689 }
13690
13691 #[tokio::test]
13692 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13693 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13694 let path = dir.join("terminals.jsonl");
13695 std::fs::create_dir(&path).unwrap();
13696 let supervisor = Supervisor::new_for_test(
13697 Arc::new(Registry::default()),
13698 RestartPolicy::new(0, Duration::ZERO),
13699 )
13700 .with_terminal_journal(path, "budget-journal-failure".into());
13701 let runtime = supervisor.runtime_config();
13702 let spec = windowed_crash_spec("budget-journal-failure");
13703 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13704 assert!(matches!(
13705 on_child_exit(
13706 &spec,
13707 runtime.restart_policy,
13708 &supervisor.registry,
13709 &snapshot,
13710 &runtime.terminal_ring,
13711 &runtime.spawn_events,
13712 &runtime.child_roster,
13713 crash_exit_report(1),
13714 true,
13715 )
13716 .await,
13717 NextAction::Stop { .. }
13718 ));
13719 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13720 let history = runtime
13721 .terminal_ring
13722 .lock()
13723 .unwrap()
13724 .durable_history(&spec.module_id);
13725 assert!(history.journal_write_failures > 0);
13726 assert_eq!(history.entries.len(), 1);
13727 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13728 }
13729
13730 #[test]
13731 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13732 let supervisor = Supervisor::new_for_test(
13733 Arc::new(Registry::default()),
13734 RestartPolicy::new(0, Duration::ZERO),
13735 );
13736 let runtime = supervisor.runtime_config();
13737 let spec = windowed_crash_spec("budget-recording-panic");
13738 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13739 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13740 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13743 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13744 tokio::runtime::Builder::new_current_thread()
13745 .enable_all()
13746 .build()
13747 .unwrap()
13748 .block_on(on_child_exit(
13749 &spec,
13750 runtime.restart_policy,
13751 &supervisor.registry,
13752 &snapshot,
13753 &runtime.terminal_ring,
13754 &runtime.spawn_events,
13755 &runtime.child_roster,
13756 crash_exit_report(1),
13757 true,
13758 ))
13759 }));
13760 let panic = result.err().expect("recording must still unwind");
13761 assert_eq!(
13762 panic.downcast_ref::<String>().map(String::as_str),
13763 Some("spawn event sequence exhausted")
13764 );
13765 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13766 }
13767
13768 #[cfg(unix)]
13771 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13772 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13773 for disable in [false, true] {
13774 let label = if disable {
13775 "none-requested-disable"
13776 } else {
13777 "none-requested-stop"
13778 };
13779 let dir = cortexkit_test_support::ScratchDir::new(label);
13780 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13781 let supervisor = Supervisor::new_for_test(
13782 Arc::new(Registry::default()),
13783 RestartPolicy::new(3, Duration::ZERO),
13784 );
13785 let module = supervisor.spawn(spec).unwrap();
13786 wait_for_file(&ready).await;
13787
13788 if disable {
13789 module.set_enabled(false).await.unwrap();
13790 } else {
13791 module.stop().await.unwrap();
13792 }
13793 assert!(
13794 marker.exists(),
13795 "{label}: the child must have left through its SIGTERM handler with exit 0"
13796 );
13797
13798 sleep(Duration::from_millis(500)).await;
13801 let status = module.status().unwrap();
13802 let expected = if disable {
13803 ModuleState::Disabled
13804 } else {
13805 ModuleState::Stopped
13806 };
13807 assert_eq!(status.state, expected, "{label}");
13808 assert_eq!(
13809 status.spawn_generation, 1,
13810 "{label}: respawned after a requested stop"
13811 );
13812 let history = module.terminal_history();
13813 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13814 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13815 assert_ne!(
13816 history.entries[0].disposition,
13817 TerminalDisposition::Restarting,
13818 "{label}"
13819 );
13820 }
13821 }
13822
13823 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13826 async fn subc_wire_clean_exit_is_still_a_stop() {
13827 let supervisor = Supervisor::new_for_test(
13828 Arc::new(Registry::default()),
13829 RestartPolicy::new(3, Duration::ZERO),
13830 );
13831 let module = supervisor
13832 .spawn(ModuleSpec {
13833 module_id: "wire-clean-exit".to_string(),
13834 program: fake_aft_stub_path(),
13835 args: Vec::new(),
13836 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13837 reserved: false,
13838 reserved_prefixes: Vec::new(),
13839 protocol: ModuleProtocol::Subc,
13840 overlap: Default::default(),
13841 carry_containment: false,
13842 })
13843 .unwrap();
13844
13845 let deadline = Instant::now() + Duration::from_secs(10);
13846 while module.terminal_history().entries.is_empty() {
13847 assert!(Instant::now() < deadline, "module never exited");
13848 sleep(Duration::from_millis(10)).await;
13849 }
13850 sleep(Duration::from_millis(500)).await;
13852 let status = module.status().unwrap();
13853 assert_eq!(status.state, ModuleState::Stopped);
13854 assert_eq!(status.spawn_generation, 1);
13855 let history = module.terminal_history();
13856 assert_eq!(history.entries.len(), 1, "{history:?}");
13857 assert_eq!(history.entries[0].exit_code, Some(0));
13858 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13859 }
13860
13861 #[cfg(unix)]
13862 #[tokio::test]
13863 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13864 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13865 let record = dir.join("live-children.json");
13866 let supervisor = Supervisor::new_for_test(
13867 Arc::new(Registry::default()),
13868 RestartPolicy::new(0, Duration::ZERO),
13869 );
13870 let mut runtime = supervisor.runtime_config();
13871 runtime.child_roster.record_to(record.clone());
13872 let gate = Arc::new(super::ReloadExitRecordGate::default());
13873 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13874 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13875 let spec = ModuleSpec {
13876 module_id: "reload-exit-roster".into(),
13877 program: fake_aft_stub_path(),
13878 args: Vec::new(),
13879 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13880 reserved: false,
13881 reserved_prefixes: Vec::new(),
13882 protocol: ModuleProtocol::Subc,
13883 overlap: Default::default(),
13884 carry_containment: false,
13885 };
13886 let mut child = None;
13887 let reload = super::finish_reload_child(
13888 &spec,
13889 &runtime,
13890 &supervisor.registry,
13891 &supervisor.process_liveness,
13892 &snapshot,
13893 &mut child,
13894 );
13895 tokio::pin!(reload);
13896 tokio::select! {
13897 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13898 _ = gate.reached.notified() => {}
13899 }
13900 assert!(runtime
13901 .terminal_ring
13902 .lock()
13903 .unwrap()
13904 .snapshot()
13905 .entries
13906 .is_empty());
13907 assert_eq!(
13908 crate::live_children::read_record(&record).unwrap().len(),
13909 1,
13910 "shutdown must still wait for the reaped child until its terminal record exists"
13911 );
13912 runtime.child_roster.close();
13913 gate.resume.notify_one();
13914 assert!(reload.await.is_err());
13915 assert!(crate::live_children::read_record(&record)
13916 .unwrap()
13917 .is_empty());
13918 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13919 assert_eq!(history.entries.len(), 1);
13920 assert_eq!(
13921 history.entries[0].disposition,
13922 TerminalDisposition::DaemonShutdown
13923 );
13924 }
13925
13926 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13930 async fn every_restart_increment_path_advances_lifetime_count() {
13931 let supervisor = Supervisor::new_for_test(
13932 Arc::new(Registry::default()),
13933 RestartPolicy::new(1, Duration::ZERO),
13934 );
13935 let runtime = supervisor.runtime_config();
13936 let spec = ModuleSpec {
13937 module_id: "lifetime-increment-path".to_string(),
13938 program: PathBuf::from("/unused/lifetime-increment-path"),
13939 args: Vec::new(),
13940 env: Vec::new(),
13941 reserved: false,
13942 reserved_prefixes: Vec::new(),
13943 protocol: ModuleProtocol::Subc,
13944 overlap: Default::default(),
13945 carry_containment: false,
13946 };
13947
13948 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13949 assert!(matches!(
13950 on_child_exit(
13951 &spec,
13952 runtime.restart_policy,
13953 &supervisor.registry,
13954 &crash_snapshot,
13955 &runtime.terminal_ring,
13956 &runtime.spawn_events,
13957 &runtime.child_roster,
13958 ExitReport {
13959 kind: ExitKind::Crash,
13960 code: Some(1),
13961 signal: None,
13962 at_ms: 1,
13963 },
13964 true,
13965 )
13966 .await,
13967 NextAction::Restart { schedule: _ }
13968 ));
13969 let (crash_restarts, crash_lifetime) = {
13970 let state = lock_snapshot(&crash_snapshot).unwrap();
13971 (state.crash_restarts.len(), state.lifetime_restarts)
13972 };
13973 assert_eq!(crash_restarts, 1);
13974 assert_eq!(crash_lifetime, 1);
13975
13976 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13977 let mut health_child = None;
13978 assert!(matches!(
13979 health_restart_child(
13980 &spec,
13981 &runtime,
13982 &supervisor.registry,
13983 &supervisor.process_liveness,
13984 &health_snapshot,
13985 &mut health_child,
13986 SupervisorHealthStatus::Failing,
13987 None,
13988 2,
13989 )
13990 .await,
13991 Ok(())
13992 ));
13993 assert!(health_child.is_none());
13994 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13995 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13996 let (health_restarts, health_lifetime) = {
13997 let state = lock_snapshot(&health_snapshot).unwrap();
13998 (state.crash_restarts.len(), state.lifetime_restarts)
13999 };
14000 assert_eq!(health_restarts, 1);
14001 assert_eq!(health_lifetime, 1);
14002
14003 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14004 let mut reload_child = None;
14005 assert!(matches!(
14006 handle_reload_spawn_failure(
14007 &spec,
14008 &runtime,
14009 &supervisor.process_liveness,
14010 &reload_snapshot,
14011 &mut reload_child,
14012 "forced reload spawn failure".to_string(),
14013 )
14014 .await,
14015 Err(SuperviseError::ReloadFailed { .. })
14016 ));
14017 let (reload_restarts, reload_lifetime) = {
14018 let state = lock_snapshot(&reload_snapshot).unwrap();
14019 (state.crash_restarts.len(), state.lifetime_restarts)
14020 };
14021 assert_eq!(reload_restarts, 1);
14022 assert_eq!(reload_lifetime, 1);
14023 }
14024
14025 #[tokio::test]
14026 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
14027 let supervisor = Supervisor::new_for_test(
14028 Arc::new(Registry::default()),
14029 RestartPolicy::new(3, Duration::ZERO),
14030 );
14031 let runtime = supervisor.runtime_config();
14032 let spec = ModuleSpec {
14033 module_id: "deliberately-severed".to_string(),
14034 program: PathBuf::from("/unused/deliberately-severed"),
14035 args: Vec::new(),
14036 env: Vec::new(),
14037 reserved: false,
14038 reserved_prefixes: Vec::new(),
14039 protocol: ModuleProtocol::Subc,
14040 overlap: Default::default(),
14041 carry_containment: false,
14042 };
14043 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14044 let process = ProcessIdentity {
14045 pid: 41,
14046 start_time: 101,
14047 };
14048 record_deliberate_severance(&snapshot, process).unwrap();
14049 let exit_report = apply_deliberate_severance_marker(
14050 &snapshot,
14051 Some(process),
14052 ExitReport {
14053 kind: ExitKind::Crash,
14054 code: Some(1),
14055 signal: None,
14056 at_ms: 1,
14057 },
14058 );
14059 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
14060
14061 assert!(matches!(
14062 on_child_exit(
14063 &spec,
14064 runtime.restart_policy,
14065 &supervisor.registry,
14066 &snapshot,
14067 &runtime.terminal_ring,
14068 &runtime.spawn_events,
14069 &runtime.child_roster,
14070 exit_report,
14071 true,
14072 )
14073 .await,
14074 NextAction::Restart { schedule: _ }
14075 ));
14076 let state = lock_snapshot(&snapshot).unwrap();
14077 assert_eq!(state.lifetime_restarts, 1);
14078 assert_eq!(state.crash_restarts.len(), 0);
14079 }
14080
14081 #[tokio::test]
14082 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
14083 let supervisor = Supervisor::new_for_test(
14084 Arc::new(Registry::default()),
14085 RestartPolicy::new(3, Duration::ZERO),
14086 );
14087 let runtime = supervisor.runtime_config();
14088 let spec = ModuleSpec {
14089 module_id: "genuine-crash".to_string(),
14090 program: PathBuf::from("/unused/genuine-crash"),
14091 args: Vec::new(),
14092 env: Vec::new(),
14093 reserved: false,
14094 reserved_prefixes: Vec::new(),
14095 protocol: ModuleProtocol::Subc,
14096 overlap: Default::default(),
14097 carry_containment: false,
14098 };
14099 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14100
14101 assert!(matches!(
14102 on_child_exit(
14103 &spec,
14104 runtime.restart_policy,
14105 &supervisor.registry,
14106 &snapshot,
14107 &runtime.terminal_ring,
14108 &runtime.spawn_events,
14109 &runtime.child_roster,
14110 ExitReport {
14111 kind: ExitKind::Crash,
14112 code: Some(1),
14113 signal: None,
14114 at_ms: 1,
14115 },
14116 true,
14117 )
14118 .await,
14119 NextAction::Restart { schedule: _ }
14120 ));
14121 let state = lock_snapshot(&snapshot).unwrap();
14122 assert_eq!(state.lifetime_restarts, 1);
14123 assert_eq!(state.crash_restarts.len(), 1);
14124 }
14125
14126 fn crash_exit_report(at_ms: u64) -> ExitReport {
14127 ExitReport {
14128 kind: ExitKind::Crash,
14129 code: Some(1),
14130 signal: None,
14131 at_ms,
14132 }
14133 }
14134
14135 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
14136 ModuleSpec {
14137 module_id: module_id.to_string(),
14138 program: PathBuf::from("/unused").join(module_id),
14139 args: Vec::new(),
14140 env: Vec::new(),
14141 reserved: false,
14142 reserved_prefixes: Vec::new(),
14143 protocol: ModuleProtocol::Subc,
14144 overlap: Default::default(),
14145 carry_containment: false,
14146 }
14147 }
14148
14149 #[tokio::test]
14155 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
14156 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
14157 let supervisor = Supervisor::new_for_test(
14158 Arc::new(Registry::default()),
14159 RestartPolicy::new(2, Duration::ZERO),
14160 );
14161 let runtime = supervisor.runtime_config();
14162 let spec = windowed_crash_spec("crash-loop-in-window");
14163 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14164
14165 for attempt in 1..=2 {
14166 assert!(
14167 matches!(
14168 on_child_exit(
14169 &spec,
14170 runtime.restart_policy,
14171 &supervisor.registry,
14172 &snapshot,
14173 &runtime.terminal_ring,
14174 &runtime.spawn_events,
14175 &runtime.child_roster,
14176 crash_exit_report(attempt),
14177 true,
14178 )
14179 .await,
14180 NextAction::Restart { schedule: _ }
14181 ),
14182 "crash {attempt} is inside the budget and must respawn"
14183 );
14184 }
14185
14186 assert!(matches!(
14187 on_child_exit(
14188 &spec,
14189 runtime.restart_policy,
14190 &supervisor.registry,
14191 &snapshot,
14192 &runtime.terminal_ring,
14193 &runtime.spawn_events,
14194 &runtime.child_roster,
14195 crash_exit_report(3),
14196 true,
14197 )
14198 .await,
14199 NextAction::Stop { .. }
14200 ));
14201
14202 {
14203 let state = lock_snapshot(&snapshot).unwrap();
14204 assert_eq!(state.state, ModuleState::Failed);
14205 assert_eq!(state.crash_restarts.len(), 2);
14206 assert_eq!(state.lifetime_restarts, 2);
14207 }
14208
14209 let history = runtime
14210 .terminal_ring
14211 .lock()
14212 .expect("terminal ring is not poisoned")
14213 .snapshot();
14214 let last = history
14215 .entries
14216 .last()
14217 .expect("the refused crash is retained");
14218 assert_eq!(last.disposition, TerminalDisposition::Failed);
14219 assert_eq!(
14220 last.disposition_detail.as_deref(),
14221 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
14222 );
14223
14224 let captured = crate::router::test_log::captured_logs(&logs);
14225 assert!(
14226 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
14227 "the stop must be logged with its window: {captured}"
14228 );
14229 }
14230
14231 #[tokio::test]
14239 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
14240 let supervisor = Supervisor::new_for_test(
14241 Arc::new(Registry::default()),
14242 RestartPolicy::new(2, Duration::ZERO),
14243 );
14244 let runtime = supervisor.runtime_config();
14245 let spec = windowed_crash_spec("crash-across-windows");
14246 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14247
14248 for attempt in 1..=2 {
14249 assert!(matches!(
14250 on_child_exit(
14251 &spec,
14252 runtime.restart_policy,
14253 &supervisor.registry,
14254 &snapshot,
14255 &runtime.terminal_ring,
14256 &runtime.spawn_events,
14257 &runtime.child_roster,
14258 crash_exit_report(attempt),
14259 true,
14260 )
14261 .await,
14262 NextAction::Restart { schedule: _ }
14263 ));
14264 }
14265
14266 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14269 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
14270 })
14271 .unwrap();
14272
14273 assert!(
14274 matches!(
14275 on_child_exit(
14276 &spec,
14277 runtime.restart_policy,
14278 &supervisor.registry,
14279 &snapshot,
14280 &runtime.terminal_ring,
14281 &runtime.spawn_events,
14282 &runtime.child_roster,
14283 crash_exit_report(3),
14284 true,
14285 )
14286 .await,
14287 NextAction::Restart { schedule: _ }
14288 ),
14289 "a crash older than the window must not hold a budget slot"
14290 );
14291
14292 let state = lock_snapshot(&snapshot).unwrap();
14293 assert_eq!(state.state, ModuleState::Restarting);
14294 assert_eq!(
14295 state.crash_restarts.len(),
14296 2,
14297 "the aged instant is dropped and the new one takes its place"
14298 );
14299 assert_eq!(
14300 state.lifetime_restarts, 3,
14301 "the ledger counts every restart, including the ones the window forgot"
14302 );
14303 }
14304
14305 #[tokio::test]
14310 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
14311 let supervisor = Supervisor::new_for_test(
14312 Arc::new(Registry::default()),
14313 RestartPolicy::new(2, Duration::ZERO),
14314 );
14315 let runtime = supervisor.runtime_config();
14316 let spec = windowed_crash_spec("operator-cleared-budget");
14317 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14318
14319 for attempt in 1..=2 {
14320 assert!(matches!(
14321 on_child_exit(
14322 &spec,
14323 runtime.restart_policy,
14324 &supervisor.registry,
14325 &snapshot,
14326 &runtime.terminal_ring,
14327 &runtime.spawn_events,
14328 &runtime.child_roster,
14329 crash_exit_report(attempt),
14330 true,
14331 )
14332 .await,
14333 NextAction::Restart { schedule: _ }
14334 ));
14335 }
14336
14337 reset_restart_count(&snapshot, &spec.module_id).unwrap();
14338 {
14339 let state = lock_snapshot(&snapshot).unwrap();
14340 assert!(
14341 state.crash_restarts.is_empty(),
14342 "an operator restart returns the full budget"
14343 );
14344 assert_eq!(
14345 state.lifetime_restarts, 2,
14346 "clearing the budget must not unmake the crashes"
14347 );
14348 }
14349
14350 assert!(
14351 matches!(
14352 on_child_exit(
14353 &spec,
14354 runtime.restart_policy,
14355 &supervisor.registry,
14356 &snapshot,
14357 &runtime.terminal_ring,
14358 &runtime.spawn_events,
14359 &runtime.child_roster,
14360 crash_exit_report(3),
14361 true,
14362 )
14363 .await,
14364 NextAction::Restart { schedule: _ }
14365 ),
14366 "the cleared budget must be spendable again"
14367 );
14368 let state = lock_snapshot(&snapshot).unwrap();
14369 assert_eq!(state.crash_restarts.len(), 1);
14370 assert_eq!(state.lifetime_restarts, 3);
14371 }
14372
14373 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14374 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
14375 let severed = ProcessIdentity {
14376 pid: 41,
14377 start_time: 101,
14378 };
14379 let successor = ProcessIdentity {
14380 pid: 41,
14381 start_time: 202,
14382 };
14383 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
14384 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
14385 state.pid = Some(successor.pid);
14386 state.process_start_time = Some(successor.start_time);
14387 })
14388 .unwrap();
14389 assert!(!module.record_deliberate_severance(severed).unwrap());
14390
14391 let exit_report = apply_deliberate_severance_marker(
14392 &module.inner.snapshot,
14393 Some(successor),
14394 ExitReport {
14395 kind: ExitKind::Crash,
14396 code: Some(1),
14397 signal: None,
14398 at_ms: 1,
14399 },
14400 );
14401
14402 assert_eq!(exit_report.kind, ExitKind::Crash);
14403 }
14404
14405 #[tokio::test]
14406 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
14407 let registry = Registry::default();
14408 let supervisor = Supervisor::new_for_test(
14409 Arc::new(Registry::default()),
14410 RestartPolicy::new(3, Duration::ZERO),
14411 );
14412 let runtime = supervisor.runtime_config();
14413 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14414 let spec = ModuleSpec {
14415 module_id: "drain-deliberate-severance".to_string(),
14416 program: fake_aft_stub_path(),
14417 args: Vec::new(),
14418 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14419 reserved: false,
14420 reserved_prefixes: Vec::new(),
14421 protocol: ModuleProtocol::Subc,
14422 overlap: Default::default(),
14423 carry_containment: false,
14424 };
14425 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14426 let process = ProcessIdentity {
14427 pid: 41,
14428 start_time: 101,
14429 };
14430 child.process_identity = Some(process);
14431 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14432 state.pid = Some(process.pid);
14433 state.process_start_time = Some(process.start_time);
14434 })
14435 .unwrap();
14436 record_deliberate_severance(&snapshot, process).unwrap();
14437
14438 drain_child_to_state(
14439 &spec.module_id,
14440 spec.protocol,
14441 StopNotice::SentOverConnection,
14444 ®istry,
14445 None,
14446 &snapshot,
14447 &runtime.terminal_ring,
14448 &runtime.spawn_events,
14449 child,
14450 Duration::from_secs(1),
14451 ModuleState::Stopped,
14452 Some(false),
14453 &super::FailureContext::operator(),
14454 )
14455 .await
14456 .unwrap();
14457
14458 let state = lock_snapshot(&snapshot).unwrap();
14459 assert_eq!(
14460 state.last_exit.as_ref().map(|exit| exit.kind),
14461 Some(ExitKind::DeliberateSeverance)
14462 );
14463 assert_eq!(state.lifetime_restarts, 1);
14464 assert_eq!(state.crash_restarts.len(), 0);
14465 drop(state);
14466 let history = runtime.terminal_ring.lock().unwrap().snapshot();
14467 assert_eq!(
14468 history.entries[0].exit_kind,
14469 subc_control::TerminalExitKind::DeliberateSeverance
14470 );
14471 }
14472
14473 #[tokio::test]
14474 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
14475 let registry = Registry::default();
14476 let supervisor = Supervisor::new_for_test(
14477 Arc::new(Registry::default()),
14478 RestartPolicy::new(3, Duration::ZERO),
14479 );
14480 let runtime = supervisor.runtime_config();
14481 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14482 let spec = ModuleSpec {
14483 module_id: "ordinary-drain".to_string(),
14484 program: fake_aft_stub_path(),
14485 args: Vec::new(),
14486 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14487 reserved: false,
14488 reserved_prefixes: Vec::new(),
14489 protocol: ModuleProtocol::Subc,
14490 overlap: Default::default(),
14491 carry_containment: false,
14492 };
14493 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14494
14495 drain_child_to_state(
14496 &spec.module_id,
14497 spec.protocol,
14498 StopNotice::SentOverConnection,
14501 ®istry,
14502 None,
14503 &snapshot,
14504 &runtime.terminal_ring,
14505 &runtime.spawn_events,
14506 child,
14507 Duration::from_secs(1),
14508 ModuleState::Stopped,
14509 Some(false),
14510 &super::FailureContext::operator(),
14511 )
14512 .await
14513 .unwrap();
14514
14515 let state = lock_snapshot(&snapshot).unwrap();
14516 assert_eq!(
14517 state.last_exit.as_ref().map(|exit| exit.kind),
14518 Some(ExitKind::Crash)
14519 );
14520 assert_eq!(state.lifetime_restarts, 0);
14521 assert_eq!(state.crash_restarts.len(), 0);
14522 }
14523
14524 #[test]
14525 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
14526 assert!(!include_str!("server.rs")
14532 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
14533 }
14534
14535 #[test]
14542 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
14543 assert!(drained_after_quiescence_wait(&Ok(true)));
14544 assert!(!drained_after_quiescence_wait(&Ok(false)));
14545 assert!(!drained_after_quiescence_wait(&Err(
14546 SuperviseError::StatePoisoned { module_id: None }
14547 )));
14548 }
14549
14550 #[test]
14559 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
14560 let ring = Arc::new(Mutex::new(TerminalRing::new(
14561 TerminalRingConfig::default(),
14562 0,
14563 )));
14564 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
14565
14566 let snapshot = ring.lock().unwrap().snapshot();
14567 assert_eq!(snapshot.entries.len(), 1);
14568 let entry = &snapshot.entries[0];
14569 assert_eq!(entry.exit_code, None);
14570 assert_eq!(entry.exit_signal, None);
14571 assert_eq!(entry.disposition, TerminalDisposition::Failed);
14572 }
14573
14574 #[test]
14575 fn wait_error_exit_path_preserves_spawn_event_density() {
14576 let feed = super::SpawnEventFeed::default();
14577 feed.configure_incarnation("wait-error-density".to_string());
14578 feed.emit_spawned("wait-error", 41, 1);
14579 let ring = Arc::new(Mutex::new(TerminalRing::new(
14580 TerminalRingConfig::default(),
14581 0,
14582 )));
14583
14584 record_wait_error_terminal("wait-error", &ring, &feed);
14585 feed.emit_spawned("after-wait-error", 42, 2);
14586
14587 let state = feed.0.lock().unwrap();
14588 let sequences = state
14589 .events
14590 .iter()
14591 .map(|event| event.cursor.seq)
14592 .collect::<Vec<_>>();
14593 assert_eq!(sequences, vec![1, 2, 3]);
14594 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
14595 assert_eq!(state.events[1].exit_code, None);
14596 assert_eq!(state.events[1].exit_signal, None);
14597 }
14598
14599 #[test]
14603 fn wait_error_exit_report_is_classified_as_a_crash() {
14604 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
14605 }
14606}
14607
14608#[cfg(all(test, unix))]
14609mod restart_ordering_tests {
14610 use super::*;
14611 use std::io::Write;
14612 use tokio::io::AsyncReadExt;
14613
14614 #[test]
14615 #[ignore = "child process fixture"]
14616 fn serving_child() {
14617 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14618 return;
14619 };
14620 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14621 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14622 for stream in listener.incoming() {
14623 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14624 }
14625 }
14626
14627 async fn restart_before_publication(requests: usize) {
14628 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14629 let address_file = home.path().join("address");
14630 let spec = ModuleSpec {
14631 module_id: "restart-serving-child".into(),
14632 program: std::env::current_exe().unwrap(),
14633 args: vec![
14634 "--exact".into(),
14635 "supervise::restart_ordering_tests::serving_child".into(),
14636 "--ignored".into(),
14637 "--nocapture".into(),
14638 ],
14639 env: vec![(
14640 "RESTART_TEST_ADDRESS_FILE".into(),
14641 address_file.display().to_string(),
14642 )],
14643 reserved: false,
14644 reserved_prefixes: vec![],
14645 protocol: ModuleProtocol::None,
14646 overlap: Default::default(),
14647 carry_containment: false,
14648 };
14649 let mut supervisor =
14650 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14651 let (spawned_tx, spawned_rx) = oneshot::channel();
14652 let spawned_tx = Mutex::new(Some(spawned_tx));
14653 let (release_tx, release_rx) = std::sync::mpsc::channel();
14654 let release_rx = Mutex::new(release_rx);
14655 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14658 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14659 release_rx
14660 .lock()
14661 .unwrap()
14662 .recv_timeout(Duration::from_secs(20))
14663 .unwrap();
14664 })));
14665 let registered = supervisor
14666 .register_configured_with_health(
14667 spec.clone(),
14668 true,
14669 HealthConfig::default(),
14670 None,
14671 RestartPolicy::new(3, Duration::ZERO),
14672 DEFAULT_REGISTER_TIMEOUT,
14673 )
14674 .unwrap();
14675 let module = registered.module.clone();
14676 let startup = tokio::task::spawn_blocking(move || {
14677 supervisor.start_registered_configured(registered, None, true)
14678 });
14679 spawned_rx.await.unwrap();
14680 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14681 let address = loop {
14682 if let Ok(address) = std::fs::read_to_string(&address_file) {
14683 if !address.is_empty() {
14684 break address;
14685 }
14686 }
14687 tokio::time::sleep(Duration::from_millis(1)).await;
14688 };
14689 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14690 let mut reply = String::new();
14691 stream.read_to_string(&mut reply).await.unwrap();
14692 reply.trim().parse::<u32>().unwrap()
14693 })
14694 .await
14695 .expect("child must serve before generation publication");
14696 assert_eq!(
14697 lock_snapshot(&module.inner.snapshot)
14698 .unwrap()
14699 .spawn_generation,
14700 0
14701 );
14702 let mut replies = Vec::new();
14703 for _ in 0..requests {
14704 let mut restart = Box::pin(module.restart(Some(0)));
14705 std::future::poll_fn(|cx| {
14708 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14709 std::task::Poll::Ready(())
14710 })
14711 .await;
14712 replies.push(restart);
14713 }
14714 release_tx.send(()).unwrap();
14715 startup.await.unwrap().unwrap();
14716 for reply in replies {
14717 reply.await.unwrap();
14718 }
14719 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14722 loop {
14723 if lock_snapshot(&module.inner.snapshot)
14724 .unwrap()
14725 .spawn_generation
14726 > 1
14727 {
14728 break;
14729 }
14730 tokio::time::sleep(Duration::from_millis(1)).await;
14731 }
14732 })
14733 .await;
14734 module.update_spec_for_test(spec).await.unwrap();
14738 let (generation, pid) = {
14739 let state = lock_snapshot(&module.inner.snapshot).unwrap();
14740 (state.spawn_generation, state.pid)
14741 };
14742 module.stop().await.unwrap();
14743 replacement.expect("restart must publish a replacement for the already-serving child");
14744 assert_ne!(
14745 pid,
14746 Some(old_pid),
14747 "a serving child predating the restart must be replaced"
14748 );
14749 assert_eq!(generation, 2, "queued restarts must share one replacement");
14750 }
14751
14752 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14753 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14754 restart_before_publication(1).await;
14755 }
14756
14757 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14758 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14759 restart_before_publication(2).await;
14760 }
14761
14762 #[tokio::test]
14763 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14764 let supervisor =
14765 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14766 let registered = supervisor
14767 .register_configured_with_health(
14768 ModuleSpec {
14769 module_id: "queued-before-replacement".into(),
14770 program: "/bin/sleep".into(),
14775 args: vec!["600".into()],
14776 env: vec![],
14777 reserved: false,
14778 reserved_prefixes: vec![],
14779 protocol: ModuleProtocol::None,
14780 overlap: Default::default(),
14781 carry_containment: false,
14782 },
14783 true,
14784 HealthConfig::default(),
14785 None,
14786 RestartPolicy::default(),
14787 DEFAULT_REGISTER_TIMEOUT,
14788 )
14789 .unwrap();
14790 let RegisteredConfiguredModule {
14791 mut spec,
14792 mut runtime,
14793 module,
14794 mut commands,
14795 } = registered;
14796 let snapshot = &module.inner.snapshot;
14797 let mut restart = Box::pin(module.restart(Some(0)));
14798 std::future::poll_fn(|cx| {
14799 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14800 std::task::Poll::Ready(())
14801 })
14802 .await;
14803 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14806 let replacement_pid = child.as_ref().unwrap().id();
14807 let command = commands.recv().await.unwrap();
14808 assert!(
14809 handle_supervisor_command(
14810 command,
14811 &mut spec,
14812 &mut runtime,
14813 &supervisor.registry,
14814 &supervisor.process_liveness,
14815 snapshot,
14816 &mut child,
14817 &mut commands,
14818 &mut VecDeque::new(),
14819 )
14820 .await
14821 );
14822 restart.await.unwrap();
14823 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14824 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14825 if let Some(mut child) = child {
14826 child.start_kill().unwrap();
14827 child.wait().await.unwrap();
14828 }
14829 assert_eq!(
14830 kept_pid, replacement_pid,
14831 "a newer replacement must survive"
14832 );
14833 assert!(
14834 !pending,
14835 "a satisfied restart must not schedule another spawn"
14836 );
14837 }
14838}
14839
14840#[cfg(all(test, unix))]
14841mod health_event_tests {
14842 use super::*;
14843 use cortexkit_test_support::ScratchDir;
14844 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14845
14846 struct Actor {
14847 module: SupervisedModule,
14848 registry: Arc<Registry>,
14849 forwarding: Arc<ForwardingTable>,
14850 spec: ModuleSpec,
14851 health: HealthConfig,
14852 rx: mpsc::Receiver<crate::router::OutboundFrame>,
14853 roster: ChildRoster,
14854 _home: ScratchDir,
14855 }
14856
14857 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14858 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14862 let tick = Instant::now();
14863 loop {
14864 if let Some(value) = observe() {
14865 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14866 return value;
14867 }
14868 assert!(
14869 std::time::Instant::now() < deadline,
14870 "timed out waiting for {reason}"
14871 );
14872 tokio::task::yield_now().await;
14873 }
14874 }
14875
14876 impl Actor {
14877 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14878 Self::start_with_policy(
14879 protocol,
14880 cadence,
14881 DEFAULT_REGISTER_TIMEOUT,
14882 RestartPolicy::new(3, Duration::ZERO),
14883 )
14884 .await
14885 }
14886
14887 async fn start_with_policy(
14888 protocol: ModuleProtocol,
14889 cadence: Duration,
14890 register_timeout: Duration,
14891 policy: RestartPolicy,
14892 ) -> Self {
14893 let home = ScratchDir::new("health-events");
14894 let registry = Arc::new(Registry::default());
14895 let forwarding = Arc::new(ForwardingTable::default());
14896 let health = HealthConfig {
14897 cadence,
14898 deadline: Duration::from_secs(3600),
14899 ..HealthConfig::default()
14900 };
14901 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14902 .with_forwarding(forwarding.clone())
14903 .with_health_config(health.clone());
14904 let spec = ModuleSpec {
14905 module_id: "event-health-child".into(),
14906 program: PathBuf::from("/bin/sleep"),
14914 args: vec!["600".into()],
14917 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14918 .into_iter()
14919 .map(|key| (key.into(), home.path().display().to_string()))
14920 .collect(),
14921 reserved: false,
14922 reserved_prefixes: vec![],
14923 protocol,
14924 overlap: Default::default(),
14925 carry_containment: false,
14926 };
14927 let module = supervisor
14928 .supervise_configured_requested(
14929 spec.clone(),
14930 true,
14931 health.clone(),
14932 Some(0),
14933 policy,
14934 register_timeout,
14935 None,
14936 )
14937 .unwrap();
14938 let (_, rx) = mpsc::channel(8);
14939 let actor = Self {
14940 module,
14941 registry,
14942 forwarding,
14943 spec,
14944 health,
14945 rx,
14946 roster: supervisor.child_roster.clone(),
14947 _home: home,
14948 };
14949 actor
14950 .wait_for_select(|checkpoint| checkpoint.generation > 0)
14951 .await;
14952 actor
14953 }
14954
14955 async fn expire_registration_wait(&self, remaining: Duration) {
14956 tokio::time::advance(remaining).await;
14957 wait_for("registration timeout TERM grace", || {
14958 lock_snapshot(&self.module.inner.snapshot)
14959 .unwrap()
14960 .actor_select
14961 .is_none()
14962 .then_some(())
14963 })
14964 .await;
14965 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14966 }
14967
14968 fn turns(&self) -> u64 {
14969 lock_snapshot(&self.module.inner.snapshot)
14970 .unwrap()
14971 .actor_turns
14972 }
14973
14974 async fn wait_for_select(
14975 &self,
14976 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14977 ) -> ActorSelectCheckpoint {
14978 wait_for("actor select checkpoint", || {
14979 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14980 state.actor_select.filter(|checkpoint| {
14981 checkpoint.generation == state.spawn_generation
14982 && state.state == ModuleState::Running
14983 && state.process_alive
14984 && state.reported_pid().is_some()
14985 && expected(checkpoint)
14986 })
14987 })
14988 .await
14989 }
14990
14991 fn hello(&mut self, connection: u64, health: bool) {
14992 let manifest =
14993 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14994 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14995 .build();
14996 let connection = ConnectionId::new(connection);
14997 let (tx, rx) = mpsc::channel(8);
14998 self.rx = rx;
14999 self.forwarding
15000 .register_module_connection(
15001 connection,
15002 self.spec.module_id.clone(),
15003 subc_protocol::PROTOCOL_VERSION,
15004 Concurrency::ModuleManaged,
15005 FrameSink::new(tx),
15006 )
15007 .unwrap();
15008 self.registry
15009 .register_with_control_ops(
15010 manifest,
15011 subc_protocol::PROTOCOL_VERSION,
15012 connection,
15013 if health {
15014 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
15015 } else {
15016 vec![]
15017 },
15018 )
15019 .unwrap();
15020 }
15021
15022 async fn probe(&mut self) -> crate::router::OutboundFrame {
15023 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
15024 Ok(frame) => Some(frame),
15025 Err(mpsc::error::TryRecvError::Empty) => None,
15026 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
15027 })
15028 .await;
15029 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15030 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15031 frame
15032 }
15033
15034 fn no_probe(&mut self) {
15035 assert!(matches!(
15036 self.rx.try_recv(),
15037 Err(mpsc::error::TryRecvError::Empty)
15038 ));
15039 }
15040
15041 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
15042 self.forwarding
15043 .complete_module_control_rpc(
15044 ConnectionId::new(connection),
15045 frame.header.corr,
15046 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15047 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15048 status: HealthStatus::Ok,
15049 detail: None,
15050 metrics: None,
15051 }),
15052 )
15053 .unwrap();
15054 }
15055 }
15056
15057 #[tokio::test(start_paused = true)]
15058 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
15059 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15060 let initial = actor.module.status().unwrap();
15061 tokio::time::advance(Duration::from_secs(119)).await;
15062 tokio::task::yield_now().await;
15063 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
15064 assert!(actor.module.terminal_history().entries.is_empty());
15065 actor.expire_registration_wait(Duration::from_secs(1)).await;
15066 actor
15067 .wait_for_select(|at| at.generation > initial.spawn_generation)
15068 .await;
15069 let status = actor.module.status().unwrap();
15070 assert_ne!(status.pid, initial.pid);
15071 assert!(!cortexkit_test_support::process_alive(
15072 initial.pid.unwrap() as i32
15073 ));
15074 assert_eq!(status.restart_count, 1);
15075 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
15076 let entries = actor.module.terminal_history().entries;
15077 assert_eq!(entries.len(), 1);
15078 assert_eq!(
15079 entries[0].disposition_detail.as_deref(),
15080 Some("registration_timeout")
15081 );
15082 actor.module.stop().await.unwrap();
15083 }
15084
15085 #[tokio::test(start_paused = true)]
15086 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
15087 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15088 let pid = actor.module.status().unwrap().pid;
15089 tokio::time::advance(Duration::from_secs(119)).await;
15090 actor.hello(1, false);
15091 let registered = actor
15092 .wait_for_select(|at| at.registered_connection.is_some())
15093 .await;
15094 assert_eq!(
15095 registered.wake_after, None,
15096 "HELLO must cancel the deadline"
15097 );
15098 tokio::time::advance(Duration::from_secs(300)).await;
15099 tokio::task::yield_now().await;
15100 assert_eq!(actor.module.status().unwrap().pid, pid);
15101 assert!(actor.module.terminal_history().entries.is_empty());
15102 actor
15104 .registry
15105 .deregister_connection(ConnectionId::new(1))
15106 .unwrap();
15107 actor
15108 .wait_for_select(|at| at.registered_connection.is_none())
15109 .await;
15110 tokio::time::advance(Duration::from_secs(300)).await;
15111 tokio::task::yield_now().await;
15112 assert_eq!(actor.module.status().unwrap().pid, pid);
15113 assert!(actor.module.terminal_history().entries.is_empty());
15114 actor.module.stop().await.unwrap();
15115
15116 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15117 let pid = transient.module.status().unwrap().pid;
15118 tokio::time::advance(Duration::from_secs(119)).await;
15119 let turns = transient.turns();
15120 transient.hello(1, false);
15123 transient
15124 .registry
15125 .deregister_connection(ConnectionId::new(1))
15126 .unwrap();
15127 let parked = transient.wait_for_select(|at| at.turn > turns).await;
15128 assert_eq!(
15129 parked.wake_after, None,
15130 "transient HELLO must cancel the deadline"
15131 );
15132 tokio::time::advance(Duration::from_secs(300)).await;
15133 tokio::task::yield_now().await;
15134 assert_eq!(transient.module.status().unwrap().pid, pid);
15135 assert!(transient.module.terminal_history().entries.is_empty());
15136 transient.module.stop().await.unwrap();
15137 }
15138
15139 #[tokio::test(start_paused = true)]
15140 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
15141 let actor = Actor::start_with_policy(
15142 ModuleProtocol::Subc,
15143 Duration::from_secs(30),
15144 Duration::from_secs(10),
15145 RestartPolicy::new(1, Duration::ZERO),
15146 )
15147 .await;
15148 let generation = actor.module.status().unwrap().spawn_generation;
15149 actor
15150 .expire_registration_wait(Duration::from_secs(10))
15151 .await;
15152 actor.wait_for_select(|at| at.generation > generation).await;
15153 actor
15154 .expire_registration_wait(Duration::from_secs(10))
15155 .await;
15156 let retry = wait_for("failed retry schedule", || {
15157 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
15158 (state.state == ModuleState::Failed)
15159 .then_some(state.failed_retry.pending)
15160 .flatten()
15161 })
15162 .await;
15163 assert_eq!(retry.delay, Duration::from_secs(600));
15164 assert_eq!(retry.attempt, 1);
15165 let entries = actor.module.terminal_history().entries;
15166 assert_eq!(entries.len(), 2);
15167 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
15168 assert!(entries[1]
15169 .disposition_detail
15170 .as_deref()
15171 .unwrap()
15172 .contains("registration_timeout"));
15173 tokio::time::advance(Duration::from_secs(599)).await;
15174 tokio::task::yield_now().await;
15175 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
15176 tokio::time::advance(Duration::from_secs(1)).await;
15177 actor
15178 .wait_for_select(|at| at.generation > generation + 1)
15179 .await;
15180 actor
15182 .expire_registration_wait(Duration::from_secs(10))
15183 .await;
15184 wait_for("slow retry registration timeout", || {
15185 (actor.module.terminal_history().entries.len() == 3).then_some(())
15186 })
15187 .await;
15188 actor.module.stop().await.unwrap();
15189 }
15190
15191 #[tokio::test(start_paused = true)]
15192 async fn registration_timeout_honours_ten_second_override() {
15193 let actor = Actor::start_with_policy(
15194 ModuleProtocol::Subc,
15195 Duration::from_secs(30),
15196 Duration::from_secs(10),
15197 RestartPolicy::new(3, Duration::ZERO),
15198 )
15199 .await;
15200 let generation = actor.module.status().unwrap().spawn_generation;
15201 tokio::time::advance(Duration::from_secs(9)).await;
15202 tokio::task::yield_now().await;
15203 assert!(actor.module.terminal_history().entries.is_empty());
15204 actor.expire_registration_wait(Duration::from_secs(1)).await;
15205 actor.wait_for_select(|at| at.generation > generation).await;
15206 assert_eq!(actor.module.terminal_history().entries.len(), 1);
15207 actor.module.stop().await.unwrap();
15208 }
15209
15210 #[tokio::test]
15211 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
15212 use std::os::unix::process::{CommandExt, ExitStatusExt};
15213
15214 struct Helper(std::process::Child);
15215 impl Drop for Helper {
15216 fn drop(&mut self) {
15217 if matches!(self.0.try_wait(), Ok(None)) {
15220 let _ = self.0.kill();
15221 let _ = self.0.wait();
15222 }
15223 }
15224 }
15225
15226 let home = ScratchDir::new("registration-timeout-helper");
15227 let supervisor = Supervisor::new_for_test(
15228 Arc::new(Registry::default()),
15229 RestartPolicy::new(0, Duration::ZERO),
15230 );
15231 let spec = ModuleSpec {
15232 module_id: "registration-timeout-group".into(),
15233 program: "/bin/sleep".into(),
15234 args: vec!["120".into()],
15235 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15236 .into_iter()
15237 .map(|key| (key.into(), home.path().display().to_string()))
15238 .collect(),
15239 reserved: false,
15240 reserved_prefixes: vec![],
15241 protocol: ModuleProtocol::Subc,
15242 overlap: Default::default(),
15243 carry_containment: false,
15244 };
15245 let module = supervisor
15246 .supervise_configured_requested(
15247 spec,
15248 true,
15249 HealthConfig::default(),
15250 None,
15251 RestartPolicy::new(0, Duration::ZERO),
15252 Duration::from_secs(10),
15253 None,
15254 )
15255 .unwrap();
15256 let leader = timeout(Duration::from_secs(5), async {
15257 loop {
15258 if let Some(pid) = module.status().unwrap().pid {
15259 break pid;
15260 }
15261 sleep(Duration::from_millis(5)).await;
15262 }
15263 })
15264 .await
15265 .expect("supervised group leader must be reported before the deadline");
15266 assert!(leader > 1);
15267 let ready = home.join("helper.ready");
15268 let mut helper = Helper(
15271 std::process::Command::new("/bin/sh")
15272 .args([
15273 "-c",
15274 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
15275 "helper",
15276 ])
15277 .arg(&ready)
15278 .process_group(i32::try_from(leader).unwrap())
15279 .stdout(std::process::Stdio::null())
15280 .stderr(std::process::Stdio::null())
15281 .spawn()
15282 .unwrap(),
15283 );
15284 timeout(Duration::from_secs(5), async {
15285 while !ready.exists() {
15286 assert!(helper.0.try_wait().unwrap().is_none());
15287 sleep(Duration::from_millis(5)).await;
15288 }
15289 })
15290 .await
15291 .expect("helper must install TERM ignore before the registration deadline");
15292 let helper_pid =
15293 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
15294 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
15295 sleep(Duration::from_millis(20)).await;
15296 assert!(
15297 helper.0.try_wait().unwrap().is_none(),
15298 "helper must ignore TERM"
15299 );
15300
15301 timeout(Duration::from_secs(20), async {
15302 while module.state().unwrap() != ModuleState::Failed {
15303 sleep(Duration::from_millis(5)).await;
15304 }
15305 })
15306 .await
15307 .expect("registration timeout must stop and reap the group leader");
15308 let helper_exit = timeout(Duration::from_secs(3), async {
15309 loop {
15310 if let Some(status) = helper.0.try_wait().unwrap() {
15311 break status;
15312 }
15313 sleep(Duration::from_millis(5)).await;
15314 }
15315 })
15316 .await
15317 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
15318 assert_eq!(helper_exit.signal(), Some(9));
15319 assert!(!cortexkit_test_support::process_alive(
15320 i32::try_from(leader).unwrap()
15321 ));
15322 assert_eq!(
15323 module.status().unwrap().last_exit.unwrap().kind,
15324 ExitKind::Crash
15325 );
15326 assert!(module.terminal_history().entries[0]
15327 .disposition_detail
15328 .as_deref()
15329 .unwrap()
15330 .contains("registration_timeout"));
15331 module.stop().await.unwrap();
15332 }
15333
15334 #[tokio::test(start_paused = true)]
15335 async fn registration_timeout_does_not_count_an_earlier_exit() {
15336 let actor = Actor::start_with_policy(
15337 ModuleProtocol::Subc,
15338 Duration::from_secs(30),
15339 DEFAULT_REGISTER_TIMEOUT,
15340 RestartPolicy::new(0, Duration::ZERO),
15341 )
15342 .await;
15343 let pid = actor.module.status().unwrap().pid.unwrap();
15344 rustix::process::kill_process(
15345 rustix::process::Pid::from_raw(pid as i32).unwrap(),
15346 rustix::process::Signal::TERM,
15347 )
15348 .unwrap();
15349 wait_for("early exit", || {
15350 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
15351 })
15352 .await;
15353 tokio::time::advance(Duration::from_secs(121)).await;
15354 tokio::task::yield_now().await;
15355 let entries = actor.module.terminal_history().entries;
15356 assert_eq!(entries.len(), 1);
15357 assert!(!entries[0]
15358 .disposition_detail
15359 .as_deref()
15360 .unwrap()
15361 .contains("registration_timeout"));
15362 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
15363 actor.module.stop().await.unwrap();
15364 }
15365
15366 #[tokio::test(start_paused = true)]
15367 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
15368 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15369 let pid = actor.module.status().unwrap().pid;
15370 tokio::time::advance(Duration::from_secs(119)).await;
15371 actor.roster.close();
15372 let turns = actor.turns();
15373 tokio::time::advance(Duration::from_secs(300)).await;
15374 let parked = actor.wait_for_select(|at| at.turn > turns).await;
15375 assert_eq!(parked.wake_after, None);
15376 assert_eq!(actor.module.status().unwrap().pid, pid);
15377 assert!(actor.module.terminal_history().entries.is_empty());
15378 actor.module.stop().await.unwrap();
15379 let status = actor.module.status().unwrap();
15380 assert_eq!(status.restart_count, 0);
15381 assert_eq!(
15382 actor.module.terminal_history().entries[0].disposition,
15383 TerminalDisposition::DaemonShutdown
15384 );
15385 }
15386
15387 #[tokio::test(start_paused = true)]
15388 async fn no_health_children_park_for_sixty_seconds() {
15389 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15390 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15391 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15392 unadvertised.hello(1, false);
15393 for _ in 0..6000 {
15398 tokio::time::advance(Duration::from_millis(10)).await;
15399 tokio::task::yield_now().await;
15400 }
15401 for actor in [&none, &unregistered, &unadvertised] {
15402 assert!(
15403 actor.turns() <= 5,
15404 "no-health actor woke {} times",
15405 actor.turns()
15406 );
15407 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
15408 }
15409 }
15410
15411 #[tokio::test(start_paused = true)]
15412 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
15413 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
15417 let tick = Instant::now();
15418 actor.hello(1, true);
15419 actor.probe().await;
15420 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
15421 }
15422
15423 #[tokio::test(start_paused = true)]
15424 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
15425 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15426 let hello_tick = Instant::now();
15427 actor.hello(1, true);
15428 let armed = actor
15429 .wait_for_select(|checkpoint| {
15430 checkpoint.registered_connection == Some(ConnectionId::new(1))
15431 && checkpoint.next_probe_at.is_some()
15432 })
15433 .await;
15434 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15435 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
15436 let first_deadline = hello_tick + delay;
15437 assert_eq!(armed.next_probe_at, Some(first_deadline));
15438 tokio::time::advance(Duration::from_secs(29)).await;
15439 let turns = actor.turns();
15440 actor
15441 .registry
15442 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
15443 .unwrap();
15444 let updated = actor
15445 .wait_for_select(|checkpoint| checkpoint.turn > turns)
15446 .await;
15447 assert_eq!(
15448 updated.next_probe_at,
15449 Some(first_deadline),
15450 "catalog update must not reset cadence"
15451 );
15452 actor.no_probe();
15453 tokio::time::advance(first_deadline - Instant::now()).await;
15454 let frame = actor.probe().await;
15455 actor.answer(1, frame);
15456 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
15457 let next_deadline = Instant::now() + next;
15458 let rearmed = actor
15459 .wait_for_select(|checkpoint| {
15460 checkpoint
15461 .next_probe_at
15462 .is_some_and(|deadline| deadline > first_deadline)
15463 })
15464 .await;
15465 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
15466 tokio::time::advance(next - Duration::from_nanos(1)).await;
15467 actor.no_probe();
15468 tokio::time::advance(Duration::from_nanos(1)).await;
15469 actor.probe().await;
15470 }
15471
15472 #[tokio::test(start_paused = true)]
15473 async fn disconnect_disarms_and_reconnect_rearms_health() {
15474 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15475 actor.hello(1, true);
15476 actor
15477 .wait_for_select(|checkpoint| {
15478 checkpoint.registered_connection == Some(ConnectionId::new(1))
15479 && checkpoint.next_probe_at.is_some()
15480 })
15481 .await;
15482 let turns = actor.turns();
15483 let tick = Instant::now();
15484 actor
15485 .registry
15486 .deregister_connection(ConnectionId::new(1))
15487 .unwrap();
15488 let disarmed = actor
15489 .wait_for_select(|checkpoint| {
15490 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
15491 })
15492 .await;
15493 assert_eq!(disarmed.next_probe_at, None);
15494 assert_eq!(disarmed.wake_after, None);
15495 assert_eq!(
15496 Instant::now(),
15497 tick,
15498 "disconnect must disarm in the notification tick"
15499 );
15500 tokio::time::advance(Duration::from_secs(60)).await;
15501 actor.no_probe();
15502 let reconnect_tick = Instant::now();
15503 actor.hello(2, true);
15504 let rearmed = actor
15505 .wait_for_select(|checkpoint| {
15506 checkpoint.registered_connection == Some(ConnectionId::new(2))
15507 && checkpoint.next_probe_at.is_some()
15508 })
15509 .await;
15510 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15511 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
15512 tokio::time::advance(delay).await;
15513 actor.probe().await;
15514 }
15515
15516 #[tokio::test(start_paused = true)]
15517 async fn reregistration_without_health_disarms_the_old_schedule() {
15518 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15519 actor.hello(1, true);
15520 actor
15521 .wait_for_select(|checkpoint| {
15522 checkpoint.registered_connection == Some(ConnectionId::new(1))
15523 && checkpoint.next_probe_at.is_some()
15524 })
15525 .await;
15526 actor
15527 .registry
15528 .deregister_connection(ConnectionId::new(1))
15529 .unwrap();
15530 actor.hello(2, false);
15531 let disarmed = actor
15532 .wait_for_select(|checkpoint| {
15533 checkpoint.registered_connection == Some(ConnectionId::new(2))
15534 })
15535 .await;
15536 assert_eq!(disarmed.next_probe_at, None);
15537 assert_eq!(disarmed.wake_after, None);
15538 tokio::time::advance(Duration::from_secs(60)).await;
15539 actor.no_probe();
15540 }
15541
15542 #[tokio::test(start_paused = true)]
15543 async fn restart_waits_for_its_own_hello_before_rearming_health() {
15544 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15545 actor.hello(1, true);
15546 let armed = actor
15547 .wait_for_select(|checkpoint| {
15548 checkpoint.registered_connection == Some(ConnectionId::new(1))
15549 && checkpoint.next_probe_at.is_some()
15550 })
15551 .await;
15552 actor
15556 .registry
15557 .deregister_connection(ConnectionId::new(1))
15558 .unwrap();
15559 actor.module.restart(Some(0)).await.unwrap();
15560 let parked = actor
15561 .wait_for_select(|checkpoint| {
15562 checkpoint.generation > armed.generation
15563 && checkpoint.registered_connection.is_none()
15564 && checkpoint.next_probe_at.is_none()
15565 })
15566 .await;
15567 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
15568 actor.rx.close();
15569 while actor.rx.try_recv().is_ok() {}
15572 let turns = actor.turns();
15573 tokio::time::advance(Duration::from_secs(60)).await;
15574 let after = actor.turns();
15575 eprintln!(
15576 "restart park turns: before={turns} after={after} delta={}",
15577 after - turns
15578 );
15579 assert_eq!(after, turns, "replacement must park before HELLO");
15580 let hello_tick = Instant::now();
15581 actor.hello(2, true);
15582 let rearmed = actor
15583 .wait_for_select(|checkpoint| {
15584 checkpoint.registered_connection == Some(ConnectionId::new(2))
15585 && checkpoint.next_probe_at.is_some()
15586 })
15587 .await;
15588 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15589 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
15590 tokio::time::advance(delay).await;
15591 actor.probe().await;
15592 }
15593
15594 #[tokio::test(start_paused = true)]
15595 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
15596 use tokio::{
15597 io::{AsyncReadExt, AsyncWriteExt},
15598 net::TcpListener,
15599 };
15600 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15601 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15602 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
15603 let (started_tx, mut started_rx) = mpsc::channel(8);
15604 let peer = tokio::spawn(async move {
15605 loop {
15606 let (mut socket, _) = listener.accept().await.unwrap();
15607 let mut request = [0; 1024];
15608 assert!(socket.read(&mut request).await.unwrap() > 0);
15609 started_tx.send(Instant::now()).await.unwrap();
15610 socket
15611 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15612 .await
15613 .unwrap();
15614 }
15615 });
15616 actor
15617 .module
15618 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15619 .await
15620 .unwrap();
15621 let first_tick = Instant::now();
15622 let first = actor
15623 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15624 .await;
15625 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15626 assert!(started_rx.try_recv().is_err());
15627
15628 tokio::time::advance(
15631 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15632 )
15633 .await;
15634 actor.module.restart(Some(0)).await.unwrap();
15635 let spawn_tick = Instant::now();
15636 let replacement = actor
15637 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15638 .await;
15639 let deadline = replacement.next_probe_at.unwrap();
15640 assert!(
15641 deadline >= spawn_tick + actor.health.cadence,
15642 "a replacement must not inherit its predecessor's probe deadline"
15643 );
15644 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15645 assert!(
15646 started_rx.try_recv().is_err(),
15647 "no HTTP probe during startup grace"
15648 );
15649 tokio::time::advance(deadline - Instant::now()).await;
15650 let landed = wait_for("first replacement HTTP probe", || {
15651 started_rx.try_recv().ok()
15652 })
15653 .await;
15654 assert!(landed >= spawn_tick + actor.health.cadence);
15655 wait_for("healthy replacement", || {
15656 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15657 .then_some(())
15658 })
15659 .await;
15660 peer.abort();
15661 }
15662
15663 #[tokio::test(start_paused = true)]
15664 async fn rescan_adds_and_removes_http_health_without_polling() {
15665 use tokio::{
15666 io::{AsyncReadExt, AsyncWriteExt},
15667 net::TcpListener,
15668 };
15669 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15670 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15671 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15672 actor.health.cadence = Duration::from_millis(1);
15674 let (started_tx, mut started_rx) = mpsc::channel(8);
15675 let peer = tokio::spawn(async move {
15676 loop {
15677 let (mut socket, _) = listener.accept().await.unwrap();
15678 let mut request = [0; 1024];
15679 assert!(socket.read(&mut request).await.unwrap() > 0);
15680 started_tx.send(()).await.unwrap();
15681 socket
15682 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15683 .await
15684 .unwrap();
15685 }
15686 });
15687 let tick = Instant::now();
15688 let turns = actor.turns();
15689 actor
15690 .module
15691 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15692 .await
15693 .unwrap();
15694 let armed = actor
15695 .wait_for_select(|checkpoint| {
15696 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15697 })
15698 .await;
15699 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15700 assert_eq!(Instant::now(), tick);
15701 tokio::time::advance(Duration::from_millis(1)).await;
15702 let mut started = false;
15703 wait_for("successful HTTP probe", || {
15704 started |= started_rx.try_recv().is_ok();
15705 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15706 .then_some(())
15707 })
15708 .await;
15709 assert!(started, "added HTTP check must start at its first deadline");
15710 assert_eq!(
15711 actor.module.status().unwrap().health.status,
15712 SupervisorHealthStatus::Ok
15713 );
15714 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15715 actor.health.http = None;
15716 let turns = actor.turns();
15717 actor
15718 .module
15719 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15720 .await
15721 .unwrap();
15722 let parked = actor
15723 .wait_for_select(|checkpoint| {
15724 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15725 })
15726 .await;
15727 assert_eq!(parked.wake_after, None);
15728 let turns = actor.turns();
15729 tokio::time::advance(Duration::from_secs(60)).await;
15730 assert_eq!(
15731 actor.turns(),
15732 turns,
15733 "removed HTTP check must park the actor"
15734 );
15735 assert!(
15736 started_rx.try_recv().is_err(),
15737 "removed HTTP check must stay disarmed"
15738 );
15739 assert_eq!(
15740 actor.module.status().unwrap().health,
15741 ModuleHealthStatus::default()
15742 );
15743 peer.abort();
15744 }
15745}
15746
15747#[cfg(test)]
15748mod drain_busy_reprobe_tests {
15749 use super::*;
15753 use std::collections::BTreeMap;
15754 use std::sync::Mutex as StdMutex;
15755 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15756
15757 const MODULE_ID: &str = "drain-reprobe-module";
15758 const CONNECTION: u64 = 7;
15759 const GAUGE: &str = "runs_in_flight";
15760 const CADENCE: Duration = Duration::from_secs(30);
15763 const CEILING: Duration = Duration::from_secs(30);
15764
15765 #[derive(Clone, Copy)]
15767 enum Answer {
15768 BusyUntil { until: Instant, latency: Duration },
15771 OmitsGauge,
15773 Silent,
15775 }
15776
15777 struct Drain {
15778 forwarding: Arc<ForwardingTable>,
15779 runtime: SupervisorRuntimeConfig,
15780 endpoint: crate::ModuleEndpointId,
15781 probes: Arc<StdMutex<Vec<Instant>>>,
15783 responder: tokio::task::JoinHandle<()>,
15784 }
15785
15786 impl Drain {
15787 fn start(answer: Answer) -> Self {
15788 let registry = Arc::new(Registry::default());
15789 let forwarding = Arc::new(ForwardingTable::default());
15790 let health = HealthConfig {
15791 cadence: CADENCE,
15792 ..HealthConfig::default()
15793 };
15794 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15795 .with_forwarding(forwarding.clone())
15796 .with_health_config(health)
15797 .runtime_config();
15798 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15799 let endpoint = forwarding
15800 .register_module_connection(
15801 ConnectionId::new(CONNECTION),
15802 MODULE_ID.to_string(),
15803 subc_protocol::PROTOCOL_VERSION,
15804 Concurrency::ModuleManaged,
15805 FrameSink::new(tx),
15806 )
15807 .unwrap();
15808 let probes = Arc::new(StdMutex::new(Vec::new()));
15809 let responder = tokio::spawn({
15810 let forwarding = forwarding.clone();
15811 let probes = probes.clone();
15812 async move {
15813 while let Some(frame) = rx.recv().await {
15814 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15815 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15816 probes.lock().unwrap().push(Instant::now());
15817 let metrics = match answer {
15818 Answer::BusyUntil { until, latency } => {
15819 sleep(latency).await;
15820 let busy = u64::from(Instant::now() < until);
15821 serde_json::json!({ GAUGE: busy })
15822 }
15823 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15824 Answer::Silent => continue,
15825 };
15826 let _ = forwarding.complete_module_control_rpc(
15829 ConnectionId::new(CONNECTION),
15830 frame.header.corr,
15831 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15832 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15833 status: HealthStatus::Ok,
15834 detail: None,
15835 metrics: Some(metrics),
15836 }),
15837 );
15838 }
15839 }
15840 });
15841 Self {
15842 forwarding,
15843 runtime,
15844 endpoint,
15845 probes,
15846 responder,
15847 }
15848 }
15849
15850 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15853 let (drained, elapsed, _) = self.run_traced(ceiling).await;
15854 (drained, elapsed)
15855 }
15856
15857 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15859 let started = Instant::now();
15860 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15861 let drained = wait_for_forwarding_quiescence(
15862 &self.forwarding,
15863 MODULE_ID,
15864 &self.runtime,
15865 self.endpoint,
15866 started + ceiling,
15867 &[GAUGE.to_string()],
15868 DrainScope::Active,
15869 &mut trace,
15870 )
15871 .await
15872 .unwrap();
15873 (drained, started.elapsed(), trace)
15874 }
15875
15876 fn probes(&self) -> Vec<Instant> {
15877 self.probes.lock().unwrap().clone()
15878 }
15879 }
15880
15881 impl Drop for Drain {
15882 fn drop(&mut self) {
15883 self.responder.abort();
15884 }
15885 }
15886
15887 #[tokio::test(start_paused = true)]
15888 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15889 let drain = Drain::start(Answer::BusyUntil {
15890 until: Instant::now() + Duration::from_secs(1),
15891 latency: Duration::ZERO,
15892 });
15893 let (drained, elapsed) = drain.run(CEILING).await;
15894 assert!(
15895 drained,
15896 "gauges that read zero during the drain must let it finish drained, \
15897 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15898 );
15899 assert!(
15900 elapsed < Duration::from_secs(3),
15901 "drain took {elapsed:?} after the gauges went quiet at 1s"
15902 );
15903 let probes = drain.probes();
15904 assert!(
15905 probes.len() >= 2,
15906 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15907 );
15908 }
15909
15910 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15915 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15916 let drain = Drain::start(Answer::BusyUntil {
15917 until: Instant::now() + Duration::from_millis(600),
15918 latency: Duration::ZERO,
15919 });
15920 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15921 assert!(drained, "the gauge fell at 600ms: {trace}");
15922 assert!(
15923 elapsed < Duration::from_secs(3),
15924 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15925 );
15926 let probes = drain.probes();
15927 assert!(
15928 probes.len() >= 2 && trace.probes as usize == probes.len(),
15929 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15930 );
15931 }
15932
15933 #[tokio::test(start_paused = true)]
15934 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15935 let started = Instant::now();
15936 let drain = Drain::start(Answer::BusyUntil {
15937 until: started + Duration::from_secs(3600),
15938 latency: Duration::ZERO,
15939 });
15940 let (drained, elapsed) = drain.run(CEILING).await;
15941 assert!(!drained, "a module busy throughout must not drain");
15942 assert!(
15943 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15944 "a busy module is held until the ceiling and no longer: {elapsed:?}"
15945 );
15946 let probes = drain.probes();
15949 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15950 assert!(
15951 probes.len() as u128 <= max_probes,
15952 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15953 probes.len()
15954 );
15955 for pair in probes.windows(2) {
15956 assert!(
15957 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15958 "consecutive probes {:?} apart",
15959 pair[1] - pair[0]
15960 );
15961 }
15962 }
15963
15964 #[tokio::test(start_paused = true)]
15965 async fn a_slow_probe_is_never_overlapped_by_the_next() {
15966 let latency = Duration::from_secs(1);
15969 let drain = Drain::start(Answer::BusyUntil {
15970 until: Instant::now() + Duration::from_secs(3600),
15971 latency,
15972 });
15973 let (drained, _) = drain.run(Duration::from_secs(5)).await;
15974 assert!(!drained);
15975 let probes = drain.probes();
15976 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15977 for pair in probes.windows(2) {
15978 assert!(
15979 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15980 "a probe started {:?} after the previous one, before it could finish",
15981 pair[1] - pair[0]
15982 );
15983 }
15984 }
15985
15986 #[tokio::test(start_paused = true)]
15987 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15988 let drain = Drain::start(Answer::Silent);
15989 let ceiling = Duration::from_secs(2);
15990 let (drained, elapsed) = drain.run(ceiling).await;
15991 assert!(!drained, "an unanswered probe counts as busy");
15992 assert!(
15993 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15994 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15995 );
15996 assert_eq!(
15997 drain.probes().len(),
15998 1,
15999 "the unanswered probe holds the only slot until the ceiling"
16000 );
16001 }
16002
16003 #[derive(Clone, Default)]
16005 struct LogCapture {
16006 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
16007 }
16008
16009 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
16010 fn on_event(
16011 &self,
16012 event: &tracing::Event<'_>,
16013 _context: tracing_subscriber::layer::Context<'_, S>,
16014 ) {
16015 struct Fields(BTreeMap<String, String>);
16016 impl tracing::field::Visit for Fields {
16017 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
16018 self.0
16019 .insert(field.name().to_string(), format!("{value:?}"));
16020 }
16021 }
16022 let mut fields = Fields(BTreeMap::new());
16023 event.record(&mut fields);
16024 self.events.lock().unwrap().push(fields.0);
16025 }
16026 }
16027
16028 impl LogCapture {
16029 fn install(&self) -> tracing::subscriber::DefaultGuard {
16030 use tracing_subscriber::layer::SubscriberExt;
16031 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
16032 }
16033
16034 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
16035 self.events
16036 .lock()
16037 .unwrap()
16038 .iter()
16039 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
16040 .cloned()
16041 .collect()
16042 }
16043 }
16044
16045 #[tokio::test(start_paused = true)]
16050 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
16051 let capture = LogCapture::default();
16052 let _guard = capture.install();
16053 let drain = Drain::start(Answer::OmitsGauge);
16054 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
16055 assert!(!drained, "a gauge never reported cannot read quiet");
16056 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
16057
16058 let missing = capture.with_message("lacks a declared busy gauge");
16059 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
16060 assert_eq!(
16061 missing[0].get("missing_gauges").map(String::as_str),
16062 Some(GAUGE)
16063 );
16064
16065 warn_drain_timed_out(
16066 MODULE_ID,
16067 Duration::from_secs(2),
16068 RouteCloseReason::Restart,
16069 &Default::default(),
16070 &trace,
16071 );
16072 let timeouts = capture.with_message("route drain timed out");
16073 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
16074 let gauges = &timeouts[0]["busy_gauges"];
16075 assert!(
16076 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
16077 "the timeout line must name the gauge and why it read busy: {gauges}"
16078 );
16079 assert!(
16080 gauges.ends_with("; no value ever read"),
16081 "no value was ever read for the gauge: {gauges}"
16082 );
16083 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
16084 }
16085
16086 #[tokio::test(start_paused = true)]
16090 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
16091 let busy = Drain::start(Answer::BusyUntil {
16092 until: Instant::now() + Duration::from_secs(3600),
16093 latency: Duration::ZERO,
16094 });
16095 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
16096 let rendered = trace.to_string();
16097 assert!(
16098 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
16099 && rendered.contains("; last value non_zero(1) @"),
16100 "{rendered}"
16101 );
16102
16103 let silent = Drain::start(Answer::Silent);
16104 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
16105 assert_eq!(
16108 trace.to_string(),
16109 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
16110 );
16111 assert_eq!(trace.probes, 1);
16112 }
16113
16114 #[test]
16116 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
16117 let gauges = [GAUGE.to_string()];
16118 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
16119 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
16120 assert_eq!(
16121 busy_gauge_observation(&verdicts),
16122 BusyGaugeObservation::Busy
16123 );
16124 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
16125 assert_eq!(
16126 busy_gauge_observation(&zero),
16127 BusyGaugeObservation::Quiescent
16128 );
16129 let none = gauge_verdicts(None, &gauges);
16130 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
16131 }
16132}
16133
16134#[cfg(test)]
16135mod health_evidence_tests {
16136 use super::{HealthProbeError, HealthProbeEvidence};
16137 use std::collections::HashSet;
16138
16139 #[test]
16147 fn only_a_dead_lane_is_proof_of_death() {
16148 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
16149 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
16153 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
16154 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
16155 }
16156
16157 #[test]
16163 fn every_evidence_class_has_a_distinct_label() {
16164 let labels = [
16165 HealthProbeError::lane_dead("").label(),
16166 HealthProbeError::no_answer("").label(),
16167 HealthProbeError::bad_answer("").label(),
16168 HealthProbeError::misconfigured("").label(),
16169 ];
16170 let unique: HashSet<_> = labels.iter().collect();
16171 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
16172 }
16173
16174 #[test]
16180 fn classification_preserves_the_original_message() {
16181 let err = HealthProbeError::no_answer("module did not answer within 5s");
16182 assert_eq!(err.to_string(), "module did not answer within 5s");
16183 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16184 }
16185}
16186
16187#[cfg(test)]
16188mod health_tombstone_tests {
16189 use std::{path::PathBuf, sync::Arc, time::Duration};
16190
16191 use subc_protocol::{
16192 manifest::Concurrency,
16193 session::{HealthStatus, ModuleControlResponse},
16194 };
16195 use tokio::sync::mpsc;
16196
16197 use super::{
16198 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
16199 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
16200 };
16201 use crate::{
16202 control::ControlHandler,
16203 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
16204 registry::{ConnectionId, Registry},
16205 router::FrameSink,
16206 };
16207
16208 struct ProbeHarness {
16209 spec: ModuleSpec,
16210 runtime: SupervisorRuntimeConfig,
16211 forwarding: Arc<ForwardingTable>,
16212 module_connection: ConnectionId,
16213 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16214 handler: ControlHandler,
16215 module: super::SupervisedModule,
16216 }
16217
16218 fn probe_harness() -> ProbeHarness {
16219 let registry = Arc::new(Registry::default());
16220 let forwarding = Arc::new(ForwardingTable::default());
16221 let supervisor_handle = super::SupervisorHandle::new();
16222 let health = HealthConfig {
16223 http: None,
16224 cadence: Duration::from_secs(30),
16225 deadline: Duration::from_secs(5),
16226 failure_threshold: 3,
16227 on_degraded: HealthAction::Report,
16228 on_failing: HealthAction::Report,
16229 critical: false,
16230 };
16231 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16232 .with_forwarding(Arc::clone(&forwarding))
16233 .with_handle(supervisor_handle.clone())
16234 .with_health_config(health);
16235 let spec = ModuleSpec {
16236 module_id: "late-health-module".to_string(),
16237 program: PathBuf::from("disabled-module"),
16238 args: Vec::new(),
16239 env: Vec::new(),
16240 reserved: false,
16241 reserved_prefixes: Vec::new(),
16242 protocol: ModuleProtocol::Subc,
16243 overlap: Default::default(),
16244 carry_containment: false,
16245 };
16246 let module = supervisor
16247 .supervise_configured(spec.clone(), false)
16248 .unwrap();
16249 let runtime = supervisor.runtime_config();
16250 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
16251 .with_supervisor(supervisor_handle);
16252 let module_connection = ConnectionId::new(700);
16253 let (module_tx, module_rx) = mpsc::channel(8);
16254 forwarding
16255 .register_module_connection(
16256 module_connection,
16257 spec.module_id.clone(),
16258 subc_protocol::PROTOCOL_VERSION,
16259 Concurrency::ModuleManaged,
16260 FrameSink::new(module_tx),
16261 )
16262 .unwrap();
16263
16264 ProbeHarness {
16265 spec,
16266 runtime,
16267 forwarding,
16268 module_connection,
16269 module_rx,
16270 handler,
16271 module,
16272 }
16273 }
16274
16275 async fn finish_after(
16276 harness: &mut ProbeHarness,
16277 stall: Duration,
16278 ) -> ModuleControlRpcCompletion {
16279 assert!(stall > harness.runtime.health.deadline);
16280 let deadline = harness.runtime.health.deadline;
16281 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16282 let answer = async {
16283 let frame = harness.module_rx.recv().await.expect("health.check frame");
16284 tokio::time::advance(deadline).await;
16285 tokio::task::yield_now().await;
16286 tokio::time::advance(stall - deadline).await;
16287 harness
16288 .forwarding
16289 .complete_module_control_rpc(
16290 harness.module_connection,
16291 frame.header.corr,
16292 Some("health.check"),
16293 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16294 status: HealthStatus::Ok,
16295 detail: None,
16296 metrics: None,
16297 }),
16298 )
16299 .unwrap()
16300 };
16301 let (probe_result, completion) = tokio::join!(probe, answer);
16302 let err = probe_result.expect_err("probe must miss its deadline");
16303 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16304 completion
16305 }
16306
16307 async fn time_out_without_answer(harness: &mut ProbeHarness) {
16308 let deadline = harness.runtime.health.deadline;
16309 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16310 let exhaust_deadline = async {
16311 let _frame = harness.module_rx.recv().await.expect("health.check frame");
16312 tokio::time::advance(deadline).await;
16313 tokio::task::yield_now().await;
16314 };
16315 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
16316 let err = probe_result.expect_err("probe must miss its deadline");
16317 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16318 }
16319
16320 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
16321 let registry = Arc::clone(&harness.module.inner.registry);
16322 let snapshot = Arc::clone(&harness.module.inner.snapshot);
16323 let process_liveness = super::SupervisorProcessLiveness::default();
16324 let mut child = None;
16325 let cycle = super::run_health_probe_cycle(
16326 &harness.spec,
16327 &harness.runtime,
16328 ®istry,
16329 &process_liveness,
16330 &snapshot,
16331 &mut child,
16332 );
16333 let peer = async {
16334 let frame = harness.module_rx.recv().await.expect("health.check frame");
16335 if answer {
16336 harness
16337 .forwarding
16338 .complete_module_control_rpc(
16339 harness.module_connection,
16340 frame.header.corr,
16341 Some("health.check"),
16342 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16343 status: HealthStatus::Ok,
16344 detail: None,
16345 metrics: Some(serde_json::json!({"ready": true})),
16346 }),
16347 )
16348 .unwrap();
16349 } else {
16350 tokio::time::advance(harness.runtime.health.deadline).await;
16351 tokio::task::yield_now().await;
16352 }
16353 };
16354 tokio::join!(cycle, peer);
16355 }
16356
16357 #[tokio::test(start_paused = true)]
16358 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
16359 let mut harness = probe_harness();
16360 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
16364 monitor.abort();
16365 let _ = monitor.await;
16366 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
16367 state.enabled = true;
16368 state.state = super::ModuleState::Running;
16369 state.process_alive = true;
16370 })
16371 .unwrap();
16372
16373 run_probe_cycle(&mut harness, true).await;
16374 assert_eq!(
16375 harness.module.status().unwrap().health.status,
16376 super::SupervisorHealthStatus::Ok
16377 );
16378
16379 for failures in 1..harness.runtime.health.failure_threshold {
16380 run_probe_cycle(&mut harness, false).await;
16381 let status = harness.module.status().unwrap();
16382 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16383 assert_eq!(status.health.consecutive_failures, failures);
16384 assert!(status.health.last_probe_ms.is_some());
16385 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
16386 assert!(status.health.metrics.is_none());
16387 assert_eq!(status.state, super::ModuleState::Running);
16388 assert!(status.process_alive);
16389 assert_eq!(status.restart_count, 0);
16390 assert_eq!(status.lifetime_restarts, 0);
16391 assert!(status.health.last_action.is_none());
16392 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16393 }
16394
16395 run_probe_cycle(&mut harness, true).await;
16396 let recovered = harness.module.status().unwrap();
16397 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
16398 assert_eq!(recovered.health.consecutive_failures, 0);
16399 assert!(recovered.health.detail.is_none());
16400 assert_eq!(
16401 recovered.health.metrics,
16402 Some(serde_json::json!({"ready": true}))
16403 );
16404 assert_eq!(recovered.lifetime_restarts, 0);
16405
16406 for failures in 1..=harness.runtime.health.failure_threshold {
16407 run_probe_cycle(&mut harness, false).await;
16408 let status = harness.module.status().unwrap();
16409 assert_eq!(status.health.consecutive_failures, failures);
16410 if failures < harness.runtime.health.failure_threshold {
16411 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16412 assert_eq!(status.state, super::ModuleState::Running);
16413 assert_eq!(status.lifetime_restarts, 0);
16414 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16415 } else {
16416 assert_eq!(
16417 status.health.status,
16418 super::SupervisorHealthStatus::Unresponsive
16419 );
16420 assert_eq!(status.state, super::ModuleState::Restarting);
16421 assert_eq!(status.restart_count, 1);
16422 assert_eq!(status.lifetime_restarts, 1);
16423 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
16424 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
16425 }
16426 }
16427 }
16428
16429 #[tokio::test(start_paused = true)]
16430 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
16431 let mut harness = probe_harness();
16432
16433 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
16434 let first_latency = match &first {
16435 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16436 other => panic!("late answer was not retained: {other:?}"),
16437 };
16438 assert!(harness.handler.observe_module_control_completion(first));
16439
16440 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
16441 let second_latency = match &second {
16442 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16443 other => panic!("late answer was not retained: {other:?}"),
16444 };
16445 assert!(harness.handler.observe_module_control_completion(second));
16446
16447 assert_eq!(first_latency, Duration::from_secs(8));
16448 assert_eq!(
16449 second_latency - first_latency,
16450 Duration::from_secs(3),
16451 "latency must grow linearly with the additional stall"
16452 );
16453 let health = harness.module.status().unwrap().health;
16454 assert_eq!(health.late_answer_count, 2);
16455 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
16456 }
16457
16458 #[tokio::test(start_paused = true)]
16466 async fn late_answer_clears_the_consecutive_failure_streak() {
16467 let mut harness = probe_harness();
16468
16469 time_out_without_answer(&mut harness).await;
16471 harness
16472 .module
16473 .record_health_probe_failure_for_test("[no-answer] test miss")
16474 .unwrap();
16475 assert_eq!(
16476 harness.module.status().unwrap().health.consecutive_failures,
16477 1,
16478 "precondition: the miss must be on the streak before the late answer"
16479 );
16480
16481 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
16483 assert!(matches!(
16484 late,
16485 ModuleControlRpcCompletion::LateHealthAnswer { .. }
16486 ));
16487 assert!(harness.handler.observe_module_control_completion(late));
16488
16489 let health = harness.module.status().unwrap().health;
16490 assert_eq!(
16491 health.consecutive_failures, 0,
16492 "a late answer is an answer: the streak must reset"
16493 );
16494 assert_eq!(health.late_answer_count, 1);
16495 }
16496
16497 #[tokio::test(start_paused = true)]
16498 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
16499 let mut harness = probe_harness();
16500
16501 for _ in 0..20 {
16502 time_out_without_answer(&mut harness).await;
16503 assert_eq!(
16504 harness.forwarding.health_probe_tombstone_count().unwrap(),
16505 1
16506 );
16507 }
16508 }
16509}
16510
16511#[cfg(test)]
16512mod child_env_tests {
16513 #[cfg(windows)]
16514 use super::{
16515 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
16516 };
16517 #[cfg(windows)]
16518 use std::sync::{Arc, OnceLock};
16519 #[cfg(windows)]
16520 #[test]
16521 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
16522 let spec = spec(Vec::new());
16523 let mut command = Command::new("unused");
16524 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
16525 .unwrap()
16526 .unwrap();
16527 let name = command
16528 .as_std()
16529 .get_envs()
16530 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
16531 .and_then(|(_, value)| value)
16532 .unwrap();
16533 assert_eq!(name, OsStr::new(handoff.name()));
16534 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
16535 assert!(command.as_std().get_envs().any(|(key, value)| key
16536 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16537 && value == Some(OsStr::new("env"))));
16538 assert!(command
16539 .as_std()
16540 .get_envs()
16541 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
16542 }
16543
16544 #[cfg(windows)]
16545 #[test]
16546 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
16547 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
16548 snapshot.process_alive = true;
16549 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
16550 assert_eq!(
16551 observed_launch_nonce_source(&snapshot).as_deref(),
16552 Some("env")
16553 );
16554 let consumed = Arc::new(OnceLock::new());
16555 snapshot.pipe_consumed = Some(consumed.clone());
16556 assert_eq!(
16557 observed_launch_nonce_source(&snapshot).as_deref(),
16558 Some("env")
16559 );
16560 consumed.set(()).unwrap();
16561 assert_eq!(
16562 observed_launch_nonce_source(&snapshot).as_deref(),
16563 Some("pipe")
16564 );
16565 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
16566 assert_eq!(
16567 observed_launch_nonce_source(&snapshot).as_deref(),
16568 Some("env")
16569 );
16570 clear_current_process_facts(&mut snapshot);
16571 assert_eq!(observed_launch_nonce_source(&snapshot), None);
16572 }
16573 use super::{
16574 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
16575 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
16576 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
16577 };
16578 use std::{ffi::OsStr, path::PathBuf};
16579 use tokio::process::Command;
16580
16581 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
16582 ModuleSpec {
16583 module_id: "env-plan".to_string(),
16584 program: PathBuf::from("/nonexistent"),
16585 args: Vec::new(),
16586 env,
16587 reserved: false,
16588 reserved_prefixes: Vec::new(),
16589 protocol: ModuleProtocol::Subc,
16590 overlap: Default::default(),
16591 carry_containment: false,
16592 }
16593 }
16594
16595 #[test]
16596 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
16597 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
16598 let digest = LaunchSpecDigest::from_spec(&base);
16599 let mut reordered = base.clone();
16600 reordered.env.reverse();
16601 reordered.module_id = "different-id".into();
16602 reordered.reserved = true;
16603 reordered.overlap = Default::default();
16604 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
16605 let mut delimiter_collision = base.clone();
16606 delimiter_collision.env[0] = ("A=B".into(), "C".into());
16607 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
16608 let mut changed = base.clone();
16609 changed.protocol = ModuleProtocol::None;
16610 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16611 changed = base.clone();
16612 changed.args.push("--flag".into());
16613 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16614 changed = base.clone();
16615 changed.program = PathBuf::from("/other");
16616 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16617 changed = base.clone();
16618 changed.env[1].1 = "other".into();
16619 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16620 assert!(!format!("{digest:?}").contains("B=C"));
16621 assert!(!format!("{digest:?}").contains("[u8"));
16622 }
16623
16624 #[cfg(unix)]
16625 #[test]
16626 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16627 use std::os::unix::ffi::OsStringExt;
16628 let mut left = spec(Vec::new());
16629 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16630 let mut right = left.clone();
16631 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16632 assert_eq!(
16633 left.program.to_string_lossy(),
16634 right.program.to_string_lossy()
16635 );
16636 assert_ne!(
16637 LaunchSpecDigest::from_spec(&left),
16638 LaunchSpecDigest::from_spec(&right)
16639 );
16640 }
16641
16642 #[test]
16656 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16657 let mut command = Command::new("/nonexistent");
16658 apply_child_env(&mut command, &spec(Vec::new()));
16659 let removed = command
16660 .as_std()
16661 .get_envs()
16662 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16663 assert!(
16664 removed,
16665 "ambient CK_LOG must be explicitly removed for an unconfigured module"
16666 );
16667
16668 let mut configured = Command::new("/nonexistent");
16669 apply_child_env(
16670 &mut configured,
16671 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16672 );
16673 let effective = configured
16674 .as_std()
16675 .get_envs()
16676 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16677 .last()
16678 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16679 assert_eq!(
16680 effective,
16681 Some(Some("debug".to_string())),
16682 "a module's configured CK_LOG must survive the ambient removal"
16683 );
16684 }
16685
16686 #[test]
16695 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16696 let connection_file = std::path::Path::new("/run/subc-connection.json");
16697 let handle = SupervisorHandle::new();
16698
16699 let mut none_spec = spec(Vec::new());
16700 none_spec.protocol = ModuleProtocol::None;
16701 let mut none = Command::new("/nonexistent");
16702 let none_handoff =
16703 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16704 .expect("protocol-none spawn args apply");
16705 assert!(
16706 none_handoff.is_none(),
16707 "protocol:none spawn must not receive a nonce descriptor"
16708 );
16709 #[cfg(windows)]
16710 assert!(!none.as_std().get_envs().any(|(key, value)| key
16711 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16712 && value.is_some()));
16713 assert!(
16714 !none.as_std().get_envs().any(|(key, value)| key
16715 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16716 && value.is_some()),
16717 "protocol:none spawn must not name a nonce descriptor"
16718 );
16719 let none_args: Vec<String> = none
16720 .as_std()
16721 .get_args()
16722 .map(|a| a.to_string_lossy().into_owned())
16723 .collect();
16724 assert!(
16725 !none_args.iter().any(|a| a == SUBC_ARG),
16726 "protocol:none argv must not carry --subc; got {none_args:?}"
16727 );
16728 let none_has_nonce = none
16729 .as_std()
16730 .get_envs()
16731 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16732 assert!(
16733 !none_has_nonce,
16734 "protocol:none spawn must not receive a launch nonce"
16735 );
16736 let none_has_module_id = none
16737 .as_std()
16738 .get_envs()
16739 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16740 assert!(
16741 none_has_module_id,
16742 "SUBC_MODULE_ID is inert and stays on every path"
16743 );
16744 assert!(
16745 handle.spawn_nonce(&none_spec.module_id).is_none(),
16746 "no nonce record for a process that will never present one"
16747 );
16748
16749 let wire_spec = spec(Vec::new());
16751 let mut wire = Command::new("/nonexistent");
16752 let wire_handoff =
16753 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16754 .expect("subc-wire spawn args apply");
16755 let wire_fd_env = wire
16756 .as_std()
16757 .get_envs()
16758 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16759 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16760 #[cfg(unix)]
16761 assert_eq!(
16762 wire_fd_env,
16763 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16764 "a subc-wire spawn names the pipe it will receive at descriptor 3"
16765 );
16766 #[cfg(windows)]
16767 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16768 #[cfg(not(any(unix, windows)))]
16769 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16770 let wire_args: Vec<String> = wire
16771 .as_std()
16772 .get_args()
16773 .map(|a| a.to_string_lossy().into_owned())
16774 .collect();
16775 assert_eq!(
16776 wire_args,
16777 vec![
16778 SUBC_ARG.to_string(),
16779 connection_file.to_string_lossy().into_owned()
16780 ],
16781 "a subc-wire spawn still carries --subc <path>"
16782 );
16783 assert_eq!(
16784 wire.as_std()
16785 .get_envs()
16786 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16787 !cfg!(unix),
16788 "only Windows supplies the environment nonce"
16789 );
16790 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16791 }
16792
16793 #[test]
16803 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16804 let role = |command: &Command| {
16805 command
16806 .as_std()
16807 .get_envs()
16808 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16809 .last()
16810 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16811 };
16812 let forged = spec(vec![(
16813 SUBC_SPAWN_ROLE_ENV.to_string(),
16814 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16815 )]);
16816
16817 let mut plain = Command::new("/nonexistent");
16818 apply_child_env(&mut plain, &forged);
16819 apply_spawn_role(&mut plain, SpawnRole::Plain);
16820 assert_eq!(
16821 role(&plain),
16822 Some(None),
16823 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16824 );
16825
16826 let mut candidate = Command::new("/nonexistent");
16827 apply_child_env(&mut candidate, &spec(Vec::new()));
16828 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16829 assert_eq!(
16830 role(&candidate),
16831 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16832 );
16833 }
16834
16835 #[test]
16841 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16842 let mut command = Command::new("/nonexistent");
16843 apply_child_env(
16844 &mut command,
16845 &spec(vec![
16846 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16847 ("KEPT".to_string(), "yes".to_string()),
16848 ]),
16849 );
16850 let keys: Vec<String> = command
16851 .as_std()
16852 .get_envs()
16853 .filter(|(_, value)| value.is_some())
16854 .map(|(key, _)| key.to_string_lossy().into_owned())
16855 .collect();
16856 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16857 assert!(
16858 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16859 "daemon-private capture key leaked to the child: {keys:?}"
16860 );
16861 }
16862}
16863
16864#[cfg(test)]
16865mod jitter_tests {
16866 use super::jittered_health_delay;
16867 use std::{collections::HashSet, time::Duration};
16868
16869 const FLEET: [&str; 14] = [
16878 "aft",
16879 "alfonso-core",
16880 "magic-context",
16881 "broca",
16882 "thalamus",
16883 "quota",
16884 "engram",
16885 "plexus",
16886 "cerebellum",
16887 "astrocyte",
16888 "synapse",
16889 "subc-mcp",
16890 "cortexkit-credentials",
16891 "subc-federation",
16892 ];
16893
16894 #[test]
16902 fn probe_delays_disperse_across_the_fleet() {
16903 let cadence = Duration::from_secs(30);
16904 let delays: HashSet<Duration> = FLEET
16905 .iter()
16906 .map(|id| jittered_health_delay(id, 0, cadence))
16907 .collect();
16908 assert_eq!(
16909 delays.len(),
16910 FLEET.len(),
16911 "every supervised module must land on its own probe offset"
16912 );
16913 }
16914
16915 #[test]
16921 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16922 let cadence = Duration::from_secs(30);
16923 let span = cadence / 10;
16924 for id in FLEET {
16925 for probe_index in 0..8 {
16926 let delay = jittered_health_delay(id, probe_index, cadence);
16927 assert!(
16928 delay >= cadence,
16929 "{id}#{probe_index}: jitter must not shorten the cadence"
16930 );
16931 assert!(
16932 delay < cadence + span,
16933 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16934 );
16935 }
16936 }
16937 }
16938
16939 #[test]
16945 fn a_module_offset_is_stable_across_restarts() {
16946 let cadence = Duration::from_secs(30);
16947 for id in FLEET {
16948 assert_eq!(
16949 jittered_health_delay(id, 0, cadence),
16950 jittered_health_delay(id, 0, cadence),
16951 "{id}: the same module and probe index must produce the same offset"
16952 );
16953 }
16954 }
16955
16956 #[test]
16958 fn zero_cadence_yields_zero_delay() {
16959 assert_eq!(
16960 jittered_health_delay("aft", 0, Duration::ZERO),
16961 Duration::ZERO
16962 );
16963 }
16964}
16965
16966#[cfg(all(test, target_os = "linux"))]
16967mod cgroup_placement_tests {
16968 use super::{
16969 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16970 SuperviseError, SupervisedChild,
16971 };
16972 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16973 use cortexkit_test_support::ScratchDir;
16974 use std::{
16975 fs, io,
16976 path::{Path, PathBuf},
16977 sync::{Arc, Mutex},
16978 };
16979 use tokio::process::Command;
16980
16981 #[tokio::test]
16982 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16983 use super::*;
16984 let dir = ScratchDir::new("unique-spawn-cgroups");
16985 let root = PathBuf::from(format!(
16986 "/sys/fs/cgroup/subc-unique-test-{}-{}",
16987 std::process::id(),
16988 unix_ms_now()
16989 ));
16990 if let Err(error) = fs::create_dir(&root) {
16991 assert!(
16992 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16993 "required cgroup test cannot execute: {error}"
16994 );
16995 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16996 return;
16997 }
16998 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16999 let group_count = || {
17000 fs::read_dir(root.join("subc-modules"))
17001 .unwrap()
17002 .map(|entry| entry.unwrap().file_type().unwrap())
17003 .filter(|kind| kind.is_dir())
17004 .count()
17005 };
17006 let supervisor =
17007 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17008 .with_cgroup_placement(Some(placement.clone()));
17009 let runtime = supervisor.runtime_config();
17010 let mut spec = ModuleSpec {
17011 module_id: "unique-spawn".into(),
17012 program: PathBuf::from("/bin/sleep"),
17013 args: vec!["60".into()],
17014 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17015 .into_iter()
17016 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
17017 .collect(),
17018 reserved: false,
17019 reserved_prefixes: vec![],
17020 protocol: ModuleProtocol::None,
17021 overlap: Default::default(),
17022 carry_containment: false,
17023 };
17024 let spawn = |spec: &ModuleSpec| {
17025 spawn_child(
17026 spec,
17027 None,
17028 None,
17029 &runtime.stderr_ring,
17030 None,
17031 &runtime.child_roster,
17032 Some(&placement),
17033 )
17034 .unwrap()
17035 };
17036 let mut live = spawn(&spec);
17037 for _ in 0..3 {
17038 let next = spawn(&spec);
17040 assert_ne!(live.module_id, next.module_id);
17041 live.start_kill().unwrap();
17042 live.wait().await.unwrap();
17043 live = next;
17044 assert!(
17045 live.child.try_wait().unwrap().is_none(),
17046 "retiring the old slot must not kill the replacement"
17047 );
17048 assert_eq!(
17049 group_count(),
17050 1,
17051 "only the live spawn's cgroup should remain"
17052 );
17053 }
17054 supervisor.begin_daemon_shutdown();
17055 let reap = tokio::spawn(async move {
17056 live.wait().await.unwrap();
17057 });
17058 supervisor
17059 .end_children_for_daemon_shutdown(false, std::future::pending())
17060 .await;
17061 reap.await.unwrap();
17062 assert_eq!(group_count(), 0);
17063 spec.program = PathBuf::from("/bin/true");
17065 spec.args.clear();
17066 let fresh_roster = ChildRoster::default();
17067 let mut short = spawn_child(
17068 &spec,
17069 None,
17070 None,
17071 &runtime.stderr_ring,
17072 None,
17073 &fresh_roster,
17074 Some(&placement),
17075 )
17076 .unwrap();
17077 short.wait().await.unwrap();
17078 assert_eq!(group_count(), 0);
17079 spec.module_id = "_".repeat(255);
17080 let mut long_id = spawn_child(
17081 &spec,
17082 None,
17083 None,
17084 &runtime.stderr_ring,
17085 None,
17086 &fresh_roster,
17087 Some(&placement),
17088 )
17089 .unwrap();
17090 long_id.wait().await.unwrap();
17091 assert_eq!(
17092 group_count(),
17093 0,
17094 "valid long module IDs must not exceed cgroup NAME_MAX"
17095 );
17096 fs::remove_dir(root.join("subc-modules")).unwrap();
17097 fs::remove_dir(root).unwrap();
17098 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
17099 }
17100
17101 #[test]
17102 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
17103 let path = Path::new("/definitely-missing-subc-cgroup");
17104 let mut command = Command::new("true");
17105 let error = apply_cgroup_placement(
17106 &mut command,
17107 &ModuleSpec {
17108 module_id: "broken-cgroup".to_string(),
17109 program: PathBuf::from("true"),
17110 args: Vec::new(),
17111 env: Vec::new(),
17112 reserved: false,
17113 reserved_prefixes: Vec::new(),
17114 protocol: ModuleProtocol::Subc,
17115 overlap: Default::default(),
17116 carry_containment: false,
17117 },
17118 path,
17119 )
17120 .expect_err("a parent cgroup open failure must reject the supervised spawn");
17121 let reason = error.to_string();
17122
17123 assert!(
17124 matches!(error, SuperviseError::Cgroup { .. }),
17125 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
17126 );
17127 assert!(
17128 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
17129 "parent cgroup open failure must name cgroup.procs: {reason}"
17130 );
17131 }
17132
17133 #[tokio::test]
17134 async fn reaping_a_child_removes_its_empty_module_cgroup() {
17135 let root = ScratchDir::new("supervisor-reap-cgroup");
17136 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17137 let placement = subc_cgroup::prepare_at(&root)
17138 .expect("prepare scratch cgroup root")
17139 .expect("scratch root has a cgroup.procs marker");
17140 let module_id = "reaped-module";
17141 let module = placement
17142 .module_path(module_id)
17143 .expect("create scratch module cgroup");
17144 let child = Command::new("true")
17145 .env("XDG_DATA_HOME", root.path())
17146 .env("XDG_RUNTIME_DIR", root.path())
17147 .env("XDG_CONFIG_HOME", root.path())
17148 .spawn()
17149 .expect("spawn short-lived child");
17150 let pid = child.id().expect("spawned child has pid");
17151 let mut child = SupervisedChild {
17152 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
17153 child: crate::supervise::containment::ProcessChild::plain(child, pid),
17154 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
17155 protocol: ModuleProtocol::Subc,
17156 module_id: module_id.to_string(),
17157 cgroup_placement: Some(placement),
17158 stdout_pump: None,
17159 stderr_pump: None,
17160 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
17161 spawned_at: tokio::time::Instant::now(),
17162 registration_deadline: None,
17163 registration_watch: None,
17164 spawned_at_ms: 0,
17165 spawned_from: PathBuf::from("true"),
17166 launch_spec_digest: LaunchSpecDigest([0; 32]),
17167 spawned_file_identity: None,
17168 process_start_time: None,
17169 process_identity: None,
17170 pid,
17171 roster_guard: None,
17172 #[cfg(target_os = "macos")]
17173 privacy_exec: None,
17174 spawn_failure: None,
17175 };
17176
17177 child.wait().await.expect("reap short-lived child");
17178
17179 assert!(
17180 !module.exists(),
17181 "waiting for a non-carried child must remove its empty cgroup before returning"
17182 );
17183 child.finish_exit(false).await.unwrap();
17184 }
17185
17186 #[test]
17187 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
17188 let root = ScratchDir::new("supervisor-non-empty-cgroup");
17189 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17190 let placement = subc_cgroup::prepare_at(&root)
17191 .expect("prepare scratch cgroup root")
17192 .expect("scratch root has a cgroup.procs marker");
17193 let module = placement
17194 .module_path("surviving-module")
17195 .expect("create scratch module cgroup");
17196 fs::write(module.join("surviving-process"), b"still present")
17197 .expect("make scratch cgroup non-empty");
17198 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
17199
17200 remove_module_cgroup(&placement, "surviving-module");
17201
17202 let logs = crate::router::test_log::captured_logs(&logs);
17203 assert!(
17204 module.exists(),
17205 "failed removal must leave the cgroup intact"
17206 );
17207 assert!(
17208 logs.contains("could not remove module cgroup after process exit; continuing teardown")
17209 && logs.contains("surviving-module"),
17210 "best-effort removal must report the failure without returning it: {logs}"
17211 );
17212 }
17213
17214 #[test]
17215 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
17216 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
17217 let reason = SuperviseError::Spawn {
17218 program: PathBuf::from("/bin/true"),
17219 source: io::Error::from_raw_os_error(13),
17220 cgroup_path: Some(cgroup_path.clone()),
17221 }
17222 .to_string();
17223
17224 assert!(
17225 reason.contains(&cgroup_path.display().to_string()),
17226 "a pre_exec spawn failure must name the cgroup path: {reason}"
17227 );
17228 }
17229}
17230
17231#[cfg(test)]
17232mod spawn_subscriber_lag_tests {
17233 use super::*;
17234
17235 #[tokio::test]
17240 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
17241 let feed = SpawnEventFeed::default();
17242 feed.configure_incarnation("lag-incarnation".to_string());
17243 let (tx, mut rx) = mpsc::channel(1);
17246 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
17247 .expect("subscribe");
17248 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
17249 for index in 0..emitted {
17250 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
17251 tokio::task::yield_now().await;
17254 }
17255 assert_eq!(
17256 feed.subscriber_count(),
17257 0,
17258 "the lagged subscriber must be removed"
17259 );
17260
17261 let mut data = Vec::new();
17262 let mut last = None;
17263 loop {
17264 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
17265 .await
17266 .expect("the forwarder must finish once the subscriber is dropped");
17267 let Some(outbound) = next else { break };
17268 let frame = outbound.frame;
17269 if frame.header.ty == FrameType::StreamData {
17270 assert!(last.is_none(), "no data may follow the terminal frame");
17271 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
17272 data.push(event.cursor.seq);
17273 } else {
17274 assert!(last.is_none(), "exactly one terminal frame");
17275 last = Some(frame);
17276 }
17277 }
17278 assert!(!data.is_empty(), "queued frames drain before the terminal");
17279 for pair in data.windows(2) {
17280 assert_eq!(
17281 pair[1],
17282 pair[0] + 1,
17283 "queued frames arrive dense and in order"
17284 );
17285 }
17286 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
17287 assert_eq!(terminal.header.ty, FrameType::Error);
17288 assert_eq!(terminal.header.corr, 7);
17289 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
17290 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
17291 let detail = body.detail.expect("lagged error carries detail");
17292 assert_eq!(
17293 detail["first_undelivered_cursor"]["seq"],
17294 data.last().unwrap() + 1,
17295 "the named cursor is the first event the subscriber did not receive"
17296 );
17297 assert_eq!(
17298 detail["first_undelivered_cursor"]["daemon_incarnation"],
17299 "lag-incarnation"
17300 );
17301 }
17302}
17303
17304#[cfg(test)]
17305mod terminal_history_read_concurrency_tests {
17306 use super::*;
17307 use crate::terminal_journal::read_pause;
17308 use cortexkit_test_support::ScratchDir;
17309 use std::sync::mpsc as std_mpsc;
17310
17311 fn journaled_ring(
17312 journal: &Arc<crate::terminal_journal::TerminalJournal>,
17313 ) -> Arc<Mutex<TerminalRing>> {
17314 Arc::new(Mutex::new(
17315 TerminalRing::new(TerminalRingConfig::default(), 1)
17316 .with_journal(Some(Arc::clone(journal))),
17317 ))
17318 }
17319
17320 fn crash(at_ms: u64) -> ExitReport {
17321 ExitReport {
17322 kind: ExitKind::Crash,
17323 code: Some(1),
17324 signal: None,
17325 at_ms,
17326 }
17327 }
17328
17329 fn record_within(
17332 module_id: &'static str,
17333 ring: &Arc<Mutex<TerminalRing>>,
17334 at_ms: u64,
17335 bound: Duration,
17336 ) -> bool {
17337 let ring = Arc::clone(ring);
17338 let (done, done_rx) = std_mpsc::channel();
17339 std::thread::spawn(move || {
17340 record_terminal(
17341 module_id,
17342 &ring,
17343 &SpawnEventFeed::default(),
17344 &crash(at_ms),
17345 TerminalDisposition::Restarting,
17346 );
17347 let _ = done.send(());
17348 });
17349 done_rx.recv_timeout(bound).is_ok()
17350 }
17351
17352 #[test]
17357 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
17358 let dir = ScratchDir::new("terminal-history-concurrent-read");
17359 let path = dir.join("terminals.jsonl");
17360 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
17361 path.clone(),
17362 "daemon".into(),
17363 ));
17364 let reader_ring = journaled_ring(&journal);
17365 let other_ring = journaled_ring(&journal);
17366 assert!(record_within(
17367 "reader-module",
17368 &reader_ring,
17369 10,
17370 Duration::from_secs(5)
17371 ));
17372
17373 let (started, release) = read_pause::install(&path);
17374 let reading = {
17375 let ring = Arc::clone(&reader_ring);
17376 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
17377 };
17378 started
17379 .recv_timeout(Duration::from_secs(5))
17380 .expect("the history read reached its pause");
17381
17382 let bound = Duration::from_secs(1);
17383 assert!(
17384 record_within("other-module", &other_ring, 20, bound),
17385 "another module's exit waited on a history read (journal writer held)"
17386 );
17387 assert!(
17388 record_within("reader-module", &reader_ring, 30, bound),
17389 "the read module's own exit waited on its history read (ring held)"
17390 );
17391
17392 drop(release);
17393 let paused = reading.join().unwrap();
17394 assert_eq!(
17395 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17396 vec![10],
17397 "an exit recorded after the read began lands in neither half of it"
17398 );
17399 assert_eq!(paused.journal_skipped_lines, 0);
17400 assert_eq!(paused.journal_read_errors, 0);
17401
17402 let after = durable_terminal_history_of(&reader_ring, "reader-module");
17403 assert_eq!(
17404 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17405 vec![10, 30],
17406 "the next read merges ring and journal with no duplicate"
17407 );
17408 assert_eq!(after.journal_skipped_lines, 0);
17409 }
17410}
17411
17412#[cfg(test)]
17417mod stderr_settle_tests {
17418 use std::{
17419 future::Future,
17420 io,
17421 pin::Pin,
17422 sync::{Arc, Mutex},
17423 task::{Context, Poll},
17424 time::Duration,
17425 };
17426
17427 use tokio::{
17428 io::{AsyncRead, ReadBuf},
17429 sync::oneshot,
17430 time::Instant,
17431 };
17432
17433 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
17434 use crate::stderr_tail::{
17435 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
17436 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
17437 };
17438
17439 const BOUND: Duration = Duration::from_millis(250);
17440
17441 struct HeldReader {
17445 before: Option<Vec<u8>>,
17446 gate: Option<oneshot::Receiver<()>>,
17447 after: io::Cursor<Vec<u8>>,
17448 }
17449
17450 impl AsyncRead for HeldReader {
17451 fn poll_read(
17452 mut self: Pin<&mut Self>,
17453 cx: &mut Context<'_>,
17454 buf: &mut ReadBuf<'_>,
17455 ) -> Poll<io::Result<()>> {
17456 if let Some(bytes) = self.before.take() {
17457 buf.put_slice(&bytes);
17458 return Poll::Ready(Ok(()));
17459 }
17460 if let Some(gate) = self.gate.as_mut() {
17461 match Pin::new(gate).poll(cx) {
17462 Poll::Pending => return Poll::Pending,
17463 Poll::Ready(_) => self.gate = None,
17464 }
17465 }
17466 Pin::new(&mut self.after).poll_read(cx, buf)
17467 }
17468 }
17469
17470 struct DiscardSink;
17471
17472 impl OutputSink for DiscardSink {
17473 fn write_line(&mut self, _line: &[u8]) {}
17474 }
17475
17476 fn line(text: &str) -> TailEntry {
17477 TailEntry::Line {
17478 text: text.to_string(),
17479 truncated: false,
17480 at_ms: None,
17481 }
17482 }
17483
17484 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
17485 ring.lock().unwrap()
17486 }
17487
17488 fn held_pump(
17492 ring: &Arc<Mutex<StderrRing>>,
17493 before: &str,
17494 after: &str,
17495 ) -> (StderrPump, oneshot::Sender<()>) {
17496 let generation = lock(ring).begin_process();
17497 let (release, gate) = oneshot::channel();
17498 let reader = HeldReader {
17499 before: Some(before.as_bytes().to_vec()),
17500 gate: Some(gate),
17501 after: io::Cursor::new(after.as_bytes().to_vec()),
17502 };
17503 let task = tokio::spawn(pump_stderr_to(
17504 reader,
17505 Arc::clone(ring),
17506 generation,
17507 DiscardSink,
17508 ));
17509 (StderrPump { task, generation }, release)
17510 }
17511
17512 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
17513 for _ in 0..1000 {
17514 if done(&lock(ring)) {
17515 return;
17516 }
17517 tokio::time::sleep(Duration::from_millis(1)).await;
17518 }
17519 panic!(
17520 "ring never reached the expected state: {:?}",
17521 lock(ring).snapshot(None, None)
17522 );
17523 }
17524
17525 #[tokio::test(start_paused = true)]
17526 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
17527 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17528 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
17529
17530 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
17531 let before_release = lock(&ring).snapshot(None, None);
17532 assert!(
17533 matches!(before_release.capture, CaptureState::Incomplete { .. }),
17534 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
17535 );
17536
17537 let next = lock(&ring).begin_process();
17540 lock(&ring).push_line_from(next, "next process booting");
17541 release.send(()).unwrap();
17542 wait_until(&ring, |ring| {
17543 ring.snapshot(None, None).capture == CaptureState::Captured
17544 })
17545 .await;
17546
17547 assert_eq!(
17548 untimed(lock(&ring).snapshot(None, None).entries),
17549 vec![
17550 line("booting"),
17551 line("config error: missing storage"),
17552 TailEntry::ProcessStart,
17553 line("next process booting"),
17554 ],
17555 "the crash's last line must survive a slow reader and stay in the crashed process's section"
17556 );
17557 }
17558
17559 #[tokio::test(start_paused = true)]
17560 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
17561 ) {
17562 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17563 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
17566
17567 let started = Instant::now();
17568 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
17569 assert_eq!(
17570 started.elapsed(),
17571 BOUND,
17572 "the restart must wait exactly the bound for a pipe that stays open, no longer"
17573 );
17574
17575 let next = lock(&ring).begin_process();
17576 lock(&ring).push_line_from(next, "next process booting");
17577 tokio::time::sleep(Duration::from_secs(60)).await;
17578
17579 let snapshot = lock(&ring).snapshot(None, None);
17580 match &snapshot.capture {
17581 CaptureState::Incomplete { reason } => assert!(
17582 reason.contains("had not reached EOF") && reason.contains("250ms"),
17583 "the reason must say what is missing and after how long: {reason}"
17584 ),
17585 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
17586 }
17587 assert_eq!(
17588 untimed(snapshot.entries),
17589 vec![
17590 line("parent exiting"),
17591 TailEntry::ProcessStart,
17592 line("next process booting"),
17593 ]
17594 );
17595 }
17596
17597 #[tokio::test(start_paused = true)]
17598 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
17599 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17600 let (pump, release) = held_pump(&ring, "one\n", "two\n");
17601 release.send(()).unwrap();
17602
17603 settle_stderr_pump("clean", &ring, pump, BOUND).await;
17604
17605 let snapshot = lock(&ring).snapshot(None, None);
17606 assert_eq!(snapshot.capture, CaptureState::Captured);
17607 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
17608 }
17609
17610 #[tokio::test(start_paused = true)]
17611 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
17612 const BURST: usize = 150;
17613 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
17614 let path = capture.join("two-pipe.stderr.log");
17615 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
17616 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17617 let generation = lock(&ring).begin_process();
17618 let stdout = (0..BURST)
17619 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
17620 .collect::<String>();
17621 let stderr = (0..BURST)
17622 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
17623 .collect::<String>();
17624 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17629 let (release, gate) = oneshot::channel();
17630 let reader = HeldReader {
17631 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17632 gate: Some(gate),
17633 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17634 };
17635 let (finished, completion) = oneshot::channel();
17636 let stdout_sink = sink.clone();
17637 let pump = tokio::spawn(async move {
17638 pump_stdout_to(reader, stdout_sink).await;
17639 finished.send(()).unwrap();
17640 });
17641 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17642
17643 let started = Instant::now();
17644 settle_stdout_pump("two-pipe", pump, BOUND).await;
17645 assert_eq!(
17646 started.elapsed(),
17647 BOUND,
17648 "a late reader must not hold up restart"
17649 );
17650 assert_eq!(
17651 std::fs::read_to_string(&path).unwrap().lines().count(),
17652 BURST + 1
17653 );
17654
17655 release
17656 .send(())
17657 .expect("stdout reader must survive the exit drain bound");
17658 tokio::time::timeout(Duration::from_secs(1), completion)
17659 .await
17660 .expect("released stdout reader must reach EOF")
17661 .expect("stdout capture must finish at EOF, not be cancelled");
17662 let contents = std::fs::read_to_string(&path).unwrap();
17663 let mut lines = contents
17664 .lines()
17665 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17666 .collect::<Vec<_>>();
17667 let mut expected = (0..BURST)
17668 .flat_map(|i| {
17669 [
17670 format!("out-{i:04}-{}", "o".repeat(64)),
17671 format!("err-{i:04}-{}", "e".repeat(64)),
17672 ]
17673 })
17674 .collect::<Vec<_>>();
17675 lines.sort_unstable();
17676 expected.sort_unstable();
17677 assert_eq!(
17678 lines, expected,
17679 "both pipes must retain every complete line exactly once"
17680 );
17681 }
17682}
17683
17684#[cfg(all(test, windows))]
17698mod job_containment_tests {
17699 use super::*;
17700 use cortexkit_test_support::ScratchDir;
17701 use std::{
17702 path::{Path, PathBuf},
17703 sync::{Arc, Mutex},
17704 time::{Duration, Instant},
17705 };
17706
17707 fn stub_path() -> PathBuf {
17713 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
17714 assert!(
17715 path.exists(),
17716 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17717 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17718 path.display()
17719 );
17720 path
17721 }
17722
17723 fn read_grandchild_pid(path: &Path) -> u32 {
17725 let deadline = Instant::now() + Duration::from_secs(10);
17726 loop {
17727 if let Ok(contents) = std::fs::read_to_string(path) {
17728 if let Ok(pid) = contents.trim().parse() {
17729 return pid;
17730 }
17731 }
17732 assert!(
17733 Instant::now() < deadline,
17734 "the stub never recorded a grandchild pid at {}",
17735 path.display()
17736 );
17737 std::thread::sleep(Duration::from_millis(10));
17738 }
17739 }
17740
17741 struct Fixture {
17744 _dir: ScratchDir,
17745 module_id: String,
17746 grandchild: u32,
17747 child: Option<SupervisedChild>,
17748 registry: Arc<Registry>,
17749 snapshot: Arc<Mutex<SupervisorSnapshot>>,
17750 terminal_ring: Arc<Mutex<TerminalRing>>,
17751 spawn_events: SpawnEventFeed,
17752 }
17753
17754 fn fixture(label: &str, module_id: &str) -> Fixture {
17755 let dir = ScratchDir::new(label);
17756 let pid_file = dir.join("grandchild.pid");
17757 let supervisor = Supervisor::new_for_test(
17758 Arc::new(Registry::default()),
17759 RestartPolicy::new(3, Duration::ZERO),
17760 );
17761 let runtime = supervisor.runtime_config();
17762 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17763 let spec = ModuleSpec {
17764 module_id: module_id.to_string(),
17765 program: stub_path(),
17766 args: Vec::new(),
17770 env: vec![
17771 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17772 (
17773 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17774 pid_file.display().to_string(),
17775 ),
17776 ],
17777 reserved: false,
17778 reserved_prefixes: Vec::new(),
17779 protocol: ModuleProtocol::Subc,
17780 overlap: Default::default(),
17781 carry_containment: false,
17782 };
17783 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17784 .expect("spawn the supervised fixture");
17785 let grandchild = read_grandchild_pid(&pid_file);
17786 Fixture {
17787 _dir: dir,
17788 module_id: module_id.to_string(),
17789 grandchild,
17790 child: Some(child),
17791 registry: Arc::new(Registry::default()),
17792 snapshot,
17793 terminal_ring: Arc::clone(&runtime.terminal_ring),
17794 spawn_events: SpawnEventFeed::default(),
17795 }
17796 }
17797
17798 impl Fixture {
17799 async fn drain(&mut self) {
17801 let child = self
17802 .child
17803 .take()
17804 .expect("the fixture child is still present");
17805 drain_child_to_state(
17806 &self.module_id,
17807 ModuleProtocol::Subc,
17808 StopNotice::NotSent,
17811 &self.registry,
17812 None,
17813 &self.snapshot,
17814 &self.terminal_ring,
17815 &self.spawn_events,
17816 child,
17817 Duration::from_millis(500),
17818 ModuleState::Stopped,
17819 Some(false),
17820 &FailureContext::operator(),
17821 )
17822 .await
17823 .expect("drain the supervised fixture");
17824 }
17825 }
17826
17827 #[tokio::test]
17833 async fn teardown_reaps_the_grandchild() {
17834 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17835 let grandchild = fixture.grandchild;
17836
17837 assert!(
17838 subc_jobobject::process_exists(grandchild),
17839 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17840 );
17841
17842 fixture.drain().await;
17843
17844 assert!(
17845 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17846 "grandchild {grandchild} outlived module teardown: the tree was not contained"
17847 );
17848 }
17849
17850 #[test]
17863 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17864 let dir = ScratchDir::new("teardown-uncontained");
17865 let pid_file = dir.join("grandchild.pid");
17866 let mut child = std::process::Command::new(stub_path())
17867 .env("FAKE_AFT_NEVER_CONNECT", "1")
17868 .env(
17869 "FAKE_AFT_GRANDCHILD_PID_FILE",
17870 pid_file.display().to_string(),
17871 )
17872 .stdin(std::process::Stdio::null())
17873 .stdout(std::process::Stdio::null())
17874 .stderr(std::process::Stdio::null())
17875 .spawn()
17876 .expect("spawn the uncontained fixture");
17877 let grandchild = read_grandchild_pid(&pid_file);
17878
17879 child.kill().expect("kill the direct child");
17881 let _ = child.wait();
17882
17883 assert!(
17884 subc_jobobject::process_exists(grandchild),
17885 "grandchild {grandchild} died with the direct child, so this control no longer \
17886 distinguishes contained from uncontained teardown and the regression test is \
17887 passing vacuously"
17888 );
17889
17890 kill_tree(grandchild);
17893 }
17894
17895 #[tokio::test]
17904 async fn dropping_containment_reaps_the_grandchild() {
17905 let mut fixture = fixture("drop-containment", "tree-drop");
17906 let grandchild = fixture.grandchild;
17907
17908 assert!(subc_jobobject::process_exists(grandchild));
17909
17910 fixture.child.as_mut().expect("child present").job = None;
17912
17913 assert!(
17914 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17915 "grandchild {grandchild} survived the containment handle closing, so a daemon \
17916 crash would leave the tree behind"
17917 );
17918 }
17919
17920 fn kill_tree(pid: u32) {
17922 let _ = std::process::Command::new("taskkill.exe")
17923 .args(["/PID", &pid.to_string(), "/T", "/F"])
17924 .stdin(std::process::Stdio::null())
17925 .stdout(std::process::Stdio::null())
17926 .stderr(std::process::Stdio::null())
17927 .status();
17928 assert!(
17929 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17930 "could not clean up grandchild {pid}"
17931 );
17932 }
17933}
17934
17935#[cfg(test)]
17936mod privacy_trampoline_configuration_tests {
17937 #[tokio::test]
17938 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17939 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17940 #[cfg(target_os = "macos")]
17941 {
17942 let supervisor = super::Supervisor::new(
17943 std::sync::Arc::new(crate::Registry::default()),
17944 super::RestartPolicy::default(),
17945 );
17946 let error = supervisor.spawn(spec()).unwrap_err();
17947 assert!(
17948 error
17949 .to_string()
17950 .contains("no privacy trampoline configured"),
17951 "{error}"
17952 );
17953 }
17954 }
17955
17956 #[tokio::test]
17957 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17958 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17959 #[cfg(target_os = "macos")]
17960 {
17961 let supervisor = super::Supervisor::new(
17962 std::sync::Arc::new(crate::Registry::default()),
17963 super::RestartPolicy::default(),
17964 )
17965 .with_privacy_trampoline(std::env::current_exe().unwrap());
17966 let error = supervisor.spawn(spec()).unwrap_err();
17967 assert!(
17968 error
17969 .to_string()
17970 .contains("binary does not implement the privacy trampoline protocol"),
17971 "{error}"
17972 );
17973 }
17974 }
17975
17976 #[cfg(target_os = "macos")]
17977 fn spec() -> super::ModuleSpec {
17978 super::ModuleSpec {
17979 module_id: "privacy-configuration".into(),
17980 program: "/bin/sleep".into(),
17981 args: vec!["30".into()],
17982 env: vec![],
17983 reserved: false,
17984 reserved_prefixes: vec![],
17985 protocol: subc_control::ModuleProtocol::None,
17986 overlap: super::ModuleOverlap::Exclusive,
17987 carry_containment: false,
17988 }
17989 }
17990}
17991
17992#[cfg(test)]
17993mod privacy_exec_boundary_tests {
17994 #[cfg(target_os = "macos")]
17995 use super::*;
17996 #[cfg(target_os = "macos")]
17997 use std::{
17998 io::{Read, Write},
17999 net::{TcpListener, TcpStream},
18000 };
18001
18002 #[cfg(target_os = "macos")]
18005 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
18006 if let Some((_, path)) = spec
18007 .env
18008 .iter()
18009 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
18010 {
18011 let mut barrier = TcpStream::connect(path).unwrap();
18012 barrier
18013 .set_read_timeout(Some(Duration::from_secs(30)))
18014 .unwrap();
18015 barrier.write_all(&pid.to_ne_bytes()).unwrap();
18016 let mut release = [0];
18017 barrier.read_exact(&mut release).unwrap();
18018 assert_eq!(&release, b"X");
18019 }
18020 }
18021
18022 #[cfg(target_os = "macos")]
18023 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
18024 ModuleSpec {
18025 module_id: "privacy-boundary".into(),
18026 program: program.into(),
18027 args: args.iter().map(|arg| (*arg).into()).collect(),
18028 env: vec![],
18029 reserved: false,
18030 reserved_prefixes: vec![],
18031 protocol: ModuleProtocol::None,
18032 overlap: ModuleOverlap::Exclusive,
18033 carry_containment: false,
18034 }
18035 }
18036
18037 #[cfg(target_os = "macos")]
18038 async fn accept(listener: TcpListener) -> TcpStream {
18039 let listener = tokio::net::TcpListener::from_std({
18041 listener.set_nonblocking(true).unwrap();
18042 listener
18043 })
18044 .unwrap();
18045 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
18046 .await
18047 .unwrap()
18048 .unwrap();
18049 let stream = stream.into_std().unwrap();
18050 stream.set_nonblocking(false).unwrap();
18051 stream
18052 .set_read_timeout(Some(Duration::from_secs(30)))
18053 .unwrap();
18054 stream
18055 }
18056
18057 #[tokio::test]
18058 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18059 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
18060 #[cfg(target_os = "macos")]
18061 {
18062 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
18063 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18066 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18067 let record = root.join("live-children.json");
18068 let supervisor =
18069 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18070 .with_live_children_record(&record);
18071 let runtime = supervisor.runtime_config();
18072 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18073 let mut spec = spec("/bin/sleep", &["30"]);
18074 spec.env = vec![
18075 (
18076 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
18077 exec_listener.local_addr().unwrap().to_string(),
18078 ),
18079 (
18080 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
18081 sample_listener.local_addr().unwrap().to_string(),
18082 ),
18083 ];
18084 let spawn = tokio::task::spawn_blocking(move || {
18085 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
18086 });
18087 let mut sample = accept(sample_listener).await;
18088 let mut pid = [0; 4];
18089 sample.read_exact(&mut pid).unwrap();
18090 let pid = u32::from_ne_bytes(pid);
18091 let mut exec = accept(exec_listener).await;
18092 let mut ready = [0];
18093 exec.read_exact(&mut ready).unwrap();
18094 assert_eq!(&ready, b"R");
18095 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
18098 assert_eq!(
18099 observe_spawned_image(pid).unwrap().executable,
18100 Some(trampoline)
18101 );
18102 sample.write_all(b"X").unwrap();
18103 let mut child = spawn.await.unwrap();
18104 let early = crate::live_children::read_record(&record).unwrap();
18105 assert_eq!(early.len(), 1);
18106 assert_eq!(early[0].pid, pid);
18107 assert_eq!(
18108 early[0].executable, None,
18109 "unconfirmed trampoline image entered the roster"
18110 );
18111 assert!(child.report_ready.get().is_none());
18112 child.privacy_exec.as_mut().unwrap().deadline =
18115 tokio::time::Instant::now() + Duration::from_secs(30);
18116 exec.write_all(b"X").unwrap();
18117 child.confirm_privacy_exec().await;
18118 assert_eq!(child.spawn_failure, None);
18119 assert!(child.report_ready.get().is_some());
18120 let confirmed = crate::live_children::read_record(&record).unwrap();
18121 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
18122 assert_ne!(module, trampoline);
18123 assert_eq!(confirmed[0].executable, Some(module.into()));
18124 child.start_kill().unwrap();
18125 child.wait().await.unwrap();
18126 child.release_roster();
18127 }
18128 }
18129
18130 #[tokio::test]
18131 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18132 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
18133 #[cfg(target_os = "macos")]
18134 {
18135 let registry = Arc::new(Registry::default());
18136 let policy = RestartPolicy::new(0, Duration::ZERO);
18137 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
18138 let runtime = supervisor.runtime_config();
18139 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18140 let spec = spec("/bin/sh", &["-c", "exit 121"]);
18141 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
18145 let pid = child.pid;
18146 tokio::task::spawn_blocking(move || {
18147 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
18148 })
18149 .await
18150 .unwrap()
18151 .unwrap();
18152 child.privacy_exec.as_mut().unwrap().deadline =
18153 tokio::time::Instant::now() + Duration::from_secs(30);
18154 let status = child.wait().await.unwrap();
18155 assert_eq!(status.code(), Some(121));
18156 assert!(child.privacy_exec.is_none());
18157 assert!(
18158 child.report_ready.get().is_none(),
18159 "an exited module must not publish a live pid"
18160 );
18161 let report = classify_reaped_child_exit(&snapshot, &child, &status);
18162 on_child_exit(
18163 &spec,
18164 policy,
18165 ®istry,
18166 &snapshot,
18167 &runtime.terminal_ring,
18168 &runtime.spawn_events,
18169 &runtime.child_roster,
18170 report,
18171 true,
18172 )
18173 .await;
18174 let state = lock_snapshot(&snapshot).unwrap();
18175 assert_eq!(state.state, ModuleState::Failed);
18176 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
18177 assert_eq!(state.reported_pid(), None);
18178 drop(state);
18179 let history = runtime.terminal_ring.lock().unwrap().snapshot();
18180 assert_eq!(history.entries.len(), 1);
18181 let terminal = &history.entries[0];
18182 assert_eq!(terminal.exit_code, Some(121));
18183 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
18184 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
18185 assert_eq!(
18186 terminal.disposition_detail.as_deref(),
18187 Some(policy.budget_exhausted_detail().as_str()),
18188 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
18189 );
18190 assert_eq!(child.spawn_failure, None);
18191 child.release_roster();
18192 }
18193 }
18194}
18195
18196#[cfg(all(test, unix))]
18200mod launch_nonce_descriptor_tests {
18201 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
18202 use crate::stderr_tail::{StderrRing, StderrTailConfig};
18203 use cortexkit_test_support::ScratchDir;
18204 use std::{
18205 path::PathBuf,
18206 sync::{Arc, Mutex},
18207 time::{Duration, Instant},
18208 };
18209
18210 async fn probe(role: super::SpawnRole) {
18211 let scratch = ScratchDir::new("launch-nonce-descriptor");
18212 let fd_copy = scratch.join("from-descriptor");
18213 let env_copy = scratch.join("environment");
18214 let script = format!(
18215 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
18216 fd = fd_copy.display(), env = env_copy.display(),
18217 );
18218 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
18219 let spec = ModuleSpec {
18220 module_id: "nonce-descriptor-probe".to_string(),
18221 program: PathBuf::from("/bin/sh"),
18222 args: vec!["-c".to_string(), script],
18223 env: vec![
18224 xdg("XDG_DATA_HOME"),
18225 xdg("XDG_RUNTIME_DIR"),
18226 xdg("XDG_CONFIG_HOME"),
18227 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
18228 ],
18229 reserved: true,
18230 reserved_prefixes: Vec::new(),
18231 protocol: ModuleProtocol::Subc,
18232 overlap: Default::default(),
18233 carry_containment: false,
18234 };
18235 let handle = SupervisorHandle::new();
18236 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
18237 let roster = ChildRoster::default();
18238 #[cfg(target_os = "macos")]
18239 {
18240 let path = super::test_privacy_trampoline();
18241 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
18242 }
18243 let child = super::spawn_child_in_slot(
18244 &spec,
18245 None,
18246 Some(&handle),
18247 &ring,
18248 None,
18249 &roster,
18250 #[cfg(target_os = "linux")]
18251 None,
18252 role,
18253 matches!(role, super::SpawnRole::SwapCandidate),
18254 )
18255 .expect("spawn probe");
18256 let deadline = Instant::now() + Duration::from_secs(10);
18257 while !(fd_copy.exists() && env_copy.exists()) {
18258 assert!(Instant::now() < deadline, "probe never wrote its copies");
18259 tokio::time::sleep(Duration::from_millis(20)).await;
18260 }
18261 let nonce = std::fs::read_to_string(fd_copy).unwrap();
18262 assert!(!nonce.is_empty());
18263 let environment = std::fs::read_to_string(env_copy).unwrap();
18264 assert!(environment
18265 .lines()
18266 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
18267 let copy = environment
18268 .lines()
18269 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
18270 assert_eq!(
18271 copy, None,
18272 "Unix children must never receive the environment nonce"
18273 );
18274 if matches!(role, super::SpawnRole::Plain) {
18275 assert_eq!(
18276 handle.spawn_nonce(&spec.module_id).as_deref(),
18277 Some(nonce.as_str())
18278 );
18279 }
18280 drop(child);
18281 }
18282
18283 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18284 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
18285 probe(super::SpawnRole::Plain).await;
18286 }
18287
18288 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18289 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
18290 probe(super::SpawnRole::SwapCandidate).await;
18291 }
18292}
18293
18294#[cfg(all(test, target_os = "linux"))]
18295#[path = "supervise/cgroup_reap_tests.rs"]
18296mod cgroup_reap_tests;
18297
18298#[cfg(all(test, target_os = "linux"))]
18299mod cgroup_containment_tests {
18300 use super::*;
18301 use cortexkit_test_support::ScratchDir;
18302
18303 #[tokio::test]
18304 async fn linux_subtree_memory_covers_module_children() {
18305 let dir = ScratchDir::new("subtree-memory");
18306 let root = PathBuf::from(format!(
18307 "/sys/fs/cgroup/subc-memory-test-{}-{}",
18308 std::process::id(),
18309 unix_ms_now()
18310 ));
18311 if let Err(error) = std::fs::create_dir(&root) {
18312 assert!(
18313 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18314 "required cgroup test cannot execute: {error}"
18315 );
18316 eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
18317 return;
18318 }
18319 let placement = subc_cgroup::prepare_at(&root)
18320 .unwrap()
18321 .expect("delegated cgroup");
18322 let supervisor = Supervisor::new_for_test(
18323 Arc::new(Registry::default()),
18324 RestartPolicy::new(3, Duration::ZERO),
18325 )
18326 .with_cgroup_placement(Some(placement));
18327 let ready = dir.join("ready");
18328 let module = supervisor
18329 .spawn(ModuleSpec {
18330 module_id: "memory-tree".into(),
18331 program: PathBuf::from("/bin/sh"),
18332 args: vec![
18333 "-c".into(),
18334 "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
18335 "fixture".into(),
18336 ready.display().to_string(),
18337 ],
18338 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18339 .into_iter()
18340 .map(|key| (key.into(), dir.display().to_string()))
18341 .collect(),
18342 reserved: false,
18343 reserved_prefixes: Vec::new(),
18344 protocol: ModuleProtocol::None,
18345 overlap: Default::default(),
18346 carry_containment: false,
18347 })
18348 .unwrap();
18349 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18350 while !ready.exists() {
18351 assert!(
18352 tokio::time::Instant::now() < deadline,
18353 "children never started"
18354 );
18355 tokio::time::sleep(Duration::from_millis(10)).await;
18356 }
18357 let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
18358 let reading = &readings[0];
18359 assert!(
18360 reading.process_count.is_some_and(|count| count >= 4),
18361 "{reading:?}"
18362 );
18363 assert!(
18364 reading.memory_bytes.is_some_and(|bytes| bytes > 0),
18365 "{reading:?}"
18366 );
18367 module.stop().await.unwrap();
18368 drop(module);
18369 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18370 std::fs::remove_dir(&root).unwrap();
18371 eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
18372 }
18373
18374 fn running(pid: u32) -> bool {
18375 std::fs::read_to_string(format!("/proc/{pid}/stat"))
18377 .ok()
18378 .and_then(|stat| {
18379 stat.rsplit_once(") ")
18380 .map(|(_, rest)| rest.starts_with('Z'))
18381 })
18382 .is_some_and(|zombie| !zombie)
18383 }
18384
18385 #[tokio::test]
18386 async fn linux_teardown_reaps_the_grandchild() {
18387 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
18388 }
18389
18390 #[tokio::test]
18391 async fn linux_shutdown_straggler_reaps_the_grandchild() {
18392 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
18393 }
18394
18395 async fn teardown_tree(test_name: &str, shutdown: bool) {
18396 let dir = ScratchDir::new(test_name);
18397 let root = PathBuf::from(format!(
18398 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
18399 std::process::id(),
18400 unix_ms_now()
18401 ));
18402 if let Err(error) = std::fs::create_dir(&root) {
18403 assert!(
18404 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18405 "required cgroup test cannot execute: {error}"
18406 );
18407 eprintln!(
18408 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
18409 root.display()
18410 );
18411 return;
18412 }
18413 let placement = subc_cgroup::prepare_at(&root)
18414 .expect("prepare isolated kernel cgroup")
18415 .expect("isolated cgroup is delegated");
18416 let module_id = "tree-teardown";
18417 let module = placement
18418 .module_path(module_id)
18419 .expect("create isolated module cgroup");
18420 if !module.join("cgroup.kill").exists() {
18421 std::fs::remove_dir(&module).unwrap();
18422 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18423 std::fs::remove_dir(&root).unwrap();
18424 assert!(
18425 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18426 "required cgroup.kill interface unavailable"
18427 );
18428 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
18429 return;
18430 }
18431 let supervisor = Supervisor::new_for_test(
18432 Arc::new(Registry::default()),
18433 RestartPolicy::new(3, Duration::ZERO),
18434 )
18435 .with_cgroup_placement(Some(placement));
18436 let mut runtime = supervisor.runtime_config();
18437 runtime.child_roster = runtime
18438 .child_roster
18439 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
18440 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18441 let pid_file = dir.join("grandchild.pid");
18442 let spec = ModuleSpec {
18443 module_id: module_id.to_string(),
18444 program: PathBuf::from("/bin/sh"),
18445 args: vec![
18446 "-c".into(),
18447 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
18448 "fixture".into(),
18449 pid_file.display().to_string(),
18450 ],
18451 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18452 .into_iter()
18453 .map(|key| (key.to_string(), dir.display().to_string()))
18454 .collect(),
18455 reserved: false,
18456 reserved_prefixes: Vec::new(),
18457 protocol: ModuleProtocol::None,
18458 overlap: Default::default(),
18459 carry_containment: false,
18460 };
18461 let child =
18462 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
18463 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18464 let grandchild: u32 = loop {
18465 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
18466 if let Ok(pid) = contents.trim().parse() {
18467 break pid;
18468 }
18469 }
18470 assert!(
18471 tokio::time::Instant::now() < deadline,
18472 "grandchild pid was not recorded"
18473 );
18474 tokio::time::sleep(Duration::from_millis(10)).await;
18475 };
18476 assert!(
18477 running(grandchild),
18478 "grandchild must be alive before teardown"
18479 );
18480 if shutdown {
18481 let mut child = child;
18482 crate::child_roster::end_children_for_daemon_shutdown(
18483 &runtime.child_roster,
18484 false,
18485 std::future::pending(),
18486 )
18487 .await;
18488 child.wait().await.expect("reap shutdown straggler");
18489 } else {
18490 drain_child_to_state(
18491 module_id,
18492 ModuleProtocol::None,
18493 StopNotice::NotSent,
18494 &Registry::default(),
18495 None,
18496 &snapshot,
18497 &runtime.terminal_ring,
18498 &SpawnEventFeed::default(),
18499 child,
18500 Duration::from_millis(100),
18501 ModuleState::Stopped,
18502 Some(false),
18503 &FailureContext::operator(),
18504 )
18505 .await
18506 .expect("real supervisor teardown");
18507 }
18508 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
18509 while running(grandchild) && tokio::time::Instant::now() < deadline {
18510 tokio::time::sleep(Duration::from_millis(10)).await;
18511 }
18512 let survived = running(grandchild);
18513 if survived {
18515 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
18516 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
18517 tokio::time::sleep(Duration::from_millis(100)).await;
18518 }
18519 if module.exists() {
18520 std::fs::remove_dir(&module).expect("remove empty module cgroup");
18521 }
18522 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18523 std::fs::remove_dir(&root).unwrap();
18524 assert!(
18525 !survived,
18526 "grandchild {grandchild} outlived module teardown"
18527 );
18528 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
18529 }
18530}
18531
18532#[cfg(all(test, unix))]
18535mod record_write_stall_tests {
18536 use super::*;
18537 use std::sync::atomic::{AtomicBool, Ordering};
18538 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
18539
18540 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
18543 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
18547
18548 fn sleeping_spec() -> ModuleSpec {
18560 ModuleSpec {
18561 module_id: "record-write-stall".into(),
18562 program: "/bin/sleep".into(),
18563 args: vec!["600".into()],
18564 env: vec![],
18565 reserved: false,
18566 reserved_prefixes: vec![],
18567 protocol: ModuleProtocol::None,
18568 overlap: Default::default(),
18569 carry_containment: false,
18570 }
18571 }
18572
18573 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18574 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
18575 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
18576 let supervisor =
18577 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18578 .with_live_children_record(home.join("live-children.json"));
18579 let module = supervisor.spawn(sleeping_spec()).unwrap();
18580 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18585 loop {
18586 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
18587 .unwrap()
18588 .reported_pid()
18589 {
18590 return pid;
18591 }
18592 tokio::time::sleep(Duration::from_millis(10)).await;
18593 }
18594 })
18595 .await
18596 .expect("the first process was admitted as the module");
18597
18598 let key = vec![0x42; 32];
18601 let daemon_id = [0x24; 16];
18602 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
18603 let port = listener.local_addr().unwrap().port();
18604 let server = tokio::spawn(crate::server::serve_listener(
18605 listener,
18606 Arc::new(crate::router::Router::with_default_self_handler()),
18607 crate::server::ServerAuth::with_limits(
18608 key.clone(),
18609 daemon_id,
18610 "test",
18611 crate::server::DEFAULT_AUTH_DEADLINE,
18612 4,
18613 ),
18614 ));
18615
18616 let (held_tx, held_rx) = std::sync::mpsc::channel();
18619 let armed = AtomicBool::new(true);
18620 supervisor
18621 .child_roster
18622 .set_record_write_hook_for_test(move |live| {
18623 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
18624 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
18625 let _ = held_tx.send(());
18626 std::thread::sleep(HELD_RECORD_WRITE);
18627 }
18628 });
18629
18630 let module_process = std::thread::spawn(move || {
18633 held_rx
18634 .recv_timeout(Duration::from_secs(30))
18635 .expect("the restart reached the replacement's record write");
18636 let conn = ConnectionInfo {
18637 schema: SCHEMA_VERSION,
18638 wire_version: None,
18639 endpoints: vec![Endpoint {
18640 host: "127.0.0.1".to_owned(),
18641 port,
18642 }],
18643 key,
18644 daemon_id,
18645 pid: std::process::id(),
18646 daemon_ver: "test".to_owned(),
18647 };
18648 tokio::runtime::Builder::new_current_thread()
18649 .enable_all()
18650 .build()
18651 .unwrap()
18652 .block_on(async {
18653 let started = Instant::now();
18654 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
18655 .await
18656 .unwrap();
18657 let result =
18658 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
18659 (result, started.elapsed())
18660 })
18661 });
18662
18663 module.restart(Some(0)).await.unwrap();
18664 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
18665 .await
18666 .unwrap();
18667 assert!(
18668 auth.is_ok(),
18669 "the daemon must answer a module's handshake while a restart waits on the \
18670 live-children record write; got {auth:?} after {elapsed:?}"
18671 );
18672
18673 tokio::time::timeout(Duration::from_secs(20), async {
18674 while lock_snapshot(&module.inner.snapshot)
18675 .unwrap()
18676 .reported_pid()
18677 .is_none_or(|pid| pid == old_pid)
18678 {
18679 tokio::time::sleep(Duration::from_millis(10)).await;
18680 }
18681 })
18682 .await
18683 .expect("the restart published its replacement");
18684 module.stop().await.unwrap();
18685 server.abort();
18686 }
18687}
18688
18689#[cfg(all(test, unix))]
18690mod retained_status_tests {
18691 use super::*;
18692
18693 #[tokio::test]
18694 async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
18695 use crate::live_children::{ContainerIdentity, RetainedReason};
18696 for (reason, carried, expected) in [
18697 (
18698 RetainedReason::ContainerSurvivors(vec![4242]),
18699 true,
18700 Some("waiting: container survivors (pids [4242])".to_owned()),
18701 ),
18702 (
18703 RetainedReason::ContainerUnverifiable("member inspection denied".into()),
18704 true,
18705 Some(
18706 "waiting: container survivors could not be verified: member inspection denied"
18707 .to_owned(),
18708 ),
18709 ),
18710 (RetainedReason::Survived, false, None),
18711 ] {
18712 let supervisor =
18713 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18714 let spec = ModuleSpec {
18715 module_id: "retained-status-fixture".into(),
18716 program: PathBuf::from("/not-spawned"),
18717 args: Vec::new(),
18718 env: Vec::new(),
18719 reserved: false,
18720 reserved_prefixes: Vec::new(),
18721 protocol: ModuleProtocol::None,
18722 overlap: ModuleOverlap::Exclusive,
18723 carry_containment: carried,
18724 };
18725 let module = supervisor
18726 .supervise_configured(spec.clone(), false)
18727 .unwrap();
18728 let guard = supervisor
18732 .child_roster
18733 .admit(
18734 spec.module_id,
18735 0,
18736 ModuleProtocol::None,
18737 None,
18738 crate::child_roster::RecordedIdentity {
18739 container: carried.then_some(ContainerIdentity {
18740 pgid: Some(0),
18741 created_start_time: Some(0),
18742 cgroup_path: None,
18743 ..ContainerIdentity::test_metadata()
18744 }),
18745 ..crate::child_roster::RecordedIdentity::default()
18746 },
18747 )
18748 .unwrap();
18749 guard.retain(reason);
18750 let status = module.status().unwrap();
18751 assert_eq!(status.waiting_for_previous_detail, expected.clone());
18752 assert_eq!(status.health.detail, expected);
18753 assert!(!status.process_alive);
18754 }
18755 }
18756}