1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62mod slow_retry;
63use slow_retry::{FailedRetry, FailureContext};
64
65#[cfg(all(test, unix))]
66mod slow_retry_tests;
67
68pub const SUBC_ARG: &str = "--subc";
74
75const DEFAULT_MAX_RESTARTS: u32 = 3;
76const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
77const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
78const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
82pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
93pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
94const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
95const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
96const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
108const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
131pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
133const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
134pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
139
140#[cfg(target_os = "macos")]
141struct PrivacyExec {
142 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
143 deadline: tokio::time::Instant,
144 expected: Option<subc_os::FileIdentity>,
145 trampoline: Option<subc_os::FileIdentity>,
146 script: bool,
147 module_id: String,
148 #[cfg(test)]
149 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
150}
151
152#[cfg(all(test, target_os = "macos"))]
153pub(crate) fn test_privacy_trampoline() -> PathBuf {
154 let path = std::env::current_exe()
157 .unwrap()
158 .parent()
159 .unwrap()
160 .parent()
161 .unwrap()
162 .join("privacy-trampoline-fixture");
163 assert!(
167 path.exists(),
168 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
169 --bins --features test-support` or `cargo test -p subc-daemon` first",
170 path.display()
171 );
172 path
173}
174
175#[cfg(target_os = "macos")]
176fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
177 use std::io::Read;
178 let mut probe = std::process::Command::new(path)
179 .args(["__disclaim-exec", "--probe"])
180 .stdin(Stdio::null())
181 .stdout(Stdio::piped())
182 .stderr(Stdio::piped())
183 .spawn()
184 .map_err(|error| {
185 format!(
186 "privacy trampoline probe failed for {}: {error}",
187 path.display()
188 )
189 })?;
190 let deadline = std::time::Instant::now() + Duration::from_secs(5);
191 let status = loop {
192 match probe.try_wait() {
193 Ok(Some(status)) => break status,
194 Ok(None) if std::time::Instant::now() < deadline => {
195 std::thread::sleep(Duration::from_millis(5))
196 }
197 result => {
198 let _ = probe.kill();
199 let _ = probe.wait();
200 return Err(format!(
201 "privacy trampoline probe failed or timed out for {}: {result:?}",
202 path.display()
203 ));
204 }
205 }
206 };
207 let mut answer = String::new();
208 if let Some(stdout) = probe.stdout.take() {
209 let _ = stdout.take(256).read_to_string(&mut answer);
210 }
211 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
212 return Ok(());
213 }
214 let mut diagnostic = String::new();
215 if let Some(stderr) = probe.stderr.take() {
216 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
217 }
218 let cause = diagnostic
219 .trim()
220 .strip_prefix("ck-subc: own privacy identity refused: ")
221 .unwrap_or("binary does not implement the privacy trampoline protocol");
222 Err(format!(
223 "{cause}: probe of {} exited {status}",
224 path.display()
225 ))
226}
227
228#[cfg(target_os = "macos")]
229fn privacy_command(
230 spec: &ModuleSpec,
231 roster: &ChildRoster,
232) -> Result<
233 (
234 Command,
235 Option<PrivacyExec>,
236 subc_os::privacy_identity::ExecAcknowledgement,
237 ),
238 SuperviseError,
239> {
240 let failure = |cause: String| {
241 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
242 SuperviseError::Spawn {
243 program: spec.program.clone(),
244 source: io::Error::other(cause),
245 cgroup_path: None,
246 }
247 };
248 let trampoline = roster.privacy_trampoline().map_err(failure)?;
249 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
254 let path = spec
255 .env
256 .iter()
257 .find(|(key, _)| key == "PATH")
258 .map(|(_, value)| std::ffi::OsString::from(value))
259 .or_else(|| std::env::var_os("PATH"))
260 .unwrap_or_else(|| "/usr/bin:/bin".into());
261 std::env::split_paths(&path)
262 .map(|dir| dir.join(&spec.program))
263 .find(|path| path.is_file())
264 .unwrap_or_else(|| spec.program.clone())
265 } else {
266 spec.program.clone()
267 };
268 let expected = subc_os::file_identity(&program);
269 let trampoline_image = subc_os::file_identity(&trampoline);
270 let script = {
271 use std::io::Read;
272 let mut prefix = [0u8; 2];
273 std::fs::File::open(&program)
274 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
275 };
276 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
277 return Err(failure(
278 "privacy identity module executable is missing or is the trampoline itself".to_string(),
279 ));
280 }
281 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
282 .map_err(|error| failure(error.to_string()))?;
283 let reader =
284 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
285 let mut command = Command::new(&trampoline);
286 command
287 .arg("__disclaim-exec")
288 .arg(ack.fd().to_string())
289 .arg(&program);
290 ack.install(command.as_std_mut());
291 Ok((
292 command,
293 Some(PrivacyExec {
294 reader,
295 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
296 expected,
297 trampoline: trampoline_image,
298 script,
299 module_id: spec.module_id.clone(),
300 #[cfg(test)]
301 confirmation_pause: spec
302 .env
303 .iter()
304 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
305 .map(|(_, address)| (address.clone(), None)),
306 }),
307 ack,
308 ))
309}
310
311struct SupervisedChild {
312 child: Child,
313 exec: crate::activity::ExecOwner,
314 #[cfg(windows)]
315 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
316 #[cfg(target_os = "macos")]
317 privacy_exec: Option<PrivacyExec>,
318 #[cfg(target_os = "macos")]
325 report_ready: Arc<OnceLock<()>>,
326 spawn_failure: Option<String>,
329 protocol: ModuleProtocol,
334 #[cfg(target_os = "linux")]
340 module_id: String,
341 #[cfg(target_os = "linux")]
342 cgroup_placement: Option<subc_cgroup::Placement>,
343 #[cfg(windows)]
365 job: Option<subc_jobobject::JobObject>,
366 stdout_pump: Option<JoinHandle<()>>,
367 stderr_pump: Option<StderrPump>,
368 stderr_ring: Arc<Mutex<StderrRing>>,
369 spawned_at: Instant,
372 registration_deadline: Option<Instant>,
375 registration_watch: Option<RegistrationWatch>,
376 spawned_at_ms: u64,
377 spawned_from: PathBuf,
378 launch_spec_digest: LaunchSpecDigest,
379 spawned_file_identity: Option<SpawnedFileIdentity>,
380 process_start_time: Option<u64>,
381 process_identity: Option<ProcessIdentity>,
382 pid: u32,
383 roster_guard: Option<crate::child_roster::RosterGuard>,
386}
387
388impl SupervisedChild {
389 fn registered_since_spawn(&self) -> bool {
390 self.registration_watch
391 .as_ref()
392 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
393 }
394
395 fn id(&self) -> Option<u32> {
396 Some(self.pid)
397 }
398
399 fn process_identity(&self) -> Option<ProcessIdentity> {
400 self.process_identity
401 }
402
403 async fn wait(&mut self) -> io::Result<ExitStatus> {
404 #[cfg(target_os = "macos")]
405 self.confirm_privacy_exec().await;
406 let result = self.child.wait().await;
414 #[cfg(target_os = "linux")]
415 if result.is_ok() {
416 if let Some(placement) = self.cgroup_placement.as_ref() {
417 cleanup_reaped_cgroup(placement, &self.module_id).await;
418 self.cgroup_placement = None;
421 }
422 }
423 result
424 }
425
426 #[cfg(target_os = "macos")]
427 async fn confirm_privacy_exec(&mut self) {
428 let Some(pending) = &mut self.privacy_exec else {
429 return;
430 };
431 let result = tokio::time::timeout_at(pending.deadline, async {
432 let mut record = Vec::new();
433 loop {
434 let mut ready = pending.reader.readable().await?;
435 let read = ready.try_io(|reader| {
436 use std::io::Read;
437 let mut reader = reader.get_ref();
438 let mut buffer = [0u8; 256];
439 reader.read(&mut buffer).map(|count| (count, buffer))
440 });
441 match read {
442 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
443 Ok(Ok((count, buffer))) => {
444 if record.len() + count > 1024 {
445 return Err(io::Error::other(
446 "privacy exec refusal record is too long",
447 ));
448 }
449 record.extend_from_slice(&buffer[..count]);
450 }
451 Ok(Err(error)) => return Err(error),
452 Err(_) => continue,
453 }
454 }
455 })
456 .await;
457 #[cfg(test)]
458 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
459 use tokio::io::{AsyncReadExt, AsyncWriteExt};
460 if stream.is_none() {
461 *stream = Some(
462 tokio::net::TcpStream::connect(address.as_str())
463 .await
464 .unwrap(),
465 );
466 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
467 stream
468 .as_mut()
469 .unwrap()
470 .write_all(&self.pid.to_ne_bytes())
471 .await
472 .unwrap();
473 }
474 let mut reply = [0];
475 stream
476 .as_mut()
477 .unwrap()
478 .read_exact(&mut reply)
479 .await
480 .unwrap();
481 pending.confirmation_pause = None;
482 if &reply == b"F" {
483 Ok(Err(io::Error::other("injected late exec refusal")))
484 } else {
485 assert_eq!(&reply, b"X");
486 result
487 }
488 } else {
489 result
490 };
491 let pending = self.privacy_exec.as_ref().expect("pending exec");
494 let cause = match result {
495 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
496 Ok(Err(error)) => Some(format!(
497 "privacy identity exec acknowledgement failed: {error}"
498 )),
499 Ok(Ok(record)) if !record.is_empty() => Some(
500 std::str::from_utf8(&record)
501 .ok()
502 .and_then(|record| {
503 record
504 .trim()
505 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
506 })
507 .filter(|cause| !cause.is_empty())
508 .unwrap_or("invalid privacy exec refusal record")
509 .to_string(),
510 ),
511 Ok(Ok(_)) => match self.child.try_wait() {
512 Ok(Some(_status)) => None,
516 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
517 Ok(None) => {
518 let image = observe_spawned_image(self.pid);
519 if let Some(image) = image.filter(|image| {
520 image.executable.is_some()
521 && image.executable != pending.trampoline
522 && (image.executable == pending.expected || pending.script)
523 }) {
524 if let Err(error) = self
525 .roster_guard
526 .as_ref()
527 .map(|guard| guard.confirm_executable(image))
528 .transpose()
529 {
530 Some(format!(
531 "could not durably record confirmed module image: {error}"
532 ))
533 } else {
534 let _ = self.report_ready.set(());
535 info!(module_id = %pending.module_id, pid = self.pid,
536 "module spawned with own privacy identity (responsibility disclaimed)");
537 None
538 }
539 } else if image.is_none()
540 || image.is_some_and(|image| image.executable.is_none())
541 {
542 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
549 Ok(Ok(_status)) => None,
550 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
551 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
552 }
553 } else {
554 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
555 }
556 }
557 },
558 };
559 let pending = self.privacy_exec.take().expect("pending exec");
560 self.exec.0.finish(if cause.is_some() {
563 crate::activity::ExecState::Refused
564 } else {
565 crate::activity::ExecState::CommittedExec
566 });
567 if let Some(cause) = cause {
568 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
569 self.spawn_failure = Some(cause);
570 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
573 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
574 }
575 let _ = self.child.start_kill();
576 }
577 }
578
579 fn release_roster(&mut self) {
583 self.roster_guard = None;
584 }
585
586 fn start_kill(&mut self) -> io::Result<()> {
599 #[cfg(windows)]
600 if let Some(job) = &self.job {
601 if let Err(error) = job.terminate() {
602 debug!(
603 error = %error,
604 "job termination failed; the direct-child kill still owns the outcome"
605 );
606 }
607 }
608 #[cfg(target_os = "linux")]
609 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
610 self.child.start_kill()
611 }
612
613 async fn drain_stderr(&mut self, module_id: &str) {
614 if let Some(pump) = self.stdout_pump.take() {
615 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
616 }
617
618 let Some(pump) = self.stderr_pump.take() else {
619 return;
620 };
621 settle_stderr_pump(
622 module_id,
623 &self.stderr_ring,
624 pump,
625 STDERR_PUMP_DRAIN_TIMEOUT,
626 )
627 .await;
628 }
629}
630
631async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
635 match timeout(bound, &mut pump).await {
636 Ok(Ok(())) => {}
637 Ok(Err(error)) => {
638 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
639 }
640 Err(_) => {
641 drop(pump);
642 warn!(
643 module_id,
644 waited = ?bound,
645 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
646 );
647 }
648 }
649}
650
651struct StderrPump {
654 task: JoinHandle<()>,
655 generation: u64,
656}
657
658async fn settle_stderr_pump(
664 module_id: &str,
665 ring: &Arc<Mutex<StderrRing>>,
666 pump: StderrPump,
667 bound: Duration,
668) {
669 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
670 let StderrPump {
671 mut task,
672 generation,
673 } = pump;
674 lock().retire_pump(generation);
675 match timeout(bound, &mut task).await {
676 Ok(Ok(())) => {}
677 Ok(Err(err)) => {
678 let mut ring = lock();
679 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
680 ring.finish_pump(generation);
681 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
682 }
683 Err(_) => {
684 drop(task);
686 lock().mark_pump_late(
687 generation,
688 format!(
689 "stderr of the exited process had not reached EOF {bound:?} after it was \
690 retired (a descendant may still hold the pipe open); lines it still \
691 writes are kept in that process's section"
692 ),
693 );
694 warn!(
695 module_id,
696 waited = ?bound,
697 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
698 );
699 }
700 }
701}
702
703fn registration_release_events() -> &'static watch::Sender<u64> {
704 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
705 EVENTS.get_or_init(|| {
706 let (sender, _receiver) = watch::channel(0);
707 sender
708 })
709}
710
711pub(crate) fn notify_registration_release() {
712 let events = registration_release_events();
713 let next_generation = (*events.borrow()).wrapping_add(1);
714 events.send_replace(next_generation);
715}
716
717#[derive(Debug, Clone, PartialEq, Eq)]
719pub struct ModuleSpec {
720 pub module_id: String,
721 pub program: PathBuf,
722 pub args: Vec<String>,
723 pub env: Vec<(String, String)>,
724 pub reserved: bool,
729 pub reserved_prefixes: Vec<String>,
734 pub protocol: ModuleProtocol,
753 pub overlap: ModuleOverlap,
758}
759
760#[derive(Clone, PartialEq, Eq)]
763pub(crate) struct LaunchSpecDigest([u8; 32]);
764
765impl fmt::Debug for LaunchSpecDigest {
766 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
767 f.write_str("LaunchSpecDigest([redacted])")
768 }
769}
770
771impl LaunchSpecDigest {
772 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
773 fn field(hash: &mut Sha256, bytes: &[u8]) {
774 hash.update((bytes.len() as u64).to_le_bytes());
775 hash.update(bytes);
776 }
777 let mut hash = Sha256::new();
778 if let Some(program) = spec.program.to_str() {
779 field(&mut hash, b"utf8-program");
780 field(&mut hash, program.as_bytes());
781 } else {
782 #[cfg(unix)]
785 {
786 use std::os::unix::ffi::OsStrExt;
787 field(&mut hash, b"unix-program");
788 field(&mut hash, spec.program.as_os_str().as_bytes());
789 }
790 #[cfg(windows)]
791 {
792 use std::os::windows::ffi::OsStrExt;
793 field(&mut hash, b"windows-program");
794 let bytes = spec
795 .program
796 .as_os_str()
797 .encode_wide()
798 .flat_map(u16::to_le_bytes)
799 .collect::<Vec<_>>();
800 field(&mut hash, &bytes);
801 }
802 #[cfg(not(any(unix, windows)))]
803 field(&mut hash, spec.program.to_string_lossy().as_bytes());
804 }
805 hash.update((spec.args.len() as u64).to_le_bytes());
806 for arg in &spec.args {
807 field(&mut hash, arg.as_bytes());
808 }
809 let mut env = spec.env.iter().collect::<Vec<_>>();
810 env.sort_unstable();
811 hash.update((env.len() as u64).to_le_bytes());
812 for (key, value) in env {
813 field(&mut hash, key.as_bytes());
814 field(&mut hash, value.as_bytes());
815 }
816 field(
817 &mut hash,
818 match spec.protocol {
819 ModuleProtocol::Subc => b"subc",
820 ModuleProtocol::None => b"none",
821 },
822 );
823 Self(hash.finalize().into())
824 }
825}
826
827#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
834pub enum ModuleOverlap {
835 #[default]
837 Exclusive,
838 Safe,
850}
851
852impl ModuleOverlap {
853 pub fn as_str(self) -> &'static str {
854 match self {
855 Self::Exclusive => "exclusive",
856 Self::Safe => "safe",
857 }
858 }
859}
860
861pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
871pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
873pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
878
879#[derive(Debug, Clone, Copy, PartialEq, Eq)]
897pub struct RestartPolicy {
898 pub max_restarts: u32,
899 pub backoff: Duration,
902 pub max_backoff: Duration,
904 pub window: Duration,
908}
909
910impl RestartPolicy {
911 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
915 Self {
916 max_restarts,
917 backoff,
918 max_backoff: DEFAULT_MAX_BACKOFF,
919 window: DEFAULT_RESTART_WINDOW,
920 }
921 }
922
923 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
924 self.max_backoff = max_backoff;
925 self
926 }
927
928 pub fn with_window(mut self, window: Duration) -> Self {
929 self.window = window;
930 self
931 }
932
933 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
938 if self.backoff.is_zero() || self.max_backoff.is_zero() {
939 return Duration::ZERO;
940 }
941
942 let mut delay = self.backoff;
943 for _ in 0..restart_in_window {
944 if delay >= self.max_backoff {
945 return self.max_backoff;
946 }
947 delay = delay
948 .checked_mul(10)
949 .unwrap_or(self.max_backoff)
950 .min(self.max_backoff);
951 }
952 delay.min(self.max_backoff)
953 }
954
955 fn budget_exhausted_detail(&self) -> String {
960 format!(
961 "crash budget exhausted: max_restarts={} within window_secs={}",
962 self.max_restarts,
963 self.window.as_secs()
964 )
965 }
966}
967
968impl Default for RestartPolicy {
969 fn default() -> Self {
970 Self {
971 max_restarts: DEFAULT_MAX_RESTARTS,
972 backoff: DEFAULT_BACKOFF,
973 max_backoff: DEFAULT_MAX_BACKOFF,
974 window: DEFAULT_RESTART_WINDOW,
975 }
976 }
977}
978
979#[derive(Debug, Clone, Copy, PartialEq, Eq)]
980struct CrashRestartSchedule {
981 restart_in_window: u32,
982 delay: Duration,
983}
984
985fn daemon_will_restart(
992 state: &mut SupervisorSnapshot,
993 policy: &RestartPolicy,
994 now: Instant,
995) -> bool {
996 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
997}
998
999const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1000const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1001const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1002const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1003
1004#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1005pub enum HealthAction {
1006 Report,
1007 Restart,
1008 Alert,
1009}
1010
1011impl fmt::Display for HealthAction {
1012 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1013 f.write_str(match self {
1014 Self::Report => "report",
1015 Self::Restart => "restart",
1016 Self::Alert => "alert",
1017 })
1018 }
1019}
1020
1021#[derive(Debug, Clone, PartialEq, Eq)]
1022pub struct HealthConfig {
1023 pub http: Option<String>,
1027 pub cadence: Duration,
1028 pub deadline: Duration,
1029 pub failure_threshold: u32,
1030 pub on_degraded: HealthAction,
1031 pub on_failing: HealthAction,
1032 pub critical: bool,
1033}
1034
1035impl Default for HealthConfig {
1036 fn default() -> Self {
1037 Self {
1038 http: None,
1039 cadence: DEFAULT_HEALTH_CADENCE,
1040 deadline: DEFAULT_HEALTH_DEADLINE,
1041 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1042 on_degraded: HealthAction::Report,
1043 on_failing: HealthAction::Report,
1044 critical: false,
1045 }
1046 }
1047}
1048
1049#[derive(Debug, Clone, PartialEq)]
1067pub struct ModuleHealthStatus {
1068 pub status: SupervisorHealthStatus,
1069 pub last_probe_ms: Option<u64>,
1070 pub detail: Option<String>,
1071 pub metrics: Option<Value>,
1072 pub consecutive_failures: u32,
1073 pub late_answer_count: u64,
1076 pub last_late_answer_latency_ms: Option<u64>,
1078 pub last_action: Option<String>,
1079 pub last_action_ms: Option<u64>,
1083}
1084
1085impl Default for ModuleHealthStatus {
1086 fn default() -> Self {
1087 Self {
1088 status: SupervisorHealthStatus::Unknown,
1089 last_probe_ms: None,
1090 detail: None,
1091 metrics: None,
1092 consecutive_failures: 0,
1093 late_answer_count: 0,
1094 last_late_answer_latency_ms: None,
1095 last_action: None,
1096 last_action_ms: None,
1097 }
1098 }
1099}
1100
1101#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1103pub enum ModuleState {
1104 Starting,
1105 Running,
1106 Unresponsive,
1107 Restarting,
1108 Draining,
1109 Stopped,
1110 Failed,
1111 Disabled,
1112}
1113
1114impl fmt::Display for ModuleState {
1115 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1116 f.write_str(match self {
1117 Self::Starting => "starting",
1118 Self::Running => "running",
1119 Self::Unresponsive => "unresponsive",
1120 Self::Restarting => "restarting",
1121 Self::Draining => "draining",
1122 Self::Stopped => "stopped",
1123 Self::Failed => "failed",
1124 Self::Disabled => "disabled",
1125 })
1126 }
1127}
1128
1129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1131pub enum ExitKind {
1132 Clean,
1133 Crash,
1134 DeliberateSeverance,
1135}
1136
1137impl From<ExitKind> for TerminalExitKind {
1138 fn from(kind: ExitKind) -> Self {
1139 match kind {
1140 ExitKind::Clean => Self::Clean,
1141 ExitKind::Crash => Self::Crash,
1142 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1143 }
1144 }
1145}
1146
1147#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1150pub(crate) struct ProcessIdentity {
1151 pub(crate) pid: u32,
1152 pub(crate) start_time: u64,
1153}
1154
1155#[derive(Debug, Clone, PartialEq, Eq)]
1157pub struct ExitReport {
1158 pub kind: ExitKind,
1159 pub code: Option<i32>,
1160 pub signal: Option<i32>,
1161 pub at_ms: u64,
1162}
1163
1164#[derive(Debug, Clone, PartialEq)]
1167pub struct ModuleStatus {
1168 pub module_id: String,
1169 pub waiting_for_previous_pid: Option<u32>,
1171 pub waiting_for_previous_detail: Option<String>,
1173 pub launch_nonce_source: Option<String>,
1178 pub state: ModuleState,
1179 pub enabled: bool,
1180 pub process_alive: bool,
1181 pub registration_active: bool,
1182 pub protocol: ModuleProtocol,
1187 pub live: bool,
1198 pub restart_count: u32,
1202 pub lifetime_restarts: u32,
1206 pub spawn_generation: u64,
1207 pub max_restarts: u32,
1212 pub restart_window: Duration,
1216 pub drain_timeout: Duration,
1220 pub restart_backoff: Duration,
1221 pub restart_max_backoff: Duration,
1222 pub pid: Option<u32>,
1227 pub spawned_at_ms: Option<u64>,
1228 pub spawned_from: Option<PathBuf>,
1229 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1230 pub process_start_time: Option<u64>,
1231 pub last_exit: Option<ExitReport>,
1232 pub health: ModuleHealthStatus,
1233}
1234
1235#[cfg(test)]
1236#[derive(Debug, Clone, Copy, PartialEq)]
1237struct ActorSelectCheckpoint {
1238 generation: u64,
1239 turn: u64,
1240 registered_connection: Option<ConnectionId>,
1241 next_probe_at: Option<Instant>,
1242 wake_after: Option<Duration>,
1243}
1244
1245#[derive(Debug, Clone, PartialEq)]
1246struct SupervisorSnapshot {
1247 failed_retry: FailedRetry,
1248 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1249 #[cfg(windows)]
1250 pipe_consumed: Option<Arc<OnceLock<()>>>,
1251 #[cfg(test)]
1253 actor_turns: u64,
1254 #[cfg(test)]
1257 actor_select: Option<ActorSelectCheckpoint>,
1258 state: ModuleState,
1259 enabled: bool,
1260 process_alive: bool,
1261 spawned_protocol: Option<ModuleProtocol>,
1262 spawn_failure: Option<String>,
1263 waiting_for_previous_pid: Option<u32>,
1264 waiting_for_previous_detail: Option<String>,
1265 crash_restarts: VecDeque<Instant>,
1271 lifetime_restarts: u32,
1272 spawn_generation: u64,
1281 pid: Option<u32>,
1282 #[cfg(target_os = "macos")]
1283 report_ready: Option<Arc<OnceLock<()>>>,
1284 reaped_pid: Option<u32>,
1286 respawn_pending: bool,
1288 coalesced_restart_pending: bool,
1290 spawned_at_ms: Option<u64>,
1291 spawned_from: Option<PathBuf>,
1292 launch_spec_digest: Option<LaunchSpecDigest>,
1293 spawned_file_identity: Option<SpawnedFileIdentity>,
1294 process_start_time: Option<u64>,
1295 deliberate_severance: Option<ProcessIdentity>,
1296 last_exit: Option<ExitReport>,
1297 drain_disposition_detail: Option<String>,
1299 health: ModuleHealthStatus,
1300 in_alternate_slot: bool,
1305 draining_to_replace: bool,
1312 configuration_updated_since_spawn: bool,
1318}
1319
1320impl SupervisorSnapshot {
1321 fn reported_pid(&self) -> Option<u32> {
1326 #[cfg(target_os = "macos")]
1327 if self
1328 .report_ready
1329 .as_ref()
1330 .is_some_and(|ready| ready.get().is_none())
1331 {
1332 return None;
1333 }
1334 self.pid
1335 }
1336
1337 fn starting() -> Self {
1338 Self::new(ModuleState::Starting, true)
1339 }
1340
1341 fn disabled() -> Self {
1342 Self::new(ModuleState::Disabled, false)
1343 }
1344
1345 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1349 while let Some(oldest) = self.crash_restarts.front() {
1350 if now.duration_since(*oldest) > window {
1351 self.crash_restarts.pop_front();
1352 } else {
1353 break;
1354 }
1355 }
1356 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1357 }
1358
1359 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1365 self.crash_restarts.push_back(now);
1366 while self.crash_restarts.len() > policy.max_restarts as usize {
1367 self.crash_restarts.pop_front();
1368 }
1369 self.lifetime_restarts += 1;
1370 }
1371
1372 fn next_crash_restart(
1376 &mut self,
1377 policy: &RestartPolicy,
1378 now: Instant,
1379 ) -> Option<CrashRestartSchedule> {
1380 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1381 if restart_in_window >= policy.max_restarts {
1382 return None;
1383 }
1384 self.record_crash_restart(policy, now);
1385 Some(CrashRestartSchedule {
1386 restart_in_window,
1387 delay: policy.delay_for_restart(restart_in_window),
1388 })
1389 }
1390
1391 fn clear_crash_restarts(&mut self) {
1396 self.crash_restarts.clear();
1397 }
1398
1399 fn new(state: ModuleState, enabled: bool) -> Self {
1400 Self {
1401 failed_retry: FailedRetry::default(),
1402 exec_outcome: None,
1403 #[cfg(test)]
1404 actor_turns: 0,
1405 #[cfg(test)]
1406 actor_select: None,
1407 state,
1408 enabled,
1409 process_alive: false,
1410 #[cfg(windows)]
1411 pipe_consumed: None,
1412 spawned_protocol: None,
1413 spawn_failure: None,
1414 waiting_for_previous_pid: None,
1415 waiting_for_previous_detail: None,
1416 crash_restarts: VecDeque::new(),
1417 lifetime_restarts: 0,
1418 spawn_generation: 0,
1419 pid: None,
1420 #[cfg(target_os = "macos")]
1421 report_ready: None,
1422 reaped_pid: None,
1423 respawn_pending: false,
1424 coalesced_restart_pending: false,
1425 spawned_at_ms: None,
1426 spawned_from: None,
1427 launch_spec_digest: None,
1428 spawned_file_identity: None,
1429 process_start_time: None,
1430 deliberate_severance: None,
1431 last_exit: None,
1432 drain_disposition_detail: None,
1433 health: ModuleHealthStatus::default(),
1434 in_alternate_slot: false,
1435 draining_to_replace: false,
1436 configuration_updated_since_spawn: false,
1437 }
1438 }
1439}
1440
1441type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1442
1443type SpawnSubscriberKey = (ConnectionId, u64);
1444
1445#[derive(Debug)]
1446struct SpawnSubscriber {
1447 version: u8,
1448 frames: mpsc::Sender<Frame>,
1449 lagged: Option<oneshot::Sender<SpawnCursor>>,
1453}
1454
1455#[derive(Debug)]
1456struct SpawnEventState {
1457 daemon_incarnation: String,
1458 seq: u64,
1459 capacity: usize,
1460 live: HashMap<String, LiveSpawn>,
1461 generations: HashMap<String, u64>,
1462 events: VecDeque<SpawnEvent>,
1463 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1464}
1465
1466impl Default for SpawnEventState {
1467 fn default() -> Self {
1468 Self {
1469 daemon_incarnation: "unconfigured".to_string(),
1470 seq: 0,
1471 capacity: SPAWN_EVENT_RING_CAPACITY,
1472 live: HashMap::new(),
1473 generations: HashMap::new(),
1474 events: VecDeque::new(),
1475 subscribers: HashMap::new(),
1476 }
1477 }
1478}
1479
1480#[derive(Debug, Clone, Default)]
1481struct SpawnEventFeed(
1482 Arc<Mutex<SpawnEventState>>,
1483 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1484);
1485
1486#[derive(Debug, Clone, PartialEq, Eq)]
1487pub(crate) enum SpawnSubscribeRefusal {
1488 ForeignIncarnation { current: String },
1489 TooOld { oldest: SpawnCursor },
1490 Frame(String),
1491}
1492
1493impl SpawnEventFeed {
1494 fn configure_incarnation(&self, daemon_incarnation: String) {
1495 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1496 state.daemon_incarnation = daemon_incarnation;
1497 state.seq = 0;
1498 state.live.clear();
1499 state.generations.clear();
1500 state.events.clear();
1501 state.subscribers.clear();
1502 }
1503
1504 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1505 SpawnCursor {
1506 daemon_incarnation: state.daemon_incarnation.clone(),
1507 seq: state.seq,
1508 }
1509 }
1510
1511 fn snapshot(&self) -> SpawnSnapshot {
1512 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1513 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1514 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1515 SpawnSnapshot {
1516 cursor: Self::cursor(&state),
1517 ring_bound: state.capacity as u64,
1518 live,
1519 }
1520 }
1521
1522 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1523 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1524 let generation = state
1525 .generations
1526 .get(module_id)
1527 .copied()
1528 .unwrap_or(0)
1529 .checked_add(1)
1530 .expect("spawn generation exhausted");
1531 state.generations.insert(module_id.to_string(), generation);
1532 let live = LiveSpawn {
1533 module_id: module_id.to_string(),
1534 spawn_generation: generation,
1535 pid,
1536 spawned_at_ms,
1537 };
1538 state.live.insert(module_id.to_string(), live);
1539 Self::emit_locked(
1540 &mut state,
1541 SpawnEventKind::Spawned,
1542 module_id.to_string(),
1543 generation,
1544 pid,
1545 None,
1546 None,
1547 );
1548 generation
1549 }
1550
1551 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1552 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1553 let Some(live) = state.live.remove(module_id) else {
1554 warn!(
1555 module_id,
1556 "terminal record had no live spawn event identity"
1557 );
1558 return;
1559 };
1560 Self::emit_locked(
1561 &mut state,
1562 SpawnEventKind::Exited,
1563 module_id.to_string(),
1564 live.spawn_generation,
1565 live.pid,
1566 exit_code,
1567 exit_signal,
1568 );
1569 }
1570
1571 fn emit_superseded_exited(
1578 &self,
1579 module_id: &str,
1580 spawn_generation: u64,
1581 pid: u32,
1582 exit_code: Option<i32>,
1583 exit_signal: Option<i32>,
1584 ) {
1585 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1586 if state
1587 .live
1588 .get(module_id)
1589 .is_some_and(|live| live.spawn_generation == spawn_generation)
1590 {
1591 state.live.remove(module_id);
1592 }
1593 Self::emit_locked(
1594 &mut state,
1595 SpawnEventKind::Exited,
1596 module_id.to_string(),
1597 spawn_generation,
1598 pid,
1599 exit_code,
1600 exit_signal,
1601 );
1602 }
1603
1604 #[allow(clippy::too_many_arguments)]
1605 fn emit_locked(
1606 state: &mut SpawnEventState,
1607 kind: SpawnEventKind,
1608 module_id: String,
1609 spawn_generation: u64,
1610 pid: u32,
1611 exit_code: Option<i32>,
1612 exit_signal: Option<i32>,
1613 ) {
1614 state.seq = state
1615 .seq
1616 .checked_add(1)
1617 .expect("spawn event sequence exhausted");
1618 let event = SpawnEvent {
1619 cursor: Self::cursor(state),
1620 kind,
1621 module_id,
1622 spawn_generation,
1623 pid,
1624 exit_code,
1625 exit_signal,
1626 };
1627 state.events.push_back(event.clone());
1628 while state.events.len() > state.capacity {
1629 state.events.pop_front();
1630 }
1631 let body = match serde_json::to_vec(&event) {
1632 Ok(body) => body,
1633 Err(error) => {
1634 error!(%error, "failed to serialize supervisor spawn event");
1635 return;
1636 }
1637 };
1638 state.subscribers.retain(|(connection_id, corr), subscriber| {
1639 let frame = Frame::build_with_version(
1640 subscriber.version,
1641 FrameType::StreamData,
1642 control_flags(),
1643 0,
1644 0,
1645 *corr,
1646 body.clone(),
1647 );
1648 match frame {
1649 Ok(frame) => {
1650 if subscriber.frames.try_send(frame).is_ok() {
1651 true
1652 } else {
1653 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1654 if let Some(lagged) = subscriber.lagged.take() {
1655 let _ = lagged.send(event.cursor.clone());
1656 }
1657 false
1658 }
1659 }
1660 Err(error) => {
1661 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1662 false
1663 }
1664 }
1665 });
1666 }
1667
1668 fn subscribe(
1669 &self,
1670 connection_id: ConnectionId,
1671 corr: u64,
1672 version: u8,
1673 since: Option<SpawnCursor>,
1674 sink: FrameSink,
1675 ) -> Result<(), SpawnSubscribeRefusal> {
1676 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1677 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1678 {
1679 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1680 let replay = if let Some(since) = since {
1681 if since.daemon_incarnation != state.daemon_incarnation {
1682 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1683 current: state.daemon_incarnation.clone(),
1684 });
1685 }
1686 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1687 if since.seq < oldest.seq.saturating_sub(1) {
1688 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1689 }
1690 }
1691 state
1692 .events
1693 .iter()
1694 .filter(|event| event.cursor.seq > since.seq)
1695 .cloned()
1696 .collect::<Vec<_>>()
1697 } else {
1698 Vec::new()
1699 };
1700 for event in replay {
1701 let body = serde_json::to_vec(&event)
1702 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1703 let frame = Frame::build_with_version(
1704 version,
1705 FrameType::StreamData,
1706 control_flags(),
1707 0,
1708 0,
1709 corr,
1710 body,
1711 )
1712 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1713 frames
1714 .try_send(frame)
1715 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1716 }
1717 state.subscribers.insert(
1718 (connection_id, corr),
1719 SpawnSubscriber {
1720 version,
1721 frames,
1722 lagged: Some(lagged),
1723 },
1724 );
1725 }
1726 tokio::spawn(async move {
1737 while let Some(frame) = receiver.recv().await {
1738 if sink.send(frame).await.is_err() {
1739 return;
1740 }
1741 }
1742 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1743 return;
1744 };
1745 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1746 Ok(frame) => {
1747 let _ = sink.send(frame).await;
1748 }
1749 Err(error) => {
1750 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1751 }
1752 }
1753 });
1754 Ok(())
1755 }
1756
1757 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1758 let Some(subscriber) = self
1759 .0
1760 .lock()
1761 .unwrap_or_else(|p| p.into_inner())
1762 .subscribers
1763 .remove(&(connection_id, corr))
1764 else {
1765 return false;
1766 };
1767 if let Ok(frame) = Frame::build_with_version(
1768 subscriber.version,
1769 FrameType::StreamEnd,
1770 control_flags(),
1771 0,
1772 0,
1773 corr,
1774 Vec::new(),
1775 ) {
1776 tokio::spawn(async move {
1777 let _ = subscriber.frames.send(frame).await;
1778 });
1779 }
1780 true
1781 }
1782
1783 fn remove_connection(&self, connection_id: ConnectionId) {
1784 self.0
1785 .lock()
1786 .unwrap_or_else(|p| p.into_inner())
1787 .subscribers
1788 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1789 }
1790
1791 #[cfg(any(test, feature = "test-support"))]
1792 fn set_capacity(&self, capacity: usize) {
1793 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1794 }
1795
1796 #[cfg(any(test, feature = "test-support"))]
1797 fn subscriber_count(&self) -> usize {
1798 self.0
1799 .lock()
1800 .unwrap_or_else(|p| p.into_inner())
1801 .subscribers
1802 .len()
1803 }
1804}
1805
1806fn spawn_subscriber_lagged_frame(
1809 version: u8,
1810 corr: u64,
1811 first_undelivered: SpawnCursor,
1812) -> Result<Frame, String> {
1813 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1815 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1816 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1817 .to_string(),
1818 detail: Some(serde_json::json!({
1819 "first_undelivered_cursor": first_undelivered
1820 })),
1821 })
1822 .map_err(|error| error.to_string())?;
1823 Frame::build_with_version(
1824 version,
1825 FrameType::Error,
1826 crate::router::daemon_error_flags(),
1827 0,
1828 0,
1829 corr,
1830 body,
1831 )
1832 .map_err(|error| error.to_string())
1833}
1834
1835pub trait ModuleProcessLiveness: Send + Sync {
1836 fn process_live(&self, module_id: &str) -> Option<bool>;
1837
1838 fn process_replacing(&self, _module_id: &str) -> bool {
1844 false
1845 }
1846}
1847
1848#[derive(Debug, Clone, Default)]
1850pub struct SupervisorProcessLiveness {
1851 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1852}
1853
1854impl SupervisorProcessLiveness {
1855 pub fn new() -> Self {
1856 Self::default()
1857 }
1858
1859 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1860 let mut snapshots = self
1861 .snapshots
1862 .lock()
1863 .unwrap_or_else(|poisoned| poisoned.into_inner());
1864 snapshots.insert(module_id, snapshot);
1865 }
1866
1867 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1868 let mut snapshots = self
1869 .snapshots
1870 .lock()
1871 .unwrap_or_else(|poisoned| poisoned.into_inner());
1872 let is_current = snapshots
1873 .get(module_id)
1874 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1875 .unwrap_or(false);
1876 if is_current {
1877 snapshots.remove(module_id);
1878 }
1879 }
1880}
1881
1882impl ModuleProcessLiveness for SupervisorProcessLiveness {
1883 fn process_live(&self, module_id: &str) -> Option<bool> {
1884 let snapshot = {
1885 let snapshots = self
1886 .snapshots
1887 .lock()
1888 .unwrap_or_else(|poisoned| poisoned.into_inner());
1889 snapshots.get(module_id).cloned()
1890 }?;
1891 let snapshot = snapshot
1892 .lock()
1893 .unwrap_or_else(|poisoned| poisoned.into_inner());
1894 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1895 }
1896
1897 fn process_replacing(&self, module_id: &str) -> bool {
1898 let Some(snapshot) = self
1899 .snapshots
1900 .lock()
1901 .unwrap_or_else(|poisoned| poisoned.into_inner())
1902 .get(module_id)
1903 .cloned()
1904 else {
1905 return false;
1906 };
1907 let snapshot = snapshot
1908 .lock()
1909 .unwrap_or_else(|poisoned| poisoned.into_inner());
1910 snapshot.enabled
1911 && match snapshot.state {
1912 ModuleState::Restarting => true,
1913 ModuleState::Draining => snapshot.draining_to_replace,
1914 ModuleState::Starting
1915 | ModuleState::Running
1916 | ModuleState::Unresponsive
1917 | ModuleState::Stopped
1918 | ModuleState::Failed
1919 | ModuleState::Disabled => false,
1920 }
1921 }
1922}
1923
1924#[cfg(test)]
1925#[derive(Debug, Default)]
1926struct ReloadExitRecordGate {
1927 reached: tokio::sync::Notify,
1928 resume: tokio::sync::Notify,
1929}
1930
1931#[derive(Debug, Clone, Copy)]
1932enum RespawnKind {
1933 Spawn,
1934 Reload,
1935}
1936
1937#[derive(Debug, Clone, Copy)]
1938struct PendingRespawn {
1939 deadline: Instant,
1940 kind: RespawnKind,
1941 daemon_initiated: bool,
1942}
1943
1944type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1945
1946#[derive(Debug, Clone)]
1947struct SupervisorRuntimeConfig {
1948 registry: Arc<Registry>,
1949 recovery_is_daemon: bool,
1951 start_requester: Option<Principal>,
1952 pending_activity_restart: bool,
1953 provenance_probe: ExecutableIdentityProbe,
1954 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1956 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1958 restart_policy: RestartPolicy,
1959 register_timeout: Duration,
1960 drain_timeout: Duration,
1963 effective_drain_timeout: Arc<Mutex<Duration>>,
1966 default_drain_timeout: Duration,
1969 health: HealthConfig,
1970 connection_file_path: Option<PathBuf>,
1971 capture_logs_dir: Option<PathBuf>,
1972 forwarding: Option<Arc<ForwardingTable>>,
1973 supervisor_handle: Option<SupervisorHandle>,
1976 stderr_ring: Arc<Mutex<StderrRing>>,
1983 terminal_ring: Arc<Mutex<TerminalRing>>,
1984 spawn_events: SpawnEventFeed,
1985 child_roster: ChildRoster,
1986 #[cfg(target_os = "linux")]
1987 cgroup_placement: Option<subc_cgroup::Placement>,
1988 #[cfg(test)]
1989 test_seed_stale_facts_before_enable_spawn: bool,
1990 #[cfg(test)]
1991 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1992}
1993
1994impl SupervisorRuntimeConfig {
1995 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
1996 FailureContext {
1997 daemon_initiated,
1998 shutdown: self.child_roster.shutdown_flag(),
1999 }
2000 }
2001}
2002
2003#[derive(Debug, Clone, PartialEq, Eq)]
2004struct SupervisedConfiguration {
2005 spec: ModuleSpec,
2006 health: HealthConfig,
2007}
2008
2009#[derive(Debug, Clone, Default)]
2015pub struct SupervisorHandle {
2016 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2017 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2018 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2019 configured_ids: Arc<Mutex<HashSet<String>>>,
2031 spawn_events: SpawnEventFeed,
2032 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2043 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2049 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2053 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2061 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2067 promotion_observer: PromotionObserverSlot,
2069 operation_lock: Arc<AsyncMutex<()>>,
2073}
2074
2075pub(crate) trait SwapPromotionObserver: Send + Sync {
2084 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2085}
2086
2087#[derive(Clone, Default)]
2091struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2092
2093impl fmt::Debug for PromotionObserverSlot {
2094 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2095 f.write_str("PromotionObserverSlot")
2096 }
2097}
2098
2099#[derive(Debug, Clone)]
2101struct OpenSwap {
2102 candidate_nonce: String,
2105 incumbent_nonce: Option<String>,
2110 candidate_admitted: bool,
2114}
2115
2116#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2119pub(crate) enum SwapHelloAdmission {
2120 NotSwapping,
2123 Candidate,
2125 Refused,
2128}
2129
2130#[derive(Debug, Clone, PartialEq, Eq)]
2131pub(crate) enum ReservedHelloRejection {
2132 Exact {
2133 module_id: String,
2134 },
2135 Prefix {
2136 prefix: String,
2137 owner_module_id: String,
2138 },
2139}
2140
2141impl SupervisorHandle {
2142 pub fn new() -> Self {
2143 Self::default()
2144 }
2145
2146 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2147 self.spawn_events.snapshot()
2148 }
2149
2150 pub(crate) fn subscribe_spawns(
2151 &self,
2152 connection_id: ConnectionId,
2153 corr: u64,
2154 version: u8,
2155 since: Option<SpawnCursor>,
2156 sink: FrameSink,
2157 ) -> Result<(), SpawnSubscribeRefusal> {
2158 self.spawn_events
2159 .subscribe(connection_id, corr, version, since, sink)
2160 }
2161
2162 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2163 self.spawn_events.cancel(connection_id, corr)
2164 }
2165
2166 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2167 self.spawn_events.remove_connection(connection_id);
2168 }
2169
2170 #[cfg(any(test, feature = "test-support"))]
2171 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2172 assert!(capacity > 0, "spawn event capacity must be non-zero");
2173 self.spawn_events.set_capacity(capacity);
2174 }
2175
2176 #[cfg(any(test, feature = "test-support"))]
2177 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2178 self.spawn_events.subscriber_count()
2179 }
2180
2181 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2184 self.spawn_nonces
2185 .lock()
2186 .unwrap_or_else(|poisoned| poisoned.into_inner())
2187 .insert(module_id.to_string(), nonce);
2188 }
2189
2190 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2193 self.reserved_nonces
2194 .lock()
2195 .unwrap_or_else(|poisoned| poisoned.into_inner())
2196 .insert(module_id.to_string(), Some(nonce));
2197 }
2198
2199 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2201 let mut owners = self
2202 .reserved_prefix_owners
2203 .lock()
2204 .unwrap_or_else(|poisoned| poisoned.into_inner());
2205 owners.retain(|_, owner| owner != owner_module_id);
2206 for prefix in prefixes {
2207 owners.insert(prefix.clone(), owner_module_id.to_string());
2208 }
2209 }
2210
2211 #[cfg(test)]
2213 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2214 self.spawn_nonces
2215 .lock()
2216 .unwrap_or_else(|poisoned| poisoned.into_inner())
2217 .get(module_id)
2218 .cloned()
2219 }
2220
2221 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2222 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2223 let spawn_nonce = self
2224 .spawn_nonces
2225 .lock()
2226 .unwrap_or_else(|poisoned| poisoned.into_inner())
2227 .get(&spec.module_id)
2228 .cloned();
2229 let mut reserved_nonces = self
2230 .reserved_nonces
2231 .lock()
2232 .unwrap_or_else(|poisoned| poisoned.into_inner());
2233 if spec.reserved {
2234 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2239 }
2240 drop(reserved_nonces);
2241 self.removal_tombstones
2245 .lock()
2246 .unwrap_or_else(|poisoned| poisoned.into_inner())
2247 .remove(&spec.module_id);
2248 }
2249
2250 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2255 self.reserved_hello_verification(module_id, presented)
2256 .is_ok()
2257 }
2258
2259 pub(crate) fn reserved_hello_verification(
2262 &self,
2263 module_id: &str,
2264 presented: Option<&str>,
2265 ) -> Result<bool, ReservedHelloRejection> {
2266 let nonces = self
2267 .reserved_nonces
2268 .lock()
2269 .unwrap_or_else(|poisoned| poisoned.into_inner());
2270 if let Some(expected) = nonces.get(module_id) {
2271 let authorized = match expected {
2275 Some(expected) => {
2276 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2277 }
2278 None => false,
2279 };
2280 if authorized {
2281 return Ok(true);
2282 }
2283 return Err(ReservedHelloRejection::Exact {
2284 module_id: module_id.to_string(),
2285 });
2286 }
2287 drop(nonces);
2288
2289 let matched_prefix = self
2290 .reserved_prefix_owners
2291 .lock()
2292 .unwrap_or_else(|poisoned| poisoned.into_inner())
2293 .iter()
2294 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2295 .max_by_key(|(prefix, _)| prefix.len())
2296 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2297 let Some((prefix, owner_module_id)) = matched_prefix else {
2298 return Ok(false);
2299 };
2300
2301 let authorized = presented.is_some_and(|presented| {
2302 self.spawn_nonces
2303 .lock()
2304 .unwrap_or_else(|poisoned| poisoned.into_inner())
2305 .get(&owner_module_id)
2306 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2307 || self.swap_nonce_matches(&owner_module_id, presented)
2310 });
2311 if authorized {
2312 Ok(true)
2313 } else {
2314 Err(ReservedHelloRejection::Prefix {
2315 prefix,
2316 owner_module_id,
2317 })
2318 }
2319 }
2320
2321 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2326 if presented.is_empty() {
2327 return false;
2328 }
2329 let nonces = self
2330 .spawn_nonces
2331 .lock()
2332 .unwrap_or_else(|poisoned| poisoned.into_inner());
2333 let current = nonces
2334 .get(module_id)
2335 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2336 drop(nonces);
2337 current || self.swap_nonce_matches(module_id, presented)
2342 }
2343
2344 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2346 let swaps = self
2347 .swaps
2348 .lock()
2349 .unwrap_or_else(|poisoned| poisoned.into_inner());
2350 swaps.get(module_id).is_some_and(|swap| {
2351 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2352 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2353 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2354 })
2355 })
2356 }
2357
2358 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2361 let incumbent_nonce = self
2362 .spawn_nonces
2363 .lock()
2364 .unwrap_or_else(|poisoned| poisoned.into_inner())
2365 .get(module_id)
2366 .cloned();
2367 self.swaps
2368 .lock()
2369 .unwrap_or_else(|poisoned| poisoned.into_inner())
2370 .insert(
2371 module_id.to_string(),
2372 OpenSwap {
2373 candidate_nonce,
2374 incumbent_nonce,
2375 candidate_admitted: false,
2376 },
2377 );
2378 }
2379
2380 pub(crate) fn close_swap(&self, module_id: &str) {
2383 self.swaps
2384 .lock()
2385 .unwrap_or_else(|poisoned| poisoned.into_inner())
2386 .remove(module_id);
2387 }
2388
2389 pub(crate) fn set_swap_promotion_observer(
2392 &self,
2393 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2394 ) {
2395 *self
2396 .promotion_observer
2397 .0
2398 .lock()
2399 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2400 }
2401
2402 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2405 let observer = self
2406 .promotion_observer
2407 .0
2408 .lock()
2409 .unwrap_or_else(|poisoned| poisoned.into_inner())
2410 .as_ref()
2411 .and_then(std::sync::Weak::upgrade);
2412 if let Some(observer) = observer {
2413 observer.swap_promoted(registration);
2414 }
2415 }
2416
2417 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2419 self.swaps
2420 .lock()
2421 .unwrap_or_else(|poisoned| poisoned.into_inner())
2422 .contains_key(module_id)
2423 }
2424
2425 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2430 let candidate_nonce = self
2431 .swaps
2432 .lock()
2433 .unwrap_or_else(|poisoned| poisoned.into_inner())
2434 .get(module_id)
2435 .map(|swap| swap.candidate_nonce.clone());
2436 let Some(nonce) = candidate_nonce else {
2437 return;
2438 };
2439 self.set_spawn_nonce(module_id, nonce.clone());
2440 if reserved {
2441 self.set_reserved_nonce(module_id, nonce);
2442 }
2443 }
2444
2445 pub(crate) fn swap_hello_admission(
2460 &self,
2461 module_id: &str,
2462 presented: Option<&str>,
2463 ) -> SwapHelloAdmission {
2464 let swaps = self
2465 .swaps
2466 .lock()
2467 .unwrap_or_else(|poisoned| poisoned.into_inner());
2468 let Some(swap) = swaps.get(module_id) else {
2469 return SwapHelloAdmission::NotSwapping;
2470 };
2471 let Some(presented) = presented else {
2472 return SwapHelloAdmission::Refused;
2473 };
2474 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2475 return if swap.candidate_admitted {
2476 SwapHelloAdmission::Refused
2477 } else {
2478 SwapHelloAdmission::Candidate
2479 };
2480 }
2481 if swap
2482 .incumbent_nonce
2483 .as_deref()
2484 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2485 {
2486 return SwapHelloAdmission::NotSwapping;
2487 }
2488 SwapHelloAdmission::Refused
2489 }
2490
2491 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2494 if let Some(swap) = self
2495 .swaps
2496 .lock()
2497 .unwrap_or_else(|poisoned| poisoned.into_inner())
2498 .get_mut(module_id)
2499 {
2500 swap.candidate_admitted = true;
2501 }
2502 }
2503
2504 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2506 self.spawn_nonces
2507 .lock()
2508 .unwrap_or_else(|poisoned| poisoned.into_inner())
2509 .get(module_id)
2510 .cloned()
2511 }
2512
2513 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2515 self.reserved_nonces
2516 .lock()
2517 .unwrap_or_else(|poisoned| poisoned.into_inner())
2518 .get(module_id)
2519 .cloned()
2520 .flatten()
2521 }
2522
2523 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2524 self.mark_configured(module.module_id());
2528 let mut modules = self
2529 .modules
2530 .lock()
2531 .unwrap_or_else(|poisoned| poisoned.into_inner());
2532 modules.insert(module.module_id().to_string(), module)
2533 }
2534
2535 fn mark_configured(&self, module_id: &str) {
2539 self.configured_ids
2540 .lock()
2541 .unwrap_or_else(|poisoned| poisoned.into_inner())
2542 .insert(module_id.to_string());
2543 }
2544
2545 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2549 let modules = self
2550 .modules
2551 .lock()
2552 .unwrap_or_else(|poisoned| poisoned.into_inner());
2553 if !modules.contains_key(module_id) {
2554 self.configured_ids
2555 .lock()
2556 .unwrap_or_else(|poisoned| poisoned.into_inner())
2557 .remove(module_id);
2558 }
2559 }
2560
2561 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2569 self.configured_ids
2570 .lock()
2571 .unwrap_or_else(|poisoned| poisoned.into_inner())
2572 .contains(module_id)
2573 }
2574
2575 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2576 let modules = self
2577 .modules
2578 .lock()
2579 .unwrap_or_else(|poisoned| poisoned.into_inner());
2580 modules.get(module_id).cloned()
2581 }
2582
2583 pub(crate) fn record_late_health_answer(
2584 &self,
2585 module_id: &str,
2586 latency_ms: u64,
2587 ) -> Result<bool, SuperviseError> {
2588 let Some(module) = self.get(module_id) else {
2589 return Ok(false);
2590 };
2591 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2592 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2593 state.health.last_late_answer_latency_ms = Some(latency_ms);
2594 state.health.consecutive_failures = 0;
2602 })?;
2603 Ok(true)
2604 }
2605
2606 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2612 let Some(module) = self.get(module_id) else {
2613 return Ok(false);
2614 };
2615 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2616 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2617 return Ok(false);
2618 };
2619 drop(snapshot);
2620 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2621 }
2622
2623 pub fn list(&self) -> Vec<SupervisedModule> {
2624 let modules = self
2625 .modules
2626 .lock()
2627 .unwrap_or_else(|poisoned| poisoned.into_inner());
2628 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2629 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2630 modules
2631 }
2632
2633 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2634 self.activity_images
2635 .lock()
2636 .unwrap_or_else(|p| p.into_inner())
2637 .remove(module_id);
2638 self.spawn_nonces
2639 .lock()
2640 .unwrap_or_else(|poisoned| poisoned.into_inner())
2641 .remove(module_id);
2642 self.close_swap(module_id);
2643 let mut reserved_nonces = self
2644 .reserved_nonces
2645 .lock()
2646 .unwrap_or_else(|poisoned| poisoned.into_inner());
2647 if reserved_nonces.contains_key(module_id) {
2648 reserved_nonces.insert(module_id.to_string(), None);
2651 }
2652 drop(reserved_nonces);
2653 self.reserved_prefix_owners
2654 .lock()
2655 .unwrap_or_else(|poisoned| poisoned.into_inner())
2656 .retain(|_, owner| owner != module_id);
2657 let removed = self
2658 .modules
2659 .lock()
2660 .unwrap_or_else(|poisoned| poisoned.into_inner())
2661 .remove(module_id);
2662 self.configured_ids
2663 .lock()
2664 .unwrap_or_else(|poisoned| poisoned.into_inner())
2665 .remove(module_id);
2666 removed
2667 }
2668
2669 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2672 self.removal_tombstones
2673 .lock()
2674 .unwrap_or_else(|poisoned| poisoned.into_inner())
2675 .insert(module_id.to_string(), unix_ms_now());
2676 }
2677
2678 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2680 self.removal_tombstones
2681 .lock()
2682 .unwrap_or_else(|poisoned| poisoned.into_inner())
2683 .get(module_id)
2684 .copied()
2685 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2686 }
2687
2688 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2693 if self.get(module_id).is_some() {
2694 return false;
2695 }
2696 let mut reserved_nonces = self
2697 .reserved_nonces
2698 .lock()
2699 .unwrap_or_else(|poisoned| poisoned.into_inner());
2700 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2701 return false;
2702 }
2703 reserved_nonces.remove(module_id);
2704 true
2705 }
2706
2707 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2708 Arc::clone(&self.operation_lock)
2709 }
2710}
2711
2712#[derive(Debug, Clone)]
2714pub struct Supervisor {
2715 registry: Arc<Registry>,
2716 restart_policy: RestartPolicy,
2717 drain_timeout: Duration,
2718 connection_file_path: Option<PathBuf>,
2719 capture_logs_dir: Option<PathBuf>,
2720 forwarding: Option<Arc<ForwardingTable>>,
2721 process_liveness: Arc<SupervisorProcessLiveness>,
2722 supervisor_handle: Option<SupervisorHandle>,
2723 health: HealthConfig,
2724 daemon_start_clock: crate::clock::StartClock,
2725 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2726 spawn_events: SpawnEventFeed,
2727 provenance_probe: ExecutableIdentityProbe,
2728 child_roster: ChildRoster,
2731 #[cfg(target_os = "linux")]
2732 cgroup_placement: Option<subc_cgroup::Placement>,
2733 #[cfg(test)]
2734 test_after_first_spawn: AfterFirstSpawnHook,
2735}
2736
2737pub(crate) struct RegisteredConfiguredModule {
2741 spec: ModuleSpec,
2742 runtime: SupervisorRuntimeConfig,
2743 module: SupervisedModule,
2744 commands: mpsc::Receiver<SupervisorCommand>,
2745}
2746
2747#[cfg(test)]
2753#[derive(Clone, Default)]
2754struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2755
2756#[cfg(test)]
2757type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2758
2759#[cfg(test)]
2760impl fmt::Debug for AfterFirstSpawnHook {
2761 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2762 f.write_str("AfterFirstSpawnHook")
2763 }
2764}
2765
2766#[cfg(test)]
2767impl AfterFirstSpawnHook {
2768 fn run(&self, module_id: &str) {
2769 if let Some(hook) = &self.0 {
2770 hook(module_id);
2771 }
2772 }
2773}
2774
2775impl Supervisor {
2776 #[cfg(test)]
2777 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2778 let supervisor = Self::new(registry, policy);
2779 #[cfg(target_os = "macos")]
2780 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2781 supervisor
2782 }
2783 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2788 let path = path.into();
2789 #[cfg(target_os = "macos")]
2790 {
2791 let result = probe_privacy_trampoline(&path).map(|()| path);
2792 if let Err(cause) = &result {
2793 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2794 }
2795 self.child_roster.set_privacy_trampoline(result);
2796 }
2797 #[cfg(not(target_os = "macos"))]
2798 let _ = path;
2799 self
2800 }
2801 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2804 self.child_roster.shutdown_flag()
2805 }
2806
2807 #[cfg(unix)]
2818 pub(crate) fn begin_daemon_shutdown(&self) {
2819 self.child_roster.close();
2820 if let Some(journal) = &self.terminal_journal {
2821 journal.stamp_shutdown();
2822 }
2823 }
2824
2825 #[cfg(unix)]
2829 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2830 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2831 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2832 let Some(forwarding) = &self.forwarding else {
2833 return Ok(());
2834 };
2835 let notice_started = Instant::now();
2836 let modules = forwarding
2837 .begin_daemon_drain()
2838 .map_err(SuperviseError::Forwarding)?;
2839 let deadline_ms =
2840 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2841 let mut notices = tokio::task::JoinSet::new();
2842 let mut drains = Vec::new();
2843 for (module_id, endpoint) in modules {
2844 let Some(target) = forwarding
2845 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2846 .map_err(SuperviseError::Forwarding)?
2847 else {
2848 continue;
2849 };
2850 let routes = forwarding
2851 .endpoint_routes(target.endpoint)
2852 .map_err(SuperviseError::Forwarding)?;
2853 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2859 reason: RouteCloseReason::Restart,
2860 deadline_ms,
2861 })
2862 .expect("module draining serializes");
2863 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2864 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2865 for route in routes {
2866 let client = route.goodbye_target;
2867 if let Some((_, channels)) = clients
2868 .iter_mut()
2869 .find(|(existing, _)| existing.connection_id == client.connection_id)
2870 {
2871 channels.push(client.channel);
2872 } else {
2873 let channel = client.channel;
2874 clients.push((client, vec![channel]));
2875 }
2876 }
2877 for (client, mut channels) in clients {
2878 channels.sort_unstable();
2879 channels.dedup();
2880 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2881 module_id: module_id.clone(),
2882 channels,
2883 reason: RouteCloseReason::Restart,
2884 })
2885 .expect("route closing serializes");
2886 recipients.push((client.sink, client.negotiated_ver, closing));
2887 }
2888 for (sink, version, body) in recipients {
2889 notices.spawn(async move {
2890 let frame = Frame::build_with_version(
2891 version,
2892 FrameType::Push,
2893 control_flags(),
2894 0,
2895 0,
2896 0,
2897 body,
2898 )
2899 .expect("bounded lifecycle notice frame builds");
2900 sink.send_flushed(frame).await
2901 });
2902 }
2903 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2904 drains.push((module_id, target.endpoint, gauges));
2905 }
2906 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2909 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2910 if !matches!(result, Ok(Ok(()))) {
2911 warn!(?result, "daemon shutdown notice delivery failed");
2912 }
2913 }
2914 notices.abort_all();
2915 let notice_elapsed = notice_started.elapsed();
2916 let drain_started = Instant::now();
2917 let deadline = drain_started + DRAIN_BUDGET;
2918 let mut waits = tokio::task::JoinSet::new();
2919 for (module_id, endpoint, gauges) in drains {
2920 let forwarding = Arc::clone(forwarding);
2921 let mut runtime = self.runtime_config();
2922 runtime.health.cadence = Duration::from_millis(100);
2923 waits.spawn(async move {
2924 let wait_result = wait_for_forwarding_quiescence(
2925 &forwarding,
2926 &module_id,
2927 &runtime,
2928 endpoint,
2929 deadline,
2930 &gauges,
2931 DrainScope::Endpoint(endpoint),
2932 &mut DrainGaugeTrace::new(&gauges),
2933 )
2934 .await;
2935 forwarding
2939 .record_daemon_drain_result(
2940 endpoint,
2941 drained_after_quiescence_wait(&wait_result),
2942 )
2943 .map_err(SuperviseError::Forwarding)?;
2944 wait_result
2945 });
2946 }
2947 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2948 if !matches!(result, Ok(Ok(true))) {
2949 warn!(?result, "daemon shutdown drain did not reach quiescence");
2950 }
2951 }
2952 info!(
2955 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
2956 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
2957 "daemon shutdown forwarding drain complete"
2958 );
2959 Ok(())
2960 }
2961
2962 #[cfg(unix)]
2974 pub(crate) async fn end_children_for_daemon_shutdown(
2975 &self,
2976 already_escalated: bool,
2977 escalate: impl std::future::Future<Output = ()>,
2978 ) {
2979 tokio::pin!(escalate);
2980 let mut escalated = already_escalated;
2981 if let Some(forwarding) = &self.forwarding {
2982 let reason = CloseReason::new(
2983 "daemon_shutdown",
2984 "the daemon is exiting after its shutdown notice and drain",
2985 );
2986 if escalated {
2987 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2990 } else {
2991 tokio::select! {
2992 biased;
2993 _ = escalate.as_mut() => {
2994 info!("second SIGTERM: abandoning module GOODBYE delivery");
2995 escalated = true;
2996 }
2997 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2998 }
2999 }
3000 let closed = forwarding.close_all_connections(&reason);
3001 debug!(closed, "closed established connections for daemon shutdown");
3002 }
3003 let escalated_here = escalated && !already_escalated;
3007 let remaining_escalate = async move {
3008 if escalated_here {
3009 std::future::pending::<()>().await;
3010 } else {
3011 escalate.await;
3012 }
3013 };
3014 crate::child_roster::end_children_for_daemon_shutdown(
3015 &self.child_roster,
3016 escalated,
3017 remaining_escalate,
3018 )
3019 .await;
3020 }
3021
3022 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3023 Self {
3024 registry,
3025 restart_policy,
3026 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3027 connection_file_path: None,
3028 capture_logs_dir: None,
3029 forwarding: None,
3030 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3031 supervisor_handle: None,
3032 health: HealthConfig::default(),
3033 daemon_start_clock: crate::clock::StartClock::capture(),
3034 terminal_journal: None,
3035 spawn_events: SpawnEventFeed::default(),
3036 provenance_probe: ExecutableIdentityProbe::default(),
3037 child_roster: ChildRoster::default(),
3038 #[cfg(target_os = "linux")]
3039 cgroup_placement: None,
3040 #[cfg(test)]
3041 test_after_first_spawn: AfterFirstSpawnHook::default(),
3042 }
3043 }
3044
3045 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3046 self.drain_timeout = drain_timeout;
3047 self
3048 }
3049
3050 pub(crate) fn default_drain_timeout(&self) -> Duration {
3052 self.drain_timeout
3053 }
3054
3055 pub fn with_process_liveness(
3056 mut self,
3057 process_liveness: Arc<SupervisorProcessLiveness>,
3058 ) -> Self {
3059 self.process_liveness = process_liveness;
3060 self
3061 }
3062
3063 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3064 self.connection_file_path = Some(connection_file_path.into());
3065 self
3066 }
3067
3068 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3070 self.capture_logs_dir = Some(logs_dir.into());
3071 self
3072 }
3073
3074 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3077 self.spawn_events.configure_incarnation(daemon_incarnation);
3081 self
3082 }
3083
3084 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3087 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3088 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3089 path,
3090 daemon_incarnation,
3091 )));
3092 this
3093 }
3094
3095 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3096 self.forwarding = Some(forwarding);
3097 self
3098 }
3099
3100 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3101 self.spawn_events = supervisor_handle.spawn_events.clone();
3102 self.supervisor_handle = Some(supervisor_handle);
3103 self
3104 }
3105
3106 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3107 self.health = health;
3108 self
3109 }
3110
3111 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3115 self.child_roster.record_to(path.into());
3116 self
3117 }
3118
3119 #[cfg(target_os = "linux")]
3120 pub fn with_cgroup_placement(
3121 mut self,
3122 cgroup_placement: Option<subc_cgroup::Placement>,
3123 ) -> Self {
3124 self.cgroup_placement = cgroup_placement;
3125 self
3126 }
3127
3128 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3134 validate_spec(&spec)?;
3135 self.establish_identity(&spec);
3136
3137 let runtime = self.runtime_config();
3138 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3139 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3140 let registration_watch = watch_registration(&self.registry, &spec);
3141 let spawned = spawn_child(
3142 &spec,
3143 runtime.connection_file_path.as_deref(),
3144 self.supervisor_handle.as_ref(),
3145 &runtime.stderr_ring,
3146 runtime.capture_logs_dir.as_deref(),
3147 &runtime.child_roster,
3148 #[cfg(target_os = "linux")]
3149 runtime.cgroup_placement.as_ref(),
3150 );
3151 #[cfg(test)]
3152 self.test_after_first_spawn.run(&spec.module_id);
3153 let mut child = match spawned {
3154 Ok(child) => child,
3155 Err(err) => {
3156 self.abandon_unrostered(&spec.module_id);
3159 return Err(err);
3160 }
3161 };
3162 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3163 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3164 if let Some(spawn) = activity_spawn {
3165 spawn.complete(&snapshot);
3166 }
3167
3168 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3169 }
3170
3171 fn establish_identity(&self, spec: &ModuleSpec) {
3184 if let Some(supervisor_handle) = &self.supervisor_handle {
3185 supervisor_handle.apply_identity_configuration(spec);
3186 supervisor_handle.mark_configured(&spec.module_id);
3187 }
3188 }
3189
3190 fn abandon_unrostered(&self, module_id: &str) {
3193 if let Some(supervisor_handle) = &self.supervisor_handle {
3194 supervisor_handle.unmark_configured_unless_rostered(module_id);
3195 }
3196 }
3197
3198 fn mark_first_process_running(
3201 &self,
3202 spec: &ModuleSpec,
3203 runtime: &SupervisorRuntimeConfig,
3204 snapshot: &SharedSnapshot,
3205 child: &mut SupervisedChild,
3206 ) -> Result<(), SuperviseError> {
3207 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3208 self.abandon_unrostered(&spec.module_id);
3209 return Err(err);
3210 }
3211 self.process_liveness
3212 .track(spec.module_id.clone(), Arc::clone(snapshot));
3213 Ok(())
3214 }
3215
3216 pub fn supervise_configured(
3222 &self,
3223 spec: ModuleSpec,
3224 enabled: bool,
3225 ) -> Result<SupervisedModule, SuperviseError> {
3226 self.supervise_configured_requested(
3227 spec,
3228 enabled,
3229 self.health.clone(),
3230 None,
3231 self.restart_policy,
3232 DEFAULT_REGISTER_TIMEOUT,
3233 None,
3234 )
3235 }
3236
3237 pub fn supervise_configured_with_health(
3243 &self,
3244 spec: ModuleSpec,
3245 enabled: bool,
3246 health: HealthConfig,
3247 drain_timeout_ms: Option<u64>,
3248 restart_policy: RestartPolicy,
3249 ) -> Result<SupervisedModule, SuperviseError> {
3250 self.supervise_configured_requested(
3251 spec,
3252 enabled,
3253 health,
3254 drain_timeout_ms,
3255 restart_policy,
3256 DEFAULT_REGISTER_TIMEOUT,
3257 None,
3258 )
3259 }
3260
3261 #[allow(clippy::too_many_arguments)]
3262 pub(crate) fn supervise_configured_requested(
3263 &self,
3264 spec: ModuleSpec,
3265 enabled: bool,
3266 health: HealthConfig,
3267 drain_timeout_ms: Option<u64>,
3268 restart_policy: RestartPolicy,
3269 register_timeout: Duration,
3270 requester: Option<Principal>,
3271 ) -> Result<SupervisedModule, SuperviseError> {
3272 let registered = self.register_configured_with_health(
3273 spec,
3274 enabled,
3275 health,
3276 drain_timeout_ms,
3277 restart_policy,
3278 register_timeout,
3279 )?;
3280 let daemon_initiated = requester.is_none();
3281 self.start_registered_configured(registered, requester, daemon_initiated)
3282 }
3283
3284 pub(crate) fn register_configured_with_health(
3289 &self,
3290 spec: ModuleSpec,
3291 enabled: bool,
3292 health: HealthConfig,
3293 drain_timeout_ms: Option<u64>,
3294 restart_policy: RestartPolicy,
3295 register_timeout: Duration,
3296 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3297 validate_spec(&spec)?;
3298 self.establish_identity(&spec);
3299
3300 let mut runtime = self.runtime_config();
3301 runtime.health = health;
3302 runtime.restart_policy = restart_policy;
3303 runtime.register_timeout = register_timeout;
3304 if let Some(ms) = drain_timeout_ms {
3305 runtime.drain_timeout = Duration::from_millis(ms);
3306 *runtime
3307 .effective_drain_timeout
3308 .lock()
3309 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3310 }
3311 let snapshot = if enabled {
3312 SupervisorSnapshot::starting()
3313 } else {
3314 SupervisorSnapshot::disabled()
3315 };
3316 let snapshot = {
3317 let mut snapshot = snapshot;
3318 if enabled {
3319 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3320 {
3321 snapshot.waiting_for_previous_pid = Some(pid);
3322 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3323 }
3324 }
3325 snapshot
3326 };
3327 let (module, commands) =
3328 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3329 Ok(RegisteredConfiguredModule {
3330 spec,
3331 runtime,
3332 module,
3333 commands,
3334 })
3335 }
3336
3337 pub(crate) fn start_registered_configured(
3340 &self,
3341 registered: RegisteredConfiguredModule,
3342 requester: Option<Principal>,
3343 daemon_initiated: bool,
3344 ) -> Result<SupervisedModule, SuperviseError> {
3345 let RegisteredConfiguredModule {
3346 spec,
3347 mut runtime,
3348 module,
3349 commands,
3350 } = registered;
3351 runtime.start_requester = requester;
3352 runtime.recovery_is_daemon = daemon_initiated;
3353 let snapshot = &module.inner.snapshot;
3354 let should_spawn = {
3355 let state = lock_snapshot(snapshot)?;
3356 state.enabled && state.waiting_for_previous_pid.is_none()
3357 };
3358 let child = if should_spawn {
3359 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3360 let registration_watch = watch_registration(&self.registry, &spec);
3361 let spawned = spawn_child(
3362 &spec,
3363 runtime.connection_file_path.as_deref(),
3364 self.supervisor_handle.as_ref(),
3365 &runtime.stderr_ring,
3366 runtime.capture_logs_dir.as_deref(),
3367 &runtime.child_roster,
3368 #[cfg(target_os = "linux")]
3369 runtime.cgroup_placement.as_ref(),
3370 );
3371 #[cfg(test)]
3372 self.test_after_first_spawn.run(&spec.module_id);
3373 match spawned {
3374 Ok(mut child) => {
3375 arm_registration_deadline(
3376 &mut child,
3377 runtime.register_timeout,
3378 registration_watch,
3379 );
3380 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3381 if let Some(spawn) = activity_spawn {
3382 spawn.complete(snapshot);
3383 }
3384 Some(child)
3385 }
3386 Err(err) => {
3387 if runtime.health.critical {
3388 error!(
3389 module_id = %spec.module_id,
3390 program = %spec.program.display(),
3391 error = %err,
3392 "critical configured module failed to spawn; marking failed and alerting"
3393 );
3394 } else {
3395 error!(
3396 module_id = %spec.module_id,
3397 program = %spec.program.display(),
3398 error = %err,
3399 "configured module failed to spawn; marking failed and continuing"
3400 );
3401 }
3402 fail_snapshot(
3403 snapshot,
3404 Some(&spec.module_id),
3405 None,
3406 &runtime.failure_context(daemon_initiated),
3407 );
3408 None
3409 }
3410 }
3411 } else {
3412 None
3413 };
3414 self.start_monitor(spec, runtime, &module, child, commands);
3415 Ok(module)
3416 }
3417
3418 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3419 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3420 SupervisorRuntimeConfig {
3421 registry: self.registry.clone(),
3422 recovery_is_daemon: true,
3423 start_requester: None,
3424 pending_activity_restart: false,
3425 provenance_probe: self.provenance_probe.clone(),
3426 scheduled_respawn: Arc::default(),
3427 deferred_reload_reply: Arc::default(),
3428 restart_policy: self.restart_policy,
3429 register_timeout: DEFAULT_REGISTER_TIMEOUT,
3430 drain_timeout: self.drain_timeout,
3431 child_roster: self
3434 .child_roster
3435 .for_module(Arc::clone(&effective_drain_timeout)),
3436 effective_drain_timeout,
3437 default_drain_timeout: self.drain_timeout,
3438 health: self.health.clone(),
3439 connection_file_path: self.connection_file_path.clone(),
3440 capture_logs_dir: self.capture_logs_dir.clone(),
3441 forwarding: self.forwarding.clone(),
3442 supervisor_handle: self.supervisor_handle.clone(),
3443 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3444 terminal_ring: Arc::new(Mutex::new(
3445 TerminalRing::new(
3446 TerminalRingConfig::default(),
3447 self.daemon_start_clock.started_at_ms(),
3448 )
3449 .with_start_clock(self.daemon_start_clock)
3450 .with_journal(self.terminal_journal.clone())
3451 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3452 )),
3453 spawn_events: self.spawn_events.clone(),
3454 #[cfg(target_os = "linux")]
3455 cgroup_placement: self.cgroup_placement.clone(),
3456 #[cfg(test)]
3457 test_seed_stale_facts_before_enable_spawn: false,
3458 #[cfg(test)]
3459 test_reload_exit_record_gate: None,
3460 }
3461 }
3462
3463 fn supervised_module(
3464 &self,
3465 spec: ModuleSpec,
3466 runtime: SupervisorRuntimeConfig,
3467 snapshot: SharedSnapshot,
3468 child: Option<SupervisedChild>,
3469 ) -> SupervisedModule {
3470 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3471 self.start_monitor(spec, runtime, &module, child, commands);
3472 module
3473 }
3474
3475 fn register_module(
3476 &self,
3477 spec: &ModuleSpec,
3478 runtime: &SupervisorRuntimeConfig,
3479 snapshot: SharedSnapshot,
3480 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3481 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3482 spec: spec.clone(),
3483 health: runtime.health.clone(),
3484 }));
3485 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3486 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3487 let restart_policy = runtime.restart_policy;
3491 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3492 let (tx, rx) = mpsc::channel(4);
3493 let module_id = spec.module_id.clone();
3494 let module = SupervisedModule {
3495 inner: Arc::new(SupervisedModuleInner {
3496 module_id: module_id.clone(),
3497 registry: Arc::clone(&self.registry),
3498 snapshot,
3499 configuration,
3500 stderr_ring,
3501 terminal_ring,
3502 commands: tx,
3503 monitor: Mutex::new(None),
3504 restart_policy,
3505 effective_drain_timeout,
3506 provenance_probe: self.provenance_probe.clone(),
3507 }),
3508 };
3509 if let Some(supervisor_handle) = &self.supervisor_handle {
3512 supervisor_handle.insert(module.clone());
3513 }
3514 (module, rx)
3515 }
3516
3517 fn start_monitor(
3518 &self,
3519 spec: ModuleSpec,
3520 runtime: SupervisorRuntimeConfig,
3521 module: &SupervisedModule,
3522 child: Option<SupervisedChild>,
3523 commands: mpsc::Receiver<SupervisorCommand>,
3524 ) {
3525 let monitor = tokio::spawn(supervise_loop(
3526 spec,
3527 runtime,
3528 Arc::clone(&self.registry),
3529 Arc::clone(&self.process_liveness),
3530 Arc::clone(&module.inner.snapshot),
3531 child,
3532 commands,
3533 ));
3534 *module
3535 .inner
3536 .monitor
3537 .lock()
3538 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3539 }
3540}
3541
3542impl Default for Supervisor {
3543 fn default() -> Self {
3544 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3545 }
3546}
3547
3548#[derive(Clone)]
3550pub struct SupervisedModule {
3551 inner: Arc<SupervisedModuleInner>,
3552}
3553
3554struct SupervisedModuleInner {
3555 module_id: String,
3556 registry: Arc<Registry>,
3557 snapshot: SharedSnapshot,
3558 configuration: Arc<Mutex<SupervisedConfiguration>>,
3559 stderr_ring: Arc<Mutex<StderrRing>>,
3560 terminal_ring: Arc<Mutex<TerminalRing>>,
3561 commands: mpsc::Sender<SupervisorCommand>,
3562 monitor: Mutex<Option<JoinHandle<()>>>,
3563 restart_policy: RestartPolicy,
3567 effective_drain_timeout: Arc<Mutex<Duration>>,
3568 provenance_probe: ExecutableIdentityProbe,
3569}
3570
3571impl fmt::Debug for SupervisedModule {
3572 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3573 f.debug_struct("SupervisedModule")
3574 .field("module_id", &self.inner.module_id)
3575 .field("status", &self.status())
3576 .finish_non_exhaustive()
3577 }
3578}
3579
3580impl SupervisedModule {
3581 pub fn module_id(&self) -> &str {
3582 &self.inner.module_id
3583 }
3584
3585 #[cfg(test)]
3589 pub(crate) fn record_health_probe_failure_for_test(
3590 &self,
3591 detail: &str,
3592 ) -> Result<(), SuperviseError> {
3593 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3594 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3595 state.health.detail = Some(detail.to_string());
3596 })
3597 }
3598
3599 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3600 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3601 }
3602
3603 pub(crate) fn registration_end_reason(
3604 &self,
3605 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3606 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3607 Ok(match snapshot.state {
3608 ModuleState::Draining if snapshot.draining_to_replace => {
3609 Some(RegistrationEndReason::SupervisorRestart)
3610 }
3611 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3612 ModuleState::Restarting if snapshot.draining_to_replace => {
3613 Some(RegistrationEndReason::SupervisorRestart)
3614 }
3615 _ => None,
3616 })
3617 }
3618
3619 pub fn stderr_tail(
3626 &self,
3627 max_lines: Option<usize>,
3628 max_bytes: Option<usize>,
3629 ) -> StderrTailSnapshot {
3630 self.inner
3631 .stderr_ring
3632 .lock()
3633 .unwrap_or_else(|poisoned| poisoned.into_inner())
3634 .snapshot(max_lines, max_bytes)
3635 }
3636
3637 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3642 self.inner
3643 .terminal_ring
3644 .lock()
3645 .unwrap_or_else(|poisoned| poisoned.into_inner())
3646 .snapshot()
3647 }
3648
3649 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3654 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3655 }
3656
3657 pub(crate) async fn read_durable_terminal_history(
3662 &self,
3663 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3664 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3665 let module_id = self.inner.module_id.clone();
3666 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3667 .await
3668 }
3669
3670 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3671 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3672 .map(|(status, _)| status)
3673 }
3674
3675 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3676 let state = lock_snapshot(&self.inner.snapshot)?;
3677 Ok(state
3678 .failed_retry
3679 .pending
3680 .filter(|_| state.state == ModuleState::Failed && state.enabled)
3681 .map(|retry| {
3682 unix_ms_now().saturating_add(
3683 u64::try_from(
3684 retry
3685 .deadline
3686 .saturating_duration_since(Instant::now())
3687 .as_millis(),
3688 )
3689 .unwrap_or(u64::MAX),
3690 )
3691 }))
3692 }
3693
3694 pub(crate) fn record_deliberate_severance(
3695 &self,
3696 identity: ProcessIdentity,
3697 ) -> Result<bool, SuperviseError> {
3698 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3699 if snapshot.pid != Some(identity.pid)
3700 || snapshot.process_start_time != Some(identity.start_time)
3701 {
3702 return Ok(false);
3703 }
3704 snapshot.deliberate_severance = Some(identity);
3705 Ok(true)
3706 }
3707
3708 pub(crate) fn status_for_control(
3713 &self,
3714 caller: &'static str,
3715 ) -> Result<ModuleStatus, SuperviseError> {
3716 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3717 .map(|(status, _)| status)
3718 }
3719
3720 fn status_with_snapshot_lock(
3721 &self,
3722 snapshot: &SharedSnapshot,
3723 caller: Option<&'static str>,
3724 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3725 let mut guard = match caller {
3726 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3727 None => lock_snapshot(snapshot)?,
3728 };
3729 let restart_count =
3732 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3733 let snapshot = guard.clone();
3734 drop(guard);
3735 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3736 SuperviseError::StatePoisoned {
3737 module_id: Some(self.inner.module_id.clone()),
3738 }
3739 })?;
3740 let registration_active = self
3741 .inner
3742 .registry
3743 .get_module(&self.inner.module_id)
3744 .map_err(SuperviseError::Registry)?
3745 .is_some();
3746 let protocol = snapshot
3747 .spawned_protocol
3748 .unwrap_or(self.declared_protocol()?);
3749 let running_process =
3750 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3751 let live = match protocol {
3757 ModuleProtocol::Subc => running_process && registration_active,
3758 ModuleProtocol::None => running_process,
3759 };
3760
3761 Ok((
3762 ModuleStatus {
3763 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3764 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3765 module_id: self.inner.module_id.clone(),
3766 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3767 state: snapshot.state,
3768 enabled: snapshot.enabled,
3769 process_alive: snapshot.process_alive,
3770 registration_active,
3771 protocol,
3772 live,
3773 restart_count,
3774 lifetime_restarts: snapshot.lifetime_restarts,
3775 spawn_generation: snapshot.spawn_generation,
3776 max_restarts: self.inner.restart_policy.max_restarts,
3777 restart_window: self.inner.restart_policy.window,
3778 drain_timeout,
3779 restart_backoff: self.inner.restart_policy.backoff,
3780 restart_max_backoff: self.inner.restart_policy.max_backoff,
3781 pid: snapshot.reported_pid(),
3782 spawned_at_ms: snapshot.spawned_at_ms,
3783 spawned_from: snapshot.spawned_from,
3784 launch_spec_digest: snapshot.launch_spec_digest,
3785 process_start_time: snapshot.process_start_time,
3786 last_exit: snapshot.last_exit,
3787 health: snapshot.health,
3788 },
3789 snapshot.spawned_file_identity.clone(),
3790 ))
3791 }
3792
3793 #[cfg(test)]
3794 pub(crate) fn hold_snapshot_for_test(
3795 &self,
3796 acquired: std::sync::mpsc::Sender<()>,
3797 hold: Duration,
3798 ) -> std::thread::JoinHandle<()> {
3799 let snapshot = Arc::clone(&self.inner.snapshot);
3800 std::thread::spawn(move || {
3801 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3802 acquired
3803 .send(())
3804 .expect("test receiver waits for snapshot lock");
3805 std::thread::sleep(hold);
3806 })
3807 }
3808
3809 pub(crate) async fn status_and_running_image_agreement(
3815 &self,
3816 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3817 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3818 let image = self
3819 .inner
3820 .provenance_probe
3821 .observe(
3822 status.pid,
3823 status.spawned_from.as_deref(),
3824 identity,
3825 status.process_start_time,
3826 )
3827 .await;
3828 Ok((status, image))
3829 }
3830
3831 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3832 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3833 Ok(snapshot) => snapshot.clone(),
3834 Err(_) => {
3835 return subc_control::RunningImageAgreement::Unavailable {
3836 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3837 };
3838 }
3839 };
3840 self.inner
3841 .provenance_probe
3842 .observe(
3843 snapshot.reported_pid(),
3844 snapshot.spawned_from.as_deref(),
3845 snapshot.spawned_file_identity.clone(),
3846 snapshot.process_start_time,
3847 )
3848 .await
3849 }
3850
3851 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3854 #[cfg(windows)]
3855 {
3856 let image = lock_snapshot(&self.inner.snapshot)
3857 .ok()
3858 .and_then(|snapshot| {
3859 snapshot
3860 .spawned_file_identity
3861 .as_ref()
3862 .and_then(|identity| identity.windows_image.clone())
3863 });
3864 match image {
3865 Some(image) => {
3866 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3867 }
3868 None => subc_control::ChildResourceUsage::Unavailable {
3869 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3870 },
3871 }
3872 }
3873 #[cfg(not(windows))]
3874 {
3875 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3876 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3877 Err(_) => {
3878 return subc_control::ChildResourceUsage::Unavailable {
3879 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3880 }
3881 }
3882 };
3883 crate::child_resources::read(pid, start_time)
3884 }
3885 }
3886
3887 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3888 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3889 Ok(match snapshot.state {
3890 ModuleState::Restarting => true,
3891 ModuleState::Failed | ModuleState::Disabled => false,
3892 _ if snapshot.enabled && snapshot.respawn_pending => true,
3893 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3894 })
3895 }
3896
3897 #[cfg(test)]
3898 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3899 self.is_warming_with_snapshot_lock(None)
3900 }
3901
3902 pub(crate) fn is_warming_for_control(
3903 &self,
3904 caller: &'static str,
3905 ) -> Result<bool, SuperviseError> {
3906 self.is_warming_with_snapshot_lock(Some(caller))
3907 }
3908
3909 fn is_warming_with_snapshot_lock(
3910 &self,
3911 caller: Option<&'static str>,
3912 ) -> Result<bool, SuperviseError> {
3913 let snapshot = match caller {
3914 Some(caller) => {
3915 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3916 }
3917 None => lock_snapshot(&self.inner.snapshot)?,
3918 }
3919 .clone();
3920 Ok(matches!(
3921 snapshot.state,
3922 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3923 ))
3924 }
3925
3926 pub async fn drain(&self) -> Result<(), SuperviseError> {
3928 self.stop().await
3929 }
3930
3931 pub(crate) async fn retire_requested(
3932 &self,
3933 requester: Option<Principal>,
3934 ) -> Result<(), SuperviseError> {
3935 match self.state()? {
3936 ModuleState::Stopped => return Ok(()),
3937 ModuleState::Failed
3938 if lock_snapshot(&self.inner.snapshot)?
3939 .failed_retry
3940 .pending
3941 .is_none() =>
3942 {
3943 return Ok(())
3944 }
3945 ModuleState::Failed
3946 | ModuleState::Starting
3947 | ModuleState::Running
3948 | ModuleState::Unresponsive
3949 | ModuleState::Restarting
3950 | ModuleState::Draining
3951 | ModuleState::Disabled => {}
3952 }
3953
3954 let (reply_tx, reply_rx) = oneshot::channel();
3955 self.inner
3956 .commands
3957 .send(SupervisorCommand::Retire {
3958 requester,
3959 reply: reply_tx,
3960 })
3961 .await
3962 .map_err(|_| SuperviseError::CommandClosed {
3963 module_id: self.inner.module_id.clone(),
3964 })?;
3965 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3966 module_id: self.inner.module_id.clone(),
3967 })?
3968 }
3969
3970 pub async fn stop(&self) -> Result<(), SuperviseError> {
3971 self.stop_requested(None).await
3972 }
3973
3974 pub(crate) async fn stop_requested(
3975 &self,
3976 requester: Option<Principal>,
3977 ) -> Result<(), SuperviseError> {
3978 match self.state()? {
3979 ModuleState::Stopped => return Ok(()),
3980 ModuleState::Failed
3981 if lock_snapshot(&self.inner.snapshot)?
3982 .failed_retry
3983 .pending
3984 .is_none() =>
3985 {
3986 return Ok(())
3987 }
3988 ModuleState::Failed
3989 | ModuleState::Starting
3990 | ModuleState::Running
3991 | ModuleState::Unresponsive
3992 | ModuleState::Restarting
3993 | ModuleState::Draining
3994 | ModuleState::Disabled => {}
3995 }
3996
3997 let (reply_tx, reply_rx) = oneshot::channel();
3998 self.inner
3999 .commands
4000 .send(SupervisorCommand::Drain {
4001 requester,
4002 reply: reply_tx,
4003 })
4004 .await
4005 .map_err(|_| SuperviseError::CommandClosed {
4006 module_id: self.inner.module_id.clone(),
4007 })?;
4008 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4009 module_id: self.inner.module_id.clone(),
4010 })?
4011 }
4012
4013 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4014 self.restart_requested(drain_timeout_ms, None).await
4015 }
4016
4017 pub(crate) async fn restart_requested(
4018 &self,
4019 drain_timeout_ms: Option<u64>,
4020 requester: Option<Principal>,
4021 ) -> Result<(), SuperviseError> {
4022 let queued_at = Instant::now();
4023 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4024 let (reply_tx, reply_rx) = oneshot::channel();
4025 self.inner
4026 .commands
4027 .send(SupervisorCommand::Restart {
4028 requester,
4029 drain_timeout_ms,
4030 received_at_generation,
4031 queued_at,
4032 reply: reply_tx,
4033 })
4034 .await
4035 .map_err(|_| SuperviseError::CommandClosed {
4036 module_id: self.inner.module_id.clone(),
4037 })?;
4038 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4039 module_id: self.inner.module_id.clone(),
4040 })?
4041 }
4042
4043 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4048 self.swap_requested(ready_timeout, None).await
4049 }
4050
4051 pub(crate) async fn swap_requested(
4052 &self,
4053 ready_timeout: Option<Duration>,
4054 requester: Option<Principal>,
4055 ) -> Result<(), SuperviseError> {
4056 let (reply_tx, reply_rx) = oneshot::channel();
4057 self.inner
4058 .commands
4059 .send(SupervisorCommand::Swap {
4060 requester,
4061 ready_timeout,
4062 reply: reply_tx,
4063 })
4064 .await
4065 .map_err(|_| SuperviseError::CommandClosed {
4066 module_id: self.inner.module_id.clone(),
4067 })?;
4068 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4069 module_id: self.inner.module_id.clone(),
4070 })?
4071 }
4072
4073 pub async fn reload(&self) -> Result<(), SuperviseError> {
4074 self.reload_requested(None).await
4075 }
4076
4077 pub(crate) async fn reload_requested(
4078 &self,
4079 requester: Option<Principal>,
4080 ) -> Result<(), SuperviseError> {
4081 let (reply_tx, reply_rx) = oneshot::channel();
4082 self.inner
4083 .commands
4084 .send(SupervisorCommand::Reload {
4085 requester,
4086 reply: reply_tx,
4087 })
4088 .await
4089 .map_err(|_| SuperviseError::CommandClosed {
4090 module_id: self.inner.module_id.clone(),
4091 })?;
4092 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4093 module_id: self.inner.module_id.clone(),
4094 })?
4095 }
4096
4097 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4098 self.set_enabled_requested(enabled, None).await
4099 }
4100
4101 pub(crate) async fn set_enabled_requested(
4102 &self,
4103 enabled: bool,
4104 requester: Option<Principal>,
4105 ) -> Result<bool, SuperviseError> {
4106 let (reply_tx, reply_rx) = oneshot::channel();
4107 self.inner
4108 .commands
4109 .send(SupervisorCommand::SetEnabled {
4110 requester,
4111 enabled,
4112 reply: reply_tx,
4113 })
4114 .await
4115 .map_err(|_| SuperviseError::CommandClosed {
4116 module_id: self.inner.module_id.clone(),
4117 })?;
4118 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4119 module_id: self.inner.module_id.clone(),
4120 })?
4121 }
4122
4123 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4127 let configured = self
4128 .inner
4129 .configuration
4130 .lock()
4131 .map_err(|_| SuperviseError::StatePoisoned {
4132 module_id: Some(self.inner.module_id.clone()),
4133 })?
4134 .spec
4135 .protocol;
4136 let state = lock_snapshot(&self.inner.snapshot)?;
4137 Ok(state.spawned_protocol.unwrap_or(configured))
4138 }
4139
4140 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4141 let configuration =
4142 self.inner
4143 .configuration
4144 .lock()
4145 .map_err(|_| SuperviseError::StatePoisoned {
4146 module_id: Some(self.inner.module_id.clone()),
4147 })?;
4148 Ok((configuration.spec.clone(), configuration.health.clone()))
4149 }
4150
4151 #[cfg(any(test, feature = "test-support"))]
4155 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4156 let (_, health) = self.configuration()?;
4157 let drain_timeout_ms = u64::try_from(
4158 self.inner
4159 .effective_drain_timeout
4160 .lock()
4161 .unwrap_or_else(|poisoned| poisoned.into_inner())
4162 .as_millis(),
4163 )
4164 .ok();
4165 self.update_configuration(spec, health, drain_timeout_ms)
4166 .await
4167 }
4168
4169 pub(crate) async fn update_configuration(
4170 &self,
4171 spec: ModuleSpec,
4172 health: HealthConfig,
4173 drain_timeout_ms: Option<u64>,
4174 ) -> Result<(), SuperviseError> {
4175 if spec.module_id != self.inner.module_id {
4176 return Err(SuperviseError::InvalidSpec {
4177 reason: "a supervised module's module_id cannot be changed".to_string(),
4178 });
4179 }
4180 validate_spec(&spec)?;
4181 let (reply_tx, reply_rx) = oneshot::channel();
4182 self.inner
4183 .commands
4184 .send(SupervisorCommand::UpdateConfiguration {
4185 spec: spec.clone(),
4186 health: health.clone(),
4187 drain_timeout_ms,
4188 reply: reply_tx,
4189 })
4190 .await
4191 .map_err(|_| SuperviseError::CommandClosed {
4192 module_id: self.inner.module_id.clone(),
4193 })?;
4194 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4195 module_id: self.inner.module_id.clone(),
4196 })?;
4197 let mut configuration =
4198 self.inner
4199 .configuration
4200 .lock()
4201 .map_err(|_| SuperviseError::StatePoisoned {
4202 module_id: Some(self.inner.module_id.clone()),
4203 })?;
4204 configuration.spec = spec;
4205 configuration.health = health;
4206 Ok(())
4207 }
4208}
4209
4210impl Drop for SupervisedModuleInner {
4211 fn drop(&mut self) {
4212 let Ok(mut monitor) = self.monitor.lock() else {
4213 return;
4214 };
4215 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4216 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4217 state.state = ModuleState::Stopped;
4218 clear_current_process_facts(state);
4219 });
4220 monitor.abort();
4221 }
4222 let _ = monitor.take();
4223 }
4224}
4225
4226#[derive(Debug)]
4227enum SupervisorCommand {
4228 Drain {
4229 requester: Option<Principal>,
4230 reply: oneshot::Sender<Result<(), SuperviseError>>,
4231 },
4232 Retire {
4233 requester: Option<Principal>,
4234 reply: oneshot::Sender<Result<(), SuperviseError>>,
4235 },
4236 Restart {
4237 requester: Option<Principal>,
4238 drain_timeout_ms: Option<u64>,
4243 received_at_generation: u64,
4247 queued_at: Instant,
4250 reply: oneshot::Sender<Result<(), SuperviseError>>,
4251 },
4252 Reload {
4253 requester: Option<Principal>,
4254 reply: oneshot::Sender<Result<(), SuperviseError>>,
4255 },
4256 SetEnabled {
4257 requester: Option<Principal>,
4258 enabled: bool,
4259 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4260 },
4261 UpdateConfiguration {
4262 spec: ModuleSpec,
4263 health: HealthConfig,
4264 drain_timeout_ms: Option<u64>,
4267 reply: oneshot::Sender<()>,
4268 },
4269 Swap {
4270 requester: Option<Principal>,
4271 ready_timeout: Option<Duration>,
4274 reply: oneshot::Sender<Result<(), SuperviseError>>,
4276 },
4277}
4278
4279#[derive(Debug)]
4280pub enum SuperviseError {
4281 InvalidSpec {
4282 reason: String,
4283 },
4284 Spawn {
4285 program: PathBuf,
4286 source: io::Error,
4287 cgroup_path: Option<PathBuf>,
4288 },
4289 Cgroup {
4290 module_id: String,
4291 source: io::Error,
4292 },
4293 LaunchNonce {
4296 reason: String,
4297 },
4298 Wait {
4299 module_id: String,
4300 source: io::Error,
4301 },
4302 Kill {
4303 module_id: String,
4304 source: io::Error,
4305 },
4306 Forwarding(ForwardingError),
4307 Registry(RegistryError),
4308 ReloadUnavailable {
4309 module_id: String,
4310 reason: String,
4311 },
4312 Disabled {
4317 module_id: String,
4318 },
4319 ReloadFailed {
4320 module_id: String,
4321 reason: String,
4322 },
4323 RegistrationStillActive {
4324 module_id: String,
4325 waited: Duration,
4326 },
4327 StatePoisoned {
4328 module_id: Option<String>,
4329 },
4330 CommandClosed {
4331 module_id: String,
4332 },
4333 SwapInProgress {
4337 module_id: String,
4338 },
4339 SwapRefused {
4341 module_id: String,
4342 reason: SwapRefusal,
4343 },
4344 SwapFailed {
4348 module_id: String,
4349 arm: SwapFailureArm,
4350 detail: String,
4351 candidate_exit: Option<ExitReport>,
4354 },
4355}
4356
4357#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4359pub enum SwapRefusal {
4360 OverlapExclusive,
4362 NotRegistered,
4365 ProtocolNone,
4368 NotConfigured,
4371 AlreadySwapping,
4373}
4374
4375impl SwapRefusal {
4376 pub fn as_str(self) -> &'static str {
4377 match self {
4378 Self::OverlapExclusive => "overlap_exclusive",
4379 Self::NotRegistered => "not_registered",
4380 Self::ProtocolNone => "protocol_none",
4381 Self::NotConfigured => "not_configured",
4382 Self::AlreadySwapping => "already_swapping",
4383 }
4384 }
4385}
4386
4387#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4390pub enum SwapFailureArm {
4391 SpawnFailed,
4393 NeverRegistered,
4395 NeverReady,
4397 CandidateExited,
4399 CandidateUnhealthy,
4401 Interrupted,
4405 CutoverLost,
4410}
4411
4412impl SwapFailureArm {
4413 pub fn as_str(self) -> &'static str {
4414 match self {
4415 Self::SpawnFailed => "spawn_failed",
4416 Self::NeverRegistered => "never_registered",
4417 Self::NeverReady => "never_ready",
4418 Self::CandidateExited => "candidate_exited",
4419 Self::CandidateUnhealthy => "candidate_unhealthy",
4420 Self::Interrupted => "interrupted",
4421 Self::CutoverLost => "cutover_lost",
4422 }
4423 }
4424}
4425
4426impl fmt::Display for SuperviseError {
4427 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4428 match self {
4429 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4430 Self::Spawn {
4431 program,
4432 source,
4433 cgroup_path: Some(cgroup_path),
4434 } => write!(
4435 f,
4436 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4437 cgroup_path.display(),
4438 program.display()
4439 ),
4440 Self::Spawn {
4441 program,
4442 source,
4443 cgroup_path: None,
4444 } => write!(
4445 f,
4446 "failed to spawn module '{}': {source}",
4447 program.display()
4448 ),
4449 Self::Cgroup { module_id, source } => {
4450 write!(
4451 f,
4452 "failed to prepare cgroup for module '{module_id}': {source}"
4453 )
4454 }
4455 Self::LaunchNonce { reason } => {
4456 write!(
4457 f,
4458 "failed to generate reserved-module launch nonce: {reason}"
4459 )
4460 }
4461 Self::Wait { module_id, source } => {
4462 write!(f, "failed to wait for module '{module_id}': {source}")
4463 }
4464 Self::Kill { module_id, source } => {
4465 write!(f, "failed to kill module '{module_id}': {source}")
4466 }
4467 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4468 Self::Registry(err) => write!(f, "registry error: {err}"),
4469 Self::ReloadUnavailable { module_id, reason } => {
4470 write!(f, "reload unavailable for module '{module_id}': {reason}")
4471 }
4472 Self::Disabled { module_id } => {
4473 write!(
4474 f,
4475 "module '{module_id}' is disabled; enable it before restart or reload"
4476 )
4477 }
4478 Self::ReloadFailed { module_id, reason } => {
4479 write!(f, "reload failed for module '{module_id}': {reason}")
4480 }
4481 Self::RegistrationStillActive { module_id, waited } => write!(
4482 f,
4483 "module '{module_id}' registration remained active after waiting {waited:?}"
4484 ),
4485 Self::StatePoisoned { module_id } => match module_id {
4486 Some(module_id) => {
4487 write!(f, "supervisor state for module '{module_id}' was poisoned")
4488 }
4489 None => write!(f, "supervisor state was poisoned"),
4490 },
4491 Self::CommandClosed { module_id } => {
4492 write!(
4493 f,
4494 "supervisor command channel for module '{module_id}' is closed"
4495 )
4496 }
4497 Self::SwapInProgress { module_id } => write!(
4498 f,
4499 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4500 ),
4501 Self::SwapRefused { module_id, reason } => match reason {
4502 SwapRefusal::OverlapExclusive => write!(
4503 f,
4504 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4505 ),
4506 SwapRefusal::NotRegistered => write!(
4507 f,
4508 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4509 ),
4510 SwapRefusal::ProtocolNone => write!(
4511 f,
4512 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4513 ),
4514 SwapRefusal::NotConfigured => write!(
4515 f,
4516 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4517 ),
4518 SwapRefusal::AlreadySwapping => {
4519 write!(f, "module '{module_id}' is already being swapped")
4520 }
4521 },
4522 Self::SwapFailed {
4523 module_id,
4524 arm,
4525 detail,
4526 ..
4527 } => write!(
4528 f,
4529 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4530 arm.as_str()
4531 ),
4532 }
4533 }
4534}
4535
4536impl Error for SuperviseError {
4537 fn source(&self) -> Option<&(dyn Error + 'static)> {
4538 match self {
4539 Self::Spawn { source, .. }
4540 | Self::Cgroup { source, .. }
4541 | Self::Wait { source, .. }
4542 | Self::Kill { source, .. } => Some(source),
4543 Self::Forwarding(err) => Some(err),
4544 Self::Registry(err) => Some(err),
4545 Self::LaunchNonce { .. }
4546 | Self::InvalidSpec { .. }
4547 | Self::ReloadUnavailable { .. }
4548 | Self::Disabled { .. }
4549 | Self::ReloadFailed { .. }
4550 | Self::RegistrationStillActive { .. }
4551 | Self::StatePoisoned { .. }
4552 | Self::CommandClosed { .. }
4553 | Self::SwapInProgress { .. }
4554 | Self::SwapRefused { .. }
4555 | Self::SwapFailed { .. } => None,
4556 }
4557 }
4558}
4559
4560pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4561 if spec.module_id.trim().is_empty() {
4562 return Err(SuperviseError::InvalidSpec {
4563 reason: "module_id must not be empty".to_string(),
4564 });
4565 }
4566
4567 Ok(())
4568}
4569
4570#[derive(Debug, Default)]
4571struct HealthProbeRuntime {
4572 spawn_generation: Option<u64>,
4573 configured_health: Option<HealthConfig>,
4574 registered_connection: Option<crate::ConnectionId>,
4575 advertised: bool,
4576 next_probe_at: Option<Instant>,
4577 probe_index: u64,
4578}
4579
4580fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4581 lock_snapshot(snapshot)
4582 .ok()
4583 .and_then(|state| state.spawned_protocol)
4584 .unwrap_or(spec.protocol)
4585}
4586
4587impl HealthProbeRuntime {
4588 fn refresh_registration(
4589 &mut self,
4590 spec: &ModuleSpec,
4591 runtime: &SupervisorRuntimeConfig,
4592 registry: &Registry,
4593 snapshot: &SharedSnapshot,
4594 ) {
4595 let generation = lock_snapshot(snapshot)
4596 .ok()
4597 .map(|state| state.spawn_generation);
4598 if self.spawn_generation != generation {
4599 *self = Self {
4603 spawn_generation: generation,
4604 ..Self::default()
4605 };
4606 }
4607 if self.configured_health.as_ref() != Some(&runtime.health) {
4608 self.configured_health = Some(runtime.health.clone());
4609 self.next_probe_at = None;
4610 self.registered_connection = None;
4611 self.probe_index = 0;
4612 }
4613 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4617 self.registered_connection = None;
4618 self.advertised = runtime.health.http.is_some();
4619 if !self.advertised {
4620 self.next_probe_at = None;
4621 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4622 let unknown = ModuleHealthStatus::default();
4623 if state.health != unknown {
4624 state.health = unknown;
4625 }
4626 });
4627 } else if self.next_probe_at.is_none() {
4628 self.next_probe_at = Some(
4629 Instant::now()
4630 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4631 );
4632 }
4633 return;
4634 }
4635
4636 let registration = match registry.get_module(&spec.module_id) {
4637 Ok(registration) => registration,
4638 Err(err) => {
4639 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4640 self.advertised = false;
4641 self.next_probe_at = None;
4642 return;
4643 }
4644 };
4645
4646 let Some(registration) = registration else {
4647 self.registered_connection = None;
4648 self.advertised = false;
4649 self.next_probe_at = None;
4650 return;
4651 };
4652
4653 let advertised = registration
4654 .control_ops
4655 .iter()
4656 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4657 if !advertised {
4658 self.registered_connection = Some(registration.connection_id);
4659 self.advertised = false;
4660 self.next_probe_at = None;
4661 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4662 if state.health.status != SupervisorHealthStatus::Unknown
4663 || state.health.consecutive_failures != 0
4664 || state.health.last_probe_ms.is_some()
4665 || state.health.detail.is_some()
4666 || state.health.metrics.is_some()
4667 {
4668 state.health.status = SupervisorHealthStatus::Unknown;
4669 state.health.consecutive_failures = 0;
4670 state.health.last_probe_ms = None;
4671 state.health.detail = None;
4672 state.health.metrics = None;
4673 }
4674 });
4675 return;
4676 }
4677
4678 let reregistered = self.registered_connection != Some(registration.connection_id);
4679 self.registered_connection = Some(registration.connection_id);
4680 self.advertised = true;
4681 if reregistered || self.next_probe_at.is_none() {
4682 self.probe_index = 0;
4683 self.next_probe_at = Some(
4684 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4685 );
4686 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4687 state.health.status = SupervisorHealthStatus::Unknown;
4688 state.health.consecutive_failures = 0;
4689 state.health.detail = None;
4690 state.health.metrics = None;
4691 });
4692 }
4693 }
4694
4695 fn wake_at(&self) -> Option<Instant> {
4696 if !self.advertised {
4697 return None;
4698 }
4699 self.next_probe_at
4700 }
4701
4702 fn due(&self) -> bool {
4703 self.advertised
4704 && self
4705 .next_probe_at
4706 .is_some_and(|next| Instant::now() >= next)
4707 }
4708
4709 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4710 self.probe_index = self.probe_index.wrapping_add(1);
4711 self.next_probe_at = Some(
4712 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4713 );
4714 }
4715}
4716
4717#[derive(Debug)]
4752enum HealthProbeEvidence {
4753 LaneDead,
4755 NoAnswer,
4757 BadAnswer,
4759 Misconfigured,
4761}
4762
4763#[derive(Debug)]
4764struct HealthProbeError {
4765 evidence: HealthProbeEvidence,
4766 message: String,
4767}
4768
4769impl HealthProbeError {
4770 fn lane_dead(message: impl Into<String>) -> Self {
4771 Self::with(HealthProbeEvidence::LaneDead, message)
4772 }
4773
4774 fn no_answer(message: impl Into<String>) -> Self {
4775 Self::with(HealthProbeEvidence::NoAnswer, message)
4776 }
4777
4778 fn bad_answer(message: impl Into<String>) -> Self {
4779 Self::with(HealthProbeEvidence::BadAnswer, message)
4780 }
4781
4782 fn misconfigured(message: impl Into<String>) -> Self {
4783 Self::with(HealthProbeEvidence::Misconfigured, message)
4784 }
4785
4786 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4787 Self {
4788 evidence,
4789 message: message.into(),
4790 }
4791 }
4792
4793 #[allow(dead_code)]
4807 fn is_proof_of_death(&self) -> bool {
4808 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4809 }
4810
4811 fn label(&self) -> &'static str {
4819 match self.evidence {
4820 HealthProbeEvidence::LaneDead => "lane-dead",
4821 HealthProbeEvidence::NoAnswer => "no-answer",
4822 HealthProbeEvidence::BadAnswer => "bad-answer",
4823 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4824 }
4825 }
4826}
4827
4828impl fmt::Display for HealthProbeError {
4829 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4830 f.write_str(&self.message)
4831 }
4832}
4833
4834async fn run_health_probe_cycle(
4835 spec: &ModuleSpec,
4836 runtime: &SupervisorRuntimeConfig,
4837 registry: &Registry,
4838 process_liveness: &SupervisorProcessLiveness,
4839 snapshot: &SharedSnapshot,
4840 child: &mut Option<SupervisedChild>,
4841) {
4842 let now_ms = unix_ms_now();
4843 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4844 .then_some(runtime.health.http.as_deref())
4845 .flatten();
4846 let result = match http {
4847 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4848 None => probe_module_health(&spec.module_id, runtime, None).await,
4849 };
4850 match result {
4851 Ok(report) => {
4852 handle_health_report(
4853 spec,
4854 runtime,
4855 registry,
4856 process_liveness,
4857 snapshot,
4858 child,
4859 report,
4860 now_ms,
4861 )
4862 .await;
4863 }
4864 Err(err) => {
4865 if http.is_some() {
4866 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4867 state.health.status = SupervisorHealthStatus::Failing;
4868 });
4869 }
4870 handle_health_probe_failure(
4871 spec,
4872 runtime,
4873 registry,
4874 process_liveness,
4875 snapshot,
4876 child,
4877 err,
4878 now_ms,
4879 )
4880 .await;
4881 }
4882 }
4883}
4884
4885pub(crate) struct HttpProbeTarget<'a> {
4886 address: std::net::SocketAddr,
4887 localhost: bool,
4888 authority: &'a str,
4889 path: String,
4890}
4891
4892pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4895 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4896 return Err("must not contain whitespace, controls, or a fragment".into());
4897 }
4898 let rest = url
4899 .strip_prefix("http://")
4900 .ok_or("must use plain http://")?;
4901 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4902 let (authority, suffix) = rest.split_at(split);
4903 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4904 ("::1", rest)
4905 } else {
4906 let split = authority.find(':').unwrap_or(authority.len());
4907 authority.split_at(split)
4908 };
4909 let ip: std::net::IpAddr = match host {
4910 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4911 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4912 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4913 };
4914 let port = if port.is_empty() {
4915 80
4916 } else {
4917 port.strip_prefix(':')
4918 .and_then(|p| p.parse::<u16>().ok())
4919 .filter(|p| *p > 0)
4920 .ok_or("must have a valid nonzero TCP port")?
4921 };
4922 let path = if suffix.is_empty() {
4923 "/".into()
4924 } else if suffix.starts_with('?') {
4925 format!("/{suffix}")
4926 } else {
4927 suffix.into()
4928 };
4929 Ok(HttpProbeTarget {
4930 address: std::net::SocketAddr::new(ip, port),
4931 localhost: host == "localhost",
4932 authority,
4933 path,
4934 })
4935}
4936
4937async fn probe_http_health(
4938 url: &str,
4939 deadline: Duration,
4940) -> Result<HealthReport, HealthProbeError> {
4941 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4942 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4943 let mut response_status = String::new();
4946 let mut body = Vec::new();
4947 let probe = async {
4948 let connection = match tokio::net::TcpStream::connect(target.address).await {
4951 Err(_) if target.localhost => {
4952 tokio::net::TcpStream::connect((
4953 std::net::Ipv6Addr::LOCALHOST,
4954 target.address.port(),
4955 ))
4956 .await
4957 }
4958 result => result,
4959 };
4960 let mut stream = connection.map_err(|error| {
4961 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4962 })?;
4963 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4964 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4965 let mut reader = BufReader::new(stream);
4966 let mut budget = 16 * 1024;
4967 let status = http_line(&mut reader, &mut budget).await?;
4968 let mut words = status.split_ascii_whitespace();
4969 let version = words.next();
4970 let code = words
4971 .next()
4972 .filter(|word| word.len() == 3)
4973 .and_then(|word| word.parse::<u16>().ok());
4974 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4975 || !code.is_some_and(|code| (100..600).contains(&code))
4976 {
4977 return Err(HealthProbeError::bad_answer(format!(
4978 "invalid HTTP status: {status}"
4979 )));
4980 }
4981 let code = code.expect("validated status code");
4982 response_status = status.clone();
4983 let mut length = None;
4984 let mut chunked = false;
4985 loop {
4986 let line = http_line(&mut reader, &mut budget).await?;
4987 if line.is_empty() {
4988 break;
4989 }
4990 if let Some((name, value)) = line.split_once(':') {
4991 if name.eq_ignore_ascii_case("content-length") {
4992 length = Some(value.trim().parse::<u64>().map_err(|_| {
4993 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4994 })?);
4995 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4996 chunked = value.trim().eq_ignore_ascii_case("chunked");
4997 }
4998 }
4999 }
5000 if chunked {
5001 while body.len() < 200 {
5002 let line = http_line(&mut reader, &mut budget).await?;
5003 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5004 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5005 if size == 0 {
5006 break;
5007 }
5008 let count = size.min((200 - body.len()) as u64) as usize;
5009 let start = body.len();
5010 (&mut reader)
5011 .take(count as u64)
5012 .read_to_end(&mut body)
5013 .await
5014 .map_err(|error| {
5015 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5016 })?;
5017 if body.len() - start != count {
5018 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5019 }
5020 if size > count as u64 || body.len() == 200 {
5021 break;
5022 }
5023 if !http_line(&mut reader, &mut budget).await?.is_empty() {
5024 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5025 }
5026 }
5027 } else {
5028 reader
5029 .take(length.unwrap_or(200).min(200))
5030 .read_to_end(&mut body)
5031 .await
5032 .map_err(|error| {
5033 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5034 })?;
5035 }
5036 if (200..300).contains(&code) {
5037 Ok(HealthReport::ok())
5038 } else {
5039 Err(HealthProbeError::bad_answer(
5040 "HTTP health endpoint returned non-2xx",
5041 ))
5042 }
5043 };
5044 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5045 Err(HealthProbeError::no_answer(format!(
5046 "HTTP probe timed out after {deadline:?}"
5047 )))
5048 });
5049 if let Err(error) = &mut result {
5050 if !response_status.is_empty() {
5051 error.message = format!(
5052 "{}; {response_status}: {}",
5053 error.message,
5054 String::from_utf8_lossy(&body)
5055 );
5056 }
5057 }
5058 result
5059}
5060
5061async fn http_line(
5062 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5063 remaining: &mut usize,
5064) -> Result<String, HealthProbeError> {
5065 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5066 let mut line = Vec::new();
5067 (&mut *reader)
5068 .take(*remaining as u64)
5069 .read_until(b'\n', &mut line)
5070 .await
5071 .map_err(|error| {
5072 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5073 })?;
5074 *remaining -= line.len();
5075 if !line.ends_with(b"\r\n") {
5076 return Err(HealthProbeError::bad_answer(
5077 "HTTP headers are incomplete or exceed 16 KiB",
5078 ));
5079 }
5080 line.truncate(line.len() - 2);
5081 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5082}
5083
5084async fn probe_module_health(
5085 module_id: &str,
5086 runtime: &SupervisorRuntimeConfig,
5087 drain_deadline: Option<Instant>,
5088) -> Result<HealthReport, HealthProbeError> {
5089 let Some(forwarding) = runtime.forwarding.as_ref() else {
5090 return Err(HealthProbeError::misconfigured(
5091 "supervisor was not configured with a forwarding table",
5092 ));
5093 };
5094 let probe_started_at = Instant::now();
5095 let mut deadline = probe_started_at + runtime.health.deadline;
5096 if let Some(drain_deadline) = drain_deadline {
5097 deadline = deadline.min(drain_deadline);
5098 }
5099 let pending = if drain_deadline.is_some() {
5100 forwarding.begin_drain_health_probe_rpc_for(
5101 module_id,
5102 MODULE_CONTROL_OP_HEALTH_CHECK,
5103 probe_started_at,
5104 deadline,
5105 )
5106 } else {
5107 forwarding.begin_health_probe_rpc_for(
5108 module_id,
5109 MODULE_CONTROL_OP_HEALTH_CHECK,
5110 probe_started_at,
5111 deadline,
5112 )
5113 }
5114 .map_err(|err| {
5115 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5118 })?;
5119 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5120}
5121
5122async fn probe_endpoint_health(
5129 endpoint: crate::ModuleEndpointId,
5130 runtime: &SupervisorRuntimeConfig,
5131 deadline_cap: Option<Instant>,
5132) -> Result<HealthReport, HealthProbeError> {
5133 let Some(forwarding) = runtime.forwarding.as_ref() else {
5134 return Err(HealthProbeError::misconfigured(
5135 "supervisor was not configured with a forwarding table",
5136 ));
5137 };
5138 let probe_started_at = Instant::now();
5139 let mut deadline = probe_started_at + runtime.health.deadline;
5140 if let Some(cap) = deadline_cap {
5141 deadline = deadline.min(cap);
5142 }
5143 let pending = forwarding
5144 .begin_endpoint_health_probe_rpc_for(
5145 endpoint,
5146 MODULE_CONTROL_OP_HEALTH_CHECK,
5147 probe_started_at,
5148 deadline,
5149 )
5150 .map_err(|err| {
5151 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5152 })?;
5153 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5154}
5155
5156async fn await_health_probe(
5158 forwarding: &ForwardingTable,
5159 pending: PendingModuleControlRpc,
5160 deadline: Instant,
5161 probe_budget: Duration,
5162) -> Result<HealthReport, HealthProbeError> {
5163 let PendingModuleControlRpc {
5164 endpoint,
5165 module_sink,
5166 negotiated_ver,
5167 corr,
5168 receiver,
5169 } = pending;
5170 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5171 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5172 })?;
5173 let frame = Frame::build_with_version(
5174 negotiated_ver,
5175 FrameType::Request,
5176 control_flags(),
5177 0,
5178 0,
5179 corr,
5180 body,
5181 )
5182 .map_err(|err| {
5183 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5184 })?;
5185
5186 match timeout_at(deadline, module_sink.send(frame)).await {
5192 Ok(Ok(())) => {}
5193 Ok(Err(err)) => {
5194 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5195 return Err(HealthProbeError::lane_dead(format!(
5198 "failed to send health.check: {err}"
5199 )));
5200 }
5201 Err(_elapsed) => {
5202 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5203 return Err(HealthProbeError::no_answer(
5207 "health.check send timed out before enqueue (module egress full)",
5208 ));
5209 }
5210 }
5211
5212 match timeout_at(deadline, receiver).await {
5213 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5217 response.health_report().ok_or_else(|| {
5218 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5219 })
5220 }
5221 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5222 format!("health.check rejected: {}", body.message),
5223 )),
5224 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5225 Err(HealthProbeError::lane_dead(message))
5226 }
5227 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5228 Err(HealthProbeError::bad_answer(message))
5229 }
5230 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5231 Err(HealthProbeError::bad_answer(format!(
5232 "expected module-control op '{expected}', got '{actual}'"
5233 )))
5234 }
5235 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5239 "module answered health.check after its daemon deadline",
5240 )),
5241 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5242 "health.check waiter was canceled before the module responded",
5243 )),
5244 Err(_) => {
5245 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5246 Err(HealthProbeError::no_answer(format!(
5247 "module did not answer health.check within {probe_budget:?}"
5248 )))
5249 }
5250 }
5251}
5252
5253#[allow(clippy::too_many_arguments)]
5254async fn handle_health_report(
5255 spec: &ModuleSpec,
5256 runtime: &SupervisorRuntimeConfig,
5257 registry: &Registry,
5258 process_liveness: &SupervisorProcessLiveness,
5259 snapshot: &SharedSnapshot,
5260 child: &mut Option<SupervisedChild>,
5261 report: HealthReport,
5262 now_ms: u64,
5263) {
5264 let status = supervisor_health_status(report.status);
5265 let detail = report.detail.clone();
5266 let metrics = truncate_health_metrics(report.metrics);
5267 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5268 state.health.status = status;
5269 state.health.last_probe_ms = Some(now_ms);
5270 state.health.detail = detail.clone();
5271 state.health.metrics = metrics.clone();
5272 state.health.consecutive_failures = 0;
5273 state.failed_retry.observe_health(
5274 report.status == HealthStatus::Ok,
5275 runtime.restart_policy.window,
5276 );
5277 });
5278
5279 let action = match report.status {
5280 HealthStatus::Ok => return,
5281 HealthStatus::Degraded => runtime.health.on_degraded,
5282 HealthStatus::Failing => runtime.health.on_failing,
5283 };
5284 apply_l3_health_action(
5285 spec,
5286 runtime,
5287 registry,
5288 process_liveness,
5289 snapshot,
5290 child,
5291 status,
5292 detail.as_deref(),
5293 action,
5294 now_ms,
5295 )
5296 .await;
5297}
5298
5299#[allow(clippy::too_many_arguments)]
5300async fn handle_health_probe_failure(
5301 spec: &ModuleSpec,
5302 runtime: &SupervisorRuntimeConfig,
5303 registry: &Registry,
5304 process_liveness: &SupervisorProcessLiveness,
5305 snapshot: &SharedSnapshot,
5306 child: &mut Option<SupervisedChild>,
5307 err: HealthProbeError,
5308 now_ms: u64,
5309) {
5310 let threshold = runtime.health.failure_threshold.max(1);
5311 let mut failures = 0;
5312 let detail = format!("[{}] {err}", err.label());
5317 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5318 state
5319 .failed_retry
5320 .observe_health(false, runtime.restart_policy.window);
5321 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5324 state.health.status = SupervisorHealthStatus::Unknown;
5325 }
5326 state.health.last_probe_ms = Some(now_ms);
5327 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5328 state.health.detail = Some(detail.clone());
5329 state.health.metrics = None;
5330 failures = state.health.consecutive_failures;
5331 });
5332
5333 if failures < threshold {
5334 warn!(
5335 module_id = %spec.module_id,
5336 consecutive_failures = failures,
5337 threshold,
5338 evidence = err.label(),
5339 detail = %detail,
5340 "health.check probe failed"
5341 );
5342 return;
5343 }
5344
5345 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5346 state.state = ModuleState::Unresponsive;
5347 state.health.status = SupervisorHealthStatus::Unresponsive;
5348 });
5349 if runtime.health.critical {
5353 error!(
5354 module_id = %spec.module_id,
5355 status = "unresponsive",
5356 evidence = err.label(),
5357 detail = %detail,
5358 "critical module health alert"
5359 );
5360 } else {
5361 warn!(
5362 module_id = %spec.module_id,
5363 status = "unresponsive",
5364 evidence = err.label(),
5365 detail = %detail,
5366 "module health threshold breached"
5367 );
5368 }
5369 if let Err(err) = health_restart_child(
5370 spec,
5371 runtime,
5372 registry,
5373 process_liveness,
5374 snapshot,
5375 child,
5376 SupervisorHealthStatus::Unresponsive,
5377 Some(&detail),
5378 now_ms,
5379 )
5380 .await
5381 {
5382 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5383 }
5384}
5385
5386#[allow(clippy::too_many_arguments)]
5387async fn apply_l3_health_action(
5388 spec: &ModuleSpec,
5389 runtime: &SupervisorRuntimeConfig,
5390 registry: &Registry,
5391 process_liveness: &SupervisorProcessLiveness,
5392 snapshot: &SharedSnapshot,
5393 child: &mut Option<SupervisedChild>,
5394 status: SupervisorHealthStatus,
5395 detail: Option<&str>,
5396 action: HealthAction,
5397 now_ms: u64,
5398) {
5399 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5400 match action {
5401 HealthAction::Report => {
5402 info!(
5403 module_id = %spec.module_id,
5404 status = ?status,
5405 detail,
5406 "module reported non-ok health"
5407 );
5408 }
5409 HealthAction::Alert => {
5410 error!(
5411 module_id = %spec.module_id,
5412 status = ?status,
5413 detail,
5414 "module health alert"
5415 );
5416 }
5417 HealthAction::Restart => {
5418 if let Err(err) = health_restart_child(
5419 spec,
5420 runtime,
5421 registry,
5422 process_liveness,
5423 snapshot,
5424 child,
5425 status,
5426 detail,
5427 now_ms,
5428 )
5429 .await
5430 {
5431 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5432 }
5433 }
5434 }
5435}
5436
5437#[allow(clippy::too_many_arguments)]
5438async fn health_restart_child(
5439 spec: &ModuleSpec,
5440 runtime: &SupervisorRuntimeConfig,
5441 registry: &Registry,
5442 process_liveness: &SupervisorProcessLiveness,
5443 snapshot: &SharedSnapshot,
5444 child: &mut Option<SupervisedChild>,
5445 status: SupervisorHealthStatus,
5446 detail: Option<&str>,
5447 now_ms: u64,
5448) -> Result<(), SuperviseError> {
5449 let (enabled, schedule) = {
5450 let mut state = lock_snapshot(snapshot)?;
5451 let enabled = state.enabled;
5452 let schedule = if enabled {
5453 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5454 } else {
5455 None
5456 };
5457 (enabled, schedule)
5458 };
5459
5460 if !enabled {
5461 return Err(SuperviseError::Disabled {
5462 module_id: spec.module_id.clone(),
5463 });
5464 }
5465
5466 if schedule.is_none() {
5467 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5468 error!(
5469 module_id = %spec.module_id,
5470 status = ?status,
5471 detail,
5472 max_restarts = runtime.restart_policy.max_restarts,
5473 window_secs = runtime.restart_policy.window.as_secs(),
5474 reason = %runtime.restart_policy.budget_exhausted_detail(),
5475 "health restart budget exhausted; marking module failed"
5476 );
5477 let stop_notice = begin_forwarding_drain_if_configured(
5478 spec,
5479 runtime,
5480 registry,
5481 snapshot,
5482 Some(true),
5483 RouteCloseReason::Disable,
5484 )
5485 .await?;
5486 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5487 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5488 })?;
5489 let result = drain_optional_child(
5490 &spec.module_id,
5491 spec.protocol,
5492 stop_notice,
5493 registry,
5494 runtime.forwarding.as_deref(),
5495 snapshot,
5496 &runtime.terminal_ring,
5497 &runtime.spawn_events,
5498 child,
5499 runtime.drain_timeout,
5500 ModuleState::Failed,
5501 Some(true),
5502 &runtime.failure_context(true),
5503 )
5504 .await;
5505 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5506 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5507 state.failed_retry.schedule(&spec.module_id);
5508 }
5509 })?;
5510 result?;
5511 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5512 return Ok(());
5513 }
5514
5515 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5516 let mut restart_count = 0;
5517 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5518 restart_count = state.crash_restarts.len();
5519 state.state = ModuleState::Unresponsive;
5520 state.health.status = status;
5521 state.health.last_action = Some(HealthAction::Restart.to_string());
5522 state.health.last_action_ms = Some(now_ms);
5523 })?;
5524 warn!(
5525 module_id = %spec.module_id,
5526 status = ?status,
5527 detail,
5528 restart_count,
5529 restart_in_window = schedule.restart_in_window,
5530 delay_ms = schedule.delay.as_millis() as u64,
5531 "health-triggered module restart"
5532 );
5533
5534 let stop_notice = begin_forwarding_drain_if_configured(
5535 spec,
5536 runtime,
5537 registry,
5538 snapshot,
5539 Some(true),
5540 RouteCloseReason::Restart,
5541 )
5542 .await?;
5543 drain_optional_child(
5544 &spec.module_id,
5545 spec.protocol,
5546 stop_notice,
5547 registry,
5548 runtime.forwarding.as_deref(),
5549 snapshot,
5550 &runtime.terminal_ring,
5551 &runtime.spawn_events,
5552 child,
5553 runtime.drain_timeout,
5554 ModuleState::Restarting,
5555 Some(true),
5556 &runtime.failure_context(true),
5557 )
5558 .await?;
5559 schedule_respawn(
5560 runtime,
5561 snapshot,
5562 &spec.module_id,
5563 schedule.delay,
5564 RespawnKind::Spawn,
5565 true,
5566 )
5567}
5568
5569fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5570 if let Some(reply) = runtime
5571 .deferred_reload_reply
5572 .lock()
5573 .unwrap_or_else(|p| p.into_inner())
5574 .take()
5575 {
5576 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5577 module_id: module_id.to_string(),
5578 reason: reason.to_string(),
5579 }));
5580 }
5581}
5582
5583fn schedule_respawn(
5584 runtime: &SupervisorRuntimeConfig,
5585 snapshot: &SharedSnapshot,
5586 module_id: &str,
5587 delay: Duration,
5588 kind: RespawnKind,
5589 daemon_initiated: bool,
5590) -> Result<(), SuperviseError> {
5591 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5592 update_snapshot(snapshot, Some(module_id), |state| {
5593 state.respawn_pending = true
5594 })?;
5595 *runtime
5596 .scheduled_respawn
5597 .lock()
5598 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5599 deadline: Instant::now() + delay,
5600 kind,
5601 daemon_initiated,
5602 });
5603 Ok(())
5604}
5605
5606fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5607 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5608 state.health.last_action = Some(action);
5609 state.health.last_action_ms = Some(now_ms);
5610 });
5611}
5612
5613fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5614 match status {
5615 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5616 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5617 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5618 }
5619}
5620
5621fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5633 let metrics = metrics?;
5634 match serde_json::to_vec(&metrics) {
5635 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5636 "truncated": true,
5637 "original_bytes": encoded.len(),
5638 })),
5639 Ok(_) | Err(_) => Some(metrics),
5640 }
5641}
5642
5643fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5649 if cadence.is_zero() {
5650 return Duration::ZERO;
5651 }
5652 let cadence_ms = cadence.as_millis() as u64;
5653 if cadence_ms == 0 {
5669 return cadence;
5670 }
5671 let jitter_span = (cadence_ms / 10).max(1);
5686 let hash = module_id.as_bytes().iter().fold(
5687 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5688 |acc, byte| {
5689 acc.wrapping_mul(1099511628211)
5690 .wrapping_add(u64::from(*byte))
5691 },
5692 );
5693 cadence + Duration::from_millis(hash % jitter_span)
5694}
5695
5696#[cfg(test)]
5697mod tests {
5698 use super::*;
5699
5700 #[test]
5701 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5702 let handle = SupervisorHandle::new();
5703 let module_id = "readded-tombstone";
5704 handle.record_rescan_removal(module_id);
5705 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5706
5707 handle.apply_identity_configuration(&ModuleSpec {
5708 module_id: module_id.to_string(),
5709 program: PathBuf::from("/test/module"),
5710 args: Vec::new(),
5711 env: Vec::new(),
5712 reserved: false,
5713 reserved_prefixes: Vec::new(),
5714 protocol: ModuleProtocol::Subc,
5715 overlap: Default::default(),
5716 });
5717
5718 assert!(
5719 handle.removal_tombstone_age_ms(module_id).is_none(),
5720 "a re-added module must not retain a stale removal tombstone"
5721 );
5722 }
5723
5724 #[derive(Debug, PartialEq, Eq)]
5727 struct OwnerInSpawnWindow {
5728 module_id: String,
5729 configured: bool,
5730 on_roster: bool,
5731 admission_refusal: Option<&'static str>,
5732 }
5733
5734 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5745 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5746 use crate::scopes::ScopeTable;
5747 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5748
5749 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5750 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5751 module_id: module_id.to_string(),
5752 program,
5753 args: Vec::new(),
5754 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5755 .into_iter()
5756 .map(|key| (key.to_string(), dir.path().display().to_string()))
5757 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5758 .collect(),
5759 reserved: false,
5760 reserved_prefixes: Vec::new(),
5761 protocol: ModuleProtocol::Subc,
5762 overlap: Default::default(),
5763 };
5764 let live = super::terminal_history_tests::fake_aft_stub_path();
5765 let missing = dir.path().join("definitely-missing-module");
5766
5767 let handle = SupervisorHandle::new();
5768 let mut supervisor =
5769 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5770 .with_handle(handle.clone());
5771 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5772 let hook_handle = handle.clone();
5773 let hook_observed = Arc::clone(&observed);
5774 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5775 let configured = hook_handle.is_configured(module_id);
5778 let selector = ScopeSelector::new(
5779 Principal::Reserved {
5780 module_id: module_id.to_string(),
5781 },
5782 "s".to_string(),
5783 Some(1),
5784 );
5785 let carrier = Principal::Reserved {
5786 module_id: "carrier".to_string(),
5787 };
5788 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5789 .admit(&carrier, module_id, &selector, configured)
5790 {
5791 Ok(_) => None,
5792 Err(refusal) => Some(refusal.code),
5793 };
5794 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5795 module_id: module_id.to_string(),
5796 configured,
5797 on_roster: hook_handle.get(module_id).is_some(),
5798 admission_refusal,
5799 });
5800 })));
5801
5802 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5803 let configured = supervisor
5804 .supervise_configured(stub("configured", live.clone()), true)
5805 .unwrap();
5806 let with_health = supervisor
5807 .supervise_configured_with_health(
5808 stub("with-health", live.clone()),
5809 true,
5810 HealthConfig::default(),
5811 None,
5812 RestartPolicy::default(),
5813 )
5814 .unwrap();
5815 let failed = supervisor
5818 .supervise_configured_with_health(
5819 stub("failed-spawn", missing.clone()),
5820 true,
5821 HealthConfig::default(),
5822 None,
5823 RestartPolicy::default(),
5824 )
5825 .unwrap();
5826 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5829
5830 let expected = [
5831 "plain",
5832 "configured",
5833 "with-health",
5834 "failed-spawn",
5835 "spawn-error",
5836 ]
5837 .into_iter()
5838 .map(|module_id| OwnerInSpawnWindow {
5839 module_id: module_id.to_string(),
5840 configured: true,
5841 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5842 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5843 })
5844 .collect::<Vec<_>>();
5845 assert_eq!(*observed.lock().unwrap(), expected);
5846
5847 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5848 assert!(
5849 handle.get(module_id).is_some(),
5850 "{module_id} is on the roster"
5851 );
5852 assert!(
5853 handle.is_configured(module_id),
5854 "{module_id} stays configured"
5855 );
5856 }
5857 assert!(handle.get("spawn-error").is_none());
5858 assert!(
5859 !handle.is_configured("spawn-error"),
5860 "a plain spawn that failed must not leave its module marked configured"
5861 );
5862
5863 handle.retire("failed-spawn");
5865 assert!(!handle.is_configured("failed-spawn"));
5866
5867 for module in [plain, configured, with_health] {
5868 module.stop().await.unwrap();
5869 }
5870 drop(failed);
5871 }
5872
5873 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5880 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5881 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5882 let spec = ModuleSpec {
5883 module_id: "spawned-not-admitted".to_string(),
5884 program: super::terminal_history_tests::fake_aft_stub_path(),
5885 args: Vec::new(),
5886 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5887 .into_iter()
5888 .map(|key| (key.to_string(), dir.path().display().to_string()))
5889 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5890 .collect(),
5891 reserved: false,
5892 reserved_prefixes: Vec::new(),
5893 protocol: ModuleProtocol::Subc,
5894 overlap: Default::default(),
5895 };
5896 let handle = SupervisorHandle::new();
5897 let mut supervisor =
5898 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5899 .with_handle(handle.clone());
5900 let observed = Arc::new(Mutex::new(None));
5901 let hook_handle = handle.clone();
5902 let hook_observed = Arc::clone(&observed);
5903 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5904 let state = hook_handle
5905 .get(module_id)
5906 .expect("a configured module is on the roster before it is spawned")
5907 .state()
5908 .unwrap();
5909 *hook_observed.lock().unwrap() = Some(state);
5910 })));
5911
5912 let module = supervisor.supervise_configured(spec, true).unwrap();
5913 assert_eq!(
5914 *observed.lock().unwrap(),
5915 Some(ModuleState::Starting),
5916 "between the spawn and its admission the supervisor still reports starting"
5917 );
5918 assert_eq!(module.state().unwrap(), ModuleState::Running);
5919 module.stop().await.unwrap();
5920 }
5921
5922 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5923 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5924 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5925 snapshot.process_alive = true;
5926 snapshot.pid = Some(41);
5927 snapshot.spawned_at_ms = Some(42);
5928 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5929 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5930 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5931 device: 43,
5932 inode: 44,
5933 #[cfg(windows)]
5934 windows_image: None,
5935 });
5936 })
5937 .unwrap();
5938 snapshot
5939 }
5940
5941 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5942 let snapshot = lock_snapshot(snapshot).unwrap();
5943 assert!(!snapshot.process_alive);
5944 assert_eq!(snapshot.pid, None);
5945 assert_eq!(snapshot.spawned_at_ms, None);
5946 assert_eq!(snapshot.spawned_from, None);
5947 assert_eq!(snapshot.launch_spec_digest, None);
5948 assert_eq!(snapshot.spawned_file_identity, None);
5949 }
5950
5951 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5952 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5953 let supervisor =
5954 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5955 let mut runtime = supervisor.runtime_config();
5956 runtime.test_seed_stale_facts_before_enable_spawn = true;
5957 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5958 let mut child = None;
5959 let spec = ModuleSpec {
5960 module_id: "failed-enable-clears-facts".to_string(),
5961 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5962 args: Vec::new(),
5963 env: Vec::new(),
5964 reserved: false,
5965 reserved_prefixes: Vec::new(),
5966 protocol: ModuleProtocol::Subc,
5967 overlap: Default::default(),
5968 };
5969
5970 let result = set_child_enabled(
5971 &spec,
5972 &runtime,
5973 &supervisor.registry,
5974 &supervisor.process_liveness,
5975 &snapshot,
5976 &mut child,
5977 true,
5978 &runtime.failure_context(false),
5979 )
5980 .await;
5981
5982 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5983 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5984 assert_snapshot_process_facts_cleared(&snapshot);
5985 }
5986
5987 #[tokio::test]
5988 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5989 let supervisor =
5990 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5991 let runtime = supervisor.runtime_config();
5992 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5993 ModuleState::Restarting,
5994 true,
5995 )));
5996 let spec = ModuleSpec {
5997 module_id: "start-stranded-restarting".to_string(),
5998 program: super::terminal_history_tests::fake_aft_stub_path(),
5999 args: Vec::new(),
6000 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6001 reserved: false,
6002 reserved_prefixes: Vec::new(),
6003 protocol: ModuleProtocol::None,
6004 overlap: Default::default(),
6005 };
6006 let mut child = None;
6007 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6008 assert!(!super::set_child_enabled(
6009 &spec,
6010 &runtime,
6011 &Registry::default(),
6012 &supervisor.process_liveness,
6013 &snapshot,
6014 &mut child,
6015 true,
6016 &runtime.failure_context(false),
6017 )
6018 .await
6019 .unwrap());
6020 assert!(child.is_none());
6021 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6022 assert!(super::set_child_enabled(
6023 &spec,
6024 &runtime,
6025 &Registry::default(),
6026 &supervisor.process_liveness,
6027 &snapshot,
6028 &mut child,
6029 true,
6030 &runtime.failure_context(false),
6031 )
6032 .await
6033 .unwrap());
6034 assert_eq!(
6035 lock_snapshot(&snapshot).unwrap().state,
6036 ModuleState::Running
6037 );
6038 let mut child = child.unwrap();
6039 child.start_kill().unwrap();
6040 child.wait().await.unwrap();
6041 }
6042
6043 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6044 async fn failed_reload_spawn_clears_current_process_facts() {
6045 let supervisor =
6046 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6047 let mut runtime = supervisor.runtime_config();
6048 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6049 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6050 let mut child = None;
6051 let spec = ModuleSpec {
6052 module_id: "failed-reload-clears-facts".to_string(),
6053 program: PathBuf::from("/unused/failed-reload-module"),
6054 args: Vec::new(),
6055 env: Vec::new(),
6056 reserved: false,
6057 reserved_prefixes: Vec::new(),
6058 protocol: ModuleProtocol::Subc,
6059 overlap: Default::default(),
6060 };
6061
6062 let result = handle_reload_spawn_failure(
6063 &spec,
6064 &runtime,
6065 &supervisor.process_liveness,
6066 &snapshot,
6067 &mut child,
6068 "forced reload spawn failure".to_string(),
6069 )
6070 .await;
6071
6072 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6073 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6074 assert_snapshot_process_facts_cleared(&snapshot);
6075 }
6076
6077 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6078 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6079 let supervisor =
6080 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6081 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6082 let module = supervisor.supervised_module(
6083 ModuleSpec {
6084 module_id: "drop-clears-facts".to_string(),
6085 program: PathBuf::from("/unused/drop-module"),
6086 args: Vec::new(),
6087 env: Vec::new(),
6088 reserved: false,
6089 reserved_prefixes: Vec::new(),
6090 protocol: ModuleProtocol::Subc,
6091 overlap: Default::default(),
6092 },
6093 supervisor.runtime_config(),
6094 Arc::clone(&snapshot),
6095 None,
6096 );
6097 assert!(!module
6098 .inner
6099 .monitor
6100 .lock()
6101 .unwrap()
6102 .as_ref()
6103 .unwrap()
6104 .is_finished());
6105
6106 drop(module);
6107
6108 assert_eq!(
6109 lock_snapshot(&snapshot).unwrap().state,
6110 ModuleState::Stopped
6111 );
6112 assert_snapshot_process_facts_cleared(&snapshot);
6113 }
6114
6115 #[cfg(unix)]
6116 #[tokio::test]
6117 async fn rescan_preserves_running_protocol_until_respawn() {
6118 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6119 let initial = ModuleSpec {
6120 module_id: "rescan-protocol".into(),
6121 program: PathBuf::from("/bin/sleep"),
6122 args: vec!["60".into()],
6123 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6124 .into_iter()
6125 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6126 .collect(),
6127 reserved: false,
6128 reserved_prefixes: vec![],
6129 protocol: ModuleProtocol::None,
6130 overlap: Default::default(),
6131 };
6132 let supervisor =
6133 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6134 let module = supervisor.spawn(initial.clone()).unwrap();
6135 assert!(module.status().unwrap().live);
6136 let mut next = initial;
6137 next.protocol = ModuleProtocol::Subc;
6138 module
6139 .update_configuration(next.clone(), HealthConfig::default(), None)
6140 .await
6141 .unwrap();
6142 assert!(
6143 module.status().unwrap().live,
6144 "rescan must not require HELLO from the old non-wire process"
6145 );
6146 let runtime = supervisor.runtime_config();
6147 let action = on_child_exit(
6148 &next,
6149 RestartPolicy::default(),
6150 &supervisor.registry,
6151 &module.inner.snapshot,
6152 &runtime.terminal_ring,
6153 &runtime.spawn_events,
6154 &runtime.child_roster,
6155 ExitReport {
6156 kind: ExitKind::Clean,
6157 code: Some(0),
6158 signal: None,
6159 at_ms: unix_ms_now(),
6160 },
6161 true,
6162 )
6163 .await;
6164 assert!(
6165 matches!(action, NextAction::Restart { .. }),
6166 "the old non-wire process's clean exit must restart"
6167 );
6168 module.drain().await.unwrap();
6169 }
6170
6171 #[cfg(unix)]
6172 #[tokio::test]
6173 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6174 use std::os::unix::fs::PermissionsExt;
6175 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6176 let script = dir.join("module.sh");
6177 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6178 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6179 let record_path = dir.join("live-children.json");
6180 let supervisor =
6181 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6182 .with_live_children_record(&record_path);
6183 for (program, args) in [
6184 (PathBuf::from("sleep"), vec!["60".into()]),
6185 (script, vec![]),
6186 ] {
6187 let spec = ModuleSpec {
6188 module_id: "image-identity".into(),
6189 program: program.clone(),
6190 args,
6191 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6192 .into_iter()
6193 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6194 .collect(),
6195 reserved: false,
6196 reserved_prefixes: vec![],
6197 protocol: ModuleProtocol::None,
6198 overlap: Default::default(),
6199 };
6200 let module = supervisor.spawn(spec).unwrap();
6201 #[cfg(target_os = "macos")]
6202 {
6203 let deadline = Instant::now() + Duration::from_secs(5);
6206 while crate::live_children::read_record(&record_path)
6207 .unwrap()
6208 .iter()
6209 .all(|entry| entry.executable.is_none())
6210 {
6211 assert!(Instant::now() < deadline, "module image was not confirmed");
6212 tokio::time::sleep(Duration::from_millis(5)).await;
6213 }
6214 }
6215 let entry = crate::live_children::read_record(&record_path)
6216 .unwrap()
6217 .pop()
6218 .unwrap();
6219 let observed = subc_os::Process::open(entry.pid)
6220 .unwrap()
6221 .unwrap()
6222 .observe()
6223 .unwrap();
6224 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6225 module.drain().await.unwrap();
6226 assert_eq!(
6227 verdict,
6228 crate::live_children::IdentityVerdict::Matches,
6229 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6230 );
6231 }
6232 }
6233
6234 #[cfg(unix)]
6235 fn http_fixture(
6236 dir: &std::path::Path,
6237 url: &str,
6238 threshold: u32,
6239 ) -> crate::daemon_config::ConfiguredModule {
6240 let path = dir.join("subc.jsonc");
6241 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6242 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6243 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6244 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6245 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6246 }}}).to_string()).unwrap();
6247 crate::daemon_config::load(&path)
6248 .unwrap()
6249 .unwrap()
6250 .modules
6251 .pop()
6252 .unwrap()
6253 }
6254
6255 #[cfg(unix)]
6256 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6257 timeout(Duration::from_secs(5), async {
6258 loop {
6259 if module.status().unwrap().health.status == status {
6260 break;
6261 }
6262 sleep(Duration::from_millis(5)).await;
6263 }
6264 })
6265 .await
6266 .unwrap_or_else(|_| {
6267 panic!(
6268 "expected {status:?}, got {:?}",
6269 module.status().unwrap().health
6270 )
6271 });
6272 }
6273
6274 #[cfg(unix)]
6275 #[tokio::test]
6276 async fn http_health_status_flips_ok_failing_ok() {
6277 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6278 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6279 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6280 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6281 let serving_status = status.clone();
6282 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6283 let server = tokio::spawn(async move {
6284 loop {
6285 let (mut stream, _) = listener.accept().await.unwrap();
6286 let mut request = [0u8; 2048];
6287 let count = stream.read(&mut request).await.unwrap();
6288 assert!(count > 0, "a probe must send an HTTP request");
6289 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6290 let body = if code == 200 {
6291 "ready"
6292 } else {
6293 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6294 };
6295 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6296 let _ = stream.write_all(response.as_bytes()).await;
6297 }
6298 });
6299 let configured = http_fixture(&dir, &url, 1000);
6300 let module =
6301 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6302 .supervise_configured_with_health(
6303 configured.module_spec(),
6304 true,
6305 configured.health,
6306 configured.drain_timeout_ms,
6307 configured.restart,
6308 )
6309 .unwrap();
6310 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6311 status.store(503, std::sync::atomic::Ordering::SeqCst);
6312 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6313 assert!(module
6314 .status()
6315 .unwrap()
6316 .health
6317 .detail
6318 .unwrap()
6319 .contains("scratch failure"));
6320 status.store(200, std::sync::atomic::Ordering::SeqCst);
6321 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6322 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6323 let before = module.status().unwrap();
6324 let (spec, mut health) = module.configuration().unwrap();
6325 health.http = None;
6326 module
6327 .update_configuration(spec.clone(), health.clone(), Some(10))
6328 .await
6329 .unwrap();
6330 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6331 health.http = Some(url);
6332 module
6333 .update_configuration(spec, health, Some(10))
6334 .await
6335 .unwrap();
6336 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6337 assert_eq!(
6338 module.status().unwrap().pid,
6339 before.pid,
6340 "changing a probe must apply live, not restart its process"
6341 );
6342 let (spec, mut health) = module.configuration().unwrap();
6343 health.failure_threshold = 2;
6344 module
6345 .update_configuration(spec, health, Some(10))
6346 .await
6347 .unwrap();
6348 status.store(503, std::sync::atomic::Ordering::SeqCst);
6349 timeout(Duration::from_secs(5), async {
6350 while module.status().unwrap().spawn_generation == before.spawn_generation {
6351 sleep(Duration::from_millis(5)).await;
6352 }
6353 })
6354 .await
6355 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6356 module.drain().await.unwrap();
6357 server.abort();
6358 }
6359
6360 #[cfg(unix)]
6361 #[tokio::test]
6362 async fn http_health_replacement_recovers_after_one_restart() {
6363 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6364 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6365 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6366 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6367 let mut configured = http_fixture(&dir, &url, 2);
6368 configured.restart.max_restarts = 2;
6369 let module =
6370 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6371 .supervise_configured_with_health(
6372 configured.module_spec(),
6373 true,
6374 configured.health,
6375 configured.drain_timeout_ms,
6376 configured.restart,
6377 )
6378 .unwrap();
6379 let first_generation = module.status().unwrap().spawn_generation;
6380 let observed = module.clone();
6381 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6382 let server = tokio::spawn(async move {
6383 let mut generation = first_generation;
6384 let mut probes = 0;
6385 loop {
6386 let (mut stream, _) = listener.accept().await.unwrap();
6387 let mut request = [0; 2048];
6388 assert!(stream.read(&mut request).await.unwrap() > 0);
6389 let current = observed.status().unwrap().spawn_generation;
6390 if current != generation {
6391 generation = current;
6392 probes = 0;
6393 }
6394 probes += 1;
6395 let healthy = generation != first_generation && probes > 1;
6398 let code = if healthy { 200 } else { 503 };
6399 let response = format!(
6400 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6401 );
6402 let _ = stream.write_all(response.as_bytes()).await;
6403 if healthy {
6404 healthy_tx.send(()).await.unwrap();
6405 }
6406 }
6407 });
6408 timeout(Duration::from_secs(5), async {
6409 let mut healthy_probes = 0;
6410 loop {
6411 if module.state().unwrap() == ModuleState::Failed {
6412 break;
6413 }
6414 while healthy_rx.try_recv().is_ok() {
6415 healthy_probes += 1;
6416 }
6417 if healthy_probes >= 3
6418 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6419 {
6420 break;
6421 }
6422 sleep(Duration::from_millis(5)).await;
6423 }
6424 })
6425 .await
6426 .expect("the replacement must settle healthy without exhausting its restart budget");
6427 let status = module.status().unwrap();
6428 module.drain().await.unwrap();
6429 server.abort();
6430 assert_eq!(status.state, ModuleState::Running);
6431 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6432 assert_eq!(status.health.consecutive_failures, 0);
6433 assert_eq!(status.lifetime_restarts, 1);
6434 assert_eq!(status.restart_count, 1);
6435 }
6436
6437 #[cfg(unix)]
6438 #[tokio::test]
6439 async fn http_health_sustained_failure_exhausts_restart_budget() {
6440 use std::collections::BTreeMap;
6441 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6442 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6443 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6444 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6445 let mut configured = http_fixture(&dir, &url, 2);
6446 configured.restart.max_restarts = 2;
6447 let module =
6448 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6449 .supervise_configured_with_health(
6450 configured.module_spec(),
6451 true,
6452 configured.health,
6453 configured.drain_timeout_ms,
6454 configured.restart,
6455 )
6456 .unwrap();
6457 let observed = module.clone();
6458 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6459 let served = probes.clone();
6460 let server = tokio::spawn(async move {
6461 loop {
6462 let (mut stream, _) = listener.accept().await.unwrap();
6463 let mut request = [0; 2048];
6464 assert!(stream.read(&mut request).await.unwrap() > 0);
6465 *served
6466 .lock()
6467 .unwrap()
6468 .entry(observed.status().unwrap().spawn_generation)
6469 .or_default() += 1;
6470 let _ = stream
6471 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6472 .await;
6473 }
6474 });
6475 timeout(Duration::from_secs(5), async {
6476 while module.state().unwrap() != ModuleState::Failed {
6477 sleep(Duration::from_millis(5)).await;
6478 }
6479 })
6480 .await
6481 .expect("sustained failures must still exhaust the health restart budget");
6482 let status = module.status().unwrap();
6483 server.abort();
6484 assert_eq!(status.restart_count, 2);
6485 assert_eq!(status.lifetime_restarts, 2);
6486 assert!(!status.process_alive);
6487 assert_eq!(
6488 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6489 vec![2, 2, 2]
6490 );
6491 }
6492
6493 #[cfg(unix)]
6494 #[tokio::test]
6495 async fn spawn_clears_the_previous_process_health() {
6496 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6497 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6498 configured.health.cadence = Duration::from_secs(3600);
6499 let module =
6500 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6501 .supervise_configured_with_health(
6502 configured.module_spec(),
6503 false,
6504 configured.health,
6505 configured.drain_timeout_ms,
6506 configured.restart,
6507 )
6508 .unwrap();
6509 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6510 state.health.status = SupervisorHealthStatus::Unresponsive;
6511 state.health.consecutive_failures = 3;
6512 state.health.detail = Some("old process refused connections".into());
6513 state.health.metrics = Some(serde_json::json!({"old": true}));
6514 state.health.last_probe_ms = Some(123);
6515 state.health.last_action = Some("restart".into());
6516 state.health.last_action_ms = Some(124);
6517 })
6518 .unwrap();
6519 module.set_enabled(true).await.unwrap();
6520 let health = module.status().unwrap().health;
6521 module.drain().await.unwrap();
6522 assert_eq!(health.consecutive_failures, 0);
6523 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6524 assert_eq!(health.detail, None);
6525 assert_eq!(health.metrics, None);
6526 assert_eq!(health.last_probe_ms, None);
6527 assert_eq!(health.last_action.as_deref(), Some("restart"));
6528 assert_eq!(health.last_action_ms, Some(124));
6529 }
6530
6531 #[cfg(unix)]
6532 #[tokio::test]
6533 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6534 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6535 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6536 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6537 drop(unused);
6538 let configured = http_fixture(&dir, &url, 2);
6539 let module =
6540 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6541 .supervise_configured_with_health(
6542 configured.module_spec(),
6543 true,
6544 configured.health,
6545 configured.drain_timeout_ms,
6546 configured.restart,
6547 )
6548 .unwrap();
6549 let before = module.status().unwrap().spawn_generation;
6550 timeout(Duration::from_secs(5), async {
6551 loop {
6552 let status = module.status().unwrap();
6553 if status.spawn_generation > before {
6554 assert!(status.lifetime_restarts > 0);
6555 break;
6556 }
6557 sleep(Duration::from_millis(5)).await;
6558 }
6559 })
6560 .await
6561 .expect("sustained HTTP refusal must trigger the health restart policy");
6562 module.drain().await.unwrap();
6563 }
6564
6565 #[cfg(unix)]
6566 #[tokio::test]
6567 async fn http_health_timeout_honours_deadline() {
6568 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6569 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6570 let server = tokio::spawn(async move {
6571 let _held = listener.accept().await.unwrap();
6572 std::future::pending::<()>().await;
6573 });
6574 let error = timeout(
6575 Duration::from_secs(1),
6576 probe_http_health(&url, Duration::from_millis(10)),
6577 )
6578 .await
6579 .expect("the probe must enforce its own deadline")
6580 .unwrap_err();
6581 server.abort();
6582 assert!(error.to_string().contains("timed out"));
6583 }
6584
6585 #[cfg(unix)]
6586 #[tokio::test]
6587 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6588 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6589 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6590 let url = format!(
6591 "http://localhost:{}/healthz",
6592 listener.local_addr().unwrap().port()
6593 );
6594 let server = tokio::spawn(async move {
6595 let (mut stream, _) = listener.accept().await.unwrap();
6596 let mut request = [0u8; 2048];
6597 assert!(stream.read(&mut request).await.unwrap() > 0);
6598 stream
6599 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6600 .await
6601 .unwrap();
6602 });
6603 assert_eq!(
6608 probe_http_health(&url, Duration::from_secs(10))
6609 .await
6610 .unwrap()
6611 .status,
6612 HealthStatus::Ok
6613 );
6614 server.await.unwrap();
6615 }
6616
6617 #[cfg(unix)]
6618 #[tokio::test]
6619 async fn http_health_timeout_keeps_partial_status_and_body() {
6620 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6621 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6622 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6623 let server = tokio::spawn(async move {
6624 let (mut stream, _) = listener.accept().await.unwrap();
6625 let mut request = [0u8; 2048];
6626 assert!(stream.read(&mut request).await.unwrap() > 0);
6627 stream
6628 .write_all(
6629 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6630 )
6631 .await
6632 .unwrap();
6633 std::future::pending::<()>().await;
6634 });
6635 let error = probe_http_health(&url, Duration::from_secs(1))
6636 .await
6637 .unwrap_err()
6638 .to_string();
6639 server.abort();
6640 assert!(
6641 error.contains("timed out")
6642 && error.contains("503 Unavailable")
6643 && error.contains("partial diagnostic"),
6644 "{error}"
6645 );
6646 }
6647
6648 #[cfg(unix)]
6649 #[tokio::test]
6650 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6651 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6652 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6653 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6654 let server = tokio::spawn(async move {
6655 let (mut stream, _) = listener.accept().await.unwrap();
6656 let mut request = [0u8; 2048];
6657 assert!(stream.read(&mut request).await.unwrap() > 0);
6658 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6659 let response = format!(
6660 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6661 body.len()
6662 );
6663 stream.write_all(response.as_bytes()).await.unwrap();
6664 });
6665 let error = probe_http_health(&url, Duration::from_secs(1))
6666 .await
6667 .unwrap_err()
6668 .to_string();
6669 server.await.unwrap();
6670 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6671 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6672 assert!(!error.contains("not-in-diagnostic"));
6673 }
6674
6675 #[cfg(unix)]
6676 #[tokio::test]
6677 async fn http_health_real_nats_server_monitoring() {
6678 if std::process::Command::new("nats-server")
6679 .arg("--version")
6680 .env("XDG_DATA_HOME", std::env::temp_dir())
6681 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6682 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6683 .output()
6684 .is_err()
6685 {
6686 eprintln!(
6687 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6688 );
6689 return;
6690 }
6691 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6692 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6693 let port = monitor.local_addr().unwrap().port();
6694 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6695 let client_port = client.local_addr().unwrap().port();
6696 let config = dir.join("server.conf");
6697 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6698 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6699 configured.program = PathBuf::from("nats-server");
6700 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6701 drop(monitor);
6702 drop(client);
6703 let module =
6704 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6705 .supervise_configured_with_health(
6706 configured.module_spec(),
6707 true,
6708 configured.health,
6709 configured.drain_timeout_ms,
6710 configured.restart,
6711 )
6712 .unwrap();
6713 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6714 module.drain().await.unwrap();
6715 }
6716
6717 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6718 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6719 let supervisor =
6720 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6721 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6722 let initial = ModuleSpec {
6723 module_id: "rescan-preserves-spawn-facts".to_string(),
6724 program: PathBuf::from("/spawned/module"),
6725 args: Vec::new(),
6726 env: Vec::new(),
6727 reserved: false,
6728 reserved_prefixes: Vec::new(),
6729 protocol: ModuleProtocol::Subc,
6730 overlap: Default::default(),
6731 };
6732 let module = supervisor.supervised_module(
6733 initial.clone(),
6734 supervisor.runtime_config(),
6735 snapshot,
6736 None,
6737 );
6738 let before = module.status().unwrap();
6739 let mut replacement = initial;
6740 replacement.program = PathBuf::from("/rescanned/replacement-module");
6741
6742 module
6743 .update_configuration(replacement, HealthConfig::default(), None)
6744 .await
6745 .unwrap();
6746
6747 let after = module.status().unwrap();
6748 assert_eq!(after.pid, before.pid);
6749 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6750 assert_eq!(after.spawned_from, before.spawned_from);
6751 drop(module);
6752 }
6753}
6754
6755fn unix_ms_now() -> u64 {
6756 SystemTime::now()
6757 .duration_since(UNIX_EPOCH)
6758 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6759 .unwrap_or(0)
6760}
6761
6762async fn supervise_loop(
6763 mut spec: ModuleSpec,
6764 mut runtime: SupervisorRuntimeConfig,
6765 registry: Arc<Registry>,
6766 process_liveness: Arc<SupervisorProcessLiveness>,
6767 snapshot: SharedSnapshot,
6768 mut child: Option<SupervisedChild>,
6769 mut commands: mpsc::Receiver<SupervisorCommand>,
6770) {
6771 let mut health_probe = HealthProbeRuntime::default();
6772 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6776 Ok(changes) => Some(changes),
6777 Err(err) => {
6778 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6781 None
6782 }
6783 };
6784 let mut pending_respawn: Option<PendingRespawn> = None;
6788 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6791 let mut previous_instance_recheck = tokio::time::interval_at(
6792 Instant::now() + Duration::from_secs(5),
6793 Duration::from_secs(5),
6794 );
6795 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6796 loop {
6797 #[cfg(test)]
6798 {
6799 lock_snapshot(&snapshot).unwrap().actor_select = None;
6800 }
6801 #[cfg(target_os = "macos")]
6802 if let Some(active) = child.as_mut() {
6803 active.confirm_privacy_exec().await;
6804 }
6805 if let Some(scheduled) = runtime
6806 .scheduled_respawn
6807 .lock()
6808 .unwrap_or_else(|p| p.into_inner())
6809 .take()
6810 {
6811 runtime.recovery_is_daemon = scheduled.daemon_initiated;
6812 pending_respawn = Some(scheduled);
6813 }
6814 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6815 pending_respawn = None;
6816 cancel_deferred_reload(
6817 &runtime,
6818 &spec.module_id,
6819 "respawn cancelled by a supervisor command",
6820 );
6821 }
6822 if child.is_none()
6823 && pending_respawn.is_none()
6824 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6825 {
6826 cancel_deferred_reload(
6827 &runtime,
6828 &spec.module_id,
6829 "respawn cancelled before a replacement was spawned",
6830 );
6831 let mut stranded = false;
6832 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6833 state.respawn_pending = false;
6834 state.coalesced_restart_pending = false;
6835 if matches!(
6836 state.state,
6837 ModuleState::Restarting
6838 | ModuleState::Starting
6839 | ModuleState::Draining
6840 | ModuleState::Unresponsive
6841 ) {
6842 error!(module_id = %spec.module_id, state = ?state.state,
6843 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6844 stranded = true;
6845 }
6846 });
6847 if stranded {
6848 fail_snapshot(
6849 &snapshot,
6850 Some(&spec.module_id),
6851 None,
6852 &runtime.failure_context(runtime.recovery_is_daemon),
6853 );
6854 }
6855 }
6856 if let Some(command) = requeued.pop_front() {
6857 if !handle_supervisor_command(
6858 command,
6859 &mut spec,
6860 &mut runtime,
6861 ®istry,
6862 &process_liveness,
6863 &snapshot,
6864 &mut child,
6865 &mut commands,
6866 &mut requeued,
6867 )
6868 .await
6869 {
6870 return;
6871 }
6872 if child.is_some() || !respawn_still_pending(&snapshot) {
6873 pending_respawn = None;
6874 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6875 state.respawn_pending = false
6876 });
6877 }
6878 continue;
6879 }
6880 if child.is_some() {
6881 runtime.recovery_is_daemon = true;
6882 #[cfg(test)]
6883 {
6884 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6885 }
6886 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6887 let active = child.as_mut().expect("child checked above");
6888 if runtime.child_roster.is_closed() || active.registered_since_spawn() {
6889 active.registration_deadline = None;
6890 }
6891 let registration_deadline = active.registration_deadline;
6892 let no_checks = runtime.health.http.is_none()
6893 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
6894 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
6895 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
6896 state
6897 .failed_retry
6898 .uptime_reset_at(no_checks, runtime.restart_policy.window)
6899 });
6900 let wake_at = match (health_probe.wake_at(), uptime_reset) {
6901 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
6902 (probe, None) => probe,
6903 };
6904 let wake_at = match (wake_at, registration_deadline) {
6905 (Some(wake), Some(register)) => Some(wake.min(register)),
6906 (wake, register) => wake.or(register),
6907 };
6908 #[cfg(test)]
6909 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
6910 let probe_sleep = supervisor_sleep(wake_at);
6911 tokio::pin!(probe_sleep);
6912 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6917 #[cfg(test)]
6918 {
6919 let registration_pending = wire_child
6920 && registration_changes
6921 .as_ref()
6922 .is_some_and(|changes| changes.has_changed().unwrap());
6923 let mut state = lock_snapshot(&snapshot).unwrap();
6924 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6925 generation: state.spawn_generation,
6926 turn: state.actor_turns,
6927 registered_connection: health_probe.registered_connection,
6928 next_probe_at: health_probe.next_probe_at,
6929 wake_after,
6930 });
6931 }
6932 let active_child = child.as_mut().expect("child checked above");
6933 tokio::select! {
6934 wait_result = active_child.wait() => {
6935 if let Ok(mut state) = lock_snapshot(&snapshot) {
6936 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
6937 }
6938 #[cfg(test)]
6939 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6940 let exit_report = match wait_result {
6949 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6950 Err(err) => {
6951 active_child.drain_stderr(&spec.module_id).await;
6952 fail_after_wait_error(
6953 &spec.module_id,
6954 &snapshot,
6955 &runtime.terminal_ring,
6956 &runtime.spawn_events,
6957 format!("failed to wait for supervised module: {err}"),
6958 &runtime.failure_context(true),
6959 );
6960 untrack_if_registration_released(
6961 &process_liveness,
6962 ®istry,
6963 &spec.module_id,
6964 &snapshot,
6965 );
6966 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6967 child = None;
6968 continue;
6969 }
6970 };
6971 active_child.drain_stderr(&spec.module_id).await;
6972
6973 let next = on_child_exit(
6974 &spec,
6975 runtime.restart_policy,
6976 ®istry,
6977 &snapshot,
6978 &runtime.terminal_ring,
6979 &runtime.spawn_events,
6980 &runtime.child_roster,
6981 exit_report,
6982 true,
6983 ).await;
6984 runtime.start_requester = None;
6985 runtime.pending_activity_restart = false;
6986 active_child.release_roster();
6989 match next {
6990 NextAction::Stop { registration_released } => {
6991 if registration_released {
6992 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6993 }
6994 child = None;
6995 }
6996 NextAction::Restart { schedule } => {
6997 let delay = schedule.map_or(
6998 runtime.restart_policy.delay_for_restart(0),
6999 |schedule| schedule.delay,
7000 );
7001 if let Some(schedule) = schedule {
7002 log_crash_respawn(&spec.module_id, schedule);
7003 }
7004 child = None;
7012 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7013 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7014 }
7015 }
7016 }
7017 command = commands.recv() => {
7018 #[cfg(test)]
7019 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7020 let Some(command) = command else {
7021 return;
7022 };
7023 if !handle_supervisor_command(
7024 command,
7025 &mut spec,
7026 &mut runtime,
7027 ®istry,
7028 &process_liveness,
7029 &snapshot,
7030 &mut child,
7031 &mut commands,
7032 &mut requeued,
7033 ).await {
7034 return;
7035 }
7036 }
7037 _ = async {
7038 match registration_changes.as_mut() {
7039 Some(changes) => { let _ = changes.changed().await; }
7040 None => std::future::pending().await,
7041 }
7042 }, if wire_child => {
7043 #[cfg(test)]
7044 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7045 }
7046 _ = &mut probe_sleep => {
7047 #[cfg(test)]
7048 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7049 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7050 active_child.registration_deadline = None;
7053 if !runtime.child_roster.is_closed()
7054 && !active_child.registered_since_spawn()
7055 && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7056 {
7057 warn!(module_id = %spec.module_id, pid = active_child.pid,
7058 wait_secs = runtime.register_timeout.as_secs(),
7059 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7060 active_child.spawn_failure = Some("registration_timeout".into());
7061 if let Err(error) = stop_unregistered_child(active_child).await {
7062 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7063 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7064 &runtime.spawn_events, format!("registration_timeout: {error}"),
7065 &runtime.failure_context(true));
7066 child = None;
7067 }
7068 continue;
7071 }
7072 }
7073 if health_probe.due() {
7074 run_health_probe_cycle(
7075 &spec,
7076 &runtime,
7077 ®istry,
7078 &process_liveness,
7079 &snapshot,
7080 &mut child,
7081 ).await;
7082 if child.is_none() {
7083 runtime.start_requester = None;
7084 runtime.pending_activity_restart = false;
7085 }
7086 if child.is_some() {
7087 health_probe.schedule_next(&spec, runtime.health.cadence);
7088 }
7089 }
7090 }
7091 }
7092 } else if let Some(pending) = pending_respawn {
7093 tokio::select! {
7094 _ = sleep_until(pending.deadline) => {
7095 pending_respawn = None;
7096 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7097 if !respawn_still_pending(&snapshot) {
7101 continue;
7102 }
7103 if runtime.child_roster.is_closed() {
7108 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7109 state.state = ModuleState::Stopped;
7110 });
7111 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7112 continue;
7113 }
7114 if let Err(err) = release_dead_registration(
7115 ®istry,
7116 runtime.forwarding.as_deref(),
7117 &snapshot,
7118 &spec.module_id,
7119 &runtime.failure_context(runtime.recovery_is_daemon),
7120 ).await {
7121 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7122 continue;
7123 }
7124
7125 if matches!(pending.kind, RespawnKind::Reload) {
7126 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7127 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
7128 if result.is_ok() && runtime.pending_activity_restart {
7129 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7130 }
7131 runtime.pending_activity_restart = false;
7132 if let Some(reply) = reply { let _ = reply.send(result); }
7133 continue;
7134 }
7135 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7136 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7137 Ok(next_child) => {
7138 child = Some(next_child);
7139 if runtime.pending_activity_restart {
7140 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7141 runtime.pending_activity_restart = false;
7142 }
7143 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7144 }
7145 Err(err) => {
7146 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7147 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7148 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7149 }
7150 }
7151 }
7152 command = commands.recv() => {
7153 let Some(command) = command else {
7154 return;
7155 };
7156 if !handle_supervisor_command(
7157 command,
7158 &mut spec,
7159 &mut runtime,
7160 ®istry,
7161 &process_liveness,
7162 &snapshot,
7163 &mut child,
7164 &mut commands,
7165 &mut requeued,
7166 ).await {
7167 return;
7168 }
7169 if child.is_some() || !respawn_still_pending(&snapshot) {
7174 pending_respawn = None;
7175 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7176 }
7177 }
7178 }
7179 } else {
7180 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7181 (state.enabled && state.state == ModuleState::Failed)
7182 .then_some(state.failed_retry.pending)
7183 .flatten()
7184 });
7185 let command = if lock_snapshot(&snapshot)
7186 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7187 {
7188 tokio::select! {
7189 command = commands.recv() => command,
7190 _ = previous_instance_recheck.tick() => {
7191 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7192 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7193 if enabled && !runtime.child_roster.is_closed() {
7194 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7195 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7196 Ok(next_child) => child = Some(next_child),
7197 Err(error) => {
7198 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7199 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7200 }
7201 }
7202 } else {
7203 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7204 state.waiting_for_previous_pid = None;
7205 state.waiting_for_previous_detail = None;
7206 });
7207 }
7208 }
7209 continue;
7210 }
7211 }
7212 } else {
7213 tokio::select! {
7214 biased;
7215 command = commands.recv() => command,
7216 _ = async {
7217 match slow_retry {
7218 Some(retry) => sleep_until(retry.deadline).await,
7219 None => std::future::pending().await,
7220 }
7221 } => {
7222 let retry = slow_retry.expect("only a scheduled slow retry can wake");
7223 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7224 state.failed_retry.pending = None;
7225 });
7226 if runtime.child_roster.is_closed() {
7227 continue;
7228 }
7229 info!(module_id = %spec.module_id, attempt = retry.attempt,
7230 delay_secs = retry.delay.as_secs(), "starting failed module retry");
7231 if let Err(error) = set_child_enabled(
7232 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
7233 &runtime.failure_context(true),
7234 ).await {
7235 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7236 }
7237 continue;
7238 }
7239 }
7240 };
7241 let Some(command) = command else {
7242 return;
7243 };
7244 if !handle_supervisor_command(
7245 command,
7246 &mut spec,
7247 &mut runtime,
7248 ®istry,
7249 &process_liveness,
7250 &snapshot,
7251 &mut child,
7252 &mut commands,
7253 &mut requeued,
7254 )
7255 .await
7256 {
7257 return;
7258 }
7259 }
7260 }
7261}
7262
7263async fn supervisor_sleep(wake_at: Option<Instant>) {
7266 match wake_at {
7267 Some(deadline) => sleep_until(deadline).await,
7268 None => std::future::pending().await,
7269 }
7270}
7271
7272fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7273 info!(
7274 module_id,
7275 restart_in_window = schedule.restart_in_window,
7276 delay_ms = schedule.delay.as_millis() as u64,
7277 "respawning after crash"
7278 );
7279}
7280
7281fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7287 matches!(
7288 lock_snapshot(snapshot),
7289 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7290 )
7291}
7292
7293enum NextAction {
7294 Stop {
7295 registration_released: bool,
7296 },
7297 Restart {
7298 schedule: Option<CrashRestartSchedule>,
7299 },
7300}
7301
7302#[allow(clippy::too_many_arguments)]
7303async fn handle_supervisor_command(
7304 command: SupervisorCommand,
7305 spec: &mut ModuleSpec,
7306 runtime: &mut SupervisorRuntimeConfig,
7307 registry: &Arc<Registry>,
7308 process_liveness: &SupervisorProcessLiveness,
7309 snapshot: &SharedSnapshot,
7310 child: &mut Option<SupervisedChild>,
7311 commands: &mut mpsc::Receiver<SupervisorCommand>,
7312 requeued: &mut VecDeque<SupervisorCommand>,
7313) -> bool {
7314 if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7315 runtime.recovery_is_daemon = false;
7316 }
7317 if matches!(
7318 command,
7319 SupervisorCommand::Drain { .. }
7320 | SupervisorCommand::Retire { .. }
7321 | SupervisorCommand::SetEnabled { .. }
7322 | SupervisorCommand::Restart { .. }
7323 ) {
7324 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7325 state.failed_retry.cancel()
7326 });
7327 }
7328 let command =
7329 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7330 match command {
7331 SupervisorCommand::Restart { reply, .. }
7332 | SupervisorCommand::Reload { reply, .. }
7333 | SupervisorCommand::Swap { reply, .. } => {
7334 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7335 return true;
7336 }
7337 SupervisorCommand::SetEnabled {
7338 enabled: true,
7339 reply,
7340 ..
7341 } => {
7342 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7343 return true;
7344 }
7345 other => other,
7346 }
7347 } else {
7348 command
7349 };
7350 match command {
7351 SupervisorCommand::Drain { requester, reply } => {
7352 let result = drain_optional_child(
7355 &spec.module_id,
7356 spec.protocol,
7357 StopNotice::NotSent,
7358 registry,
7359 runtime.forwarding.as_deref(),
7360 snapshot,
7361 &runtime.terminal_ring,
7362 &runtime.spawn_events,
7363 child,
7364 runtime.drain_timeout,
7365 ModuleState::Stopped,
7366 None,
7367 &FailureContext::operator(),
7368 )
7369 .await;
7370 let registration_released = result.is_ok();
7371 if registration_released {
7372 runtime.record_activity_transition(
7373 "subc.module_stopped",
7374 &spec.module_id,
7375 requester,
7376 );
7377 }
7378 let _ = reply.send(result);
7379 if registration_released {
7380 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7381 }
7382 false
7383 }
7384 SupervisorCommand::Retire { requester, reply } => {
7385 let result = async {
7386 let stop_notice = begin_forwarding_drain_if_configured(
7387 spec,
7388 runtime,
7389 registry,
7390 snapshot,
7391 None,
7392 RouteCloseReason::Disable,
7393 )
7394 .await?;
7395 drain_optional_child(
7396 &spec.module_id,
7397 spec.protocol,
7398 stop_notice,
7399 registry,
7400 runtime.forwarding.as_deref(),
7401 snapshot,
7402 &runtime.terminal_ring,
7403 &runtime.spawn_events,
7404 child,
7405 runtime.drain_timeout,
7406 ModuleState::Stopped,
7407 None,
7408 &FailureContext::operator(),
7409 )
7410 .await
7411 }
7412 .await;
7413 let registration_released = result.is_ok();
7414 if registration_released {
7415 runtime.record_activity_transition(
7416 "subc.module_stopped",
7417 &spec.module_id,
7418 requester,
7419 );
7420 }
7421 let _ = reply.send(result);
7422 if registration_released {
7423 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7424 }
7425 false
7426 }
7427 SupervisorCommand::Restart {
7428 requester,
7429 drain_timeout_ms,
7430 received_at_generation,
7431 queued_at,
7432 reply,
7433 } => {
7434 info!(
7438 module_id = %spec.module_id,
7439 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7440 "restart command dequeued"
7441 );
7442 let validation = match lock_snapshot(snapshot) {
7454 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7455 module_id: spec.module_id.clone(),
7456 }),
7457 Ok(_) => Ok(()),
7458 Err(err) => Err(err),
7459 };
7460 let initiated = validation.is_ok();
7461 let _ = reply.send(validation);
7462 let satisfied_by_generation = if initiated
7476 && child
7477 .as_ref()
7478 .is_some_and(|child| child.spawned_at > queued_at)
7479 {
7480 lock_snapshot(snapshot).ok().and_then(|state| {
7481 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7482 })
7483 } else {
7484 None
7485 };
7486 let satisfied_by_pending = initiated
7487 && child.is_none()
7488 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7489 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7493 if pending {
7494 state.coalesced_restart_pending = true;
7495 }
7496 pending
7497 });
7498 if satisfied_by_pending {
7499 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7500 } else if let Some(generation) = satisfied_by_generation {
7501 info!(
7502 module_id = %spec.module_id,
7503 received_at_generation,
7504 "restart already satisfied by generation {generation}; not restarting again"
7505 );
7506 } else if initiated {
7507 let drain_timeout = drain_timeout_ms
7510 .map(Duration::from_millis)
7511 .unwrap_or(runtime.drain_timeout);
7512 if let Err(err) = restart_child(
7513 spec,
7514 runtime,
7515 registry,
7516 process_liveness,
7517 snapshot,
7518 child,
7519 drain_timeout,
7520 )
7521 .await
7522 {
7523 warn!(
7524 module_id = %spec.module_id,
7525 error = %err,
7526 "operator restart failed after initiation ack; module state carries the outcome"
7527 );
7528 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7529 state.state = ModuleState::Failed;
7530 clear_current_process_facts(state);
7531 });
7532 } else {
7533 runtime.start_requester = requester;
7534 runtime.pending_activity_restart = true;
7535 }
7536 }
7537 true
7538 }
7539 SupervisorCommand::Reload { requester, reply } => {
7540 let result =
7541 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7542 if result.is_ok() {
7543 runtime.start_requester = requester;
7544 runtime.pending_activity_restart = true;
7545 }
7546 if result.is_ok()
7547 && runtime
7548 .scheduled_respawn
7549 .lock()
7550 .unwrap_or_else(|p| p.into_inner())
7551 .as_ref()
7552 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7553 {
7554 *runtime
7555 .deferred_reload_reply
7556 .lock()
7557 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7558 } else {
7559 let _ = reply.send(result);
7560 }
7561 true
7562 }
7563 SupervisorCommand::SetEnabled {
7564 enabled,
7565 requester,
7566 reply,
7567 } => {
7568 let previous_requester = runtime.start_requester.clone();
7569 if enabled {
7570 runtime.start_requester = requester.clone();
7571 }
7572 let result = set_child_enabled(
7573 spec,
7574 runtime,
7575 registry,
7576 process_liveness,
7577 snapshot,
7578 child,
7579 enabled,
7580 &runtime.failure_context(false),
7581 )
7582 .await;
7583 if matches!(result, Ok(true)) {
7584 runtime.pending_activity_restart = false;
7585 if enabled {
7586 runtime.start_requester = requester;
7587 } else {
7588 runtime.start_requester = None;
7589 runtime.record_activity_transition(
7590 "subc.module_disabled",
7591 &spec.module_id,
7592 requester,
7593 );
7594 }
7595 } else {
7596 runtime.start_requester = previous_requester;
7597 }
7598 let _ = reply.send(result);
7599 true
7600 }
7601 SupervisorCommand::UpdateConfiguration {
7602 spec: next_spec,
7603 health,
7604 drain_timeout_ms,
7605 reply,
7606 } => {
7607 if let Some(handle) = &runtime.supervisor_handle {
7608 handle.apply_identity_configuration(&next_spec);
7609 }
7610 *spec = next_spec;
7611 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7612 state.configuration_updated_since_spawn = true;
7613 });
7614 let health_changed = runtime.health != health;
7615 runtime.health = health;
7616 if health_changed {
7619 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7620 state.health = ModuleHealthStatus::default();
7621 });
7622 }
7623 runtime.drain_timeout = drain_timeout_ms
7624 .map(Duration::from_millis)
7625 .unwrap_or(runtime.default_drain_timeout);
7626 *runtime
7627 .effective_drain_timeout
7628 .lock()
7629 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7630 let _ = reply.send(());
7631 true
7632 }
7633 SupervisorCommand::Swap {
7634 requester,
7635 ready_timeout,
7636 reply,
7637 } => {
7638 let mut swap_runtime = runtime.clone();
7639 swap_runtime.start_requester = requester;
7640 let before = lock_snapshot(snapshot)
7641 .ok()
7642 .map(|state| state.spawn_generation);
7643 let end = swap::run_swap(
7644 spec,
7645 &swap_runtime,
7646 registry,
7647 process_liveness,
7648 snapshot,
7649 child,
7650 commands,
7651 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7652 reply,
7653 )
7654 .await;
7655 if before
7656 != lock_snapshot(snapshot)
7657 .ok()
7658 .map(|state| state.spawn_generation)
7659 {
7660 runtime.start_requester = swap_runtime.start_requester;
7661 }
7662 requeued.extend(end.requeue);
7663 true
7664 }
7665 }
7666}
7667
7668async fn restart_child(
7669 spec: &ModuleSpec,
7670 runtime: &SupervisorRuntimeConfig,
7671 registry: &Registry,
7672 process_liveness: &SupervisorProcessLiveness,
7673 snapshot: &SharedSnapshot,
7674 child: &mut Option<SupervisedChild>,
7675 drain_timeout: Duration,
7676) -> Result<(), SuperviseError> {
7677 if !lock_snapshot(snapshot)?.enabled {
7679 return Err(SuperviseError::Disabled {
7680 module_id: spec.module_id.clone(),
7681 });
7682 }
7683 let stop_notice = begin_forwarding_drain_with_timeout(
7684 spec,
7685 runtime,
7686 registry,
7687 snapshot,
7688 None,
7689 RouteCloseReason::Restart,
7690 drain_timeout,
7691 )
7692 .await?;
7693
7694 if child.is_some() {
7695 drain_optional_child(
7696 &spec.module_id,
7697 spec.protocol,
7698 stop_notice,
7699 registry,
7700 runtime.forwarding.as_deref(),
7701 snapshot,
7702 &runtime.terminal_ring,
7703 &runtime.spawn_events,
7704 child,
7705 drain_timeout,
7706 ModuleState::Restarting,
7707 Some(true),
7708 &FailureContext::operator(),
7709 )
7710 .await?;
7711 } else {
7712 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7713 state.enabled = true;
7714 state.state = ModuleState::Restarting;
7715 clear_current_process_facts(state);
7716 })?;
7717 release_dead_registration(
7718 registry,
7719 runtime.forwarding.as_deref(),
7720 snapshot,
7721 &spec.module_id,
7722 &runtime.failure_context(false),
7723 )
7724 .await?;
7725 }
7726
7727 reset_restart_count(snapshot, &spec.module_id)?;
7728 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7729 schedule_respawn(
7730 runtime,
7731 snapshot,
7732 &spec.module_id,
7733 runtime.restart_policy.backoff,
7734 RespawnKind::Spawn,
7735 false,
7736 )
7737}
7738
7739async fn reload_child(
7740 spec: &ModuleSpec,
7741 runtime: &SupervisorRuntimeConfig,
7742 registry: &Registry,
7743 process_liveness: &SupervisorProcessLiveness,
7744 snapshot: &SharedSnapshot,
7745 child: &mut Option<SupervisedChild>,
7746) -> Result<(), SuperviseError> {
7747 if !lock_snapshot(snapshot)?.enabled {
7749 return Err(SuperviseError::Disabled {
7750 module_id: spec.module_id.clone(),
7751 });
7752 }
7753 let stop_notice = begin_forwarding_drain(
7754 spec,
7755 runtime,
7756 registry,
7757 snapshot,
7758 Some(true),
7759 RouteCloseReason::Reload,
7760 )
7761 .await?;
7762
7763 if child.is_some() {
7764 drain_optional_child(
7765 &spec.module_id,
7766 spec.protocol,
7767 stop_notice,
7768 registry,
7769 runtime.forwarding.as_deref(),
7770 snapshot,
7771 &runtime.terminal_ring,
7772 &runtime.spawn_events,
7773 child,
7774 runtime.drain_timeout,
7775 ModuleState::Restarting,
7776 Some(true),
7777 &FailureContext::operator(),
7778 )
7779 .await?;
7780 } else {
7781 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7782 state.enabled = true;
7783 state.state = ModuleState::Restarting;
7784 clear_current_process_facts(state);
7785 })?;
7786 release_dead_registration(
7787 registry,
7788 runtime.forwarding.as_deref(),
7789 snapshot,
7790 &spec.module_id,
7791 &runtime.failure_context(false),
7792 )
7793 .await?;
7794 }
7795
7796 reset_restart_count(snapshot, &spec.module_id)?;
7797 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7798 schedule_respawn(
7799 runtime,
7800 snapshot,
7801 &spec.module_id,
7802 runtime.restart_policy.backoff,
7803 RespawnKind::Reload,
7804 false,
7805 )
7806}
7807
7808async fn finish_reload_child(
7809 spec: &ModuleSpec,
7810 runtime: &SupervisorRuntimeConfig,
7811 registry: &Registry,
7812 process_liveness: &SupervisorProcessLiveness,
7813 snapshot: &SharedSnapshot,
7814 child: &mut Option<SupervisedChild>,
7815) -> Result<(), SuperviseError> {
7816 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7817 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7818 Ok(next_child) => next_child,
7819 Err(err) => {
7820 return handle_reload_spawn_failure(
7821 spec,
7822 runtime,
7823 process_liveness,
7824 snapshot,
7825 child,
7826 format!("new child failed to spawn: {err}"),
7827 )
7828 .await;
7829 }
7830 };
7831 *child = Some(next_child);
7832
7833 let wait_outcome = {
7834 let active_child = child.as_mut().expect("new reload child was just stored");
7835 wait_for_registration_after_reload(
7836 registry,
7837 &spec.module_id,
7838 snapshot,
7839 active_child,
7840 REGISTRY_RELEASE_TIMEOUT,
7841 )
7842 .await?
7843 };
7844
7845 match wait_outcome {
7846 RegistrationWaitOutcome::Registered => {
7847 child
7848 .as_mut()
7849 .expect("registered reload child")
7850 .registration_deadline = None;
7851 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7852 Ok(())
7853 }
7854 RegistrationWaitOutcome::Exited(exit_report) => {
7855 if let Some(active_child) = child.as_mut() {
7856 active_child.drain_stderr(&spec.module_id).await;
7857 }
7858 let mut exited_child = child.take().expect("exited reload child is still stored");
7861 #[cfg(test)]
7862 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7863 gate.reached.notify_one();
7864 gate.resume.notified().await;
7865 }
7866 let result = handle_reload_child_registration_failure(
7867 spec,
7868 runtime,
7869 registry,
7870 process_liveness,
7871 snapshot,
7872 child,
7873 ReloadRegistrationFailure {
7874 exit_report: registration_failure_exit_report(exit_report),
7875 reason: exited_child
7876 .spawn_failure
7877 .clone()
7878 .unwrap_or_else(|| "new child exited before registering".to_string()),
7879 },
7880 )
7881 .await;
7882 exited_child.release_roster();
7883 result
7884 }
7885 RegistrationWaitOutcome::TimedOut => {
7886 let mut timed_out_child = child
7887 .take()
7888 .expect("timed-out reload child is still running");
7889 timed_out_child
7890 .start_kill()
7891 .map_err(|source| SuperviseError::Kill {
7892 module_id: spec.module_id.clone(),
7893 source,
7894 })?;
7895 let status = timed_out_child
7896 .wait()
7897 .await
7898 .map_err(|source| SuperviseError::Wait {
7899 module_id: spec.module_id.clone(),
7900 source,
7901 })?;
7902 timed_out_child.drain_stderr(&spec.module_id).await;
7903 handle_reload_child_registration_failure(
7904 spec,
7905 runtime,
7906 registry,
7907 process_liveness,
7908 snapshot,
7909 child,
7910 ReloadRegistrationFailure {
7911 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7912 snapshot,
7913 &timed_out_child,
7914 &status,
7915 )),
7916 reason: format!(
7917 "new child did not register within {:?}",
7918 REGISTRY_RELEASE_TIMEOUT
7919 ),
7920 },
7921 )
7922 .await
7923 }
7924 }
7925}
7926
7927#[allow(clippy::too_many_arguments)]
7928async fn set_child_enabled(
7929 spec: &ModuleSpec,
7930 runtime: &SupervisorRuntimeConfig,
7931 registry: &Registry,
7932 process_liveness: &SupervisorProcessLiveness,
7933 snapshot: &SharedSnapshot,
7934 child: &mut Option<SupervisedChild>,
7935 enabled: bool,
7936 failure: &FailureContext,
7937) -> Result<bool, SuperviseError> {
7938 let (current_enabled, current_state, respawn_pending) = {
7939 let state = lock_snapshot(snapshot)?;
7940 (state.enabled, state.state, state.respawn_pending)
7941 };
7942 let revive_terminal = enabled
7950 && current_enabled
7951 && child.is_none()
7952 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7953 || (current_state == ModuleState::Restarting && !respawn_pending));
7954 if current_enabled == enabled && !revive_terminal {
7955 return Ok(false);
7956 }
7957
7958 if enabled {
7959 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7960 state.enabled = true;
7961 state.state = ModuleState::Starting;
7962 clear_current_process_facts(state);
7963 })?;
7964 #[cfg(test)]
7965 if runtime.test_seed_stale_facts_before_enable_spawn {
7966 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7967 state.process_alive = true;
7968 state.pid = Some(41);
7969 state.spawned_at_ms = Some(42);
7970 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7971 state.spawned_file_identity = Some(SpawnedFileIdentity {
7972 device: 43,
7973 inode: 44,
7974 #[cfg(windows)]
7975 windows_image: None,
7976 });
7977 })?;
7978 }
7979 release_dead_registration(
7980 registry,
7981 runtime.forwarding.as_deref(),
7982 snapshot,
7983 &spec.module_id,
7984 failure,
7985 )
7986 .await?;
7987 reset_restart_count(snapshot, &spec.module_id)?;
7988 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7989 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7990 Ok(next_child) => next_child,
7991 Err(err) => {
7992 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7993 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7994 return Err(err);
7995 }
7996 };
7997 *child = Some(next_child);
7998 debug!(module_id = %spec.module_id, "supervised module enabled");
7999 Ok(true)
8000 } else {
8001 let stop_notice = begin_forwarding_drain_if_configured(
8002 spec,
8003 runtime,
8004 registry,
8005 snapshot,
8006 Some(false),
8007 RouteCloseReason::Disable,
8008 )
8009 .await?;
8010 drain_optional_child(
8011 &spec.module_id,
8012 spec.protocol,
8013 stop_notice,
8014 registry,
8015 runtime.forwarding.as_deref(),
8016 snapshot,
8017 &runtime.terminal_ring,
8018 &runtime.spawn_events,
8019 child,
8020 runtime.drain_timeout,
8021 ModuleState::Disabled,
8022 Some(false),
8023 failure,
8024 )
8025 .await?;
8026 debug!(module_id = %spec.module_id, "supervised module disabled");
8027 Ok(true)
8028 }
8029}
8030
8031#[allow(clippy::too_many_arguments)]
8032async fn on_child_exit(
8033 spec: &ModuleSpec,
8034 policy: RestartPolicy,
8035 registry: &Registry,
8036 snapshot: &SharedSnapshot,
8037 terminal_ring: &Arc<Mutex<TerminalRing>>,
8038 spawn_events: &SpawnEventFeed,
8039 roster: &ChildRoster,
8040 exit_report: ExitReport,
8041 daemon_initiated: bool,
8042) -> NextAction {
8043 if roster.is_closed() {
8049 return on_child_exit_during_daemon_shutdown(
8050 spec,
8051 registry,
8052 snapshot,
8053 terminal_ring,
8054 spawn_events,
8055 exit_report,
8056 )
8057 .await;
8058 }
8059 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8075 && running_protocol(spec, snapshot) == ModuleProtocol::None;
8076 match exit_report.kind {
8077 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8078 info!(
8079 module_id = %spec.module_id,
8080 exit_code = ?exit_report.code,
8081 exit_signal = ?exit_report.signal,
8082 "supervised module exited cleanly"
8083 );
8084 if let Err(err) = publish_after_terminal_record(
8085 &spec.module_id,
8086 || {
8087 record_terminal(
8088 &spec.module_id,
8089 terminal_ring,
8090 spawn_events,
8091 &exit_report,
8092 TerminalDisposition::Stopped,
8093 )
8094 },
8095 || {
8096 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8097 state.state = ModuleState::Stopped;
8098 clear_current_process_facts(state);
8099 state.last_exit = Some(exit_report.clone());
8100 })
8101 },
8102 ) {
8103 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8104 }
8105 let registration_released = match wait_for_registration_release(
8106 registry,
8107 &spec.module_id,
8108 REGISTRY_RELEASE_TIMEOUT,
8109 )
8110 .await
8111 {
8112 Ok(()) => true,
8113 Err(err) => {
8114 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8115 false
8116 }
8117 };
8118 NextAction::Stop {
8119 registration_released,
8120 }
8121 }
8122 ExitKind::Clean | ExitKind::Crash => {
8123 if unrequested_clean_exit_of_protocol_none {
8124 warn!(
8125 module_id = %spec.module_id,
8126 exit_code = ?exit_report.code,
8127 exit_signal = ?exit_report.signal,
8128 "protocol-none module exited cleanly without a stop request; handling it as a crash"
8129 );
8130 } else {
8131 warn!(
8132 module_id = %spec.module_id,
8133 exit_code = ?exit_report.code,
8134 exit_signal = ?exit_report.signal,
8135 "supervised module exited abnormally (crash)"
8136 );
8137 }
8138 let mut restart_schedule = None;
8139 let mut disposition = TerminalDisposition::Disabled;
8140 let mut final_state = ModuleState::Disabled;
8141 let mut disposition_detail = lock_snapshot(snapshot)
8145 .ok()
8146 .and_then(|mut state| state.spawn_failure.take());
8147 let now = Instant::now();
8148 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8149 if state.enabled {
8150 if let Some(schedule) = state.next_crash_restart(&policy, now) {
8151 state.respawn_pending = true;
8157 final_state = ModuleState::Restarting;
8158 restart_schedule = Some(schedule);
8159 disposition = TerminalDisposition::Restarting;
8160 } else {
8161 final_state = ModuleState::Failed;
8162 disposition = TerminalDisposition::Failed;
8163 let budget = policy.budget_exhausted_detail();
8164 disposition_detail =
8165 Some(disposition_detail.take().map_or_else(
8166 || budget.clone(),
8167 |cause| format!("{cause}; {budget}"),
8168 ));
8169 }
8170 } else {
8171 disposition = TerminalDisposition::Disabled;
8172 }
8173 }) {
8174 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8175 return NextAction::Stop {
8176 registration_released: false,
8177 };
8178 }
8179 if disposition == TerminalDisposition::Failed {
8180 error!(
8185 module_id = %spec.module_id,
8186 max_restarts = policy.max_restarts,
8187 window_secs = policy.window.as_secs(),
8188 "module stopped: {}",
8189 policy.budget_exhausted_detail()
8190 );
8191 }
8192 if let Err(err) = publish_after_terminal_record(
8193 &spec.module_id,
8194 || {
8195 record_terminal_with_detail(
8196 &spec.module_id,
8197 terminal_ring,
8198 spawn_events,
8199 &exit_report,
8200 disposition,
8201 disposition_detail,
8202 )
8203 },
8204 || {
8205 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8206 state.state = final_state;
8207 clear_current_process_facts(state);
8208 if final_state == ModuleState::Failed
8209 && daemon_initiated
8210 && !roster.is_closed()
8211 {
8212 state.failed_retry.schedule(&spec.module_id);
8213 }
8214 state.last_exit = Some(exit_report.clone());
8215 })
8216 },
8217 ) {
8218 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8219 }
8220
8221 if let Some(schedule) = restart_schedule {
8222 NextAction::Restart {
8223 schedule: Some(schedule),
8224 }
8225 } else {
8226 let registration_released = match wait_for_registration_release(
8227 registry,
8228 &spec.module_id,
8229 REGISTRY_RELEASE_TIMEOUT,
8230 )
8231 .await
8232 {
8233 Ok(()) => true,
8234 Err(err) => {
8235 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8236 false
8237 }
8238 };
8239 NextAction::Stop {
8240 registration_released,
8241 }
8242 }
8243 }
8244 ExitKind::DeliberateSeverance => {
8245 warn!(
8246 module_id = %spec.module_id,
8247 exit_code = ?exit_report.code,
8248 exit_signal = ?exit_report.signal,
8249 "supervised module exited after deliberate connection severance"
8250 );
8251 let should_restart = match lock_snapshot(snapshot) {
8252 Ok(state) => state.enabled,
8253 Err(err) => {
8254 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8255 return NextAction::Stop {
8256 registration_released: false,
8257 };
8258 }
8259 };
8260 let final_state = if should_restart {
8261 ModuleState::Restarting
8262 } else {
8263 ModuleState::Disabled
8264 };
8265 if let Err(err) = publish_after_terminal_record(
8266 &spec.module_id,
8267 || {
8268 record_terminal(
8269 &spec.module_id,
8270 terminal_ring,
8271 spawn_events,
8272 &exit_report,
8273 terminal_disposition(final_state),
8274 )
8275 },
8276 || {
8277 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8278 state.state = final_state;
8279 clear_current_process_facts(state);
8280 state.last_exit = Some(exit_report.clone());
8281 state.lifetime_restarts += 1;
8282 })
8283 },
8284 ) {
8285 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8286 return NextAction::Stop {
8287 registration_released: false,
8288 };
8289 }
8290
8291 if should_restart {
8292 NextAction::Restart { schedule: None }
8293 } else {
8294 let registration_released = match wait_for_registration_release(
8295 registry,
8296 &spec.module_id,
8297 REGISTRY_RELEASE_TIMEOUT,
8298 )
8299 .await
8300 {
8301 Ok(()) => true,
8302 Err(err) => {
8303 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8304 false
8305 }
8306 };
8307 NextAction::Stop {
8308 registration_released,
8309 }
8310 }
8311 }
8312 }
8313}
8314
8315async fn on_child_exit_during_daemon_shutdown(
8316 spec: &ModuleSpec,
8317 registry: &Registry,
8318 snapshot: &SharedSnapshot,
8319 terminal_ring: &Arc<Mutex<TerminalRing>>,
8320 spawn_events: &SpawnEventFeed,
8321 exit_report: ExitReport,
8322) -> NextAction {
8323 info!(
8324 module_id = %spec.module_id,
8325 exit_code = ?exit_report.code,
8326 exit_signal = ?exit_report.signal,
8327 exit_kind = ?exit_report.kind,
8328 "supervised module exited during daemon shutdown; not restarting it"
8329 );
8330 if let Err(err) = publish_after_terminal_record(
8331 &spec.module_id,
8332 || {
8333 record_terminal(
8334 &spec.module_id,
8335 terminal_ring,
8336 spawn_events,
8337 &exit_report,
8338 TerminalDisposition::DaemonShutdown,
8339 )
8340 },
8341 || {
8342 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8343 state.state = ModuleState::Stopped;
8344 clear_current_process_facts(state);
8345 state.last_exit = Some(exit_report.clone());
8346 })
8347 },
8348 ) {
8349 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8350 }
8351 let registration_released =
8352 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8353 .await
8354 .is_ok();
8355 NextAction::Stop {
8356 registration_released,
8357 }
8358}
8359
8360fn publish_after_terminal_record<T>(
8365 module_id: &str,
8366 record: impl FnOnce(),
8367 publish: impl FnOnce() -> T,
8368) -> T {
8369 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8370 let published = publish();
8371 if let Err(panic) = recorded {
8372 error!(
8373 module_id,
8374 "terminal recording panicked; published final state before resuming unwind"
8375 );
8376 std::panic::resume_unwind(panic);
8377 }
8378 published
8379}
8380
8381fn fail_after_wait_error(
8382 module_id: &str,
8383 snapshot: &SharedSnapshot,
8384 terminal_ring: &Arc<Mutex<TerminalRing>>,
8385 spawn_events: &SpawnEventFeed,
8386 detail: String,
8387 failure: &FailureContext,
8388) {
8389 let report = wait_error_exit_report();
8390 publish_after_terminal_record(
8391 module_id,
8392 || {
8393 record_terminal_with_detail(
8394 module_id,
8395 terminal_ring,
8396 spawn_events,
8397 &report,
8398 TerminalDisposition::Failed,
8399 Some(detail),
8400 )
8401 },
8402 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8403 );
8404}
8405
8406#[cfg(test)]
8407fn record_wait_error_terminal(
8408 module_id: &str,
8409 terminal_ring: &Arc<Mutex<TerminalRing>>,
8410 spawn_events: &SpawnEventFeed,
8411) {
8412 record_terminal(
8413 module_id,
8414 terminal_ring,
8415 spawn_events,
8416 &wait_error_exit_report(),
8417 TerminalDisposition::Failed,
8418 );
8419}
8420
8421fn record_terminal(
8422 module_id: &str,
8423 terminal_ring: &Arc<Mutex<TerminalRing>>,
8424 spawn_events: &SpawnEventFeed,
8425 exit_report: &ExitReport,
8426 disposition: TerminalDisposition,
8427) {
8428 record_terminal_with_detail(
8429 module_id,
8430 terminal_ring,
8431 spawn_events,
8432 exit_report,
8433 disposition,
8434 None,
8435 );
8436}
8437
8438fn durable_terminal_history_of(
8442 terminal_ring: &Mutex<TerminalRing>,
8443 module_id: &str,
8444) -> subc_control::TerminalHistory {
8445 let read = terminal_ring
8446 .lock()
8447 .unwrap_or_else(|p| p.into_inner())
8448 .capture_durable_history();
8449 read.read(module_id)
8450}
8451
8452fn record_terminal_with_detail(
8453 module_id: &str,
8454 terminal_ring: &Arc<Mutex<TerminalRing>>,
8455 spawn_events: &SpawnEventFeed,
8456 exit_report: &ExitReport,
8457 disposition: TerminalDisposition,
8458 disposition_detail: Option<String>,
8459) {
8460 #[cfg(test)]
8461 terminal_history_tests::before_terminal_record(spawn_events);
8462 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8463 let record = TerminalRecord {
8464 exit_code: exit_report.code,
8465 exit_signal: exit_report.signal,
8466 at_ms: exit_report.at_ms,
8467 disposition,
8468 exit_kind: exit_report.kind.into(),
8469 disposition_detail,
8470 };
8471 terminal_ring
8472 .lock()
8473 .unwrap_or_else(|poisoned| poisoned.into_inner())
8474 .record_exit(module_id, record);
8475}
8476
8477fn untrack_if_registration_released(
8478 process_liveness: &SupervisorProcessLiveness,
8479 registry: &Registry,
8480 module_id: &str,
8481 snapshot: &SharedSnapshot,
8482) {
8483 match registry.get_module(module_id) {
8484 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8485 Ok(Some(_)) => {}
8486 Err(err) => {
8487 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8488 }
8489 }
8490}
8491
8492#[cfg(test)]
8506fn apply_wire_spawn_args(
8507 command: &mut Command,
8508 spec: &ModuleSpec,
8509 connection_file_path: Option<&std::path::Path>,
8510 handle: Option<&SupervisorHandle>,
8511) -> Result<Option<NonceHandoff>, SuperviseError> {
8512 apply_wire_spawn_args_for_role(
8513 command,
8514 spec,
8515 connection_file_path,
8516 handle,
8517 SpawnRole::Plain,
8518 )
8519}
8520
8521#[cfg(unix)]
8526type NonceHandoff = subc_os::LaunchNonceHandoff;
8527#[cfg(windows)]
8528type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8529#[cfg(not(any(unix, windows)))]
8530type NonceHandoff = std::convert::Infallible;
8531
8532fn apply_wire_spawn_args_for_role(
8553 command: &mut Command,
8554 spec: &ModuleSpec,
8555 connection_file_path: Option<&std::path::Path>,
8556 handle: Option<&SupervisorHandle>,
8557 role: SpawnRole,
8558) -> Result<Option<NonceHandoff>, SuperviseError> {
8559 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8560 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8565 #[cfg(windows)]
8566 {
8567 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8568 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8569 }
8570 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8572 if spec.protocol == ModuleProtocol::None {
8573 return Ok(None);
8574 }
8575 if let Some(connection_file_path) = connection_file_path {
8576 command.arg(SUBC_ARG).arg(connection_file_path);
8577 }
8578
8579 let nonce = generate_launch_nonce()?;
8583 if let Some(handle) = handle {
8584 match role {
8585 SpawnRole::Plain => {
8586 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8587 if spec.reserved {
8588 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8589 }
8590 }
8591 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8592 }
8593 }
8594 #[cfg(unix)]
8595 let handoff = {
8596 let handoff =
8597 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8598 program: spec.program.clone(),
8599 source,
8600 cgroup_path: None,
8601 })?;
8602 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8603 Some(handoff)
8604 };
8605 #[cfg(windows)]
8606 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8607 #[cfg(not(any(unix, windows)))]
8608 let handoff = None;
8609 #[cfg(not(unix))]
8613 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8614 Ok(handoff)
8615}
8616
8617fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8618 command.env_remove(CK_LOG_ENV);
8619 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8626 for (key, value) in &spec.env {
8627 if matches!(
8631 key.as_str(),
8632 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8633 ) || key == SUBC_SPAWN_ROLE_ENV
8634 {
8635 continue;
8636 }
8637 command.env(key, value);
8638 }
8639}
8640
8641#[cfg(windows)]
8642fn prepare_windows_nonce_pipe(
8643 command: &mut Command,
8644 spec: &ModuleSpec,
8645 nonce: &str,
8646) -> Option<NonceHandoff> {
8647 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8648 Ok(handoff) => {
8649 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8650 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8655 Some(handoff)
8656 }
8657 Err(error) => {
8658 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8659 None
8660 }
8661 }
8662}
8663
8664#[cfg(windows)]
8665fn start_windows_nonce_pipe(
8666 handoff: Option<NonceHandoff>,
8667 pid: u32,
8668 spec: &ModuleSpec,
8669) -> Option<subc_os::LaunchNoncePipeDelivery> {
8670 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8674 Ok(delivery) => Some(delivery),
8675 Err(error) => {
8676 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8677 None
8678 }
8679 })
8680}
8681
8682fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8683 #[cfg(windows)]
8684 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8685 return Some(
8686 if snapshot
8687 .pipe_consumed
8688 .as_ref()
8689 .is_some_and(|consumed| consumed.get().is_some())
8690 {
8691 "pipe"
8692 } else {
8693 "env"
8694 }
8695 .to_owned(),
8696 );
8697 }
8698 let _ = snapshot;
8699 None
8700}
8701
8702#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8705enum SpawnRole {
8706 Plain,
8707 SwapCandidate,
8708}
8709
8710fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8713 if role == SpawnRole::SwapCandidate {
8714 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8715 }
8716}
8717
8718fn previous_instance_error(
8719 spec: &ModuleSpec,
8720 pid: u32,
8721 reason: crate::live_children::RetainedReason,
8722) -> SuperviseError {
8723 SuperviseError::Spawn {
8724 program: spec.program.clone(),
8725 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8726 cgroup_path: None,
8727 }
8728}
8729
8730fn spawn_child(
8731 spec: &ModuleSpec,
8732 connection_file_path: Option<&std::path::Path>,
8733 handle: Option<&SupervisorHandle>,
8734 ring: &Arc<Mutex<StderrRing>>,
8735 capture_logs_dir: Option<&std::path::Path>,
8736 roster: &ChildRoster,
8737 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8738) -> Result<SupervisedChild, SuperviseError> {
8739 spawn_child_in_slot(
8740 spec,
8741 connection_file_path,
8742 handle,
8743 ring,
8744 capture_logs_dir,
8745 roster,
8746 #[cfg(target_os = "linux")]
8747 cgroup_placement,
8748 SpawnRole::Plain,
8749 false,
8750 )
8751}
8752
8753#[allow(clippy::too_many_arguments)]
8766fn spawn_child_in_slot(
8767 spec: &ModuleSpec,
8768 connection_file_path: Option<&std::path::Path>,
8769 handle: Option<&SupervisorHandle>,
8770 ring: &Arc<Mutex<StderrRing>>,
8771 capture_logs_dir: Option<&std::path::Path>,
8772 roster: &ChildRoster,
8773 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8774 role: SpawnRole,
8775 alternate_slot: bool,
8776) -> Result<SupervisedChild, SuperviseError> {
8777 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8778 return Err(previous_instance_error(spec, pid, reason));
8779 }
8780 if roster.is_closed() {
8781 return Err(SuperviseError::Spawn {
8782 program: spec.program.clone(),
8783 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8784 cgroup_path: None,
8785 });
8786 }
8787 #[cfg(target_os = "linux")]
8788 let cgroup_name = {
8789 if cgroup_placement.is_none() {
8794 swap::cgroup_name(&spec.module_id, alternate_slot)
8795 } else {
8796 let nonce = generate_launch_nonce()?;
8797 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8798 let mut end = spec.module_id.len().min(64);
8801 while !spec.module_id.is_char_boundary(end) {
8802 end -= 1;
8803 }
8804 format!(
8805 "{}_{suffix}",
8806 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8807 )
8808 }
8809 };
8810 #[cfg(not(target_os = "linux"))]
8811 let _ = alternate_slot;
8812 #[cfg(target_os = "macos")]
8813 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8814 #[cfg(windows)]
8815 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8816 program: spec.program.clone(),
8817 source,
8818 cgroup_path: None,
8819 })?;
8820 #[cfg(windows)]
8821 let mut command = Command::new(image_capture.path());
8822 #[cfg(not(any(target_os = "macos", windows)))]
8823 let mut command = Command::new(&spec.program);
8824 command.args(&spec.args);
8825 apply_child_env(&mut command, spec);
8855 apply_spawn_role(&mut command, role);
8856 let nonce_handoff =
8857 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8858
8859 #[cfg(target_os = "linux")]
8860 let cgroup_path = cgroup_placement
8861 .map(|placement| placement.module_path(&cgroup_name))
8862 .transpose()
8863 .map_err(|source| SuperviseError::Cgroup {
8864 module_id: spec.module_id.clone(),
8865 source,
8866 })?;
8867 #[cfg(not(target_os = "linux"))]
8868 let cgroup_path: Option<PathBuf> = None;
8869 #[cfg(target_os = "linux")]
8870 if let Some(path) = &cgroup_path {
8871 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8872 if let Some(placement) = cgroup_placement {
8873 remove_module_cgroup(placement, &cgroup_name);
8874 }
8875 return Err(error);
8876 }
8877 }
8878
8879 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8880 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8881 match ChildOutputSink::open(&path, capture_retention(spec)) {
8882 Ok(sink) => sink,
8883 Err(error) => {
8884 warn!(
8885 module_id = %spec.module_id,
8886 path = %path.display(),
8887 error = %error,
8888 "could not open child output capture file; forwarding to stderr"
8889 );
8890 ChildOutputSink::Stderr
8891 }
8892 }
8893 } else {
8894 ChildOutputSink::Stderr
8895 };
8896
8897 command.stdout(Stdio::piped());
8898 command.stderr(Stdio::piped());
8899 command.kill_on_drop(true);
8900 #[cfg(unix)]
8917 command.process_group(0);
8918 command.stdin(Stdio::null());
8919 #[cfg(unix)]
8923 if let Some(handoff) = nonce_handoff {
8924 handoff.install_last(command.as_std_mut());
8925 }
8926 #[cfg(not(any(unix, windows)))]
8927 let _ = nonce_handoff;
8928
8929 #[cfg(windows)]
8934 {
8935 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8936 program: spec.program.clone(),
8937 source,
8938 cgroup_path: None,
8939 })?;
8940 subc_jobobject::suspend_on_create_async(&mut command);
8941 }
8942 let spawned_at = Instant::now();
8946 let mut child = match command.spawn() {
8947 Ok(child) => child,
8948 Err(source) => {
8949 #[cfg(target_os = "linux")]
8950 if let Some(placement) = cgroup_placement {
8951 remove_module_cgroup(placement, &cgroup_name);
8952 }
8953 return Err(SuperviseError::Spawn {
8954 program: spec.program.clone(),
8955 source,
8956 cgroup_path,
8957 });
8958 }
8959 };
8960 #[cfg(test)]
8961 spawn_record_tests::after_create(&child, spec);
8962 #[cfg(target_os = "macos")]
8967 drop(exec_ack);
8968
8969 #[cfg(windows)]
8974 let windows_image = match image_capture.bind_tokio(&child) {
8975 Ok(image) => Arc::new(image),
8976 Err(source) => {
8977 kill_and_reap_rejected_child(&mut child);
8978 return Err(SuperviseError::Spawn {
8979 program: spec.program.clone(),
8980 source,
8981 cgroup_path: None,
8982 });
8983 }
8984 };
8985 #[cfg(windows)]
8988 let job = contain_spawned_child(&mut child, spec)?;
8989 let spawned_at_ms = unix_ms_now();
8990 #[cfg(not(windows))]
8991 let spawned_from = spec.program.clone();
8992 #[cfg(windows)]
8993 let spawned_from = windows_image.path().to_path_buf();
8994 #[cfg(not(windows))]
8995 let spawned_file_identity = spawned_file_identity(&spawned_from);
8996 #[cfg(windows)]
8997 let spawned_file_identity = Some(SpawnedFileIdentity {
8998 device: 0,
8999 inode: 0,
9000 windows_image: Some(Arc::clone(&windows_image)),
9001 });
9002 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9003 program: spec.program.clone(),
9004 source: io::Error::other("spawned child exposed no live pid"),
9005 cgroup_path: cgroup_path.clone(),
9006 })?;
9007 #[cfg(windows)]
9008 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9009 #[cfg(not(windows))]
9010 let process_start_time = crate::provenance::process_start_time(pid);
9011 #[cfg(windows)]
9012 let process_start_time = Some(windows_image.start_time());
9013 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9014 #[cfg(all(test, target_os = "macos"))]
9015 privacy_exec_boundary_tests::before_image_sample(spec, pid);
9016 #[cfg(not(windows))]
9021 let recorded_image = observe_spawned_image(pid);
9022 #[cfg(target_os = "macos")]
9026 let recorded_image = if privacy_exec.is_some() {
9027 None
9028 } else {
9029 recorded_image
9030 };
9031 #[cfg(target_os = "linux")]
9032 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9033 #[cfg(not(target_os = "linux"))]
9034 let recorded_cgroup_name = None;
9035 let roster_guard = roster.admit(
9036 spec.module_id.clone(),
9037 pid,
9038 spec.protocol,
9039 process_start_time,
9040 crate::child_roster::RecordedIdentity {
9041 #[cfg(not(windows))]
9042 start_time: recorded_image.map(|image| image.start_time),
9043 #[cfg(windows)]
9044 start_time: process_start_time,
9045 #[cfg(not(windows))]
9046 executable: recorded_image
9047 .and_then(|image| image.executable)
9048 .map(crate::live_children::ExecutableIdentity::from),
9049 #[cfg(windows)]
9050 executable: Some(crate::live_children::ExecutableIdentity::from(
9051 windows_image.identity(),
9052 )),
9053 cgroup_name: recorded_cgroup_name,
9054 #[cfg(target_os = "linux")]
9055 cgroup_placement: cgroup_placement.cloned(),
9056 },
9057 );
9058 let roster_guard = match roster_guard {
9059 Ok(guard) => guard,
9060 Err(failure) => {
9061 return failure.with_cleanup(|source| {
9062 #[cfg(windows)]
9063 if let Some(job) = &job {
9064 let _ = job.terminate();
9065 }
9066 #[cfg(target_os = "linux")]
9067 kill_module_cgroup(cgroup_placement, &cgroup_name);
9068 kill_and_reap_rejected_child(&mut child);
9069 #[cfg(target_os = "linux")]
9070 if let Some(placement) = cgroup_placement {
9071 remove_module_cgroup(placement, &cgroup_name);
9072 }
9073 Err(SuperviseError::Spawn {
9074 program: spec.program.clone(),
9075 source: io::Error::other(format!(
9076 "could not durably record module child: {source}"
9077 )),
9078 cgroup_path,
9079 })
9080 });
9081 }
9082 };
9083 if roster.is_closed() {
9088 #[cfg(target_os = "linux")]
9090 kill_module_cgroup(cgroup_placement, &cgroup_name);
9091 if let Err(error) = child.start_kill() {
9092 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9093 }
9094 #[cfg(target_os = "linux")]
9095 if let Some(placement) = cgroup_placement {
9096 while matches!(child.try_wait(), Ok(None)) {
9100 std::thread::yield_now();
9101 }
9102 if matches!(
9103 subc_cgroup::kill_module(Some(placement), &cgroup_name),
9104 subc_cgroup::KillOutcome::Killed
9105 ) {
9106 if let Ok(path) = placement.module_path(&cgroup_name) {
9107 while std::fs::read_to_string(path.join("cgroup.events"))
9108 .ok()
9109 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9110 {
9111 std::thread::yield_now();
9112 }
9113 }
9114 }
9115 remove_module_cgroup(placement, &cgroup_name);
9116 }
9117 drop(roster_guard);
9118 return Err(SuperviseError::Spawn {
9119 program: spec.program.clone(),
9120 source: io::Error::other(
9121 "the daemon began shutting down while this process was starting; ended it",
9122 ),
9123 cgroup_path,
9124 });
9125 }
9126
9127 #[cfg(windows)]
9128 if let Err(error) = resume_suspended_child(pid, spec) {
9129 kill_and_reap_rejected_child(&mut child);
9130 return Err(error);
9131 }
9132
9133 let stdout_pump = match child.stdout.take() {
9134 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9135 None => {
9136 warn!(
9137 module_id = %spec.module_id,
9138 "spawned child exposed no stdout pipe; file capture will be incomplete"
9139 );
9140 None
9141 }
9142 };
9143 let stderr_pump = match child.stderr.take() {
9144 Some(stderr) => {
9145 let generation = ring
9146 .lock()
9147 .unwrap_or_else(|poisoned| poisoned.into_inner())
9148 .begin_process();
9149 Some(StderrPump {
9150 task: tokio::spawn(pump_stderr_to(
9151 stderr,
9152 Arc::clone(ring),
9153 generation,
9154 output_sink,
9155 )),
9156 generation,
9157 })
9158 }
9159 None => {
9160 ring.lock()
9164 .unwrap_or_else(|poisoned| poisoned.into_inner())
9165 .mark_not_captured("stderr pipe was not available on spawn");
9166 warn!(
9167 module_id = %spec.module_id,
9168 "spawned child exposed no stderr pipe; tail will be unavailable"
9169 );
9170 None
9171 }
9172 };
9173
9174 let exec = Arc::new(crate::activity::ExecOutcome::new());
9175 #[cfg(target_os = "macos")]
9176 let pending_exec = privacy_exec.is_some();
9177 #[cfg(not(target_os = "macos"))]
9178 let pending_exec = false;
9179 if !pending_exec {
9180 exec.finish(crate::activity::ExecState::CommittedExec);
9181 }
9182 Ok(SupervisedChild {
9183 child,
9184 exec: crate::activity::ExecOwner(exec),
9185 #[cfg(windows)]
9186 nonce_delivery,
9187 protocol: spec.protocol,
9188 #[cfg(target_os = "linux")]
9189 module_id: cgroup_name,
9190 #[cfg(target_os = "linux")]
9191 cgroup_placement: cgroup_placement.cloned(),
9192 #[cfg(windows)]
9193 job,
9194 stdout_pump,
9195 stderr_pump,
9196 stderr_ring: Arc::clone(ring),
9197 spawned_at,
9198 registration_deadline: None,
9199 registration_watch: None,
9200 spawned_at_ms,
9201 spawned_from,
9202 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9203 spawned_file_identity,
9204 process_start_time,
9205 process_identity,
9206 pid,
9207 roster_guard: Some(roster_guard),
9208 #[cfg(target_os = "macos")]
9209 privacy_exec,
9210 #[cfg(target_os = "macos")]
9211 report_ready: Arc::new(OnceLock::new()),
9212 spawn_failure: None,
9213 })
9214}
9215
9216#[cfg(test)]
9217mod spawn_record_tests {
9218 use super::*;
9219
9220 thread_local! {
9221 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9222 }
9223
9224 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9225 LAST_PID.set(child.id());
9226 #[cfg(windows)]
9227 if let Some((_, address)) = _spec
9228 .env
9229 .iter()
9230 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9231 {
9232 use std::io::{Read, Write};
9233 let mut stream = std::net::TcpStream::connect(address).unwrap();
9234 stream
9235 .write_all(&child.id().unwrap().to_ne_bytes())
9236 .unwrap();
9237 let mut release = [0];
9238 stream.read_exact(&mut release).unwrap();
9239 }
9240 }
9241
9242 #[cfg(windows)]
9243 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9244 if spec
9245 .env
9246 .iter()
9247 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9248 {
9249 Err(io::Error::other("injected module job assignment failure"))
9250 } else {
9251 Ok(())
9252 }
9253 }
9254
9255 fn spec() -> ModuleSpec {
9256 ModuleSpec {
9257 module_id: "spawn-record-worker".into(),
9258 #[cfg(unix)]
9259 program: "/bin/sleep".into(),
9260 #[cfg(windows)]
9261 program: std::env::current_exe().unwrap(),
9262 #[cfg(unix)]
9263 args: vec!["60".into()],
9264 #[cfg(windows)]
9265 args: vec![
9266 "--exact".into(),
9267 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9268 "--nocapture".into(),
9269 ],
9270 env: vec![],
9271 reserved: false,
9272 reserved_prefixes: vec![],
9273 protocol: ModuleProtocol::None,
9274 overlap: Default::default(),
9275 }
9276 }
9277
9278 #[tokio::test]
9279 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9280 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9281 let supervisor =
9282 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9283 .with_live_children_record(home.join("live-children.json"));
9284 crate::live_children::fail_next_file_sync();
9285 let result = supervisor.spawn(spec());
9286 let failed = result.is_err();
9287 if let Ok(module) = result {
9288 module.stop().await.unwrap();
9289 }
9290 assert!(failed, "record failure must fail spawn");
9291 let pid = LAST_PID.get().expect("the real OS child was created");
9292 assert!(
9293 subc_os::Process::open(pid).unwrap().is_none(),
9294 "child {pid} survived rejected admission or was not reaped"
9295 );
9296 }
9297
9298 #[cfg(target_os = "macos")]
9299 #[test]
9300 #[ignore = "manual on-device latency measurement"]
9301 fn durable_record_write_latency_50() {
9302 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9303 let path = home.join("live-children.json");
9304 let children = vec![crate::live_children::LiveChild {
9305 module_id: "latency-worker".into(),
9306 pid: std::process::id(),
9307 protocol: ModuleProtocol::None,
9308 start_time: Some(1),
9309 executable: None,
9310 cgroup_name: None,
9311 }];
9312 let mut samples = Vec::new();
9313 for _ in 0..50 {
9314 let start = std::time::Instant::now();
9315 crate::live_children::write_record(&path, &children).unwrap();
9316 samples.push(start.elapsed());
9317 }
9318 samples.sort();
9319 eprintln!(
9320 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9321 (samples[24] + samples[25]) / 2,
9322 samples[47]
9323 );
9324 }
9325
9326 #[cfg(windows)]
9327 #[test]
9328 fn windows_store_touch_fixture() {
9329 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9330 std::fs::write(path, b"module ran").unwrap();
9331 }
9332 }
9333
9334 #[cfg(windows)]
9335 #[tokio::test]
9336 async fn windows_daemon_fixture() {
9337 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9338 return;
9339 };
9340 let mut spec = spec();
9341 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9342 let supervisor =
9343 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9344 let _module = supervisor.spawn(spec).unwrap();
9345 }
9346
9347 #[cfg(windows)]
9348 #[test]
9349 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9350 use std::io::Read;
9351 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9352 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9355 command
9356 .args([
9357 "--exact",
9358 "supervise::spawn_record_tests::windows_daemon_fixture",
9359 "--nocapture",
9360 ])
9361 .env(
9362 "SUBC_TEST_DAEMON_BARRIER",
9363 listener.local_addr().unwrap().to_string(),
9364 );
9365 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9366 listener.set_nonblocking(true).unwrap();
9367 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9368 let mut stream = loop {
9369 if let Ok((stream, _)) = listener.accept() {
9370 break stream;
9371 }
9372 assert!(
9373 daemon.child.try_wait().unwrap().is_none(),
9374 "daemon failed before rendezvous"
9375 );
9376 assert!(
9377 std::time::Instant::now() < deadline,
9378 "daemon did not reach CreateProcess failpoint"
9379 );
9380 std::thread::yield_now();
9381 };
9382 stream
9383 .set_read_timeout(Some(Duration::from_secs(30)))
9384 .unwrap();
9385 let mut pid = [0; 4];
9386 stream.read_exact(&mut pid).unwrap();
9387 let pid = u32::from_ne_bytes(pid);
9388 assert!(subc_jobobject::process_exists(pid));
9389 daemon.child.kill().unwrap();
9392 daemon.child.wait().unwrap();
9393 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9394 if !exited {
9395 daemon.job.terminate().unwrap();
9396 }
9397 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9398 }
9399
9400 #[cfg(windows)]
9401 #[tokio::test]
9402 async fn windows_containment_failure_never_resumes() {
9403 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9404 let sentinel = home.join("store-touched");
9405 let mut spec = spec();
9406 spec.env
9407 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9408 spec.env.push((
9409 "SUBC_TEST_STORE_TOUCH".into(),
9410 sentinel.display().to_string(),
9411 ));
9412 let supervisor =
9413 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9414 let result = supervisor.spawn(spec);
9415 if let Ok(module) = result {
9418 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9419 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9420 tokio::task::yield_now().await;
9421 }
9422 })
9423 .await;
9424 let _ = module.stop().await;
9425 panic!(
9426 "assignment failure resumed a module; store touched={}",
9427 sentinel.exists()
9428 );
9429 }
9430 assert!(
9431 !sentinel.exists(),
9432 "refused module executed store-touch sentinel"
9433 );
9434 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9435 }
9436
9437 #[cfg(windows)]
9438 #[tokio::test]
9439 async fn windows_nested_daemon_fixture() {
9440 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9441 return;
9442 };
9443 let mut spec = spec();
9444 spec.env.push((
9445 "SUBC_TEST_STORE_TOUCH".into(),
9446 sentinel.to_str().unwrap().into(),
9447 ));
9448 let supervisor =
9449 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9450 let module = supervisor
9451 .spawn(spec)
9452 .expect("per-module job must nest below daemon and host jobs");
9453 tokio::time::timeout(Duration::from_secs(10), async {
9454 while !PathBuf::from(&sentinel).exists() {
9455 tokio::task::yield_now().await;
9456 }
9457 })
9458 .await
9459 .unwrap();
9460 module.stop().await.unwrap();
9461 }
9462
9463 #[cfg(windows)]
9464 #[test]
9465 fn windows_module_job_nests_under_host_job() {
9466 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9467 let sentinel = home.join("nested-store-touched");
9468 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9469 command
9470 .args([
9471 "--exact",
9472 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9473 "--nocapture",
9474 ])
9475 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9476 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9477 assert!(daemon.child.wait().unwrap().success());
9478 assert!(sentinel.exists(), "nested protocol:none module never ran");
9479 }
9480}
9481
9482#[cfg(target_os = "linux")]
9483pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9484 use subc_cgroup::KillOutcome;
9485 match subc_cgroup::kill_module(placement, module_id) {
9486 KillOutcome::Killed => {}
9487 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9488 debug!(
9489 module_id,
9490 "cgroup tree kill unavailable; using direct-child kill"
9491 );
9492 }
9493 KillOutcome::IoError { path, error } => {
9494 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9495 }
9496 }
9497}
9498
9499#[cfg(windows)]
9507fn contain_spawned_child(
9508 child: &mut Child,
9509 spec: &ModuleSpec,
9510) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9511 let assignment = || {
9512 let job = subc_jobobject::JobObject::new()?;
9513 #[cfg(test)]
9514 spawn_record_tests::assignment_failpoint(spec)?;
9515 job.assign(child)?;
9516 Ok::<_, io::Error>(job)
9517 };
9518 match assignment() {
9519 Ok(job) => Ok(Some(job)),
9520 Err(source) => {
9521 kill_and_reap_rejected_child(child);
9522 Err(SuperviseError::Spawn {
9523 program: spec.program.clone(),
9524 source: io::Error::other(format!(
9525 "Windows module job containment failed: {source}"
9526 )),
9527 cgroup_path: None,
9528 })
9529 }
9530 }
9531}
9532
9533fn kill_and_reap_rejected_child(child: &mut Child) {
9536 #[cfg(unix)]
9537 if let Some(pid) = child
9538 .id()
9539 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9540 {
9541 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9542 }
9543 let _ = child.start_kill();
9544 while matches!(child.try_wait(), Ok(None)) {
9545 std::thread::yield_now();
9546 }
9547}
9548
9549#[cfg(windows)]
9554fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9555 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9556 let _ = std::process::Command::new("taskkill.exe")
9560 .args(["/PID", &pid.to_string(), "/T", "/F"])
9561 .stdin(Stdio::null())
9562 .stdout(Stdio::null())
9563 .stderr(Stdio::null())
9564 .status();
9565 return Err(SuperviseError::Spawn {
9566 program: spec.program.clone(),
9567 source,
9568 cgroup_path: None,
9569 });
9570 }
9571 Ok(())
9572}
9573
9574#[cfg(target_os = "linux")]
9575fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9576 match placement.remove_module(module_id) {
9577 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9578 Err(error) => warn!(
9579 module_id,
9580 error = %error,
9581 "could not remove module cgroup after process exit; continuing teardown"
9582 ),
9583 }
9584}
9585
9586#[cfg(target_os = "linux")]
9587async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9588 if matches!(
9592 subc_cgroup::kill_module(Some(placement), module_id),
9593 subc_cgroup::KillOutcome::Killed
9594 ) {
9595 if let Ok(path) = placement.module_path(module_id) {
9596 while std::fs::read_to_string(path.join("cgroup.events"))
9597 .ok()
9598 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9599 {
9600 sleep(Duration::from_millis(1)).await;
9601 }
9602 }
9603 }
9604 remove_module_cgroup(placement, module_id);
9605}
9606
9607#[cfg(target_os = "linux")]
9608fn apply_cgroup_placement(
9609 command: &mut Command,
9610 spec: &ModuleSpec,
9611 path: &std::path::Path,
9612) -> Result<(), SuperviseError> {
9613 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9614 module_id: spec.module_id.clone(),
9615 source,
9616 })
9617}
9618
9619fn capture_retention(spec: &ModuleSpec) -> Retention {
9620 let defaults = Retention::default();
9621 let value = |name: &str| {
9622 spec.env
9623 .iter()
9624 .rev()
9625 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9626 };
9627 Retention {
9628 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9629 .and_then(|value| value.parse().ok())
9630 .unwrap_or(defaults.max_file_mb),
9631 keep: value(CAPTURE_KEEP_ENV)
9632 .and_then(|value| value.parse().ok())
9633 .unwrap_or(defaults.keep),
9634 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9635 .and_then(|value| value.parse().ok())
9636 .unwrap_or(defaults.max_age_days),
9637 }
9638}
9639
9640fn generate_launch_nonce() -> Result<String, SuperviseError> {
9643 let mut bytes = [0u8; 32];
9644 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9645 reason: source.to_string(),
9646 })?;
9647 let mut hex = String::with_capacity(64);
9648 for b in bytes {
9649 use std::fmt::Write;
9650 let _ = write!(hex, "{b:02x}");
9651 }
9652 Ok(hex)
9653}
9654
9655fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9658 if a.len() != b.len() {
9659 return false;
9660 }
9661 let mut diff = 0u8;
9662 for (x, y) in a.iter().zip(b.iter()) {
9663 diff |= x ^ y;
9664 }
9665 diff == 0
9666}
9667
9668#[cfg(not(windows))]
9672fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9673 let process = subc_os::Process::open(pid).ok().flatten()?;
9674 #[cfg(target_os = "linux")]
9675 return sample_after_linux_exec(
9676 || {
9677 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9678 let committed = linux_exec_committed(&stat);
9679 #[cfg(test)]
9680 spawn_image_tests::report_exec_state(committed);
9681 committed
9682 },
9683 || process.observe(),
9684 );
9685 #[cfg(not(target_os = "linux"))]
9686 process.observe()
9687}
9688
9689#[cfg(target_os = "linux")]
9693fn linux_exec_committed(stat: &str) -> Option<bool> {
9694 const PF_FORKNOEXEC: u64 = 0x40;
9695 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9696 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9697}
9698
9699#[cfg(target_os = "linux")]
9700fn sample_after_linux_exec(
9701 mut exec_committed: impl FnMut() -> Option<bool>,
9702 observe: impl FnOnce() -> Option<subc_os::Observation>,
9703) -> Option<subc_os::Observation> {
9704 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9705 while !exec_committed()? {
9706 if std::time::Instant::now() >= deadline {
9710 return None;
9711 }
9712 std::thread::sleep(Duration::from_micros(50));
9713 }
9714 observe()
9715}
9716
9717#[cfg(all(test, target_os = "linux"))]
9718mod spawn_image_tests {
9719 use super::*;
9720 use std::cell::{Cell, RefCell};
9721
9722 enum ExecEvent {
9723 Uncommitted,
9724 Image(Option<subc_os::Observation>),
9725 }
9726
9727 thread_local! {
9728 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9729 }
9730
9731 pub(super) fn report_exec_state(committed: Option<bool>) {
9732 if committed == Some(false) {
9733 EXEC_EVENTS.with(|events| {
9734 if let Some(events) = events.borrow_mut().take() {
9735 let _ = events.send(ExecEvent::Uncommitted);
9736 }
9737 });
9738 }
9739 }
9740
9741 #[test]
9742 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9743 use std::io::{Read, Write};
9744 use std::os::fd::AsRawFd;
9745
9746 let descriptor = std::fs::File::open("/dev/null").unwrap();
9747 let mut command = std::process::Command::new("/bin/sleep");
9748 command.arg("30");
9749 let (mut ready, mut release) =
9750 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9751 .unwrap();
9752 let spawn = std::thread::spawn(move || command.spawn());
9753 let mut message = [0u8; 8];
9754 ready.read_exact(&mut message).unwrap();
9755 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9756 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9757 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9758 let (events, received) = std::sync::mpsc::channel();
9759 let observer = std::thread::spawn(move || {
9760 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9761 let image = observe_spawned_image(pid);
9762 let _ = events.send(ExecEvent::Image(image));
9763 });
9764 let first = received.recv_timeout(Duration::from_secs(5));
9768 release.write_all(&[1]).unwrap();
9769 let mut child = spawn.join().unwrap().unwrap();
9770 let after = match first {
9771 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9772 Ok(ExecEvent::Image(image)) => image,
9773 _ => None,
9774 },
9775 Ok(ExecEvent::Image(image)) => image,
9776 Err(_) => None,
9777 };
9778 observer.join().unwrap();
9779 child.kill().unwrap();
9780 child.wait().unwrap();
9781
9782 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9783 assert_eq!(
9784 inherited.unwrap().executable,
9785 subc_os::file_identity(&std::env::current_exe().unwrap())
9786 );
9787 assert_eq!(
9788 after.unwrap().executable,
9789 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9790 );
9791 }
9792
9793 #[test]
9794 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9795 let committed = Cell::new(false);
9796 let inherited = subc_os::Observation {
9797 start_time: 23,
9798 executable: Some(subc_os::FileIdentity {
9799 device: 29,
9800 inode: 31,
9801 }),
9802 };
9803 let module = subc_os::Observation {
9804 start_time: 23,
9805 executable: Some(subc_os::FileIdentity {
9806 device: 29,
9807 inode: 37,
9808 }),
9809 };
9810 let polls = Cell::new(0);
9811 let image = sample_after_linux_exec(
9812 || {
9813 polls.set(polls.get() + 1);
9814 committed.set(polls.get() == 2);
9817 Some(committed.get())
9818 },
9819 || Some(if committed.get() { module } else { inherited }),
9820 );
9821 assert_eq!(image, Some(module));
9822 assert_eq!(polls.get(), 2);
9823 }
9824
9825 #[test]
9826 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9827 assert_eq!(
9828 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9829 Some(false)
9830 );
9831 assert_eq!(
9832 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9833 Some(true)
9834 );
9835 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9836 assert_eq!(linux_exec_committed("17 (a) R"), None);
9837 }
9838
9839 #[test]
9840 fn unreadable_exec_state_never_samples_an_inherited_image() {
9841 assert_eq!(
9842 sample_after_linux_exec(
9843 || None,
9844 || panic!("must not sample before exec confirmation")
9845 ),
9846 None
9847 );
9848 }
9849
9850 #[test]
9851 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9852 assert_eq!(
9853 sample_after_linux_exec(
9854 || Some(false),
9855 || panic!("must not sample after an exec timeout")
9856 ),
9857 None
9858 );
9859 }
9860}
9861
9862#[cfg(windows)]
9865fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9866 let mut program = spec.program.clone();
9867 if program.is_absolute() || program.components().count() > 1 {
9868 match subc_os::ExecutableCapture::open(&program) {
9871 Ok(capture) => return Ok(capture),
9872 Err(error)
9873 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9874 Err(error) => return Err(error),
9875 }
9876 program.set_extension("exe");
9877 return subc_os::ExecutableCapture::open(&program);
9878 }
9879 if program.extension().is_none() {
9880 program.set_extension("exe");
9881 }
9882 let custom_path = spec
9883 .env
9884 .iter()
9885 .rev()
9886 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9887 let mut directories = Vec::new();
9888 if let Some(parent) = std::env::current_exe()?.parent() {
9891 directories.push(parent.to_path_buf());
9892 }
9893 directories.push(std::env::current_dir()?);
9894 if let Some(windows) = std::env::var_os("SystemRoot") {
9895 let windows = PathBuf::from(windows);
9896 directories.push(windows.join("System32"));
9897 directories.push(windows.clone());
9898 }
9899 let path = custom_path
9900 .map(|(_, value)| std::ffi::OsString::from(value))
9901 .or_else(|| std::env::var_os("PATH"));
9902 if let Some(path) = path {
9903 directories.extend(std::env::split_paths(&path));
9904 }
9905 for directory in directories {
9906 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9907 Ok(capture) => return Ok(capture),
9908 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9909 Err(error) => return Err(error),
9910 }
9911 }
9912 Err(io::Error::new(
9913 io::ErrorKind::NotFound,
9914 "resolved executable not found",
9915 ))
9916}
9917
9918fn spawn_and_mark_running(
9919 spec: &ModuleSpec,
9920 runtime: &SupervisorRuntimeConfig,
9921 snapshot: &SharedSnapshot,
9922) -> Result<SupervisedChild, SuperviseError> {
9923 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9924 let registration_watch = watch_registration(&runtime.registry, spec);
9925 let child = spawn_child(
9926 spec,
9927 runtime.connection_file_path.as_deref(),
9928 runtime.supervisor_handle.as_ref(),
9929 &runtime.stderr_ring,
9930 runtime.capture_logs_dir.as_deref(),
9931 &runtime.child_roster,
9932 #[cfg(target_os = "linux")]
9933 runtime.cgroup_placement.as_ref(),
9934 )?;
9935 let mut child = child;
9936 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
9937 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9938 if let Some(spawn) = activity_spawn {
9939 spawn.complete(snapshot);
9940 }
9941 Ok(child)
9942}
9943
9944struct RegistrationWatch {
9945 changes: watch::Receiver<u64>,
9946 baseline: u64,
9947}
9948
9949fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
9953 if spec.protocol != ModuleProtocol::Subc {
9954 return None;
9955 }
9956 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
9957 let baseline = *changes.borrow();
9958 Some(RegistrationWatch { changes, baseline })
9959}
9960
9961fn arm_registration_deadline(
9962 child: &mut SupervisedChild,
9963 budget: Duration,
9964 registration_watch: Option<RegistrationWatch>,
9965) {
9966 child.registration_deadline =
9967 (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
9968 child.registration_watch = registration_watch;
9969}
9970
9971async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
9972 #[cfg(unix)]
9973 {
9974 let group = unreaped_registration_group(child)?;
9977 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
9978 sleep(crate::child_roster::TERM_TO_KILL).await;
9983 let group = unreaped_registration_group(child)?;
9984 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
9985 }
9986 child.start_kill()?;
9989 child.wait().await?;
9990 Ok(())
9991}
9992
9993#[cfg(unix)]
9994fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
9995 let group = i32::try_from(child.pid)
9998 .ok()
9999 .filter(|pid| *pid > 1)
10000 .and_then(rustix::process::Pid::from_raw)
10001 .ok_or_else(|| {
10002 io::Error::new(
10003 io::ErrorKind::InvalidInput,
10004 "invalid supervised process group",
10005 )
10006 })?;
10007 if child.child.id() != Some(child.pid) {
10008 return Err(io::Error::new(
10009 io::ErrorKind::InvalidInput,
10010 "refusing process-group signal after supervised child was reaped",
10011 ));
10012 }
10013 Ok(group)
10014}
10015
10016enum RegistrationWaitOutcome {
10017 Registered,
10018 Exited(ExitReport),
10019 TimedOut,
10020}
10021
10022struct ReloadRegistrationFailure {
10023 exit_report: ExitReport,
10024 reason: String,
10025}
10026
10027#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10028enum BusyGaugeObservation {
10029 Quiescent,
10030 Busy,
10031 Omitted,
10032}
10033
10034#[derive(Debug, Clone, PartialEq, Eq)]
10036enum GaugeVerdict {
10037 Zero,
10039 NonZero(u64),
10041 NotACount(String),
10045 Missing,
10047 ProbeFailed(&'static str),
10050}
10051
10052impl GaugeVerdict {
10053 fn reads_quiet(&self) -> bool {
10054 matches!(self, Self::Zero)
10055 }
10056}
10057
10058impl fmt::Display for GaugeVerdict {
10059 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10060 match self {
10061 Self::Zero => f.write_str("zero"),
10062 Self::NonZero(value) => write!(f, "non_zero({value})"),
10063 Self::NotACount(value) => write!(f, "not_a_count({value})"),
10064 Self::Missing => f.write_str("missing_from_health_body"),
10065 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10066 }
10067 }
10068}
10069
10070const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10073
10074fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10076 let metrics = metrics.and_then(Value::as_object);
10077 gauges
10078 .iter()
10079 .map(
10080 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10081 None => GaugeVerdict::Missing,
10082 Some(value) => match value.as_u64() {
10083 Some(0) => GaugeVerdict::Zero,
10084 Some(count) => GaugeVerdict::NonZero(count),
10085 None => {
10086 let mut text = value.to_string();
10087 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10088 let mut end = GAUGE_VALUE_TEXT_LIMIT;
10089 while !text.is_char_boundary(end) {
10090 end -= 1;
10091 }
10092 text.truncate(end);
10093 text.push_str("...");
10094 }
10095 GaugeVerdict::NotACount(text)
10096 }
10097 },
10098 },
10099 )
10100 .collect()
10101}
10102
10103fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10106 if verdicts
10107 .iter()
10108 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10109 {
10110 BusyGaugeObservation::Omitted
10111 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10112 BusyGaugeObservation::Quiescent
10113 } else {
10114 BusyGaugeObservation::Busy
10115 }
10116}
10117
10118#[derive(Debug, Clone, Default, PartialEq, Eq)]
10123struct GaugeTrace {
10124 latest: Option<(GaugeVerdict, Duration)>,
10125 last_value: Option<(GaugeVerdict, Duration)>,
10126}
10127
10128#[derive(Debug, Clone, Default, PartialEq, Eq)]
10134struct DrainGaugeTrace {
10135 probes: u32,
10137 gauges: Vec<(String, GaugeTrace)>,
10139}
10140
10141impl DrainGaugeTrace {
10142 fn new(gauges: &[String]) -> Self {
10143 Self {
10144 probes: 0,
10145 gauges: gauges
10146 .iter()
10147 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10148 .collect(),
10149 }
10150 }
10151
10152 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10154 self.probes = self.probes.saturating_add(1);
10155 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10156 trace.latest = Some((verdict.clone(), at));
10157 if matches!(
10158 verdict,
10159 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10160 ) {
10161 trace.last_value = Some((verdict.clone(), at));
10162 }
10163 }
10164 }
10165}
10166
10167impl fmt::Display for DrainGaugeTrace {
10172 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10173 if self.gauges.is_empty() {
10174 return f.write_str("none declared");
10175 }
10176 for (index, (name, trace)) in self.gauges.iter().enumerate() {
10177 if index > 0 {
10178 f.write_str(" | ")?;
10179 }
10180 write!(f, "{name}: ")?;
10181 match &trace.latest {
10182 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10183 None => f.write_str("never probed")?,
10184 }
10185 match &trace.last_value {
10186 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10187 None => f.write_str("; no value ever read")?,
10188 }
10189 }
10190 Ok(())
10191 }
10192}
10193
10194pub(crate) fn declared_busy_gauges(
10195 registry: &Registry,
10196 module_id: &str,
10197) -> Result<Vec<String>, SuperviseError> {
10198 busy_gauges_of(
10199 registry
10200 .get_module(module_id)
10201 .map_err(SuperviseError::Registry)?,
10202 )
10203}
10204
10205fn declared_busy_gauges_for_connection(
10209 registry: &Registry,
10210 connection_id: ConnectionId,
10211) -> Result<Vec<String>, SuperviseError> {
10212 busy_gauges_of(
10213 registry
10214 .get_module_by_connection(connection_id)
10215 .map_err(SuperviseError::Registry)?,
10216 )
10217}
10218
10219fn busy_gauges_of(
10220 registration: Option<crate::registry::ModuleRegistration>,
10221) -> Result<Vec<String>, SuperviseError> {
10222 let Some(registration) = registration else {
10223 return Ok(Vec::new());
10224 };
10225 let Some(self_signals) = registration.manifest.self_signals else {
10226 return Ok(Vec::new());
10227 };
10228
10229 let mut gauges = Vec::new();
10230 for declaration in self_signals {
10231 if declaration.kind != SelfSignalKind::Busy {
10232 continue;
10233 }
10234 match declaration.anchored_to {
10235 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10236 gauges.extend(declared)
10237 }
10238 _ => {
10239 gauges.push(String::new());
10242 }
10243 }
10244 }
10245 Ok(gauges)
10246}
10247
10248#[allow(clippy::too_many_arguments)]
10257async fn wait_for_forwarding_quiescence(
10258 forwarding: &ForwardingTable,
10259 module_id: &str,
10260 runtime: &SupervisorRuntimeConfig,
10261 endpoint: crate::ModuleEndpointId,
10262 deadline: Instant,
10263 busy_gauges: &[String],
10264 scope: DrainScope,
10265 trace: &mut DrainGaugeTrace,
10266) -> Result<bool, SuperviseError> {
10267 let wait_started = Instant::now();
10268 let mut gauges_quiescent = busy_gauges.is_empty();
10269 let mut next_probe_at = wait_started;
10270 let mut omission_reported = false;
10271
10272 loop {
10273 let now = Instant::now();
10274 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10275 let report = match scope {
10276 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10277 DrainScope::Endpoint(endpoint) => {
10278 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10279 }
10280 };
10281 let verdicts = match &report {
10282 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10283 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10284 };
10285 trace.record(&verdicts, wait_started.elapsed());
10286 gauges_quiescent = match report {
10287 Ok(_) => match busy_gauge_observation(&verdicts) {
10288 BusyGaugeObservation::Quiescent => true,
10289 BusyGaugeObservation::Busy => false,
10290 BusyGaugeObservation::Omitted => {
10291 if !omission_reported {
10294 forwarding
10295 .counters()
10296 .increment_drains_with_undeclared_gauge();
10297 let missing = busy_gauges
10298 .iter()
10299 .zip(&verdicts)
10300 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10301 .map(|(gauge, _)| gauge.as_str())
10302 .collect::<Vec<_>>()
10303 .join(",");
10304 warn!(
10305 module_id,
10306 missing_gauges = %missing,
10307 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10308 );
10309 omission_reported = true;
10310 }
10311 false
10312 }
10313 },
10314 Err(err) => {
10315 warn!(
10316 module_id,
10317 error = %err,
10318 "drain health.check did not produce declared busy gauges; treating module as busy"
10319 );
10320 false
10321 }
10322 };
10323 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10326 }
10327
10328 let in_flight = forwarding
10329 .endpoint_in_flight_count(endpoint)
10330 .map_err(SuperviseError::Forwarding)?;
10331 if in_flight == 0 && gauges_quiescent {
10332 return Ok(true);
10333 }
10334
10335 let now = Instant::now();
10336 if now >= deadline {
10337 return Ok(false);
10338 }
10339 let mut wait = deadline
10340 .saturating_duration_since(now)
10341 .min(REGISTRY_RELEASE_POLL);
10342 if !busy_gauges.is_empty() {
10343 wait = wait.min(next_probe_at.saturating_duration_since(now));
10344 }
10345 sleep(wait).await;
10346 }
10347}
10348
10349fn drain_busy_reprobe_interval() -> Duration {
10353 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10354}
10355
10356fn warn_drain_timed_out(
10371 module_id: &str,
10372 drain_timeout: Duration,
10373 reason: RouteCloseReason,
10374 holdouts: &crate::forwarding::DrainHoldouts,
10375 gauges: &DrainGaugeTrace,
10376) {
10377 warn!(
10378 module_id = %module_id,
10379 waited = ?drain_timeout,
10380 ?reason,
10381 held_requests = holdouts.requests,
10382 held_routes = holdouts.routes,
10383 total_routes = holdouts.total_routes,
10384 top_connections = ?holdouts.top_connections,
10385 held = %holdouts
10388 .held
10389 .iter()
10390 .map(|(channel, corr)| format!("{channel}:{corr}"))
10391 .collect::<Vec<_>>()
10392 .join(","),
10393 gauge_probes = gauges.probes,
10394 busy_gauges = %gauges,
10395 "route drain timed out before request quiescence; forcing teardown"
10396 );
10397}
10398
10399fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10407 match wait_result {
10408 Ok(drained) => *drained,
10409 Err(_) => false,
10410 }
10411}
10412
10413fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10414 for released in released_routes {
10415 let frame = match Frame::build_with_version(
10416 released.negotiated_ver,
10417 FrameType::Goodbye,
10418 control_flags(),
10419 released.channel,
10420 released.epoch,
10421 0,
10422 Vec::new(),
10423 ) {
10424 Ok(frame) => frame,
10425 Err(err) => {
10426 warn!(
10427 route_channel = released.channel,
10428 error = %err,
10429 "failed to build supervisor drain route GOODBYE frame"
10430 );
10431 continue;
10432 }
10433 };
10434 if !released.close_on_delivery_failure() {
10435 crate::forwarding::send_module_route_goodbye(
10436 &forwarding.counters(),
10437 &released.sink,
10438 frame,
10439 released.module_id.as_deref(),
10440 "supervisor drain",
10441 );
10442 continue;
10443 }
10444 if let Err(err) = released.sink.try_send(frame) {
10445 warn!(
10446 target_connection_id = released.connection_id.get(),
10447 route_channel = released.channel,
10448 error = %err,
10449 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10450 );
10451 let _ = forwarding.escalate_client_delivery_failure(
10452 released.connection_id,
10453 released.channel,
10454 released.epoch,
10455 CloseReason::new(
10456 "route_goodbye_delivery_failed",
10457 format!(
10458 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10459 released.channel
10460 ),
10461 ),
10462 crate::forwarding::UndeliveredFrame {
10463 module_id: released.module_id.as_deref(),
10464 sink: &released.sink,
10465 },
10466 );
10467 }
10468 }
10469}
10470
10471fn send_module_draining(
10472 module_id: &str,
10473 reason: RouteCloseReason,
10474 deadline_ms: u64,
10475 target: &ModuleDrainTarget,
10476) {
10477 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10478 reason,
10479 deadline_ms,
10480 }) {
10481 Ok(body) => body,
10482 Err(err) => {
10483 warn!(
10484 module_id,
10485 error = %err,
10486 "failed to encode module draining command"
10487 );
10488 return;
10489 }
10490 };
10491 let frame = match Frame::build_with_version(
10492 target.negotiated_ver,
10493 FrameType::Push,
10494 control_flags(),
10495 0,
10496 0,
10497 0,
10498 body,
10499 ) {
10500 Ok(frame) => frame,
10501 Err(err) => {
10502 warn!(
10503 module_id,
10504 error = %err,
10505 "failed to build module draining command frame"
10506 );
10507 return;
10508 }
10509 };
10510 if let Err(err) = target.sink.try_send(frame) {
10511 warn!(
10512 module_id,
10513 target_connection_id = target.endpoint.connection_id.get(),
10514 error = %err,
10515 "module draining command was not delivered to peer"
10516 );
10517 }
10518}
10519
10520fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10522 match Frame::build_with_version(
10523 negotiated_ver,
10524 FrameType::Goodbye,
10525 control_flags(),
10526 0,
10527 0,
10528 0,
10529 Vec::new(),
10530 ) {
10531 Ok(frame) => Some(frame),
10532 Err(err) => {
10533 warn!(
10534 module_id,
10535 error = %err,
10536 "failed to build module GOODBYE frame"
10537 );
10538 None
10539 }
10540 }
10541}
10542
10543#[cfg(unix)]
10557async fn send_module_goodbyes_for_daemon_shutdown(
10558 forwarding: &Arc<ForwardingTable>,
10559 reason: &CloseReason,
10560 wait_for_flush: bool,
10561) {
10562 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10563 let targets = match forwarding.module_connections() {
10564 Ok(targets) => targets,
10565 Err(err) => {
10566 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10567 return;
10568 }
10569 };
10570 let deadline = Instant::now() + GOODBYE_BUDGET;
10571 let mut sends = tokio::task::JoinSet::new();
10572 for target in targets {
10573 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10574 continue;
10575 };
10576 if !wait_for_flush {
10577 if let Err(err) = target.sink.try_send(frame) {
10578 debug!(
10579 module_id = %target.module_id,
10580 error = %err,
10581 "shutdown module GOODBYE was not queued"
10582 );
10583 }
10584 continue;
10585 }
10586 let forwarding = Arc::clone(forwarding);
10587 let reason = reason.clone();
10588 sends.spawn(async move {
10589 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10590 Ok(Ok(())) => {}
10591 Ok(Err(err)) => debug!(
10592 module_id = %target.module_id,
10593 error = %err,
10594 "module connection closed before its shutdown GOODBYE was written"
10595 ),
10596 Err(_) => warn!(
10597 module_id = %target.module_id,
10598 budget = ?GOODBYE_BUDGET,
10599 "shutdown module GOODBYE was not written within its budget; closing anyway"
10600 ),
10601 }
10602 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10603 });
10604 }
10605 while sends.join_next().await.is_some() {}
10607}
10608
10609fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10610 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10611 return;
10612 };
10613 if let Err(err) = target.sink.try_send(frame) {
10614 warn!(
10615 module_id,
10616 target_connection_id = target.endpoint.connection_id.get(),
10617 error = %err,
10618 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10619 );
10620 forwarding.request_connection_close(
10621 target.endpoint.connection_id,
10622 CloseReason::new(
10623 "module_goodbye_delivery_failed",
10624 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10625 ),
10626 );
10627 }
10628}
10629
10630#[derive(Clone, Copy)]
10631struct ForwardingDrainContext<'a> {
10632 spec: &'a ModuleSpec,
10633 runtime: &'a SupervisorRuntimeConfig,
10634 registry: &'a Registry,
10635 scope: DrainScope,
10636}
10637
10638#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10640enum DrainScope {
10641 Active,
10644 Endpoint(crate::ModuleEndpointId),
10649}
10650
10651#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10659enum StopNotice {
10660 SentOverConnection,
10663 NoConnection,
10667 NotSent,
10671}
10672
10673async fn begin_forwarding_drain(
10674 spec: &ModuleSpec,
10675 runtime: &SupervisorRuntimeConfig,
10676 registry: &Registry,
10677 snapshot: &SharedSnapshot,
10678 enabled: Option<bool>,
10679 reason: RouteCloseReason,
10680) -> Result<StopNotice, SuperviseError> {
10681 let Some(forwarding) = runtime.forwarding.as_ref() else {
10682 return Err(SuperviseError::ReloadUnavailable {
10683 module_id: spec.module_id.clone(),
10684 reason: "supervisor was not configured with a forwarding table".to_string(),
10685 });
10686 };
10687
10688 begin_forwarding_drain_with(
10689 forwarding,
10690 ForwardingDrainContext {
10691 spec,
10692 runtime,
10693 registry,
10694 scope: DrainScope::Active,
10695 },
10696 snapshot,
10697 enabled,
10698 reason,
10699 runtime.drain_timeout,
10700 )
10701 .await
10702}
10703
10704async fn begin_forwarding_drain_if_configured(
10705 spec: &ModuleSpec,
10706 runtime: &SupervisorRuntimeConfig,
10707 registry: &Registry,
10708 snapshot: &SharedSnapshot,
10709 enabled: Option<bool>,
10710 reason: RouteCloseReason,
10711) -> Result<StopNotice, SuperviseError> {
10712 begin_forwarding_drain_with_timeout(
10713 spec,
10714 runtime,
10715 registry,
10716 snapshot,
10717 enabled,
10718 reason,
10719 runtime.drain_timeout,
10720 )
10721 .await
10722}
10723
10724async fn begin_forwarding_drain_with_timeout(
10728 spec: &ModuleSpec,
10729 runtime: &SupervisorRuntimeConfig,
10730 registry: &Registry,
10731 snapshot: &SharedSnapshot,
10732 enabled: Option<bool>,
10733 reason: RouteCloseReason,
10734 drain_timeout: Duration,
10735) -> Result<StopNotice, SuperviseError> {
10736 let Some(forwarding) = runtime.forwarding.as_ref() else {
10737 return Ok(StopNotice::NotSent);
10738 };
10739
10740 begin_forwarding_drain_with(
10741 forwarding,
10742 ForwardingDrainContext {
10743 spec,
10744 runtime,
10745 registry,
10746 scope: DrainScope::Active,
10747 },
10748 snapshot,
10749 enabled,
10750 reason,
10751 drain_timeout,
10752 )
10753 .await
10754}
10755
10756async fn begin_forwarding_drain_with(
10757 forwarding: &ForwardingTable,
10758 context: ForwardingDrainContext<'_>,
10759 snapshot: &SharedSnapshot,
10760 enabled: Option<bool>,
10761 reason: RouteCloseReason,
10762 drain_timeout: Duration,
10763) -> Result<StopNotice, SuperviseError> {
10764 let ForwardingDrainContext {
10765 spec,
10766 runtime,
10767 registry,
10768 scope,
10769 } = context;
10770 debug_assert_ne!(reason, RouteCloseReason::Crash);
10771 let terminal = matches!(reason, RouteCloseReason::Disable);
10772 let drain_started_at = Instant::now();
10773 let drain_deadline = drain_started_at + drain_timeout;
10774 let deadline_ms =
10775 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10776 let busy_gauges = match scope {
10777 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10778 DrainScope::Endpoint(endpoint) => {
10779 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10780 }
10781 };
10782
10783 let gate_started = Instant::now();
10786 let drain_target = match scope {
10787 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10788 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10789 }
10790 .map_err(SuperviseError::Forwarding)?;
10791 info!(
10796 module_id = %spec.module_id,
10797 ?reason,
10798 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10799 connected = drain_target.is_some(),
10800 "module drain began; route admission closed"
10801 );
10802 if scope == DrainScope::Active {
10803 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10804 state.state = ModuleState::Draining;
10805 state.draining_to_replace =
10806 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10807 if let Some(enabled) = enabled {
10808 state.enabled = enabled;
10809 }
10810 })?;
10811 }
10812
10813 let Some(target) = drain_target.as_ref() else {
10814 return Ok(StopNotice::NoConnection);
10818 };
10819 {
10820 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10821 #[cfg(feature = "test-support")]
10822 forwarding.wait_before_route_closing_for_test().await;
10823 let routes = forwarding
10824 .endpoint_routes(target.endpoint)
10825 .map_err(SuperviseError::Forwarding)?;
10826 let routes_notified = routes.len();
10827 crate::control::send_route_control_pushes(
10828 forwarding,
10829 routes.clone(),
10830 ClientControlPush::RouteClosing {
10831 module_id: spec.module_id.clone(),
10832 channels: Vec::new(),
10833 reason,
10834 },
10835 );
10836 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10837
10838 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10844 let wait_result = wait_for_forwarding_quiescence(
10845 forwarding,
10846 &spec.module_id,
10847 runtime,
10848 target.endpoint,
10849 drain_deadline,
10850 &busy_gauges,
10851 scope,
10852 &mut gauge_trace,
10853 )
10854 .await;
10855 let drained = drained_after_quiescence_wait(&wait_result);
10856 if let Err(err) = &wait_result {
10857 error!(
10858 module_id = %spec.module_id,
10859 ?reason,
10860 error = %err,
10861 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10862 );
10863 } else if !drained {
10864 let holdouts = forwarding
10866 .endpoint_drain_holdouts(target.endpoint)
10867 .unwrap_or_default();
10868 warn_drain_timed_out(
10869 &spec.module_id,
10870 drain_timeout,
10871 reason,
10872 &holdouts,
10873 &gauge_trace,
10874 );
10875 }
10876 crate::control::send_route_control_pushes(
10877 forwarding,
10878 routes,
10879 ClientControlPush::RouteClosed {
10880 module_id: spec.module_id.clone(),
10881 channels: Vec::new(),
10882 reason,
10883 drained,
10884 abandoned: target.abandoned_bindings.len() as u32,
10885 excluded_subscriptions: target.excluded_subscriptions,
10886 terminal: Some(terminal),
10887 },
10888 );
10889 wait_result?;
10890
10891 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10897 Ok(routes) => routes,
10898 Err(err) => {
10899 warn!(
10900 module_id = %spec.module_id,
10901 ?reason,
10902 error = %err,
10903 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10904 );
10905 send_module_goodbye(&spec.module_id, forwarding, target);
10906 return Err(SuperviseError::Forwarding(err));
10907 }
10908 };
10909 let route_goodbye_count = released_routes.len();
10910 send_route_goodbyes(forwarding, released_routes);
10911 send_module_goodbye(&spec.module_id, forwarding, target);
10912
10913 info!(
10919 module_id = %spec.module_id,
10920 ?reason,
10921 routes_notified,
10922 route_goodbyes = route_goodbye_count,
10923 abandoned_reservations = target.abandoned_bindings.len(),
10924 excluded_subscriptions = target.excluded_subscriptions,
10925 drained,
10926 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10927 );
10928 }
10929
10930 Ok(StopNotice::SentOverConnection)
10931}
10932
10933async fn wait_for_registration_after_reload(
10936 registry: &Registry,
10937 module_id: &str,
10938 snapshot: &SharedSnapshot,
10939 child: &mut SupervisedChild,
10940 wait: Duration,
10941) -> Result<RegistrationWaitOutcome, SuperviseError> {
10942 wait_for_slot_registration(
10943 registry,
10944 crate::registry::RegistrationSlot::Active(module_id),
10945 module_id,
10946 snapshot,
10947 child,
10948 wait,
10949 )
10950 .await
10951}
10952
10953async fn wait_for_slot_registration(
10961 registry: &Registry,
10962 slot: crate::registry::RegistrationSlot<'_>,
10963 module_id: &str,
10964 snapshot: &SharedSnapshot,
10965 child: &mut SupervisedChild,
10966 wait: Duration,
10967) -> Result<RegistrationWaitOutcome, SuperviseError> {
10968 let deadline = Instant::now() + wait;
10969 loop {
10970 if registry
10971 .registration(slot)
10972 .map_err(SuperviseError::Registry)?
10973 .is_some()
10974 {
10975 return Ok(RegistrationWaitOutcome::Registered);
10976 }
10977
10978 let now = Instant::now();
10979 if now >= deadline {
10980 return Ok(RegistrationWaitOutcome::TimedOut);
10981 }
10982 let remaining = deadline.saturating_duration_since(now);
10983 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10984
10985 tokio::select! {
10986 wait_result = child.wait() => {
10987 let status = wait_result.map_err(|source| SuperviseError::Wait {
10988 module_id: module_id.to_string(),
10989 source,
10990 })?;
10991 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10992 snapshot,
10993 child,
10994 &status,
10995 )));
10996 }
10997 _ = sleep(poll) => {}
10998 }
10999 }
11000}
11001
11002fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11003 if exit_report.kind != ExitKind::DeliberateSeverance {
11007 exit_report.kind = ExitKind::Crash;
11008 }
11009 exit_report
11010}
11011
11012async fn handle_reload_child_registration_failure(
11013 spec: &ModuleSpec,
11014 runtime: &SupervisorRuntimeConfig,
11015 registry: &Registry,
11016 process_liveness: &SupervisorProcessLiveness,
11017 snapshot: &SharedSnapshot,
11018 _child: &mut Option<SupervisedChild>,
11019 failure: ReloadRegistrationFailure,
11020) -> Result<(), SuperviseError> {
11021 let ReloadRegistrationFailure {
11022 exit_report,
11023 reason,
11024 } = failure;
11025 match on_child_exit(
11026 spec,
11027 runtime.restart_policy,
11028 registry,
11029 snapshot,
11030 &runtime.terminal_ring,
11031 &runtime.spawn_events,
11032 &runtime.child_roster,
11033 exit_report,
11034 runtime.recovery_is_daemon,
11035 )
11036 .await
11037 {
11038 NextAction::Stop {
11039 registration_released,
11040 } => {
11041 if registration_released {
11042 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11043 }
11044 }
11045 NextAction::Restart { schedule } => {
11046 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11047 schedule.delay
11048 });
11049 if let Some(schedule) = schedule {
11050 log_crash_respawn(&spec.module_id, schedule);
11051 }
11052 schedule_respawn(
11053 runtime,
11054 snapshot,
11055 &spec.module_id,
11056 delay,
11057 RespawnKind::Spawn,
11058 true,
11059 )?;
11060 }
11061 }
11062 Err(SuperviseError::ReloadFailed {
11063 module_id: spec.module_id.clone(),
11064 reason,
11065 })
11066}
11067
11068async fn handle_reload_spawn_failure(
11069 spec: &ModuleSpec,
11070 runtime: &SupervisorRuntimeConfig,
11071 process_liveness: &SupervisorProcessLiveness,
11072 snapshot: &SharedSnapshot,
11073 _child: &mut Option<SupervisedChild>,
11074 reason: String,
11075) -> Result<(), SuperviseError> {
11076 let now = Instant::now();
11077 let mut schedule = None;
11078 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11079 clear_current_process_facts(state);
11080 if state.enabled {
11081 schedule = state.next_crash_restart(&runtime.restart_policy, now);
11082 state.state = if schedule.is_some() {
11083 ModuleState::Restarting
11084 } else {
11085 if runtime
11086 .failure_context(runtime.recovery_is_daemon)
11087 .may_retry()
11088 {
11089 state.failed_retry.schedule(&spec.module_id);
11090 }
11091 ModuleState::Failed
11092 };
11093 } else {
11094 state.state = ModuleState::Disabled;
11095 }
11096 })?;
11097 if let Some(schedule) = schedule {
11098 schedule_respawn(
11099 runtime,
11100 snapshot,
11101 &spec.module_id,
11102 schedule.delay,
11103 RespawnKind::Spawn,
11104 true,
11105 )?;
11106 } else {
11107 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11108 }
11109 Err(SuperviseError::ReloadFailed {
11110 module_id: spec.module_id.clone(),
11111 reason,
11112 })
11113}
11114
11115fn control_flags() -> Flags {
11116 Flags::new(false, Priority::Passive, false)
11117}
11118
11119#[allow(clippy::too_many_arguments)]
11120async fn drain_optional_child(
11121 module_id: &str,
11122 protocol: ModuleProtocol,
11123 stop_notice: StopNotice,
11124 registry: &Registry,
11125 forwarding: Option<&ForwardingTable>,
11126 snapshot: &SharedSnapshot,
11127 terminal_ring: &Arc<Mutex<TerminalRing>>,
11128 spawn_events: &SpawnEventFeed,
11129 child: &mut Option<SupervisedChild>,
11130 drain_timeout: Duration,
11131 final_state: ModuleState,
11132 enabled: Option<bool>,
11133 failure: &FailureContext,
11134) -> Result<(), SuperviseError> {
11135 if let Some(child) = child.take() {
11136 drain_child_to_state(
11137 module_id,
11138 protocol,
11139 stop_notice,
11140 registry,
11141 forwarding,
11142 snapshot,
11143 terminal_ring,
11144 spawn_events,
11145 child,
11146 drain_timeout,
11147 final_state,
11148 enabled,
11149 failure,
11150 )
11151 .await
11152 } else {
11153 update_snapshot(snapshot, Some(module_id), |state| {
11154 state.state = final_state;
11155 if final_state == ModuleState::Failed {
11156 state.failed_retry.failed_at = Some(Instant::now());
11157 }
11158 if let Some(enabled) = enabled {
11159 state.enabled = enabled;
11160 }
11161 clear_current_process_facts(state);
11162 })?;
11163 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11164 }
11165}
11166
11167#[allow(clippy::too_many_arguments)]
11168async fn drain_child_to_state(
11169 module_id: &str,
11170 _protocol: ModuleProtocol,
11171 stop_notice: StopNotice,
11172 registry: &Registry,
11173 forwarding: Option<&ForwardingTable>,
11174 snapshot: &SharedSnapshot,
11175 terminal_ring: &Arc<Mutex<TerminalRing>>,
11176 spawn_events: &SpawnEventFeed,
11177 mut child: SupervisedChild,
11178 drain_timeout: Duration,
11179 final_state: ModuleState,
11180 enabled: Option<bool>,
11181 failure: &FailureContext,
11182) -> Result<(), SuperviseError> {
11183 let protocol = child.protocol;
11184 update_snapshot(snapshot, Some(module_id), |state| {
11185 state.state = ModuleState::Draining;
11186 state.draining_to_replace = final_state == ModuleState::Restarting;
11187 if let Some(enabled) = enabled {
11188 state.enabled = enabled;
11189 }
11190 })?;
11191
11192 if stop_notice != StopNotice::SentOverConnection {
11203 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11204 info!(
11205 module_id,
11206 pid = child.pid,
11207 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11208 "module has no connection yet; requesting stop by signal"
11209 );
11210 }
11211 request_graceful_stop(module_id, &child);
11212 }
11213
11214 let exit_report = match timeout(drain_timeout, child.wait()).await {
11215 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11216 Ok(Err(source)) => {
11217 fail_after_wait_error(
11218 module_id,
11219 snapshot,
11220 terminal_ring,
11221 spawn_events,
11222 format!("failed to wait during drain: {source}"),
11223 failure,
11224 );
11225 return Err(SuperviseError::Wait {
11226 module_id: module_id.to_string(),
11227 source,
11228 });
11229 }
11230 Err(_) => {
11231 warn!(
11244 module_id,
11245 pid = child.pid,
11246 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11247 reason = ?final_state,
11248 ?stop_notice,
11249 "drain budget expired before the module exited; killing it"
11250 );
11251 child.start_kill().map_err(|source| {
11252 fail_after_wait_error(
11253 module_id,
11254 snapshot,
11255 terminal_ring,
11256 spawn_events,
11257 format!("failed to kill during drain: {source}"),
11258 failure,
11259 );
11260 SuperviseError::Kill {
11261 module_id: module_id.to_string(),
11262 source,
11263 }
11264 })?;
11265 let status = child.wait().await.map_err(|source| {
11266 fail_after_wait_error(
11267 module_id,
11268 snapshot,
11269 terminal_ring,
11270 spawn_events,
11271 format!("failed to wait after drain kill: {source}"),
11272 failure,
11273 );
11274 SuperviseError::Wait {
11275 module_id: module_id.to_string(),
11276 source,
11277 }
11278 })?;
11279 classify_reaped_child_exit(snapshot, &child, &status)
11280 }
11281 };
11282
11283 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11284 publish_after_terminal_record(
11285 module_id,
11286 || {
11287 record_terminal_with_detail(
11288 module_id,
11289 terminal_ring,
11290 spawn_events,
11291 &exit_report,
11292 terminal_disposition(final_state),
11293 detail,
11294 )
11295 },
11296 || {
11297 update_snapshot(snapshot, Some(module_id), |state| {
11298 state.state = final_state;
11299 if final_state == ModuleState::Failed {
11300 state.failed_retry.failed_at = Some(Instant::now());
11301 }
11302 if let Some(enabled) = enabled {
11303 state.enabled = enabled;
11304 }
11305 clear_current_process_facts(state);
11306 state.last_exit = Some(exit_report.clone());
11307 if exit_report.kind == ExitKind::DeliberateSeverance {
11308 state.lifetime_restarts += 1;
11309 }
11310 })
11311 },
11312 )?;
11313 child.drain_stderr(module_id).await;
11314
11315 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11316}
11317
11318#[cfg(unix)]
11338fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
11339 let Some(pid) = child
11340 .id()
11341 .and_then(|pid| i32::try_from(pid).ok())
11342 .and_then(rustix::process::Pid::from_raw)
11343 else {
11344 debug!(
11345 module_id,
11346 "no pid to signal for teardown; falling through to the drain wait"
11347 );
11348 return;
11349 };
11350 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
11351 Ok(()) => debug!(
11352 module_id,
11353 "sent SIGTERM to a module nothing else asked to stop"
11354 ),
11355 Err(err) => debug!(
11356 module_id,
11357 error = %err,
11358 "SIGTERM to module failed; the drain wait and kill still apply"
11359 ),
11360 }
11361}
11362
11363#[cfg(not(unix))]
11371fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11372 debug!(
11373 module_id,
11374 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11375 );
11376}
11377
11378fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11379 match final_state {
11380 ModuleState::Stopped => TerminalDisposition::Stopped,
11381 ModuleState::Disabled => TerminalDisposition::Disabled,
11382 ModuleState::Restarting => TerminalDisposition::Restarting,
11383 ModuleState::Failed => TerminalDisposition::Failed,
11384 ModuleState::Starting
11385 | ModuleState::Running
11386 | ModuleState::Unresponsive
11387 | ModuleState::Draining => {
11388 unreachable!("terminal exits only finish in terminal or restarting states")
11389 }
11390 }
11391}
11392
11393async fn release_dead_registration(
11401 registry: &Registry,
11402 forwarding: Option<&ForwardingTable>,
11403 snapshot: &SharedSnapshot,
11404 module_id: &str,
11405 failure: &FailureContext,
11406) -> Result<(), SuperviseError> {
11407 let result = async {
11408 let registration = registry
11409 .get_module(module_id)
11410 .map_err(SuperviseError::Registry)?;
11411 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11412 Ok(()) => return Ok(()),
11413 Err(SuperviseError::RegistrationStillActive { .. }) => {}
11414 Err(err) => return Err(err),
11415 }
11416 let pid = lock_snapshot(snapshot)?.reaped_pid;
11417 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11418 warn!(
11419 module_id,
11420 pid,
11421 connection_id = registration.connection_id.get(),
11422 "reaped module registration outlived release grace; closing dead connection"
11423 );
11424 forwarding.request_connection_close(
11425 registration.connection_id,
11426 CloseReason::new(
11427 "supervised_process_reaped",
11428 format!("module '{module_id}' pid {pid} exited"),
11429 ),
11430 );
11431 wait_for_slot_registration_release(
11432 registry,
11433 crate::registry::RegistrationSlot::Connection(registration.connection_id),
11434 REGISTRY_RELEASE_TIMEOUT,
11435 )
11436 .await?;
11437 }
11438 wait_for_registration_release(registry, module_id, Duration::ZERO).await
11439 }
11440 .await;
11441 if let Err(err) = &result {
11442 fail_snapshot(snapshot, Some(module_id), None, failure);
11443 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11444 }
11445 result
11446}
11447
11448async fn wait_for_registration_release(
11451 registry: &Registry,
11452 module_id: &str,
11453 wait: Duration,
11454) -> Result<(), SuperviseError> {
11455 wait_for_slot_registration_release(
11456 registry,
11457 crate::registry::RegistrationSlot::Active(module_id),
11458 wait,
11459 )
11460 .await
11461}
11462
11463async fn wait_for_slot_registration_release(
11471 registry: &Registry,
11472 slot: crate::registry::RegistrationSlot<'_>,
11473 wait: Duration,
11474) -> Result<(), SuperviseError> {
11475 let deadline = Instant::now() + wait;
11476 let mut release_events = registration_release_events().subscribe();
11477 let still_active = |registration: &crate::registry::ModuleRegistration| {
11478 SuperviseError::RegistrationStillActive {
11479 module_id: registration.manifest.module_id.clone(),
11480 waited: wait,
11481 }
11482 };
11483 loop {
11484 let _observed_generation = *release_events.borrow_and_update();
11485 let Some(registration) = registry
11486 .registration(slot)
11487 .map_err(SuperviseError::Registry)?
11488 else {
11489 return Ok(());
11490 };
11491
11492 let now = Instant::now();
11493 if now >= deadline {
11494 return Err(still_active(®istration));
11495 }
11496
11497 let remaining = deadline.saturating_duration_since(now);
11498 match timeout(remaining, release_events.changed()).await {
11499 Ok(Ok(())) | Ok(Err(_)) => {}
11500 Err(_) => return Err(still_active(®istration)),
11501 }
11502 }
11503}
11504
11505#[cfg(test)]
11506mod slot_registration_wait_tests {
11507 use super::*;
11508 use crate::registry::{ConnectionId, RegistrationSlot};
11509 use subc_protocol::manifest::ModuleManifest;
11510
11511 #[tokio::test]
11512 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11513 let registry = Arc::new(Registry::default());
11514 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
11515 let runtime = supervisor.runtime_config();
11516 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11517 let spec = ModuleSpec {
11518 module_id: "enable-stale-registration".to_string(),
11519 program: PathBuf::from("/missing/enable-retry-test"),
11520 args: Vec::new(),
11521 env: Vec::new(),
11522 reserved: false,
11523 reserved_prefixes: Vec::new(),
11524 protocol: ModuleProtocol::Subc,
11525 overlap: Default::default(),
11526 };
11527 let connection = ConnectionId::new(90);
11528 registry
11529 .register_with_control_ops(
11530 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11531 1,
11532 connection,
11533 Vec::new(),
11534 )
11535 .unwrap();
11536 let mut child = None;
11537 let err = set_child_enabled(
11538 &spec,
11539 &runtime,
11540 ®istry,
11541 &supervisor.process_liveness,
11542 &snapshot,
11543 &mut child,
11544 true,
11545 &runtime.failure_context(false),
11546 )
11547 .await
11548 .unwrap_err();
11549 assert!(matches!(
11550 err,
11551 SuperviseError::RegistrationStillActive { .. }
11552 ));
11553 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11554 assert!(child.is_none());
11555 registry.deregister_connection(connection).unwrap();
11556 let err = set_child_enabled(
11557 &spec,
11558 &runtime,
11559 ®istry,
11560 &supervisor.process_liveness,
11561 &snapshot,
11562 &mut child,
11563 true,
11564 &runtime.failure_context(false),
11565 )
11566 .await
11567 .unwrap_err();
11568 assert!(
11569 matches!(err, SuperviseError::Spawn { .. }),
11570 "second enable must attempt a spawn: {err}"
11571 );
11572 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11573 }
11574
11575 const INCUMBENT: u64 = 1;
11576 const CANDIDATE: u64 = 2;
11577
11578 fn swapped_registry() -> Arc<Registry> {
11579 let registry = Arc::new(Registry::default());
11580 let manifest = ModuleManifest::builder("m", "0.1.0").build();
11581 registry
11582 .register_with_control_ops(
11583 manifest.clone(),
11584 1,
11585 ConnectionId::new(INCUMBENT),
11586 Vec::new(),
11587 )
11588 .unwrap();
11589 registry
11590 .register_candidate_with_control_ops(
11591 manifest,
11592 1,
11593 ConnectionId::new(CANDIDATE),
11594 Vec::new(),
11595 )
11596 .unwrap();
11597 registry
11598 }
11599
11600 #[tokio::test]
11604 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11605 let registry = swapped_registry();
11606 registry.promote_candidate("m").unwrap().unwrap();
11607
11608 assert!(matches!(
11609 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
11610 Err(SuperviseError::RegistrationStillActive { .. })
11611 ));
11612
11613 assert!(matches!(
11615 wait_for_slot_registration_release(
11616 ®istry,
11617 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11618 Duration::from_millis(50),
11619 )
11620 .await,
11621 Err(SuperviseError::RegistrationStillActive { .. })
11622 ));
11623
11624 let releaser = Arc::clone(®istry);
11625 let release = tokio::spawn(async move {
11626 sleep(Duration::from_millis(20)).await;
11627 releaser
11628 .deregister_connection(ConnectionId::new(INCUMBENT))
11629 .unwrap();
11630 notify_registration_release();
11631 });
11632 wait_for_slot_registration_release(
11633 ®istry,
11634 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11635 Duration::from_secs(5),
11636 )
11637 .await
11638 .expect("the incumbent's own registration is released");
11639 release.await.unwrap();
11640 assert!(registry.get_module("m").unwrap().is_some());
11641 }
11642
11643 #[tokio::test]
11646 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11647 let registry = swapped_registry();
11648 assert!(matches!(
11649 wait_for_slot_registration_release(
11650 ®istry,
11651 RegistrationSlot::Candidate("m"),
11652 Duration::from_millis(50),
11653 )
11654 .await,
11655 Err(SuperviseError::RegistrationStillActive { .. })
11656 ));
11657 registry
11658 .deregister_connection(ConnectionId::new(CANDIDATE))
11659 .unwrap();
11660 wait_for_slot_registration_release(
11661 ®istry,
11662 RegistrationSlot::Candidate("m"),
11663 Duration::from_millis(50),
11664 )
11665 .await
11666 .expect("a candidate slot with no candidate is released");
11667 assert!(registry
11668 .registration(RegistrationSlot::Active("m"))
11669 .unwrap()
11670 .is_some());
11671 }
11672}
11673
11674fn classify_exit(status: &ExitStatus) -> ExitReport {
11675 ExitReport {
11676 kind: if status.success() {
11677 ExitKind::Clean
11678 } else {
11679 ExitKind::Crash
11680 },
11681 code: status.code(),
11682 signal: exit_signal(status),
11683 at_ms: unix_ms_now(),
11684 }
11685}
11686
11687fn wait_error_exit_report() -> ExitReport {
11693 ExitReport {
11694 kind: ExitKind::Crash,
11695 code: None,
11696 signal: None,
11697 at_ms: unix_ms_now(),
11698 }
11699}
11700
11701#[cfg(unix)]
11702fn exit_signal(status: &ExitStatus) -> Option<i32> {
11703 use std::os::unix::process::ExitStatusExt;
11704
11705 status.signal()
11706}
11707
11708#[cfg(not(unix))]
11709fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11710 None
11711}
11712
11713fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11719 update_snapshot(snapshot, Some(module_id), |state| {
11720 state.clear_crash_restarts();
11721 })
11722}
11723
11724fn set_running(
11725 snapshot: &SharedSnapshot,
11726 child: &SupervisedChild,
11727 module_id: &str,
11728 spawn_events: &SpawnEventFeed,
11729) -> Result<(), SuperviseError> {
11730 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11731 module_id: Some(module_id.to_string()),
11732 })?;
11733 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11734 if std::mem::take(&mut state.coalesced_restart_pending) {
11735 let generation = state.spawn_generation;
11736 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11737 }
11738 state.drain_disposition_detail = None;
11739 state.spawn_failure = None;
11740 state.waiting_for_previous_pid = None;
11741 state.waiting_for_previous_detail = None;
11742 state.in_alternate_slot = false;
11745 state.configuration_updated_since_spawn = false;
11746 state.spawned_protocol = Some(child.protocol);
11747 state.state = ModuleState::Running;
11748 state.enabled = true;
11749 state.process_alive = true;
11750 state.pid = child.id();
11751 state.health.status = SupervisorHealthStatus::Unknown;
11754 state.health.consecutive_failures = 0;
11755 state.health.detail = None;
11756 state.health.metrics = None;
11757 state.health.last_probe_ms = None;
11758 state.failed_retry.healthy_since = None;
11759 state.failed_retry.failed_at = None;
11760 state.failed_retry.running_since = Some(child.spawned_at);
11761 #[cfg(windows)]
11762 {
11763 state.pipe_consumed = child
11764 .nonce_delivery
11765 .as_ref()
11766 .map(|delivery| delivery.consumed());
11767 }
11768 #[cfg(target_os = "macos")]
11769 {
11770 state.report_ready = Some(Arc::clone(&child.report_ready));
11771 }
11772 state.spawned_at_ms = Some(child.spawned_at_ms);
11773 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11774 state.spawned_from = Some(child.spawned_from.clone());
11775 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11776 state.spawned_file_identity = child.spawned_file_identity.clone();
11777 state.process_start_time = child.process_start_time;
11778 Ok(())
11779}
11780
11781fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11782 state.failed_retry.healthy_since = None;
11783 state.failed_retry.running_since = None;
11784 state.waiting_for_previous_pid = None;
11785 state.waiting_for_previous_detail = None;
11786 state.process_alive = false;
11787 state.spawned_protocol = None;
11788 state.pid = None;
11789 #[cfg(windows)]
11790 {
11791 state.pipe_consumed = None;
11792 }
11793 #[cfg(target_os = "macos")]
11794 {
11795 state.report_ready = None;
11796 }
11797 state.spawned_at_ms = None;
11798 state.exec_outcome = None;
11799 state.spawned_from = None;
11800 state.launch_spec_digest = None;
11801 state.spawned_file_identity = None;
11802 state.process_start_time = None;
11803 state.deliberate_severance = None;
11804}
11805
11806#[cfg(test)]
11807fn record_deliberate_severance(
11808 snapshot: &SharedSnapshot,
11809 identity: ProcessIdentity,
11810) -> Result<(), SuperviseError> {
11811 update_snapshot(snapshot, None, |state| {
11812 state.deliberate_severance = Some(identity);
11813 })
11814}
11815
11816fn apply_deliberate_severance_marker(
11817 snapshot: &SharedSnapshot,
11818 exited_identity: Option<ProcessIdentity>,
11819 mut exit_report: ExitReport,
11820) -> ExitReport {
11821 let marker = lock_snapshot(snapshot)
11822 .ok()
11823 .and_then(|mut state| state.deliberate_severance.take());
11824 if marker.is_some() && marker == exited_identity {
11825 exit_report.kind = ExitKind::DeliberateSeverance;
11826 }
11827 exit_report
11828}
11829
11830fn classify_reaped_child_exit(
11831 snapshot: &SharedSnapshot,
11832 child: &SupervisedChild,
11833 status: &ExitStatus,
11834) -> ExitReport {
11835 let _ = update_snapshot(snapshot, None, |state| {
11836 state.reaped_pid = Some(child.pid);
11837 state.spawn_failure = child.spawn_failure.clone();
11838 });
11839 if child.spawn_failure.as_deref() == Some("registration_timeout") {
11840 registration_failure_exit_report(classify_exit(status))
11841 } else {
11842 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11843 }
11844}
11845
11846fn fail_snapshot(
11847 snapshot: &SharedSnapshot,
11848 module_id: Option<&str>,
11849 last_exit: Option<ExitReport>,
11850 failure: &FailureContext,
11851) {
11852 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11853 state.state = ModuleState::Failed;
11854 clear_current_process_facts(state);
11855 if failure.may_retry() && state.enabled {
11856 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
11857 } else if failure.shutdown.is_set() {
11858 state.failed_retry.cancel();
11859 }
11860 if let Some(last_exit) = last_exit {
11861 state.last_exit = Some(last_exit);
11862 }
11863 }) {
11864 error!(error = %err, "failed to mark supervisor state failed");
11865 }
11866}
11867
11868fn update_snapshot(
11869 snapshot: &SharedSnapshot,
11870 module_id: Option<&str>,
11871 update: impl FnOnce(&mut SupervisorSnapshot),
11872) -> Result<(), SuperviseError> {
11873 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11874 module_id: module_id.map(ToOwned::to_owned),
11875 })?;
11876 update(&mut state);
11877 Ok(())
11878}
11879
11880const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11881
11882fn lock_snapshot_for_control<'a>(
11883 snapshot: &'a SharedSnapshot,
11884 module_id: &str,
11885 caller: &'static str,
11886) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11887 let started_at = Instant::now();
11888 let guard = lock_snapshot(snapshot)?;
11889 let waited = started_at.elapsed();
11890 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11891 warn!(
11892 module_id = %module_id,
11893 waited_ms = waited.as_millis() as u64,
11894 caller = %caller,
11895 "slow snapshot lock"
11896 );
11897 }
11898 Ok(guard)
11899}
11900
11901fn lock_snapshot(
11902 snapshot: &SharedSnapshot,
11903) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11904 snapshot
11905 .lock()
11906 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11907}
11908
11909#[cfg(test)]
11910mod terminal_history_tests {
11911 use std::{
11912 path::PathBuf,
11913 sync::Arc,
11914 time::{Duration, Instant},
11915 };
11916
11917 use tokio::{sync::mpsc, time::sleep};
11918
11919 use super::{
11920 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11921 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11922 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11923 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11924 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11925 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11926 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11927 SupervisorHealthStatus, SupervisorSnapshot,
11928 };
11929 use super::Instant as ClockInstant;
11934 use crate::{
11935 registry::Registry,
11936 terminal_ring::{TerminalRing, TerminalRingConfig},
11937 };
11938 use std::sync::Mutex;
11939 use subc_control::TerminalDisposition;
11940
11941 pub(super) fn fake_aft_stub_path() -> PathBuf {
11946 let mut path = std::env::current_exe().expect("current_exe available in tests");
11947 path.pop();
11948 path.pop();
11949 path.push(if cfg!(windows) {
11950 "fake-aft-stub.exe"
11951 } else {
11952 "fake-aft-stub"
11953 });
11954 assert!(
11955 path.exists(),
11956 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11957 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11958 path.display()
11959 );
11960 path
11961 }
11962
11963 #[test]
11964 fn reserved_never_spawned_refuses_every_hello() {
11965 let supervisor = SupervisorHandle::default();
11970 supervisor.apply_identity_configuration(&ModuleSpec {
11971 module_id: "never-spawned".to_string(),
11972 program: PathBuf::from("/usr/bin/false"),
11973 args: Vec::new(),
11974 env: Vec::new(),
11975 reserved: true,
11976 reserved_prefixes: Vec::new(),
11977 protocol: ModuleProtocol::Subc,
11978 overlap: Default::default(),
11979 });
11980 assert!(
11981 supervisor
11982 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11983 .is_err(),
11984 "forged nonce must refuse on a reserved never-spawned id"
11985 );
11986 assert!(
11987 supervisor
11988 .reserved_hello_verification("never-spawned", None)
11989 .is_err(),
11990 "absent nonce must refuse on a reserved never-spawned id"
11991 );
11992 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11994 supervisor.apply_identity_configuration(&ModuleSpec {
11995 module_id: "never-spawned".to_string(),
11996 program: PathBuf::from("/usr/bin/false"),
11997 args: Vec::new(),
11998 env: Vec::new(),
11999 reserved: true,
12000 reserved_prefixes: Vec::new(),
12001 protocol: ModuleProtocol::Subc,
12002 overlap: Default::default(),
12003 });
12004 assert!(supervisor
12005 .reserved_hello_verification("never-spawned", Some("minted"))
12006 .is_ok());
12007 assert!(supervisor
12008 .reserved_hello_verification("never-spawned", Some("forged"))
12009 .is_err());
12010 }
12011
12012 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12015 let now = ClockInstant::now();
12016 for _ in 0..count {
12017 state.crash_restarts.push_back(now);
12018 }
12019 }
12020
12021 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12025 let aged = state
12026 .crash_restarts
12027 .front()
12028 .expect("a crash restart must be recorded before it can be aged")
12029 .checked_sub(window + Duration::from_secs(1))
12030 .expect("the test clock is far enough from its origin to age an instant");
12031 state.crash_restarts[0] = aged;
12032 }
12033
12034 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12035 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12036 seed_crash_restarts(&mut state, count);
12037 state
12038 }
12039
12040 #[test]
12041 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12042 let policy = RestartPolicy::new(3, Duration::ZERO);
12043 let now = ClockInstant::now();
12044 assert!(daemon_will_restart(
12045 &mut snapshot_with_restarts(true, 2),
12046 &policy,
12047 now
12048 ));
12049 assert!(!daemon_will_restart(
12050 &mut snapshot_with_restarts(true, 3),
12051 &policy,
12052 now
12053 ));
12054 assert!(!daemon_will_restart(
12055 &mut snapshot_with_restarts(false, 0),
12056 &policy,
12057 now
12058 ));
12059 }
12060
12061 #[test]
12062 fn crash_restart_backoff_escalates_with_in_window_count() {
12063 let policy = RestartPolicy::new(4, Duration::from_millis(100))
12064 .with_max_backoff(Duration::from_secs(30));
12065 let now = ClockInstant::now();
12066 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12067 let schedules = (0..4)
12068 .map(|_| {
12069 state
12070 .next_crash_restart(&policy, now)
12071 .expect("the test policy allows four crash restarts")
12072 })
12073 .collect::<Vec<_>>();
12074
12075 assert_eq!(
12076 schedules
12077 .iter()
12078 .map(|schedule| schedule.restart_in_window)
12079 .collect::<Vec<_>>(),
12080 vec![0, 1, 2, 3]
12081 );
12082 assert_eq!(
12083 schedules
12084 .iter()
12085 .map(|schedule| schedule.delay)
12086 .collect::<Vec<_>>(),
12087 vec![
12088 Duration::from_millis(100),
12089 Duration::from_secs(1),
12090 Duration::from_secs(10),
12091 Duration::from_secs(30),
12092 ]
12093 );
12094 }
12095
12096 #[test]
12097 fn crash_restart_backoff_resets_after_ring_clear() {
12098 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12099 let now = ClockInstant::now();
12100 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12101 assert_eq!(
12102 state.next_crash_restart(&policy, now).unwrap().delay,
12103 Duration::from_millis(100)
12104 );
12105 assert_eq!(
12106 state.next_crash_restart(&policy, now).unwrap().delay,
12107 Duration::from_secs(1)
12108 );
12109
12110 state.clear_crash_restarts();
12111 let schedule = state
12112 .next_crash_restart(&policy, now)
12113 .expect("a cleared ring must allow another restart");
12114 assert_eq!(schedule.restart_in_window, 0);
12115 assert_eq!(schedule.delay, Duration::from_millis(100));
12116 }
12117
12118 #[test]
12119 fn crash_restart_backoff_ignores_aged_restarts() {
12120 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12121 let now = ClockInstant::now();
12122 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12123 state
12124 .next_crash_restart(&policy, now)
12125 .expect("the first restart is allowed");
12126 state
12127 .next_crash_restart(&policy, now)
12128 .expect("the second restart is allowed");
12129 state.crash_restarts[0] = now
12130 .checked_sub(policy.window + Duration::from_secs(1))
12131 .expect("the fake clock can age a restart past the window");
12132
12133 let schedule = state
12134 .next_crash_restart(&policy, now)
12135 .expect("an aged restart must release its slot");
12136 assert_eq!(schedule.restart_in_window, 1);
12137 assert_eq!(schedule.delay, Duration::from_secs(1));
12138 assert_eq!(state.crash_restarts.len(), 2);
12139 }
12140
12141 #[test]
12145 fn a_budget_spent_before_the_window_no_longer_refuses() {
12146 let policy = RestartPolicy::new(3, Duration::ZERO);
12147 let mut state = snapshot_with_restarts(true, 3);
12148 let now = ClockInstant::now();
12149 assert!(!daemon_will_restart(&mut state, &policy, now));
12150
12151 assert!(daemon_will_restart(
12152 &mut state,
12153 &policy,
12154 now + policy.window + Duration::from_secs(1)
12155 ));
12156 assert!(
12157 state.crash_restarts.is_empty(),
12158 "reading the budget must drop the instants that left the window"
12159 );
12160 }
12161
12162 fn module_with_recovery_snapshot(
12163 state: ModuleState,
12164 enabled: bool,
12165 restart_count: u32,
12166 ) -> SupervisedModule {
12167 let registry = Arc::new(Registry::default());
12168 let supervisor =
12169 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
12170 let runtime = supervisor.runtime_config();
12171 let spec = ModuleSpec {
12172 module_id: "recovery-snapshot".to_string(),
12173 program: fake_aft_stub_path(),
12174 args: Vec::new(),
12175 env: Vec::new(),
12176 reserved: false,
12177 reserved_prefixes: Vec::new(),
12178 protocol: ModuleProtocol::Subc,
12179 overlap: Default::default(),
12180 };
12181 let mut snapshot = SupervisorSnapshot::new(state, enabled);
12182 seed_crash_restarts(&mut snapshot, restart_count);
12183 let (commands, _rx) = mpsc::channel(4);
12186 SupervisedModule {
12187 inner: Arc::new(SupervisedModuleInner {
12188 module_id: spec.module_id.clone(),
12189 registry,
12190 snapshot: Arc::new(Mutex::new(snapshot)),
12191 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12192 spec,
12193 health: runtime.health,
12194 })),
12195 stderr_ring: runtime.stderr_ring,
12196 terminal_ring: runtime.terminal_ring,
12197 commands,
12198 monitor: Mutex::new(None),
12199 restart_policy: runtime.restart_policy,
12200 effective_drain_timeout: runtime.effective_drain_timeout,
12201 provenance_probe: supervisor.provenance_probe.clone(),
12202 }),
12203 }
12204 }
12205
12206 #[cfg(target_os = "linux")]
12207 #[tokio::test]
12208 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12209 let supervisor =
12210 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12211 .with_cgroup_placement(None);
12212 let result = supervisor.spawn(ModuleSpec {
12213 module_id: "no-cgroup-placement".to_string(),
12214 program: fake_aft_stub_path(),
12215 args: Vec::new(),
12216 env: Vec::new(),
12217 reserved: false,
12218 reserved_prefixes: Vec::new(),
12219 protocol: ModuleProtocol::Subc,
12220 overlap: Default::default(),
12221 });
12222
12223 assert!(
12224 result.is_ok(),
12225 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12226 );
12227 }
12228
12229 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12230 async fn undecided_snapshot_uses_shared_restart_predicate() {
12231 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12232 .will_recover_after_connection_loss()
12233 .unwrap());
12234 assert!(
12235 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12236 .will_recover_after_connection_loss()
12237 .unwrap()
12238 );
12239 }
12240
12241 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12242 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12243 assert!(
12244 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12245 .will_recover_after_connection_loss()
12246 .unwrap()
12247 );
12248 }
12249
12250 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12251 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12252 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12253 .will_recover_after_connection_loss()
12254 .unwrap());
12255 assert!(
12256 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12257 .will_recover_after_connection_loss()
12258 .unwrap()
12259 );
12260 }
12261
12262 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12263 async fn warming_snapshot_is_limited_to_startup_phases() {
12264 for state in [
12265 ModuleState::Starting,
12266 ModuleState::Running,
12267 ModuleState::Restarting,
12268 ] {
12269 assert!(
12270 module_with_recovery_snapshot(state, true, 0)
12271 .is_warming()
12272 .unwrap(),
12273 "{state:?} should be warming"
12274 );
12275 }
12276 for state in [
12277 ModuleState::Unresponsive,
12278 ModuleState::Draining,
12279 ModuleState::Stopped,
12280 ModuleState::Failed,
12281 ModuleState::Disabled,
12282 ] {
12283 assert!(
12284 !module_with_recovery_snapshot(state, true, 0)
12285 .is_warming()
12286 .unwrap(),
12287 "{state:?} should not be warming"
12288 );
12289 }
12290 }
12291
12292 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12293 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12294 let registry = Arc::new(Registry::default());
12295 let supervisor =
12296 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
12297 let module = supervisor
12298 .spawn(ModuleSpec {
12299 module_id: "terminal-history".to_string(),
12300 program: fake_aft_stub_path(),
12301 args: Vec::new(),
12302 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12303 reserved: false,
12304 reserved_prefixes: Vec::new(),
12305 protocol: ModuleProtocol::Subc,
12306 overlap: Default::default(),
12307 })
12308 .unwrap();
12309
12310 let deadline = Instant::now() + Duration::from_secs(5);
12311 loop {
12312 let history = module.terminal_history();
12313 if history.entries.len() == 2 {
12314 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12315 assert_eq!(history.dropped, 0);
12316 assert_eq!(
12317 history
12318 .entries
12319 .iter()
12320 .map(|entry| entry.exit_code)
12321 .collect::<Vec<_>>(),
12322 vec![Some(23), Some(23)]
12323 );
12324 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
12325 return;
12326 }
12327 assert!(
12328 Instant::now() < deadline,
12329 "module did not retain two terminal exits: {history:?}"
12330 );
12331 sleep(Duration::from_millis(10)).await;
12332 }
12333 }
12334
12335 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12339 async fn disable_during_crash_backoff_cancels_pending_respawn() {
12340 let backoff = Duration::from_secs(2);
12341 let supervisor = Supervisor::new_for_test(
12342 Arc::new(Registry::default()),
12343 RestartPolicy::new(10, backoff),
12344 );
12345 let module = supervisor
12346 .spawn(ModuleSpec {
12347 module_id: "disable-during-backoff".to_string(),
12348 program: fake_aft_stub_path(),
12349 args: Vec::new(),
12350 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12351 reserved: false,
12352 reserved_prefixes: Vec::new(),
12353 protocol: ModuleProtocol::Subc,
12354 overlap: Default::default(),
12355 })
12356 .unwrap();
12357
12358 let deadline = Instant::now() + Duration::from_secs(5);
12360 loop {
12361 if module.status().unwrap().state == ModuleState::Restarting {
12362 break;
12363 }
12364 assert!(
12365 Instant::now() < deadline,
12366 "module never entered the crash backoff"
12367 );
12368 sleep(Duration::from_millis(10)).await;
12369 }
12370
12371 let started = Instant::now();
12372 module.set_enabled(false).await.unwrap();
12373 let waited = started.elapsed();
12374
12375 assert!(
12376 waited < backoff / 2,
12377 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12378 );
12379 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12380
12381 sleep(backoff + Duration::from_millis(500)).await;
12383 let status = module.status().unwrap();
12384 assert_eq!(status.state, ModuleState::Disabled);
12385 assert_eq!(
12386 status.spawn_generation, 1,
12387 "module respawned after the operator disabled it"
12388 );
12389 }
12390
12391 #[cfg(unix)]
12394 fn protocol_none_sigterm_exits_clean_spec(
12395 module_id: &str,
12396 dir: &std::path::Path,
12397 ) -> (ModuleSpec, PathBuf, PathBuf) {
12398 let ready = dir.join("ready");
12399 let marker = dir.join("sigterm");
12400 let spec = ModuleSpec {
12401 module_id: module_id.to_string(),
12402 program: fake_aft_stub_path(),
12403 args: Vec::new(),
12404 env: vec![
12405 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12406 (
12407 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12408 marker.display().to_string(),
12409 ),
12410 (
12411 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12412 ready.display().to_string(),
12413 ),
12414 ],
12415 reserved: false,
12416 reserved_prefixes: Vec::new(),
12417 protocol: ModuleProtocol::None,
12418 overlap: Default::default(),
12419 };
12420 (spec, ready, marker)
12421 }
12422
12423 #[cfg(unix)]
12427 async fn wait_for_file(path: &std::path::Path) {
12428 let deadline = Instant::now() + Duration::from_secs(10);
12429 while !path.exists() {
12430 assert!(
12431 Instant::now() < deadline,
12432 "{} never appeared",
12433 path.display()
12434 );
12435 sleep(Duration::from_millis(10)).await;
12436 }
12437 }
12438
12439 #[cfg(unix)]
12443 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12444 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12445 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12446 let (spec, ready, marker) =
12447 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12448 let supervisor = Supervisor::new_for_test(
12449 Arc::new(Registry::default()),
12450 RestartPolicy::new(3, Duration::ZERO),
12451 );
12452 let module = supervisor.spawn(spec).unwrap();
12453 wait_for_file(&ready).await;
12454 let deadline = Instant::now() + Duration::from_secs(10);
12458 let first_pid = loop {
12459 let status = module.status().unwrap();
12460 if status.state == ModuleState::Running {
12461 if let Some(pid) = status.pid {
12462 break pid;
12463 }
12464 }
12465 assert!(
12466 Instant::now() < deadline,
12467 "a running module must report its pid after exec confirmation: {status:?}"
12468 );
12469 sleep(Duration::from_millis(10)).await;
12470 };
12471
12472 rustix::process::kill_process(
12473 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12474 rustix::process::Signal::TERM,
12475 )
12476 .unwrap();
12477
12478 let deadline = Instant::now() + Duration::from_secs(10);
12479 let respawned = loop {
12480 let status = module.status().unwrap();
12481 if status.state == ModuleState::Running
12482 && status.pid.is_some_and(|pid| pid != first_pid)
12483 {
12484 break status;
12485 }
12486 assert!(
12487 Instant::now() < deadline,
12488 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12489 );
12490 sleep(Duration::from_millis(10)).await;
12491 };
12492 assert_eq!(respawned.spawn_generation, 2);
12493 assert!(
12494 marker.exists(),
12495 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12496 );
12497
12498 let history = module.terminal_history();
12499 assert_eq!(history.entries.len(), 1, "{history:?}");
12500 let entry = &history.entries[0];
12501 assert_eq!(entry.exit_code, Some(0));
12502 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12503 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12504
12505 module.stop().await.unwrap();
12506 }
12507
12508 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12512 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12513 let supervisor = Supervisor::new_for_test(
12514 Arc::new(Registry::default()),
12515 RestartPolicy::new(1, Duration::ZERO),
12516 );
12517 let module = supervisor
12518 .spawn(ModuleSpec {
12519 module_id: "none-clean-exit-budget".to_string(),
12520 program: fake_aft_stub_path(),
12521 args: Vec::new(),
12522 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12523 reserved: false,
12524 reserved_prefixes: Vec::new(),
12525 protocol: ModuleProtocol::None,
12526 overlap: Default::default(),
12527 })
12528 .unwrap();
12529
12530 let deadline = Instant::now() + Duration::from_secs(10);
12533 loop {
12534 let status = module.status().unwrap();
12535 if status.state == ModuleState::Failed {
12536 break;
12537 }
12538 assert!(
12539 Instant::now() < deadline,
12540 "module never exhausted its budget: {status:?} {:?}",
12541 module.terminal_history()
12542 );
12543 sleep(Duration::from_millis(10)).await;
12544 }
12545 let history = module.terminal_history();
12546 assert_eq!(
12547 history
12548 .entries
12549 .iter()
12550 .map(|entry| (entry.exit_code, entry.disposition.clone()))
12551 .collect::<Vec<_>>(),
12552 vec![
12553 (Some(0), TerminalDisposition::Restarting),
12554 (Some(0), TerminalDisposition::Failed),
12555 ]
12556 );
12557 let detail = history.entries[1]
12558 .disposition_detail
12559 .as_deref()
12560 .expect("a budget failure names the budget");
12561 assert!(detail.contains("max_restarts=1"), "{detail}");
12562 assert_eq!(module.status().unwrap().spawn_generation, 2);
12563 }
12564
12565 #[test]
12566 fn restart_budget_failure_is_published_after_its_terminal_record() {
12567 use std::io::{Read, Write};
12568 let supervisor = Supervisor::new_for_test(
12569 Arc::new(Registry::default()),
12570 RestartPolicy::new(0, Duration::ZERO),
12571 );
12572 let runtime = supervisor.runtime_config();
12573 let spec = ModuleSpec {
12574 protocol: ModuleProtocol::None,
12575 ..windowed_crash_spec("budget-publication-order")
12576 };
12577 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12578 ModuleState::Running,
12579 true,
12580 )));
12581 lock_snapshot(&snapshot).unwrap().process_alive = true;
12582 let reaped_snapshot = snapshot.clone();
12583 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12584 let history = runtime.terminal_ring.clone();
12585 let reap = std::thread::spawn(move || {
12586 tokio::runtime::Builder::new_current_thread()
12587 .enable_all()
12588 .build()
12589 .unwrap()
12590 .block_on(on_child_exit(
12591 &spec,
12592 runtime.restart_policy,
12593 &supervisor.registry,
12594 &reaped_snapshot,
12595 &runtime.terminal_ring,
12596 &runtime.spawn_events,
12597 &runtime.child_roster,
12598 ExitReport {
12599 kind: ExitKind::Clean,
12600 code: Some(0),
12601 signal: None,
12602 at_ms: 1,
12603 },
12604 true,
12605 ))
12606 });
12607 let mut reached = [0];
12608 barrier.read_exact(&mut reached).unwrap();
12609 assert_eq!(&reached, b"R");
12610 let (before_state, before_alive) = {
12611 let state = lock_snapshot(&snapshot).unwrap();
12612 (state.state, state.process_alive)
12613 };
12614 let before_history = history.lock().unwrap().snapshot();
12615 barrier.write_all(b"X").unwrap();
12616 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12617 assert!(before_history.entries.is_empty());
12618 assert_ne!(
12619 before_state,
12620 ModuleState::Failed,
12621 "Failed was visible before its terminal record could be written"
12622 );
12623 assert!(before_alive);
12624 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12625 let history = history.lock().unwrap().snapshot();
12626 assert_eq!(history.entries.len(), 1);
12627 assert_eq!(history.entries[0].exit_code, Some(0));
12628 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12629 }
12630
12631 #[test]
12632 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12633 use std::io::{Read, Write};
12634
12635 let supervisor = Supervisor::new_for_test(
12636 Arc::new(Registry::default()),
12637 RestartPolicy::new(3, Duration::ZERO),
12638 );
12639 let mut runtime = supervisor.runtime_config();
12640 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12641 let snapshot = module.inner.snapshot.clone();
12642 lock_snapshot(&snapshot).unwrap().process_alive = true;
12643 runtime.terminal_ring = module.inner.terminal_ring.clone();
12644 let spec = windowed_crash_spec(module.module_id());
12645 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12646 let reaped_snapshot = snapshot.clone();
12647 let reap = std::thread::spawn(move || {
12648 tokio::runtime::Builder::new_current_thread()
12649 .enable_all()
12650 .build()
12651 .unwrap()
12652 .block_on(on_child_exit(
12653 &spec,
12654 runtime.restart_policy,
12655 &supervisor.registry,
12656 &reaped_snapshot,
12657 &runtime.terminal_ring,
12658 &runtime.spawn_events,
12659 &runtime.child_roster,
12660 ExitReport {
12661 kind: ExitKind::Crash,
12662 code: Some(2),
12663 signal: None,
12664 at_ms: 1,
12665 },
12666 true,
12667 ))
12668 });
12669
12670 let mut reached = [0];
12674 barrier.read_exact(&mut reached).unwrap();
12675 assert_eq!(&reached, b"R");
12676 let before_status = module.status().unwrap();
12677 let before_history = module.terminal_history();
12678 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12679 barrier.write_all(b"X").unwrap();
12680 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12681
12682 assert_eq!(before_status.state, ModuleState::Running);
12683 assert!(before_status.process_alive);
12684 assert_eq!(before_status.restart_count, 3);
12685 assert!(before_history.entries.is_empty());
12686 assert!(
12687 before_recovers,
12688 "connection cleanup would report a terminal close for a scheduled crash restart"
12689 );
12690 let restarting = module.status().unwrap();
12691 assert_eq!(restarting.state, ModuleState::Restarting);
12692 assert_eq!(restarting.restart_count, 3);
12693 assert!(module.will_recover_after_connection_loss().unwrap());
12694 let history = module.terminal_history();
12695 assert_eq!(history.entries.len(), 1);
12696 assert_eq!(
12697 history.entries[0].disposition,
12698 TerminalDisposition::Restarting
12699 );
12700
12701 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12704 state.state = ModuleState::Running;
12705 state.respawn_pending = false;
12706 })
12707 .unwrap();
12708 assert!(!module.will_recover_after_connection_loss().unwrap());
12709 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12710 state.enabled = false;
12711 state.respawn_pending = true;
12712 })
12713 .unwrap();
12714 assert!(!module.will_recover_after_connection_loss().unwrap());
12715 }
12716
12717 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12718 use std::net::{TcpListener, TcpStream};
12719 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12720 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12721 let (barrier, _) = listener.accept().unwrap();
12722 for stream in [&writer, &barrier] {
12723 stream
12724 .set_read_timeout(Some(Duration::from_secs(10)))
12725 .unwrap();
12726 stream
12727 .set_write_timeout(Some(Duration::from_secs(10)))
12728 .unwrap();
12729 }
12730 *events.1.lock().unwrap() = Some(writer);
12731 barrier
12732 }
12733
12734 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12735 use std::io::{Read, Write};
12736 let barrier = events.1.lock().unwrap().take();
12737 if let Some(mut barrier) = barrier {
12738 barrier.write_all(b"R").unwrap();
12739 let mut release = [0];
12740 barrier.read_exact(&mut release).unwrap();
12741 assert_eq!(&release, b"X");
12742 }
12743 }
12744
12745 #[test]
12746 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12747 use std::io::{Read, Write};
12748
12749 let supervisor = Supervisor::new_for_test(
12750 Arc::new(Registry::default()),
12751 RestartPolicy::new(0, Duration::ZERO),
12752 );
12753 let mut runtime = supervisor.runtime_config();
12754 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12755 let snapshot = module.inner.snapshot.clone();
12756 runtime.terminal_ring = module.inner.terminal_ring.clone();
12757 let spec = ModuleSpec {
12758 module_id: module.module_id().into(),
12759 program: fake_aft_stub_path(),
12760 protocol: ModuleProtocol::None,
12761 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12762 ..windowed_crash_spec(module.module_id())
12763 };
12764 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12765 let reap = std::thread::spawn(move || {
12766 tokio::runtime::Builder::new_current_thread()
12767 .enable_all()
12768 .build()
12769 .unwrap()
12770 .block_on(async {
12771 let mut child =
12772 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12773 health_restart_child(
12774 &spec,
12775 &runtime,
12776 &supervisor.registry,
12777 &supervisor.process_liveness,
12778 &snapshot,
12779 &mut child,
12780 SupervisorHealthStatus::Failing,
12781 Some("health probe did not answer"),
12782 1,
12783 )
12784 .await
12785 .unwrap();
12786 assert!(child.is_none());
12787 });
12788 });
12789 let mut reached = [0];
12794 barrier.read_exact(&mut reached).unwrap();
12795 assert_eq!(&reached, b"R");
12796 let before_status = module.status().unwrap();
12797 let before_history = module.terminal_history();
12798 barrier.write_all(b"X").unwrap();
12799 reap.join().unwrap();
12800 assert!(before_history.entries.is_empty());
12801 assert_ne!(
12802 before_status.state,
12803 ModuleState::Failed,
12804 "Failed was visible before its terminal record could be written"
12805 );
12806 assert!(
12807 before_status.process_alive,
12808 "process death was visible before its terminal record could be written"
12809 );
12810 let failed = module.status().unwrap();
12811 assert_eq!(failed.state, ModuleState::Failed);
12812 assert!(!failed.process_alive);
12813 let history = module.terminal_history();
12814 assert_eq!(history.entries.len(), 1);
12815 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12816 assert_eq!(
12817 history.entries[0].disposition_detail.as_deref(),
12818 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12819 );
12820 }
12821
12822 #[test]
12823 fn observed_exits_publish_process_death_after_terminal_history() {
12824 use std::io::{Read, Write};
12825 for (kind, enabled, shutdown, wait_error, expected_state) in [
12826 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12827 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12828 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12829 (
12830 ExitKind::DeliberateSeverance,
12831 true,
12832 false,
12833 false,
12834 ModuleState::Restarting,
12835 ),
12836 #[cfg(unix)]
12837 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12838 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12839 ] {
12840 let supervisor = Supervisor::new_for_test(
12841 Arc::new(Registry::default()),
12842 RestartPolicy::new(1, Duration::ZERO),
12843 );
12844 let runtime = supervisor.runtime_config();
12845 #[cfg(unix)]
12846 if shutdown {
12847 runtime.child_roster.close();
12848 }
12849 #[cfg(not(unix))]
12850 let _ = shutdown;
12851 let spec = windowed_crash_spec("exit-publication-order");
12852 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12853 ModuleState::Running,
12854 enabled,
12855 )));
12856 lock_snapshot(&snapshot).unwrap().process_alive = true;
12857 let reaped_snapshot = snapshot.clone();
12858 let history = runtime.terminal_ring.clone();
12859 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12860 let reap = std::thread::spawn(move || {
12861 if wait_error {
12862 super::fail_after_wait_error(
12863 &spec.module_id,
12864 &reaped_snapshot,
12865 &runtime.terminal_ring,
12866 &runtime.spawn_events,
12867 "wait failed".into(),
12868 &runtime.failure_context(true),
12869 );
12870 } else {
12871 tokio::runtime::Builder::new_current_thread()
12872 .enable_all()
12873 .build()
12874 .unwrap()
12875 .block_on(on_child_exit(
12876 &spec,
12877 runtime.restart_policy,
12878 &supervisor.registry,
12879 &reaped_snapshot,
12880 &runtime.terminal_ring,
12881 &runtime.spawn_events,
12882 &runtime.child_roster,
12883 ExitReport {
12884 kind,
12885 code: Some(0),
12886 signal: None,
12887 at_ms: 1,
12888 },
12889 true,
12890 ));
12891 }
12892 });
12893 let mut reached = [0];
12894 barrier.read_exact(&mut reached).unwrap();
12895 assert_eq!(&reached, b"R");
12896 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12897 let before_history = history.lock().unwrap().snapshot();
12898 barrier.write_all(b"X").unwrap();
12899 reap.join().unwrap();
12900 assert!(
12901 before_alive,
12902 "{kind:?}: death was published before recording"
12903 );
12904 assert!(before_history.entries.is_empty());
12905 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12906 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12907 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12908 }
12909 }
12910
12911 #[tokio::test]
12912 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12913 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12914 let path = dir.join("terminals.jsonl");
12915 std::fs::create_dir(&path).unwrap();
12916 let supervisor = Supervisor::new_for_test(
12917 Arc::new(Registry::default()),
12918 RestartPolicy::new(0, Duration::ZERO),
12919 )
12920 .with_terminal_journal(path, "health-budget-journal-failure".into());
12921 let runtime = supervisor.runtime_config();
12922 let spec = ModuleSpec {
12923 program: fake_aft_stub_path(),
12924 protocol: ModuleProtocol::None,
12925 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12926 ..windowed_crash_spec("health-budget-journal-failure")
12927 };
12928 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12929 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12930 health_restart_child(
12931 &spec,
12932 &runtime,
12933 &supervisor.registry,
12934 &supervisor.process_liveness,
12935 &snapshot,
12936 &mut child,
12937 SupervisorHealthStatus::Failing,
12938 None,
12939 1,
12940 )
12941 .await
12942 .unwrap();
12943 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12944 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12945 let history = runtime
12946 .terminal_ring
12947 .lock()
12948 .unwrap()
12949 .durable_history(&spec.module_id);
12950 assert!(history.journal_write_failures > 0);
12951 assert_eq!(history.entries.len(), 1);
12952 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12953 }
12954
12955 #[test]
12956 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12957 let supervisor = Supervisor::new_for_test(
12958 Arc::new(Registry::default()),
12959 RestartPolicy::new(0, Duration::ZERO),
12960 );
12961 let runtime = supervisor.runtime_config();
12962 let spec = ModuleSpec {
12963 program: fake_aft_stub_path(),
12964 protocol: ModuleProtocol::None,
12965 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12966 ..windowed_crash_spec("health-budget-recording-panic")
12967 };
12968 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12969 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12970 tokio::runtime::Builder::new_current_thread()
12971 .enable_all()
12972 .build()
12973 .unwrap()
12974 .block_on(async {
12975 let mut child =
12976 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12977 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12978 health_restart_child(
12979 &spec,
12980 &runtime,
12981 &supervisor.registry,
12982 &supervisor.process_liveness,
12983 &snapshot,
12984 &mut child,
12985 SupervisorHealthStatus::Failing,
12986 None,
12987 1,
12988 )
12989 .await
12990 .unwrap();
12991 });
12992 }));
12993 let panic = result.expect_err("recording must still unwind");
12994 assert_eq!(
12995 panic.downcast_ref::<String>().map(String::as_str),
12996 Some("spawn event sequence exhausted")
12997 );
12998 let state = lock_snapshot(&snapshot).unwrap();
12999 assert_eq!(state.state, ModuleState::Failed);
13000 assert!(!state.process_alive);
13001 assert!(state.last_exit.is_some());
13002 }
13003
13004 #[tokio::test]
13005 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13006 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13007 let path = dir.join("terminals.jsonl");
13008 std::fs::create_dir(&path).unwrap();
13009 let supervisor = Supervisor::new_for_test(
13010 Arc::new(Registry::default()),
13011 RestartPolicy::new(0, Duration::ZERO),
13012 )
13013 .with_terminal_journal(path, "budget-journal-failure".into());
13014 let runtime = supervisor.runtime_config();
13015 let spec = windowed_crash_spec("budget-journal-failure");
13016 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13017 assert!(matches!(
13018 on_child_exit(
13019 &spec,
13020 runtime.restart_policy,
13021 &supervisor.registry,
13022 &snapshot,
13023 &runtime.terminal_ring,
13024 &runtime.spawn_events,
13025 &runtime.child_roster,
13026 crash_exit_report(1),
13027 true,
13028 )
13029 .await,
13030 NextAction::Stop { .. }
13031 ));
13032 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13033 let history = runtime
13034 .terminal_ring
13035 .lock()
13036 .unwrap()
13037 .durable_history(&spec.module_id);
13038 assert!(history.journal_write_failures > 0);
13039 assert_eq!(history.entries.len(), 1);
13040 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13041 }
13042
13043 #[test]
13044 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13045 let supervisor = Supervisor::new_for_test(
13046 Arc::new(Registry::default()),
13047 RestartPolicy::new(0, Duration::ZERO),
13048 );
13049 let runtime = supervisor.runtime_config();
13050 let spec = windowed_crash_spec("budget-recording-panic");
13051 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13052 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13053 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13056 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13057 tokio::runtime::Builder::new_current_thread()
13058 .enable_all()
13059 .build()
13060 .unwrap()
13061 .block_on(on_child_exit(
13062 &spec,
13063 runtime.restart_policy,
13064 &supervisor.registry,
13065 &snapshot,
13066 &runtime.terminal_ring,
13067 &runtime.spawn_events,
13068 &runtime.child_roster,
13069 crash_exit_report(1),
13070 true,
13071 ))
13072 }));
13073 let panic = result.err().expect("recording must still unwind");
13074 assert_eq!(
13075 panic.downcast_ref::<String>().map(String::as_str),
13076 Some("spawn event sequence exhausted")
13077 );
13078 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13079 }
13080
13081 #[cfg(unix)]
13084 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13085 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13086 for disable in [false, true] {
13087 let label = if disable {
13088 "none-requested-disable"
13089 } else {
13090 "none-requested-stop"
13091 };
13092 let dir = cortexkit_test_support::ScratchDir::new(label);
13093 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13094 let supervisor = Supervisor::new_for_test(
13095 Arc::new(Registry::default()),
13096 RestartPolicy::new(3, Duration::ZERO),
13097 );
13098 let module = supervisor.spawn(spec).unwrap();
13099 wait_for_file(&ready).await;
13100
13101 if disable {
13102 module.set_enabled(false).await.unwrap();
13103 } else {
13104 module.stop().await.unwrap();
13105 }
13106 assert!(
13107 marker.exists(),
13108 "{label}: the child must have left through its SIGTERM handler with exit 0"
13109 );
13110
13111 sleep(Duration::from_millis(500)).await;
13114 let status = module.status().unwrap();
13115 let expected = if disable {
13116 ModuleState::Disabled
13117 } else {
13118 ModuleState::Stopped
13119 };
13120 assert_eq!(status.state, expected, "{label}");
13121 assert_eq!(
13122 status.spawn_generation, 1,
13123 "{label}: respawned after a requested stop"
13124 );
13125 let history = module.terminal_history();
13126 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13127 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13128 assert_ne!(
13129 history.entries[0].disposition,
13130 TerminalDisposition::Restarting,
13131 "{label}"
13132 );
13133 }
13134 }
13135
13136 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13139 async fn subc_wire_clean_exit_is_still_a_stop() {
13140 let supervisor = Supervisor::new_for_test(
13141 Arc::new(Registry::default()),
13142 RestartPolicy::new(3, Duration::ZERO),
13143 );
13144 let module = supervisor
13145 .spawn(ModuleSpec {
13146 module_id: "wire-clean-exit".to_string(),
13147 program: fake_aft_stub_path(),
13148 args: Vec::new(),
13149 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13150 reserved: false,
13151 reserved_prefixes: Vec::new(),
13152 protocol: ModuleProtocol::Subc,
13153 overlap: Default::default(),
13154 })
13155 .unwrap();
13156
13157 let deadline = Instant::now() + Duration::from_secs(10);
13158 while module.terminal_history().entries.is_empty() {
13159 assert!(Instant::now() < deadline, "module never exited");
13160 sleep(Duration::from_millis(10)).await;
13161 }
13162 sleep(Duration::from_millis(500)).await;
13164 let status = module.status().unwrap();
13165 assert_eq!(status.state, ModuleState::Stopped);
13166 assert_eq!(status.spawn_generation, 1);
13167 let history = module.terminal_history();
13168 assert_eq!(history.entries.len(), 1, "{history:?}");
13169 assert_eq!(history.entries[0].exit_code, Some(0));
13170 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13171 }
13172
13173 #[cfg(unix)]
13174 #[tokio::test]
13175 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13176 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13177 let record = dir.join("live-children.json");
13178 let supervisor = Supervisor::new_for_test(
13179 Arc::new(Registry::default()),
13180 RestartPolicy::new(0, Duration::ZERO),
13181 );
13182 let mut runtime = supervisor.runtime_config();
13183 runtime.child_roster.record_to(record.clone());
13184 let gate = Arc::new(super::ReloadExitRecordGate::default());
13185 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13186 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13187 let spec = ModuleSpec {
13188 module_id: "reload-exit-roster".into(),
13189 program: fake_aft_stub_path(),
13190 args: Vec::new(),
13191 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13192 reserved: false,
13193 reserved_prefixes: Vec::new(),
13194 protocol: ModuleProtocol::Subc,
13195 overlap: Default::default(),
13196 };
13197 let mut child = None;
13198 let reload = super::finish_reload_child(
13199 &spec,
13200 &runtime,
13201 &supervisor.registry,
13202 &supervisor.process_liveness,
13203 &snapshot,
13204 &mut child,
13205 );
13206 tokio::pin!(reload);
13207 tokio::select! {
13208 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13209 _ = gate.reached.notified() => {}
13210 }
13211 assert!(runtime
13212 .terminal_ring
13213 .lock()
13214 .unwrap()
13215 .snapshot()
13216 .entries
13217 .is_empty());
13218 assert_eq!(
13219 crate::live_children::read_record(&record).unwrap().len(),
13220 1,
13221 "shutdown must still wait for the reaped child until its terminal record exists"
13222 );
13223 runtime.child_roster.close();
13224 gate.resume.notify_one();
13225 assert!(reload.await.is_err());
13226 assert!(crate::live_children::read_record(&record)
13227 .unwrap()
13228 .is_empty());
13229 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13230 assert_eq!(history.entries.len(), 1);
13231 assert_eq!(
13232 history.entries[0].disposition,
13233 TerminalDisposition::DaemonShutdown
13234 );
13235 }
13236
13237 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13241 async fn every_restart_increment_path_advances_lifetime_count() {
13242 let supervisor = Supervisor::new_for_test(
13243 Arc::new(Registry::default()),
13244 RestartPolicy::new(1, Duration::ZERO),
13245 );
13246 let runtime = supervisor.runtime_config();
13247 let spec = ModuleSpec {
13248 module_id: "lifetime-increment-path".to_string(),
13249 program: PathBuf::from("/unused/lifetime-increment-path"),
13250 args: Vec::new(),
13251 env: Vec::new(),
13252 reserved: false,
13253 reserved_prefixes: Vec::new(),
13254 protocol: ModuleProtocol::Subc,
13255 overlap: Default::default(),
13256 };
13257
13258 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13259 assert!(matches!(
13260 on_child_exit(
13261 &spec,
13262 runtime.restart_policy,
13263 &supervisor.registry,
13264 &crash_snapshot,
13265 &runtime.terminal_ring,
13266 &runtime.spawn_events,
13267 &runtime.child_roster,
13268 ExitReport {
13269 kind: ExitKind::Crash,
13270 code: Some(1),
13271 signal: None,
13272 at_ms: 1,
13273 },
13274 true,
13275 )
13276 .await,
13277 NextAction::Restart { schedule: _ }
13278 ));
13279 let (crash_restarts, crash_lifetime) = {
13280 let state = lock_snapshot(&crash_snapshot).unwrap();
13281 (state.crash_restarts.len(), state.lifetime_restarts)
13282 };
13283 assert_eq!(crash_restarts, 1);
13284 assert_eq!(crash_lifetime, 1);
13285
13286 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13287 let mut health_child = None;
13288 assert!(matches!(
13289 health_restart_child(
13290 &spec,
13291 &runtime,
13292 &supervisor.registry,
13293 &supervisor.process_liveness,
13294 &health_snapshot,
13295 &mut health_child,
13296 SupervisorHealthStatus::Failing,
13297 None,
13298 2,
13299 )
13300 .await,
13301 Ok(())
13302 ));
13303 assert!(health_child.is_none());
13304 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13305 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13306 let (health_restarts, health_lifetime) = {
13307 let state = lock_snapshot(&health_snapshot).unwrap();
13308 (state.crash_restarts.len(), state.lifetime_restarts)
13309 };
13310 assert_eq!(health_restarts, 1);
13311 assert_eq!(health_lifetime, 1);
13312
13313 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13314 let mut reload_child = None;
13315 assert!(matches!(
13316 handle_reload_spawn_failure(
13317 &spec,
13318 &runtime,
13319 &supervisor.process_liveness,
13320 &reload_snapshot,
13321 &mut reload_child,
13322 "forced reload spawn failure".to_string(),
13323 )
13324 .await,
13325 Err(SuperviseError::ReloadFailed { .. })
13326 ));
13327 let (reload_restarts, reload_lifetime) = {
13328 let state = lock_snapshot(&reload_snapshot).unwrap();
13329 (state.crash_restarts.len(), state.lifetime_restarts)
13330 };
13331 assert_eq!(reload_restarts, 1);
13332 assert_eq!(reload_lifetime, 1);
13333 }
13334
13335 #[tokio::test]
13336 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
13337 let supervisor = Supervisor::new_for_test(
13338 Arc::new(Registry::default()),
13339 RestartPolicy::new(3, Duration::ZERO),
13340 );
13341 let runtime = supervisor.runtime_config();
13342 let spec = ModuleSpec {
13343 module_id: "deliberately-severed".to_string(),
13344 program: PathBuf::from("/unused/deliberately-severed"),
13345 args: Vec::new(),
13346 env: Vec::new(),
13347 reserved: false,
13348 reserved_prefixes: Vec::new(),
13349 protocol: ModuleProtocol::Subc,
13350 overlap: Default::default(),
13351 };
13352 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13353 let process = ProcessIdentity {
13354 pid: 41,
13355 start_time: 101,
13356 };
13357 record_deliberate_severance(&snapshot, process).unwrap();
13358 let exit_report = apply_deliberate_severance_marker(
13359 &snapshot,
13360 Some(process),
13361 ExitReport {
13362 kind: ExitKind::Crash,
13363 code: Some(1),
13364 signal: None,
13365 at_ms: 1,
13366 },
13367 );
13368 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
13369
13370 assert!(matches!(
13371 on_child_exit(
13372 &spec,
13373 runtime.restart_policy,
13374 &supervisor.registry,
13375 &snapshot,
13376 &runtime.terminal_ring,
13377 &runtime.spawn_events,
13378 &runtime.child_roster,
13379 exit_report,
13380 true,
13381 )
13382 .await,
13383 NextAction::Restart { schedule: _ }
13384 ));
13385 let state = lock_snapshot(&snapshot).unwrap();
13386 assert_eq!(state.lifetime_restarts, 1);
13387 assert_eq!(state.crash_restarts.len(), 0);
13388 }
13389
13390 #[tokio::test]
13391 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13392 let supervisor = Supervisor::new_for_test(
13393 Arc::new(Registry::default()),
13394 RestartPolicy::new(3, Duration::ZERO),
13395 );
13396 let runtime = supervisor.runtime_config();
13397 let spec = ModuleSpec {
13398 module_id: "genuine-crash".to_string(),
13399 program: PathBuf::from("/unused/genuine-crash"),
13400 args: Vec::new(),
13401 env: Vec::new(),
13402 reserved: false,
13403 reserved_prefixes: Vec::new(),
13404 protocol: ModuleProtocol::Subc,
13405 overlap: Default::default(),
13406 };
13407 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13408
13409 assert!(matches!(
13410 on_child_exit(
13411 &spec,
13412 runtime.restart_policy,
13413 &supervisor.registry,
13414 &snapshot,
13415 &runtime.terminal_ring,
13416 &runtime.spawn_events,
13417 &runtime.child_roster,
13418 ExitReport {
13419 kind: ExitKind::Crash,
13420 code: Some(1),
13421 signal: None,
13422 at_ms: 1,
13423 },
13424 true,
13425 )
13426 .await,
13427 NextAction::Restart { schedule: _ }
13428 ));
13429 let state = lock_snapshot(&snapshot).unwrap();
13430 assert_eq!(state.lifetime_restarts, 1);
13431 assert_eq!(state.crash_restarts.len(), 1);
13432 }
13433
13434 fn crash_exit_report(at_ms: u64) -> ExitReport {
13435 ExitReport {
13436 kind: ExitKind::Crash,
13437 code: Some(1),
13438 signal: None,
13439 at_ms,
13440 }
13441 }
13442
13443 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13444 ModuleSpec {
13445 module_id: module_id.to_string(),
13446 program: PathBuf::from("/unused").join(module_id),
13447 args: Vec::new(),
13448 env: Vec::new(),
13449 reserved: false,
13450 reserved_prefixes: Vec::new(),
13451 protocol: ModuleProtocol::Subc,
13452 overlap: Default::default(),
13453 }
13454 }
13455
13456 #[tokio::test]
13462 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13463 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13464 let supervisor = Supervisor::new_for_test(
13465 Arc::new(Registry::default()),
13466 RestartPolicy::new(2, Duration::ZERO),
13467 );
13468 let runtime = supervisor.runtime_config();
13469 let spec = windowed_crash_spec("crash-loop-in-window");
13470 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13471
13472 for attempt in 1..=2 {
13473 assert!(
13474 matches!(
13475 on_child_exit(
13476 &spec,
13477 runtime.restart_policy,
13478 &supervisor.registry,
13479 &snapshot,
13480 &runtime.terminal_ring,
13481 &runtime.spawn_events,
13482 &runtime.child_roster,
13483 crash_exit_report(attempt),
13484 true,
13485 )
13486 .await,
13487 NextAction::Restart { schedule: _ }
13488 ),
13489 "crash {attempt} is inside the budget and must respawn"
13490 );
13491 }
13492
13493 assert!(matches!(
13494 on_child_exit(
13495 &spec,
13496 runtime.restart_policy,
13497 &supervisor.registry,
13498 &snapshot,
13499 &runtime.terminal_ring,
13500 &runtime.spawn_events,
13501 &runtime.child_roster,
13502 crash_exit_report(3),
13503 true,
13504 )
13505 .await,
13506 NextAction::Stop { .. }
13507 ));
13508
13509 {
13510 let state = lock_snapshot(&snapshot).unwrap();
13511 assert_eq!(state.state, ModuleState::Failed);
13512 assert_eq!(state.crash_restarts.len(), 2);
13513 assert_eq!(state.lifetime_restarts, 2);
13514 }
13515
13516 let history = runtime
13517 .terminal_ring
13518 .lock()
13519 .expect("terminal ring is not poisoned")
13520 .snapshot();
13521 let last = history
13522 .entries
13523 .last()
13524 .expect("the refused crash is retained");
13525 assert_eq!(last.disposition, TerminalDisposition::Failed);
13526 assert_eq!(
13527 last.disposition_detail.as_deref(),
13528 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13529 );
13530
13531 let captured = crate::router::test_log::captured_logs(&logs);
13532 assert!(
13533 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13534 "the stop must be logged with its window: {captured}"
13535 );
13536 }
13537
13538 #[tokio::test]
13546 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13547 let supervisor = Supervisor::new_for_test(
13548 Arc::new(Registry::default()),
13549 RestartPolicy::new(2, Duration::ZERO),
13550 );
13551 let runtime = supervisor.runtime_config();
13552 let spec = windowed_crash_spec("crash-across-windows");
13553 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13554
13555 for attempt in 1..=2 {
13556 assert!(matches!(
13557 on_child_exit(
13558 &spec,
13559 runtime.restart_policy,
13560 &supervisor.registry,
13561 &snapshot,
13562 &runtime.terminal_ring,
13563 &runtime.spawn_events,
13564 &runtime.child_roster,
13565 crash_exit_report(attempt),
13566 true,
13567 )
13568 .await,
13569 NextAction::Restart { schedule: _ }
13570 ));
13571 }
13572
13573 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13576 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13577 })
13578 .unwrap();
13579
13580 assert!(
13581 matches!(
13582 on_child_exit(
13583 &spec,
13584 runtime.restart_policy,
13585 &supervisor.registry,
13586 &snapshot,
13587 &runtime.terminal_ring,
13588 &runtime.spawn_events,
13589 &runtime.child_roster,
13590 crash_exit_report(3),
13591 true,
13592 )
13593 .await,
13594 NextAction::Restart { schedule: _ }
13595 ),
13596 "a crash older than the window must not hold a budget slot"
13597 );
13598
13599 let state = lock_snapshot(&snapshot).unwrap();
13600 assert_eq!(state.state, ModuleState::Restarting);
13601 assert_eq!(
13602 state.crash_restarts.len(),
13603 2,
13604 "the aged instant is dropped and the new one takes its place"
13605 );
13606 assert_eq!(
13607 state.lifetime_restarts, 3,
13608 "the ledger counts every restart, including the ones the window forgot"
13609 );
13610 }
13611
13612 #[tokio::test]
13617 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13618 let supervisor = Supervisor::new_for_test(
13619 Arc::new(Registry::default()),
13620 RestartPolicy::new(2, Duration::ZERO),
13621 );
13622 let runtime = supervisor.runtime_config();
13623 let spec = windowed_crash_spec("operator-cleared-budget");
13624 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13625
13626 for attempt in 1..=2 {
13627 assert!(matches!(
13628 on_child_exit(
13629 &spec,
13630 runtime.restart_policy,
13631 &supervisor.registry,
13632 &snapshot,
13633 &runtime.terminal_ring,
13634 &runtime.spawn_events,
13635 &runtime.child_roster,
13636 crash_exit_report(attempt),
13637 true,
13638 )
13639 .await,
13640 NextAction::Restart { schedule: _ }
13641 ));
13642 }
13643
13644 reset_restart_count(&snapshot, &spec.module_id).unwrap();
13645 {
13646 let state = lock_snapshot(&snapshot).unwrap();
13647 assert!(
13648 state.crash_restarts.is_empty(),
13649 "an operator restart returns the full budget"
13650 );
13651 assert_eq!(
13652 state.lifetime_restarts, 2,
13653 "clearing the budget must not unmake the crashes"
13654 );
13655 }
13656
13657 assert!(
13658 matches!(
13659 on_child_exit(
13660 &spec,
13661 runtime.restart_policy,
13662 &supervisor.registry,
13663 &snapshot,
13664 &runtime.terminal_ring,
13665 &runtime.spawn_events,
13666 &runtime.child_roster,
13667 crash_exit_report(3),
13668 true,
13669 )
13670 .await,
13671 NextAction::Restart { schedule: _ }
13672 ),
13673 "the cleared budget must be spendable again"
13674 );
13675 let state = lock_snapshot(&snapshot).unwrap();
13676 assert_eq!(state.crash_restarts.len(), 1);
13677 assert_eq!(state.lifetime_restarts, 3);
13678 }
13679
13680 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13681 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13682 let severed = ProcessIdentity {
13683 pid: 41,
13684 start_time: 101,
13685 };
13686 let successor = ProcessIdentity {
13687 pid: 41,
13688 start_time: 202,
13689 };
13690 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13691 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13692 state.pid = Some(successor.pid);
13693 state.process_start_time = Some(successor.start_time);
13694 })
13695 .unwrap();
13696 assert!(!module.record_deliberate_severance(severed).unwrap());
13697
13698 let exit_report = apply_deliberate_severance_marker(
13699 &module.inner.snapshot,
13700 Some(successor),
13701 ExitReport {
13702 kind: ExitKind::Crash,
13703 code: Some(1),
13704 signal: None,
13705 at_ms: 1,
13706 },
13707 );
13708
13709 assert_eq!(exit_report.kind, ExitKind::Crash);
13710 }
13711
13712 #[tokio::test]
13713 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13714 let registry = Registry::default();
13715 let supervisor = Supervisor::new_for_test(
13716 Arc::new(Registry::default()),
13717 RestartPolicy::new(3, Duration::ZERO),
13718 );
13719 let runtime = supervisor.runtime_config();
13720 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13721 let spec = ModuleSpec {
13722 module_id: "drain-deliberate-severance".to_string(),
13723 program: fake_aft_stub_path(),
13724 args: Vec::new(),
13725 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13726 reserved: false,
13727 reserved_prefixes: Vec::new(),
13728 protocol: ModuleProtocol::Subc,
13729 overlap: Default::default(),
13730 };
13731 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13732 let process = ProcessIdentity {
13733 pid: 41,
13734 start_time: 101,
13735 };
13736 child.process_identity = Some(process);
13737 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13738 state.pid = Some(process.pid);
13739 state.process_start_time = Some(process.start_time);
13740 })
13741 .unwrap();
13742 record_deliberate_severance(&snapshot, process).unwrap();
13743
13744 drain_child_to_state(
13745 &spec.module_id,
13746 spec.protocol,
13747 StopNotice::SentOverConnection,
13750 ®istry,
13751 None,
13752 &snapshot,
13753 &runtime.terminal_ring,
13754 &runtime.spawn_events,
13755 child,
13756 Duration::from_secs(1),
13757 ModuleState::Stopped,
13758 Some(false),
13759 &super::FailureContext::operator(),
13760 )
13761 .await
13762 .unwrap();
13763
13764 let state = lock_snapshot(&snapshot).unwrap();
13765 assert_eq!(
13766 state.last_exit.as_ref().map(|exit| exit.kind),
13767 Some(ExitKind::DeliberateSeverance)
13768 );
13769 assert_eq!(state.lifetime_restarts, 1);
13770 assert_eq!(state.crash_restarts.len(), 0);
13771 drop(state);
13772 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13773 assert_eq!(
13774 history.entries[0].exit_kind,
13775 subc_control::TerminalExitKind::DeliberateSeverance
13776 );
13777 }
13778
13779 #[tokio::test]
13780 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13781 let registry = Registry::default();
13782 let supervisor = Supervisor::new_for_test(
13783 Arc::new(Registry::default()),
13784 RestartPolicy::new(3, Duration::ZERO),
13785 );
13786 let runtime = supervisor.runtime_config();
13787 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13788 let spec = ModuleSpec {
13789 module_id: "ordinary-drain".to_string(),
13790 program: fake_aft_stub_path(),
13791 args: Vec::new(),
13792 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13793 reserved: false,
13794 reserved_prefixes: Vec::new(),
13795 protocol: ModuleProtocol::Subc,
13796 overlap: Default::default(),
13797 };
13798 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13799
13800 drain_child_to_state(
13801 &spec.module_id,
13802 spec.protocol,
13803 StopNotice::SentOverConnection,
13806 ®istry,
13807 None,
13808 &snapshot,
13809 &runtime.terminal_ring,
13810 &runtime.spawn_events,
13811 child,
13812 Duration::from_secs(1),
13813 ModuleState::Stopped,
13814 Some(false),
13815 &super::FailureContext::operator(),
13816 )
13817 .await
13818 .unwrap();
13819
13820 let state = lock_snapshot(&snapshot).unwrap();
13821 assert_eq!(
13822 state.last_exit.as_ref().map(|exit| exit.kind),
13823 Some(ExitKind::Crash)
13824 );
13825 assert_eq!(state.lifetime_restarts, 0);
13826 assert_eq!(state.crash_restarts.len(), 0);
13827 }
13828
13829 #[test]
13830 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13831 assert!(!include_str!("server.rs")
13837 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13838 }
13839
13840 #[test]
13847 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13848 assert!(drained_after_quiescence_wait(&Ok(true)));
13849 assert!(!drained_after_quiescence_wait(&Ok(false)));
13850 assert!(!drained_after_quiescence_wait(&Err(
13851 SuperviseError::StatePoisoned { module_id: None }
13852 )));
13853 }
13854
13855 #[test]
13864 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13865 let ring = Arc::new(Mutex::new(TerminalRing::new(
13866 TerminalRingConfig::default(),
13867 0,
13868 )));
13869 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13870
13871 let snapshot = ring.lock().unwrap().snapshot();
13872 assert_eq!(snapshot.entries.len(), 1);
13873 let entry = &snapshot.entries[0];
13874 assert_eq!(entry.exit_code, None);
13875 assert_eq!(entry.exit_signal, None);
13876 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13877 }
13878
13879 #[test]
13880 fn wait_error_exit_path_preserves_spawn_event_density() {
13881 let feed = super::SpawnEventFeed::default();
13882 feed.configure_incarnation("wait-error-density".to_string());
13883 feed.emit_spawned("wait-error", 41, 1);
13884 let ring = Arc::new(Mutex::new(TerminalRing::new(
13885 TerminalRingConfig::default(),
13886 0,
13887 )));
13888
13889 record_wait_error_terminal("wait-error", &ring, &feed);
13890 feed.emit_spawned("after-wait-error", 42, 2);
13891
13892 let state = feed.0.lock().unwrap();
13893 let sequences = state
13894 .events
13895 .iter()
13896 .map(|event| event.cursor.seq)
13897 .collect::<Vec<_>>();
13898 assert_eq!(sequences, vec![1, 2, 3]);
13899 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13900 assert_eq!(state.events[1].exit_code, None);
13901 assert_eq!(state.events[1].exit_signal, None);
13902 }
13903
13904 #[test]
13908 fn wait_error_exit_report_is_classified_as_a_crash() {
13909 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13910 }
13911}
13912
13913#[cfg(all(test, unix))]
13914mod restart_ordering_tests {
13915 use super::*;
13916 use std::io::Write;
13917 use tokio::io::AsyncReadExt;
13918
13919 #[test]
13920 #[ignore = "child process fixture"]
13921 fn serving_child() {
13922 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13923 return;
13924 };
13925 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13926 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13927 for stream in listener.incoming() {
13928 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13929 }
13930 }
13931
13932 async fn restart_before_publication(requests: usize) {
13933 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13934 let address_file = home.path().join("address");
13935 let spec = ModuleSpec {
13936 module_id: "restart-serving-child".into(),
13937 program: std::env::current_exe().unwrap(),
13938 args: vec![
13939 "--exact".into(),
13940 "supervise::restart_ordering_tests::serving_child".into(),
13941 "--ignored".into(),
13942 "--nocapture".into(),
13943 ],
13944 env: vec![(
13945 "RESTART_TEST_ADDRESS_FILE".into(),
13946 address_file.display().to_string(),
13947 )],
13948 reserved: false,
13949 reserved_prefixes: vec![],
13950 protocol: ModuleProtocol::None,
13951 overlap: Default::default(),
13952 };
13953 let mut supervisor =
13954 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13955 let (spawned_tx, spawned_rx) = oneshot::channel();
13956 let spawned_tx = Mutex::new(Some(spawned_tx));
13957 let (release_tx, release_rx) = std::sync::mpsc::channel();
13958 let release_rx = Mutex::new(release_rx);
13959 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13962 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13963 release_rx
13964 .lock()
13965 .unwrap()
13966 .recv_timeout(Duration::from_secs(20))
13967 .unwrap();
13968 })));
13969 let registered = supervisor
13970 .register_configured_with_health(
13971 spec.clone(),
13972 true,
13973 HealthConfig::default(),
13974 None,
13975 RestartPolicy::new(3, Duration::ZERO),
13976 DEFAULT_REGISTER_TIMEOUT,
13977 )
13978 .unwrap();
13979 let module = registered.module.clone();
13980 let startup = tokio::task::spawn_blocking(move || {
13981 supervisor.start_registered_configured(registered, None, true)
13982 });
13983 spawned_rx.await.unwrap();
13984 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13985 let address = loop {
13986 if let Ok(address) = std::fs::read_to_string(&address_file) {
13987 if !address.is_empty() {
13988 break address;
13989 }
13990 }
13991 tokio::time::sleep(Duration::from_millis(1)).await;
13992 };
13993 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13994 let mut reply = String::new();
13995 stream.read_to_string(&mut reply).await.unwrap();
13996 reply.trim().parse::<u32>().unwrap()
13997 })
13998 .await
13999 .expect("child must serve before generation publication");
14000 assert_eq!(
14001 lock_snapshot(&module.inner.snapshot)
14002 .unwrap()
14003 .spawn_generation,
14004 0
14005 );
14006 let mut replies = Vec::new();
14007 for _ in 0..requests {
14008 let mut restart = Box::pin(module.restart(Some(0)));
14009 std::future::poll_fn(|cx| {
14012 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14013 std::task::Poll::Ready(())
14014 })
14015 .await;
14016 replies.push(restart);
14017 }
14018 release_tx.send(()).unwrap();
14019 startup.await.unwrap().unwrap();
14020 for reply in replies {
14021 reply.await.unwrap();
14022 }
14023 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14026 loop {
14027 if lock_snapshot(&module.inner.snapshot)
14028 .unwrap()
14029 .spawn_generation
14030 > 1
14031 {
14032 break;
14033 }
14034 tokio::time::sleep(Duration::from_millis(1)).await;
14035 }
14036 })
14037 .await;
14038 module.update_spec_for_test(spec).await.unwrap();
14042 let (generation, pid) = {
14043 let state = lock_snapshot(&module.inner.snapshot).unwrap();
14044 (state.spawn_generation, state.pid)
14045 };
14046 module.stop().await.unwrap();
14047 replacement.expect("restart must publish a replacement for the already-serving child");
14048 assert_ne!(
14049 pid,
14050 Some(old_pid),
14051 "a serving child predating the restart must be replaced"
14052 );
14053 assert_eq!(generation, 2, "queued restarts must share one replacement");
14054 }
14055
14056 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14057 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14058 restart_before_publication(1).await;
14059 }
14060
14061 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14062 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14063 restart_before_publication(2).await;
14064 }
14065
14066 #[tokio::test]
14067 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14068 let supervisor =
14069 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14070 let registered = supervisor
14071 .register_configured_with_health(
14072 ModuleSpec {
14073 module_id: "queued-before-replacement".into(),
14074 program: "/bin/sleep".into(),
14079 args: vec!["600".into()],
14080 env: vec![],
14081 reserved: false,
14082 reserved_prefixes: vec![],
14083 protocol: ModuleProtocol::None,
14084 overlap: Default::default(),
14085 },
14086 true,
14087 HealthConfig::default(),
14088 None,
14089 RestartPolicy::default(),
14090 DEFAULT_REGISTER_TIMEOUT,
14091 )
14092 .unwrap();
14093 let RegisteredConfiguredModule {
14094 mut spec,
14095 mut runtime,
14096 module,
14097 mut commands,
14098 } = registered;
14099 let snapshot = &module.inner.snapshot;
14100 let mut restart = Box::pin(module.restart(Some(0)));
14101 std::future::poll_fn(|cx| {
14102 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14103 std::task::Poll::Ready(())
14104 })
14105 .await;
14106 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14109 let replacement_pid = child.as_ref().unwrap().id();
14110 let command = commands.recv().await.unwrap();
14111 assert!(
14112 handle_supervisor_command(
14113 command,
14114 &mut spec,
14115 &mut runtime,
14116 &supervisor.registry,
14117 &supervisor.process_liveness,
14118 snapshot,
14119 &mut child,
14120 &mut commands,
14121 &mut VecDeque::new(),
14122 )
14123 .await
14124 );
14125 restart.await.unwrap();
14126 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14127 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14128 if let Some(mut child) = child {
14129 child.start_kill().unwrap();
14130 child.wait().await.unwrap();
14131 }
14132 assert_eq!(
14133 kept_pid, replacement_pid,
14134 "a newer replacement must survive"
14135 );
14136 assert!(
14137 !pending,
14138 "a satisfied restart must not schedule another spawn"
14139 );
14140 }
14141}
14142
14143#[cfg(all(test, unix))]
14144mod health_event_tests {
14145 use super::*;
14146 use cortexkit_test_support::ScratchDir;
14147 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14148
14149 struct Actor {
14150 module: SupervisedModule,
14151 registry: Arc<Registry>,
14152 forwarding: Arc<ForwardingTable>,
14153 spec: ModuleSpec,
14154 health: HealthConfig,
14155 rx: mpsc::Receiver<crate::router::OutboundFrame>,
14156 roster: ChildRoster,
14157 _home: ScratchDir,
14158 }
14159
14160 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14161 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14165 let tick = Instant::now();
14166 loop {
14167 if let Some(value) = observe() {
14168 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14169 return value;
14170 }
14171 assert!(
14172 std::time::Instant::now() < deadline,
14173 "timed out waiting for {reason}"
14174 );
14175 tokio::task::yield_now().await;
14176 }
14177 }
14178
14179 impl Actor {
14180 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14181 Self::start_with_policy(
14182 protocol,
14183 cadence,
14184 DEFAULT_REGISTER_TIMEOUT,
14185 RestartPolicy::new(3, Duration::ZERO),
14186 )
14187 .await
14188 }
14189
14190 async fn start_with_policy(
14191 protocol: ModuleProtocol,
14192 cadence: Duration,
14193 register_timeout: Duration,
14194 policy: RestartPolicy,
14195 ) -> Self {
14196 let home = ScratchDir::new("health-events");
14197 let registry = Arc::new(Registry::default());
14198 let forwarding = Arc::new(ForwardingTable::default());
14199 let health = HealthConfig {
14200 cadence,
14201 deadline: Duration::from_secs(3600),
14202 ..HealthConfig::default()
14203 };
14204 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14205 .with_forwarding(forwarding.clone())
14206 .with_health_config(health.clone());
14207 let spec = ModuleSpec {
14208 module_id: "event-health-child".into(),
14209 program: PathBuf::from("/bin/sleep"),
14217 args: vec!["600".into()],
14220 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14221 .into_iter()
14222 .map(|key| (key.into(), home.path().display().to_string()))
14223 .collect(),
14224 reserved: false,
14225 reserved_prefixes: vec![],
14226 protocol,
14227 overlap: Default::default(),
14228 };
14229 let module = supervisor
14230 .supervise_configured_requested(
14231 spec.clone(),
14232 true,
14233 health.clone(),
14234 Some(0),
14235 policy,
14236 register_timeout,
14237 None,
14238 )
14239 .unwrap();
14240 let (_, rx) = mpsc::channel(8);
14241 let actor = Self {
14242 module,
14243 registry,
14244 forwarding,
14245 spec,
14246 health,
14247 rx,
14248 roster: supervisor.child_roster.clone(),
14249 _home: home,
14250 };
14251 actor
14252 .wait_for_select(|checkpoint| checkpoint.generation > 0)
14253 .await;
14254 actor
14255 }
14256
14257 async fn expire_registration_wait(&self, remaining: Duration) {
14258 tokio::time::advance(remaining).await;
14259 wait_for("registration timeout TERM grace", || {
14260 lock_snapshot(&self.module.inner.snapshot)
14261 .unwrap()
14262 .actor_select
14263 .is_none()
14264 .then_some(())
14265 })
14266 .await;
14267 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14268 }
14269
14270 fn turns(&self) -> u64 {
14271 lock_snapshot(&self.module.inner.snapshot)
14272 .unwrap()
14273 .actor_turns
14274 }
14275
14276 async fn wait_for_select(
14277 &self,
14278 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14279 ) -> ActorSelectCheckpoint {
14280 wait_for("actor select checkpoint", || {
14281 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14282 state.actor_select.filter(|checkpoint| {
14283 checkpoint.generation == state.spawn_generation
14284 && state.state == ModuleState::Running
14285 && state.process_alive
14286 && state.reported_pid().is_some()
14287 && expected(checkpoint)
14288 })
14289 })
14290 .await
14291 }
14292
14293 fn hello(&mut self, connection: u64, health: bool) {
14294 let manifest =
14295 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14296 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14297 .build();
14298 let connection = ConnectionId::new(connection);
14299 let (tx, rx) = mpsc::channel(8);
14300 self.rx = rx;
14301 self.forwarding
14302 .register_module_connection(
14303 connection,
14304 self.spec.module_id.clone(),
14305 subc_protocol::PROTOCOL_VERSION,
14306 Concurrency::ModuleManaged,
14307 FrameSink::new(tx),
14308 )
14309 .unwrap();
14310 self.registry
14311 .register_with_control_ops(
14312 manifest,
14313 subc_protocol::PROTOCOL_VERSION,
14314 connection,
14315 if health {
14316 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
14317 } else {
14318 vec![]
14319 },
14320 )
14321 .unwrap();
14322 }
14323
14324 async fn probe(&mut self) -> crate::router::OutboundFrame {
14325 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
14326 Ok(frame) => Some(frame),
14327 Err(mpsc::error::TryRecvError::Empty) => None,
14328 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
14329 })
14330 .await;
14331 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14332 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14333 frame
14334 }
14335
14336 fn no_probe(&mut self) {
14337 assert!(matches!(
14338 self.rx.try_recv(),
14339 Err(mpsc::error::TryRecvError::Empty)
14340 ));
14341 }
14342
14343 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
14344 self.forwarding
14345 .complete_module_control_rpc(
14346 ConnectionId::new(connection),
14347 frame.header.corr,
14348 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14349 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14350 status: HealthStatus::Ok,
14351 detail: None,
14352 metrics: None,
14353 }),
14354 )
14355 .unwrap();
14356 }
14357 }
14358
14359 #[tokio::test(start_paused = true)]
14360 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
14361 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14362 let initial = actor.module.status().unwrap();
14363 tokio::time::advance(Duration::from_secs(119)).await;
14364 tokio::task::yield_now().await;
14365 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
14366 assert!(actor.module.terminal_history().entries.is_empty());
14367 actor.expire_registration_wait(Duration::from_secs(1)).await;
14368 actor
14369 .wait_for_select(|at| at.generation > initial.spawn_generation)
14370 .await;
14371 let status = actor.module.status().unwrap();
14372 assert_ne!(status.pid, initial.pid);
14373 assert!(!cortexkit_test_support::process_alive(
14374 initial.pid.unwrap() as i32
14375 ));
14376 assert_eq!(status.restart_count, 1);
14377 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
14378 let entries = actor.module.terminal_history().entries;
14379 assert_eq!(entries.len(), 1);
14380 assert_eq!(
14381 entries[0].disposition_detail.as_deref(),
14382 Some("registration_timeout")
14383 );
14384 actor.module.stop().await.unwrap();
14385 }
14386
14387 #[tokio::test(start_paused = true)]
14388 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
14389 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14390 let pid = actor.module.status().unwrap().pid;
14391 tokio::time::advance(Duration::from_secs(119)).await;
14392 actor.hello(1, false);
14393 let registered = actor
14394 .wait_for_select(|at| at.registered_connection.is_some())
14395 .await;
14396 assert_eq!(
14397 registered.wake_after, None,
14398 "HELLO must cancel the deadline"
14399 );
14400 tokio::time::advance(Duration::from_secs(300)).await;
14401 tokio::task::yield_now().await;
14402 assert_eq!(actor.module.status().unwrap().pid, pid);
14403 assert!(actor.module.terminal_history().entries.is_empty());
14404 actor
14406 .registry
14407 .deregister_connection(ConnectionId::new(1))
14408 .unwrap();
14409 actor
14410 .wait_for_select(|at| at.registered_connection.is_none())
14411 .await;
14412 tokio::time::advance(Duration::from_secs(300)).await;
14413 tokio::task::yield_now().await;
14414 assert_eq!(actor.module.status().unwrap().pid, pid);
14415 assert!(actor.module.terminal_history().entries.is_empty());
14416 actor.module.stop().await.unwrap();
14417
14418 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14419 let pid = transient.module.status().unwrap().pid;
14420 tokio::time::advance(Duration::from_secs(119)).await;
14421 let turns = transient.turns();
14422 transient.hello(1, false);
14425 transient
14426 .registry
14427 .deregister_connection(ConnectionId::new(1))
14428 .unwrap();
14429 let parked = transient.wait_for_select(|at| at.turn > turns).await;
14430 assert_eq!(
14431 parked.wake_after, None,
14432 "transient HELLO must cancel the deadline"
14433 );
14434 tokio::time::advance(Duration::from_secs(300)).await;
14435 tokio::task::yield_now().await;
14436 assert_eq!(transient.module.status().unwrap().pid, pid);
14437 assert!(transient.module.terminal_history().entries.is_empty());
14438 transient.module.stop().await.unwrap();
14439 }
14440
14441 #[tokio::test(start_paused = true)]
14442 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
14443 let actor = Actor::start_with_policy(
14444 ModuleProtocol::Subc,
14445 Duration::from_secs(30),
14446 Duration::from_secs(10),
14447 RestartPolicy::new(1, Duration::ZERO),
14448 )
14449 .await;
14450 let generation = actor.module.status().unwrap().spawn_generation;
14451 actor
14452 .expire_registration_wait(Duration::from_secs(10))
14453 .await;
14454 actor.wait_for_select(|at| at.generation > generation).await;
14455 actor
14456 .expire_registration_wait(Duration::from_secs(10))
14457 .await;
14458 let retry = wait_for("failed retry schedule", || {
14459 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
14460 (state.state == ModuleState::Failed)
14461 .then_some(state.failed_retry.pending)
14462 .flatten()
14463 })
14464 .await;
14465 assert_eq!(retry.delay, Duration::from_secs(600));
14466 assert_eq!(retry.attempt, 1);
14467 let entries = actor.module.terminal_history().entries;
14468 assert_eq!(entries.len(), 2);
14469 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
14470 assert!(entries[1]
14471 .disposition_detail
14472 .as_deref()
14473 .unwrap()
14474 .contains("registration_timeout"));
14475 tokio::time::advance(Duration::from_secs(599)).await;
14476 tokio::task::yield_now().await;
14477 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
14478 tokio::time::advance(Duration::from_secs(1)).await;
14479 actor
14480 .wait_for_select(|at| at.generation > generation + 1)
14481 .await;
14482 actor
14484 .expire_registration_wait(Duration::from_secs(10))
14485 .await;
14486 wait_for("slow retry registration timeout", || {
14487 (actor.module.terminal_history().entries.len() == 3).then_some(())
14488 })
14489 .await;
14490 actor.module.stop().await.unwrap();
14491 }
14492
14493 #[tokio::test(start_paused = true)]
14494 async fn registration_timeout_honours_ten_second_override() {
14495 let actor = Actor::start_with_policy(
14496 ModuleProtocol::Subc,
14497 Duration::from_secs(30),
14498 Duration::from_secs(10),
14499 RestartPolicy::new(3, Duration::ZERO),
14500 )
14501 .await;
14502 let generation = actor.module.status().unwrap().spawn_generation;
14503 tokio::time::advance(Duration::from_secs(9)).await;
14504 tokio::task::yield_now().await;
14505 assert!(actor.module.terminal_history().entries.is_empty());
14506 actor.expire_registration_wait(Duration::from_secs(1)).await;
14507 actor.wait_for_select(|at| at.generation > generation).await;
14508 assert_eq!(actor.module.terminal_history().entries.len(), 1);
14509 actor.module.stop().await.unwrap();
14510 }
14511
14512 #[tokio::test]
14513 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
14514 use std::os::unix::process::{CommandExt, ExitStatusExt};
14515
14516 struct Helper(std::process::Child);
14517 impl Drop for Helper {
14518 fn drop(&mut self) {
14519 if matches!(self.0.try_wait(), Ok(None)) {
14522 let _ = self.0.kill();
14523 let _ = self.0.wait();
14524 }
14525 }
14526 }
14527
14528 let home = ScratchDir::new("registration-timeout-helper");
14529 let supervisor = Supervisor::new_for_test(
14530 Arc::new(Registry::default()),
14531 RestartPolicy::new(0, Duration::ZERO),
14532 );
14533 let spec = ModuleSpec {
14534 module_id: "registration-timeout-group".into(),
14535 program: "/bin/sleep".into(),
14536 args: vec!["120".into()],
14537 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14538 .into_iter()
14539 .map(|key| (key.into(), home.path().display().to_string()))
14540 .collect(),
14541 reserved: false,
14542 reserved_prefixes: vec![],
14543 protocol: ModuleProtocol::Subc,
14544 overlap: Default::default(),
14545 };
14546 let module = supervisor
14547 .supervise_configured_requested(
14548 spec,
14549 true,
14550 HealthConfig::default(),
14551 None,
14552 RestartPolicy::new(0, Duration::ZERO),
14553 Duration::from_secs(10),
14554 None,
14555 )
14556 .unwrap();
14557 let leader = timeout(Duration::from_secs(5), async {
14558 loop {
14559 if let Some(pid) = module.status().unwrap().pid {
14560 break pid;
14561 }
14562 sleep(Duration::from_millis(5)).await;
14563 }
14564 })
14565 .await
14566 .expect("supervised group leader must be reported before the deadline");
14567 assert!(leader > 1);
14568 let ready = home.join("helper.ready");
14569 let mut helper = Helper(
14572 std::process::Command::new("/bin/sh")
14573 .args([
14574 "-c",
14575 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
14576 "helper",
14577 ])
14578 .arg(&ready)
14579 .process_group(i32::try_from(leader).unwrap())
14580 .stdout(std::process::Stdio::null())
14581 .stderr(std::process::Stdio::null())
14582 .spawn()
14583 .unwrap(),
14584 );
14585 timeout(Duration::from_secs(5), async {
14586 while !ready.exists() {
14587 assert!(helper.0.try_wait().unwrap().is_none());
14588 sleep(Duration::from_millis(5)).await;
14589 }
14590 })
14591 .await
14592 .expect("helper must install TERM ignore before the registration deadline");
14593 let helper_pid =
14594 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
14595 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
14596 sleep(Duration::from_millis(20)).await;
14597 assert!(
14598 helper.0.try_wait().unwrap().is_none(),
14599 "helper must ignore TERM"
14600 );
14601
14602 timeout(Duration::from_secs(20), async {
14603 while module.state().unwrap() != ModuleState::Failed {
14604 sleep(Duration::from_millis(5)).await;
14605 }
14606 })
14607 .await
14608 .expect("registration timeout must stop and reap the group leader");
14609 let helper_exit = timeout(Duration::from_secs(3), async {
14610 loop {
14611 if let Some(status) = helper.0.try_wait().unwrap() {
14612 break status;
14613 }
14614 sleep(Duration::from_millis(5)).await;
14615 }
14616 })
14617 .await
14618 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
14619 assert_eq!(helper_exit.signal(), Some(9));
14620 assert!(!cortexkit_test_support::process_alive(
14621 i32::try_from(leader).unwrap()
14622 ));
14623 assert_eq!(
14624 module.status().unwrap().last_exit.unwrap().kind,
14625 ExitKind::Crash
14626 );
14627 assert!(module.terminal_history().entries[0]
14628 .disposition_detail
14629 .as_deref()
14630 .unwrap()
14631 .contains("registration_timeout"));
14632 module.stop().await.unwrap();
14633 }
14634
14635 #[tokio::test(start_paused = true)]
14636 async fn registration_timeout_does_not_count_an_earlier_exit() {
14637 let actor = Actor::start_with_policy(
14638 ModuleProtocol::Subc,
14639 Duration::from_secs(30),
14640 DEFAULT_REGISTER_TIMEOUT,
14641 RestartPolicy::new(0, Duration::ZERO),
14642 )
14643 .await;
14644 let pid = actor.module.status().unwrap().pid.unwrap();
14645 rustix::process::kill_process(
14646 rustix::process::Pid::from_raw(pid as i32).unwrap(),
14647 rustix::process::Signal::TERM,
14648 )
14649 .unwrap();
14650 wait_for("early exit", || {
14651 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
14652 })
14653 .await;
14654 tokio::time::advance(Duration::from_secs(121)).await;
14655 tokio::task::yield_now().await;
14656 let entries = actor.module.terminal_history().entries;
14657 assert_eq!(entries.len(), 1);
14658 assert!(!entries[0]
14659 .disposition_detail
14660 .as_deref()
14661 .unwrap()
14662 .contains("registration_timeout"));
14663 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
14664 actor.module.stop().await.unwrap();
14665 }
14666
14667 #[tokio::test(start_paused = true)]
14668 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
14669 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14670 let pid = actor.module.status().unwrap().pid;
14671 tokio::time::advance(Duration::from_secs(119)).await;
14672 actor.roster.close();
14673 let turns = actor.turns();
14674 tokio::time::advance(Duration::from_secs(300)).await;
14675 let parked = actor.wait_for_select(|at| at.turn > turns).await;
14676 assert_eq!(parked.wake_after, None);
14677 assert_eq!(actor.module.status().unwrap().pid, pid);
14678 assert!(actor.module.terminal_history().entries.is_empty());
14679 actor.module.stop().await.unwrap();
14680 let status = actor.module.status().unwrap();
14681 assert_eq!(status.restart_count, 0);
14682 assert_eq!(
14683 actor.module.terminal_history().entries[0].disposition,
14684 TerminalDisposition::DaemonShutdown
14685 );
14686 }
14687
14688 #[tokio::test(start_paused = true)]
14689 async fn no_health_children_park_for_sixty_seconds() {
14690 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14691 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14692 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14693 unadvertised.hello(1, false);
14694 for _ in 0..6000 {
14699 tokio::time::advance(Duration::from_millis(10)).await;
14700 tokio::task::yield_now().await;
14701 }
14702 for actor in [&none, &unregistered, &unadvertised] {
14703 assert!(
14704 actor.turns() <= 5,
14705 "no-health actor woke {} times",
14706 actor.turns()
14707 );
14708 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
14709 }
14710 }
14711
14712 #[tokio::test(start_paused = true)]
14713 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
14714 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
14718 let tick = Instant::now();
14719 actor.hello(1, true);
14720 actor.probe().await;
14721 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
14722 }
14723
14724 #[tokio::test(start_paused = true)]
14725 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
14726 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14727 let hello_tick = Instant::now();
14728 actor.hello(1, true);
14729 let armed = actor
14730 .wait_for_select(|checkpoint| {
14731 checkpoint.registered_connection == Some(ConnectionId::new(1))
14732 && checkpoint.next_probe_at.is_some()
14733 })
14734 .await;
14735 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14736 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
14737 let first_deadline = hello_tick + delay;
14738 assert_eq!(armed.next_probe_at, Some(first_deadline));
14739 tokio::time::advance(Duration::from_secs(29)).await;
14740 let turns = actor.turns();
14741 actor
14742 .registry
14743 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
14744 .unwrap();
14745 let updated = actor
14746 .wait_for_select(|checkpoint| checkpoint.turn > turns)
14747 .await;
14748 assert_eq!(
14749 updated.next_probe_at,
14750 Some(first_deadline),
14751 "catalog update must not reset cadence"
14752 );
14753 actor.no_probe();
14754 tokio::time::advance(first_deadline - Instant::now()).await;
14755 let frame = actor.probe().await;
14756 actor.answer(1, frame);
14757 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
14758 let next_deadline = Instant::now() + next;
14759 let rearmed = actor
14760 .wait_for_select(|checkpoint| {
14761 checkpoint
14762 .next_probe_at
14763 .is_some_and(|deadline| deadline > first_deadline)
14764 })
14765 .await;
14766 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14767 tokio::time::advance(next - Duration::from_nanos(1)).await;
14768 actor.no_probe();
14769 tokio::time::advance(Duration::from_nanos(1)).await;
14770 actor.probe().await;
14771 }
14772
14773 #[tokio::test(start_paused = true)]
14774 async fn disconnect_disarms_and_reconnect_rearms_health() {
14775 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14776 actor.hello(1, true);
14777 actor
14778 .wait_for_select(|checkpoint| {
14779 checkpoint.registered_connection == Some(ConnectionId::new(1))
14780 && checkpoint.next_probe_at.is_some()
14781 })
14782 .await;
14783 let turns = actor.turns();
14784 let tick = Instant::now();
14785 actor
14786 .registry
14787 .deregister_connection(ConnectionId::new(1))
14788 .unwrap();
14789 let disarmed = actor
14790 .wait_for_select(|checkpoint| {
14791 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14792 })
14793 .await;
14794 assert_eq!(disarmed.next_probe_at, None);
14795 assert_eq!(disarmed.wake_after, None);
14796 assert_eq!(
14797 Instant::now(),
14798 tick,
14799 "disconnect must disarm in the notification tick"
14800 );
14801 tokio::time::advance(Duration::from_secs(60)).await;
14802 actor.no_probe();
14803 let reconnect_tick = Instant::now();
14804 actor.hello(2, true);
14805 let rearmed = actor
14806 .wait_for_select(|checkpoint| {
14807 checkpoint.registered_connection == Some(ConnectionId::new(2))
14808 && checkpoint.next_probe_at.is_some()
14809 })
14810 .await;
14811 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14812 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14813 tokio::time::advance(delay).await;
14814 actor.probe().await;
14815 }
14816
14817 #[tokio::test(start_paused = true)]
14818 async fn reregistration_without_health_disarms_the_old_schedule() {
14819 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14820 actor.hello(1, true);
14821 actor
14822 .wait_for_select(|checkpoint| {
14823 checkpoint.registered_connection == Some(ConnectionId::new(1))
14824 && checkpoint.next_probe_at.is_some()
14825 })
14826 .await;
14827 actor
14828 .registry
14829 .deregister_connection(ConnectionId::new(1))
14830 .unwrap();
14831 actor.hello(2, false);
14832 let disarmed = actor
14833 .wait_for_select(|checkpoint| {
14834 checkpoint.registered_connection == Some(ConnectionId::new(2))
14835 })
14836 .await;
14837 assert_eq!(disarmed.next_probe_at, None);
14838 assert_eq!(disarmed.wake_after, None);
14839 tokio::time::advance(Duration::from_secs(60)).await;
14840 actor.no_probe();
14841 }
14842
14843 #[tokio::test(start_paused = true)]
14844 async fn restart_waits_for_its_own_hello_before_rearming_health() {
14845 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14846 actor.hello(1, true);
14847 let armed = actor
14848 .wait_for_select(|checkpoint| {
14849 checkpoint.registered_connection == Some(ConnectionId::new(1))
14850 && checkpoint.next_probe_at.is_some()
14851 })
14852 .await;
14853 actor
14857 .registry
14858 .deregister_connection(ConnectionId::new(1))
14859 .unwrap();
14860 actor.module.restart(Some(0)).await.unwrap();
14861 let parked = actor
14862 .wait_for_select(|checkpoint| {
14863 checkpoint.generation > armed.generation
14864 && checkpoint.registered_connection.is_none()
14865 && checkpoint.next_probe_at.is_none()
14866 })
14867 .await;
14868 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
14869 actor.rx.close();
14870 while actor.rx.try_recv().is_ok() {}
14873 let turns = actor.turns();
14874 tokio::time::advance(Duration::from_secs(60)).await;
14875 let after = actor.turns();
14876 eprintln!(
14877 "restart park turns: before={turns} after={after} delta={}",
14878 after - turns
14879 );
14880 assert_eq!(after, turns, "replacement must park before HELLO");
14881 let hello_tick = Instant::now();
14882 actor.hello(2, true);
14883 let rearmed = actor
14884 .wait_for_select(|checkpoint| {
14885 checkpoint.registered_connection == Some(ConnectionId::new(2))
14886 && checkpoint.next_probe_at.is_some()
14887 })
14888 .await;
14889 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14890 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14891 tokio::time::advance(delay).await;
14892 actor.probe().await;
14893 }
14894
14895 #[tokio::test(start_paused = true)]
14896 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14897 use tokio::{
14898 io::{AsyncReadExt, AsyncWriteExt},
14899 net::TcpListener,
14900 };
14901 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14902 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14903 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14904 let (started_tx, mut started_rx) = mpsc::channel(8);
14905 let peer = tokio::spawn(async move {
14906 loop {
14907 let (mut socket, _) = listener.accept().await.unwrap();
14908 let mut request = [0; 1024];
14909 assert!(socket.read(&mut request).await.unwrap() > 0);
14910 started_tx.send(Instant::now()).await.unwrap();
14911 socket
14912 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14913 .await
14914 .unwrap();
14915 }
14916 });
14917 actor
14918 .module
14919 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14920 .await
14921 .unwrap();
14922 let first_tick = Instant::now();
14923 let first = actor
14924 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
14925 .await;
14926 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
14927 assert!(started_rx.try_recv().is_err());
14928
14929 tokio::time::advance(
14932 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
14933 )
14934 .await;
14935 actor.module.restart(Some(0)).await.unwrap();
14936 let spawn_tick = Instant::now();
14937 let replacement = actor
14938 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
14939 .await;
14940 let deadline = replacement.next_probe_at.unwrap();
14941 assert!(
14942 deadline >= spawn_tick + actor.health.cadence,
14943 "a replacement must not inherit its predecessor's probe deadline"
14944 );
14945 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
14946 assert!(
14947 started_rx.try_recv().is_err(),
14948 "no HTTP probe during startup grace"
14949 );
14950 tokio::time::advance(deadline - Instant::now()).await;
14951 let landed = wait_for("first replacement HTTP probe", || {
14952 started_rx.try_recv().ok()
14953 })
14954 .await;
14955 assert!(landed >= spawn_tick + actor.health.cadence);
14956 wait_for("healthy replacement", || {
14957 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14958 .then_some(())
14959 })
14960 .await;
14961 peer.abort();
14962 }
14963
14964 #[tokio::test(start_paused = true)]
14965 async fn rescan_adds_and_removes_http_health_without_polling() {
14966 use tokio::{
14967 io::{AsyncReadExt, AsyncWriteExt},
14968 net::TcpListener,
14969 };
14970 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14971 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14972 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
14973 actor.health.cadence = Duration::from_millis(1);
14975 let (started_tx, mut started_rx) = mpsc::channel(8);
14976 let peer = tokio::spawn(async move {
14977 loop {
14978 let (mut socket, _) = listener.accept().await.unwrap();
14979 let mut request = [0; 1024];
14980 assert!(socket.read(&mut request).await.unwrap() > 0);
14981 started_tx.send(()).await.unwrap();
14982 socket
14983 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14984 .await
14985 .unwrap();
14986 }
14987 });
14988 let tick = Instant::now();
14989 let turns = actor.turns();
14990 actor
14991 .module
14992 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14993 .await
14994 .unwrap();
14995 let armed = actor
14996 .wait_for_select(|checkpoint| {
14997 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
14998 })
14999 .await;
15000 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15001 assert_eq!(Instant::now(), tick);
15002 tokio::time::advance(Duration::from_millis(1)).await;
15003 let mut started = false;
15004 wait_for("successful HTTP probe", || {
15005 started |= started_rx.try_recv().is_ok();
15006 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15007 .then_some(())
15008 })
15009 .await;
15010 assert!(started, "added HTTP check must start at its first deadline");
15011 assert_eq!(
15012 actor.module.status().unwrap().health.status,
15013 SupervisorHealthStatus::Ok
15014 );
15015 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15016 actor.health.http = None;
15017 let turns = actor.turns();
15018 actor
15019 .module
15020 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15021 .await
15022 .unwrap();
15023 let parked = actor
15024 .wait_for_select(|checkpoint| {
15025 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15026 })
15027 .await;
15028 assert_eq!(parked.wake_after, None);
15029 let turns = actor.turns();
15030 tokio::time::advance(Duration::from_secs(60)).await;
15031 assert_eq!(
15032 actor.turns(),
15033 turns,
15034 "removed HTTP check must park the actor"
15035 );
15036 assert!(
15037 started_rx.try_recv().is_err(),
15038 "removed HTTP check must stay disarmed"
15039 );
15040 assert_eq!(
15041 actor.module.status().unwrap().health,
15042 ModuleHealthStatus::default()
15043 );
15044 peer.abort();
15045 }
15046}
15047
15048#[cfg(test)]
15049mod drain_busy_reprobe_tests {
15050 use super::*;
15054 use std::collections::BTreeMap;
15055 use std::sync::Mutex as StdMutex;
15056 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15057
15058 const MODULE_ID: &str = "drain-reprobe-module";
15059 const CONNECTION: u64 = 7;
15060 const GAUGE: &str = "runs_in_flight";
15061 const CADENCE: Duration = Duration::from_secs(30);
15064 const CEILING: Duration = Duration::from_secs(30);
15065
15066 #[derive(Clone, Copy)]
15068 enum Answer {
15069 BusyUntil { until: Instant, latency: Duration },
15072 OmitsGauge,
15074 Silent,
15076 }
15077
15078 struct Drain {
15079 forwarding: Arc<ForwardingTable>,
15080 runtime: SupervisorRuntimeConfig,
15081 endpoint: crate::ModuleEndpointId,
15082 probes: Arc<StdMutex<Vec<Instant>>>,
15084 responder: tokio::task::JoinHandle<()>,
15085 }
15086
15087 impl Drain {
15088 fn start(answer: Answer) -> Self {
15089 let registry = Arc::new(Registry::default());
15090 let forwarding = Arc::new(ForwardingTable::default());
15091 let health = HealthConfig {
15092 cadence: CADENCE,
15093 ..HealthConfig::default()
15094 };
15095 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15096 .with_forwarding(forwarding.clone())
15097 .with_health_config(health)
15098 .runtime_config();
15099 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15100 let endpoint = forwarding
15101 .register_module_connection(
15102 ConnectionId::new(CONNECTION),
15103 MODULE_ID.to_string(),
15104 subc_protocol::PROTOCOL_VERSION,
15105 Concurrency::ModuleManaged,
15106 FrameSink::new(tx),
15107 )
15108 .unwrap();
15109 let probes = Arc::new(StdMutex::new(Vec::new()));
15110 let responder = tokio::spawn({
15111 let forwarding = forwarding.clone();
15112 let probes = probes.clone();
15113 async move {
15114 while let Some(frame) = rx.recv().await {
15115 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15116 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15117 probes.lock().unwrap().push(Instant::now());
15118 let metrics = match answer {
15119 Answer::BusyUntil { until, latency } => {
15120 sleep(latency).await;
15121 let busy = u64::from(Instant::now() < until);
15122 serde_json::json!({ GAUGE: busy })
15123 }
15124 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15125 Answer::Silent => continue,
15126 };
15127 let _ = forwarding.complete_module_control_rpc(
15130 ConnectionId::new(CONNECTION),
15131 frame.header.corr,
15132 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15133 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15134 status: HealthStatus::Ok,
15135 detail: None,
15136 metrics: Some(metrics),
15137 }),
15138 );
15139 }
15140 }
15141 });
15142 Self {
15143 forwarding,
15144 runtime,
15145 endpoint,
15146 probes,
15147 responder,
15148 }
15149 }
15150
15151 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15154 let (drained, elapsed, _) = self.run_traced(ceiling).await;
15155 (drained, elapsed)
15156 }
15157
15158 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15160 let started = Instant::now();
15161 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15162 let drained = wait_for_forwarding_quiescence(
15163 &self.forwarding,
15164 MODULE_ID,
15165 &self.runtime,
15166 self.endpoint,
15167 started + ceiling,
15168 &[GAUGE.to_string()],
15169 DrainScope::Active,
15170 &mut trace,
15171 )
15172 .await
15173 .unwrap();
15174 (drained, started.elapsed(), trace)
15175 }
15176
15177 fn probes(&self) -> Vec<Instant> {
15178 self.probes.lock().unwrap().clone()
15179 }
15180 }
15181
15182 impl Drop for Drain {
15183 fn drop(&mut self) {
15184 self.responder.abort();
15185 }
15186 }
15187
15188 #[tokio::test(start_paused = true)]
15189 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15190 let drain = Drain::start(Answer::BusyUntil {
15191 until: Instant::now() + Duration::from_secs(1),
15192 latency: Duration::ZERO,
15193 });
15194 let (drained, elapsed) = drain.run(CEILING).await;
15195 assert!(
15196 drained,
15197 "gauges that read zero during the drain must let it finish drained, \
15198 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15199 );
15200 assert!(
15201 elapsed < Duration::from_secs(3),
15202 "drain took {elapsed:?} after the gauges went quiet at 1s"
15203 );
15204 let probes = drain.probes();
15205 assert!(
15206 probes.len() >= 2,
15207 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15208 );
15209 }
15210
15211 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15216 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15217 let drain = Drain::start(Answer::BusyUntil {
15218 until: Instant::now() + Duration::from_millis(600),
15219 latency: Duration::ZERO,
15220 });
15221 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15222 assert!(drained, "the gauge fell at 600ms: {trace}");
15223 assert!(
15224 elapsed < Duration::from_secs(3),
15225 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15226 );
15227 let probes = drain.probes();
15228 assert!(
15229 probes.len() >= 2 && trace.probes as usize == probes.len(),
15230 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15231 );
15232 }
15233
15234 #[tokio::test(start_paused = true)]
15235 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15236 let started = Instant::now();
15237 let drain = Drain::start(Answer::BusyUntil {
15238 until: started + Duration::from_secs(3600),
15239 latency: Duration::ZERO,
15240 });
15241 let (drained, elapsed) = drain.run(CEILING).await;
15242 assert!(!drained, "a module busy throughout must not drain");
15243 assert!(
15244 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15245 "a busy module is held until the ceiling and no longer: {elapsed:?}"
15246 );
15247 let probes = drain.probes();
15250 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15251 assert!(
15252 probes.len() as u128 <= max_probes,
15253 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15254 probes.len()
15255 );
15256 for pair in probes.windows(2) {
15257 assert!(
15258 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15259 "consecutive probes {:?} apart",
15260 pair[1] - pair[0]
15261 );
15262 }
15263 }
15264
15265 #[tokio::test(start_paused = true)]
15266 async fn a_slow_probe_is_never_overlapped_by_the_next() {
15267 let latency = Duration::from_secs(1);
15270 let drain = Drain::start(Answer::BusyUntil {
15271 until: Instant::now() + Duration::from_secs(3600),
15272 latency,
15273 });
15274 let (drained, _) = drain.run(Duration::from_secs(5)).await;
15275 assert!(!drained);
15276 let probes = drain.probes();
15277 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15278 for pair in probes.windows(2) {
15279 assert!(
15280 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15281 "a probe started {:?} after the previous one, before it could finish",
15282 pair[1] - pair[0]
15283 );
15284 }
15285 }
15286
15287 #[tokio::test(start_paused = true)]
15288 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15289 let drain = Drain::start(Answer::Silent);
15290 let ceiling = Duration::from_secs(2);
15291 let (drained, elapsed) = drain.run(ceiling).await;
15292 assert!(!drained, "an unanswered probe counts as busy");
15293 assert!(
15294 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15295 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15296 );
15297 assert_eq!(
15298 drain.probes().len(),
15299 1,
15300 "the unanswered probe holds the only slot until the ceiling"
15301 );
15302 }
15303
15304 #[derive(Clone, Default)]
15306 struct LogCapture {
15307 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
15308 }
15309
15310 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
15311 fn on_event(
15312 &self,
15313 event: &tracing::Event<'_>,
15314 _context: tracing_subscriber::layer::Context<'_, S>,
15315 ) {
15316 struct Fields(BTreeMap<String, String>);
15317 impl tracing::field::Visit for Fields {
15318 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
15319 self.0
15320 .insert(field.name().to_string(), format!("{value:?}"));
15321 }
15322 }
15323 let mut fields = Fields(BTreeMap::new());
15324 event.record(&mut fields);
15325 self.events.lock().unwrap().push(fields.0);
15326 }
15327 }
15328
15329 impl LogCapture {
15330 fn install(&self) -> tracing::subscriber::DefaultGuard {
15331 use tracing_subscriber::layer::SubscriberExt;
15332 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
15333 }
15334
15335 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
15336 self.events
15337 .lock()
15338 .unwrap()
15339 .iter()
15340 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
15341 .cloned()
15342 .collect()
15343 }
15344 }
15345
15346 #[tokio::test(start_paused = true)]
15351 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
15352 let capture = LogCapture::default();
15353 let _guard = capture.install();
15354 let drain = Drain::start(Answer::OmitsGauge);
15355 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
15356 assert!(!drained, "a gauge never reported cannot read quiet");
15357 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
15358
15359 let missing = capture.with_message("lacks a declared busy gauge");
15360 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
15361 assert_eq!(
15362 missing[0].get("missing_gauges").map(String::as_str),
15363 Some(GAUGE)
15364 );
15365
15366 warn_drain_timed_out(
15367 MODULE_ID,
15368 Duration::from_secs(2),
15369 RouteCloseReason::Restart,
15370 &Default::default(),
15371 &trace,
15372 );
15373 let timeouts = capture.with_message("route drain timed out");
15374 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
15375 let gauges = &timeouts[0]["busy_gauges"];
15376 assert!(
15377 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
15378 "the timeout line must name the gauge and why it read busy: {gauges}"
15379 );
15380 assert!(
15381 gauges.ends_with("; no value ever read"),
15382 "no value was ever read for the gauge: {gauges}"
15383 );
15384 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
15385 }
15386
15387 #[tokio::test(start_paused = true)]
15391 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
15392 let busy = Drain::start(Answer::BusyUntil {
15393 until: Instant::now() + Duration::from_secs(3600),
15394 latency: Duration::ZERO,
15395 });
15396 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
15397 let rendered = trace.to_string();
15398 assert!(
15399 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
15400 && rendered.contains("; last value non_zero(1) @"),
15401 "{rendered}"
15402 );
15403
15404 let silent = Drain::start(Answer::Silent);
15405 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
15406 assert_eq!(
15409 trace.to_string(),
15410 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
15411 );
15412 assert_eq!(trace.probes, 1);
15413 }
15414
15415 #[test]
15417 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
15418 let gauges = [GAUGE.to_string()];
15419 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
15420 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
15421 assert_eq!(
15422 busy_gauge_observation(&verdicts),
15423 BusyGaugeObservation::Busy
15424 );
15425 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
15426 assert_eq!(
15427 busy_gauge_observation(&zero),
15428 BusyGaugeObservation::Quiescent
15429 );
15430 let none = gauge_verdicts(None, &gauges);
15431 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
15432 }
15433}
15434
15435#[cfg(test)]
15436mod health_evidence_tests {
15437 use super::{HealthProbeError, HealthProbeEvidence};
15438 use std::collections::HashSet;
15439
15440 #[test]
15448 fn only_a_dead_lane_is_proof_of_death() {
15449 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
15450 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
15454 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
15455 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
15456 }
15457
15458 #[test]
15464 fn every_evidence_class_has_a_distinct_label() {
15465 let labels = [
15466 HealthProbeError::lane_dead("").label(),
15467 HealthProbeError::no_answer("").label(),
15468 HealthProbeError::bad_answer("").label(),
15469 HealthProbeError::misconfigured("").label(),
15470 ];
15471 let unique: HashSet<_> = labels.iter().collect();
15472 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
15473 }
15474
15475 #[test]
15481 fn classification_preserves_the_original_message() {
15482 let err = HealthProbeError::no_answer("module did not answer within 5s");
15483 assert_eq!(err.to_string(), "module did not answer within 5s");
15484 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15485 }
15486}
15487
15488#[cfg(test)]
15489mod health_tombstone_tests {
15490 use std::{path::PathBuf, sync::Arc, time::Duration};
15491
15492 use subc_protocol::{
15493 manifest::Concurrency,
15494 session::{HealthStatus, ModuleControlResponse},
15495 };
15496 use tokio::sync::mpsc;
15497
15498 use super::{
15499 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
15500 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
15501 };
15502 use crate::{
15503 control::ControlHandler,
15504 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
15505 registry::{ConnectionId, Registry},
15506 router::FrameSink,
15507 };
15508
15509 struct ProbeHarness {
15510 spec: ModuleSpec,
15511 runtime: SupervisorRuntimeConfig,
15512 forwarding: Arc<ForwardingTable>,
15513 module_connection: ConnectionId,
15514 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15515 handler: ControlHandler,
15516 module: super::SupervisedModule,
15517 }
15518
15519 fn probe_harness() -> ProbeHarness {
15520 let registry = Arc::new(Registry::default());
15521 let forwarding = Arc::new(ForwardingTable::default());
15522 let supervisor_handle = super::SupervisorHandle::new();
15523 let health = HealthConfig {
15524 http: None,
15525 cadence: Duration::from_secs(30),
15526 deadline: Duration::from_secs(5),
15527 failure_threshold: 3,
15528 on_degraded: HealthAction::Report,
15529 on_failing: HealthAction::Report,
15530 critical: false,
15531 };
15532 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15533 .with_forwarding(Arc::clone(&forwarding))
15534 .with_handle(supervisor_handle.clone())
15535 .with_health_config(health);
15536 let spec = ModuleSpec {
15537 module_id: "late-health-module".to_string(),
15538 program: PathBuf::from("disabled-module"),
15539 args: Vec::new(),
15540 env: Vec::new(),
15541 reserved: false,
15542 reserved_prefixes: Vec::new(),
15543 protocol: ModuleProtocol::Subc,
15544 overlap: Default::default(),
15545 };
15546 let module = supervisor
15547 .supervise_configured(spec.clone(), false)
15548 .unwrap();
15549 let runtime = supervisor.runtime_config();
15550 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
15551 .with_supervisor(supervisor_handle);
15552 let module_connection = ConnectionId::new(700);
15553 let (module_tx, module_rx) = mpsc::channel(8);
15554 forwarding
15555 .register_module_connection(
15556 module_connection,
15557 spec.module_id.clone(),
15558 subc_protocol::PROTOCOL_VERSION,
15559 Concurrency::ModuleManaged,
15560 FrameSink::new(module_tx),
15561 )
15562 .unwrap();
15563
15564 ProbeHarness {
15565 spec,
15566 runtime,
15567 forwarding,
15568 module_connection,
15569 module_rx,
15570 handler,
15571 module,
15572 }
15573 }
15574
15575 async fn finish_after(
15576 harness: &mut ProbeHarness,
15577 stall: Duration,
15578 ) -> ModuleControlRpcCompletion {
15579 assert!(stall > harness.runtime.health.deadline);
15580 let deadline = harness.runtime.health.deadline;
15581 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15582 let answer = async {
15583 let frame = harness.module_rx.recv().await.expect("health.check frame");
15584 tokio::time::advance(deadline).await;
15585 tokio::task::yield_now().await;
15586 tokio::time::advance(stall - deadline).await;
15587 harness
15588 .forwarding
15589 .complete_module_control_rpc(
15590 harness.module_connection,
15591 frame.header.corr,
15592 Some("health.check"),
15593 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15594 status: HealthStatus::Ok,
15595 detail: None,
15596 metrics: None,
15597 }),
15598 )
15599 .unwrap()
15600 };
15601 let (probe_result, completion) = tokio::join!(probe, answer);
15602 let err = probe_result.expect_err("probe must miss its deadline");
15603 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15604 completion
15605 }
15606
15607 async fn time_out_without_answer(harness: &mut ProbeHarness) {
15608 let deadline = harness.runtime.health.deadline;
15609 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15610 let exhaust_deadline = async {
15611 let _frame = harness.module_rx.recv().await.expect("health.check frame");
15612 tokio::time::advance(deadline).await;
15613 tokio::task::yield_now().await;
15614 };
15615 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
15616 let err = probe_result.expect_err("probe must miss its deadline");
15617 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15618 }
15619
15620 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
15621 let registry = Arc::clone(&harness.module.inner.registry);
15622 let snapshot = Arc::clone(&harness.module.inner.snapshot);
15623 let process_liveness = super::SupervisorProcessLiveness::default();
15624 let mut child = None;
15625 let cycle = super::run_health_probe_cycle(
15626 &harness.spec,
15627 &harness.runtime,
15628 ®istry,
15629 &process_liveness,
15630 &snapshot,
15631 &mut child,
15632 );
15633 let peer = async {
15634 let frame = harness.module_rx.recv().await.expect("health.check frame");
15635 if answer {
15636 harness
15637 .forwarding
15638 .complete_module_control_rpc(
15639 harness.module_connection,
15640 frame.header.corr,
15641 Some("health.check"),
15642 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15643 status: HealthStatus::Ok,
15644 detail: None,
15645 metrics: Some(serde_json::json!({"ready": true})),
15646 }),
15647 )
15648 .unwrap();
15649 } else {
15650 tokio::time::advance(harness.runtime.health.deadline).await;
15651 tokio::task::yield_now().await;
15652 }
15653 };
15654 tokio::join!(cycle, peer);
15655 }
15656
15657 #[tokio::test(start_paused = true)]
15658 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
15659 let mut harness = probe_harness();
15660 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
15664 monitor.abort();
15665 let _ = monitor.await;
15666 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
15667 state.enabled = true;
15668 state.state = super::ModuleState::Running;
15669 state.process_alive = true;
15670 })
15671 .unwrap();
15672
15673 run_probe_cycle(&mut harness, true).await;
15674 assert_eq!(
15675 harness.module.status().unwrap().health.status,
15676 super::SupervisorHealthStatus::Ok
15677 );
15678
15679 for failures in 1..harness.runtime.health.failure_threshold {
15680 run_probe_cycle(&mut harness, false).await;
15681 let status = harness.module.status().unwrap();
15682 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15683 assert_eq!(status.health.consecutive_failures, failures);
15684 assert!(status.health.last_probe_ms.is_some());
15685 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
15686 assert!(status.health.metrics.is_none());
15687 assert_eq!(status.state, super::ModuleState::Running);
15688 assert!(status.process_alive);
15689 assert_eq!(status.restart_count, 0);
15690 assert_eq!(status.lifetime_restarts, 0);
15691 assert!(status.health.last_action.is_none());
15692 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15693 }
15694
15695 run_probe_cycle(&mut harness, true).await;
15696 let recovered = harness.module.status().unwrap();
15697 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
15698 assert_eq!(recovered.health.consecutive_failures, 0);
15699 assert!(recovered.health.detail.is_none());
15700 assert_eq!(
15701 recovered.health.metrics,
15702 Some(serde_json::json!({"ready": true}))
15703 );
15704 assert_eq!(recovered.lifetime_restarts, 0);
15705
15706 for failures in 1..=harness.runtime.health.failure_threshold {
15707 run_probe_cycle(&mut harness, false).await;
15708 let status = harness.module.status().unwrap();
15709 assert_eq!(status.health.consecutive_failures, failures);
15710 if failures < harness.runtime.health.failure_threshold {
15711 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15712 assert_eq!(status.state, super::ModuleState::Running);
15713 assert_eq!(status.lifetime_restarts, 0);
15714 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15715 } else {
15716 assert_eq!(
15717 status.health.status,
15718 super::SupervisorHealthStatus::Unresponsive
15719 );
15720 assert_eq!(status.state, super::ModuleState::Restarting);
15721 assert_eq!(status.restart_count, 1);
15722 assert_eq!(status.lifetime_restarts, 1);
15723 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
15724 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
15725 }
15726 }
15727 }
15728
15729 #[tokio::test(start_paused = true)]
15730 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
15731 let mut harness = probe_harness();
15732
15733 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
15734 let first_latency = match &first {
15735 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15736 other => panic!("late answer was not retained: {other:?}"),
15737 };
15738 assert!(harness.handler.observe_module_control_completion(first));
15739
15740 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
15741 let second_latency = match &second {
15742 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15743 other => panic!("late answer was not retained: {other:?}"),
15744 };
15745 assert!(harness.handler.observe_module_control_completion(second));
15746
15747 assert_eq!(first_latency, Duration::from_secs(8));
15748 assert_eq!(
15749 second_latency - first_latency,
15750 Duration::from_secs(3),
15751 "latency must grow linearly with the additional stall"
15752 );
15753 let health = harness.module.status().unwrap().health;
15754 assert_eq!(health.late_answer_count, 2);
15755 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
15756 }
15757
15758 #[tokio::test(start_paused = true)]
15766 async fn late_answer_clears_the_consecutive_failure_streak() {
15767 let mut harness = probe_harness();
15768
15769 time_out_without_answer(&mut harness).await;
15771 harness
15772 .module
15773 .record_health_probe_failure_for_test("[no-answer] test miss")
15774 .unwrap();
15775 assert_eq!(
15776 harness.module.status().unwrap().health.consecutive_failures,
15777 1,
15778 "precondition: the miss must be on the streak before the late answer"
15779 );
15780
15781 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15783 assert!(matches!(
15784 late,
15785 ModuleControlRpcCompletion::LateHealthAnswer { .. }
15786 ));
15787 assert!(harness.handler.observe_module_control_completion(late));
15788
15789 let health = harness.module.status().unwrap().health;
15790 assert_eq!(
15791 health.consecutive_failures, 0,
15792 "a late answer is an answer: the streak must reset"
15793 );
15794 assert_eq!(health.late_answer_count, 1);
15795 }
15796
15797 #[tokio::test(start_paused = true)]
15798 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15799 let mut harness = probe_harness();
15800
15801 for _ in 0..20 {
15802 time_out_without_answer(&mut harness).await;
15803 assert_eq!(
15804 harness.forwarding.health_probe_tombstone_count().unwrap(),
15805 1
15806 );
15807 }
15808 }
15809}
15810
15811#[cfg(test)]
15812mod child_env_tests {
15813 #[cfg(windows)]
15814 use super::{
15815 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15816 };
15817 #[cfg(windows)]
15818 use std::sync::{Arc, OnceLock};
15819 #[cfg(windows)]
15820 #[test]
15821 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15822 let spec = spec(Vec::new());
15823 let mut command = Command::new("unused");
15824 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15825 .unwrap()
15826 .unwrap();
15827 let name = command
15828 .as_std()
15829 .get_envs()
15830 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15831 .and_then(|(_, value)| value)
15832 .unwrap();
15833 assert_eq!(name, OsStr::new(handoff.name()));
15834 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15835 assert!(command.as_std().get_envs().any(|(key, value)| key
15836 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15837 && value == Some(OsStr::new("env"))));
15838 assert!(command
15839 .as_std()
15840 .get_envs()
15841 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15842 }
15843
15844 #[cfg(windows)]
15845 #[test]
15846 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15847 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15848 snapshot.process_alive = true;
15849 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15850 assert_eq!(
15851 observed_launch_nonce_source(&snapshot).as_deref(),
15852 Some("env")
15853 );
15854 let consumed = Arc::new(OnceLock::new());
15855 snapshot.pipe_consumed = Some(consumed.clone());
15856 assert_eq!(
15857 observed_launch_nonce_source(&snapshot).as_deref(),
15858 Some("env")
15859 );
15860 consumed.set(()).unwrap();
15861 assert_eq!(
15862 observed_launch_nonce_source(&snapshot).as_deref(),
15863 Some("pipe")
15864 );
15865 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15866 assert_eq!(
15867 observed_launch_nonce_source(&snapshot).as_deref(),
15868 Some("env")
15869 );
15870 clear_current_process_facts(&mut snapshot);
15871 assert_eq!(observed_launch_nonce_source(&snapshot), None);
15872 }
15873 use super::{
15874 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15875 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15876 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15877 };
15878 use std::{ffi::OsStr, path::PathBuf};
15879 use tokio::process::Command;
15880
15881 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15882 ModuleSpec {
15883 module_id: "env-plan".to_string(),
15884 program: PathBuf::from("/nonexistent"),
15885 args: Vec::new(),
15886 env,
15887 reserved: false,
15888 reserved_prefixes: Vec::new(),
15889 protocol: ModuleProtocol::Subc,
15890 overlap: Default::default(),
15891 }
15892 }
15893
15894 #[test]
15895 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15896 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15897 let digest = LaunchSpecDigest::from_spec(&base);
15898 let mut reordered = base.clone();
15899 reordered.env.reverse();
15900 reordered.module_id = "different-id".into();
15901 reordered.reserved = true;
15902 reordered.overlap = Default::default();
15903 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15904 let mut delimiter_collision = base.clone();
15905 delimiter_collision.env[0] = ("A=B".into(), "C".into());
15906 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15907 let mut changed = base.clone();
15908 changed.protocol = ModuleProtocol::None;
15909 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15910 changed = base.clone();
15911 changed.args.push("--flag".into());
15912 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15913 changed = base.clone();
15914 changed.program = PathBuf::from("/other");
15915 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15916 changed = base.clone();
15917 changed.env[1].1 = "other".into();
15918 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15919 assert!(!format!("{digest:?}").contains("B=C"));
15920 assert!(!format!("{digest:?}").contains("[u8"));
15921 }
15922
15923 #[cfg(unix)]
15924 #[test]
15925 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
15926 use std::os::unix::ffi::OsStringExt;
15927 let mut left = spec(Vec::new());
15928 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
15929 let mut right = left.clone();
15930 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
15931 assert_eq!(
15932 left.program.to_string_lossy(),
15933 right.program.to_string_lossy()
15934 );
15935 assert_ne!(
15936 LaunchSpecDigest::from_spec(&left),
15937 LaunchSpecDigest::from_spec(&right)
15938 );
15939 }
15940
15941 #[test]
15955 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
15956 let mut command = Command::new("/nonexistent");
15957 apply_child_env(&mut command, &spec(Vec::new()));
15958 let removed = command
15959 .as_std()
15960 .get_envs()
15961 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
15962 assert!(
15963 removed,
15964 "ambient CK_LOG must be explicitly removed for an unconfigured module"
15965 );
15966
15967 let mut configured = Command::new("/nonexistent");
15968 apply_child_env(
15969 &mut configured,
15970 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
15971 );
15972 let effective = configured
15973 .as_std()
15974 .get_envs()
15975 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
15976 .last()
15977 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15978 assert_eq!(
15979 effective,
15980 Some(Some("debug".to_string())),
15981 "a module's configured CK_LOG must survive the ambient removal"
15982 );
15983 }
15984
15985 #[test]
15994 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
15995 let connection_file = std::path::Path::new("/run/subc-connection.json");
15996 let handle = SupervisorHandle::new();
15997
15998 let mut none_spec = spec(Vec::new());
15999 none_spec.protocol = ModuleProtocol::None;
16000 let mut none = Command::new("/nonexistent");
16001 let none_handoff =
16002 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16003 .expect("protocol-none spawn args apply");
16004 assert!(
16005 none_handoff.is_none(),
16006 "protocol:none spawn must not receive a nonce descriptor"
16007 );
16008 #[cfg(windows)]
16009 assert!(!none.as_std().get_envs().any(|(key, value)| key
16010 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16011 && value.is_some()));
16012 assert!(
16013 !none.as_std().get_envs().any(|(key, value)| key
16014 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16015 && value.is_some()),
16016 "protocol:none spawn must not name a nonce descriptor"
16017 );
16018 let none_args: Vec<String> = none
16019 .as_std()
16020 .get_args()
16021 .map(|a| a.to_string_lossy().into_owned())
16022 .collect();
16023 assert!(
16024 !none_args.iter().any(|a| a == SUBC_ARG),
16025 "protocol:none argv must not carry --subc; got {none_args:?}"
16026 );
16027 let none_has_nonce = none
16028 .as_std()
16029 .get_envs()
16030 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16031 assert!(
16032 !none_has_nonce,
16033 "protocol:none spawn must not receive a launch nonce"
16034 );
16035 let none_has_module_id = none
16036 .as_std()
16037 .get_envs()
16038 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16039 assert!(
16040 none_has_module_id,
16041 "SUBC_MODULE_ID is inert and stays on every path"
16042 );
16043 assert!(
16044 handle.spawn_nonce(&none_spec.module_id).is_none(),
16045 "no nonce record for a process that will never present one"
16046 );
16047
16048 let wire_spec = spec(Vec::new());
16050 let mut wire = Command::new("/nonexistent");
16051 let wire_handoff =
16052 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16053 .expect("subc-wire spawn args apply");
16054 let wire_fd_env = wire
16055 .as_std()
16056 .get_envs()
16057 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16058 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16059 #[cfg(unix)]
16060 assert_eq!(
16061 wire_fd_env,
16062 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16063 "a subc-wire spawn names the pipe it will receive at descriptor 3"
16064 );
16065 #[cfg(windows)]
16066 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16067 #[cfg(not(any(unix, windows)))]
16068 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16069 let wire_args: Vec<String> = wire
16070 .as_std()
16071 .get_args()
16072 .map(|a| a.to_string_lossy().into_owned())
16073 .collect();
16074 assert_eq!(
16075 wire_args,
16076 vec![
16077 SUBC_ARG.to_string(),
16078 connection_file.to_string_lossy().into_owned()
16079 ],
16080 "a subc-wire spawn still carries --subc <path>"
16081 );
16082 assert_eq!(
16083 wire.as_std()
16084 .get_envs()
16085 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16086 !cfg!(unix),
16087 "only Windows supplies the environment nonce"
16088 );
16089 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16090 }
16091
16092 #[test]
16102 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16103 let role = |command: &Command| {
16104 command
16105 .as_std()
16106 .get_envs()
16107 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16108 .last()
16109 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16110 };
16111 let forged = spec(vec![(
16112 SUBC_SPAWN_ROLE_ENV.to_string(),
16113 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16114 )]);
16115
16116 let mut plain = Command::new("/nonexistent");
16117 apply_child_env(&mut plain, &forged);
16118 apply_spawn_role(&mut plain, SpawnRole::Plain);
16119 assert_eq!(
16120 role(&plain),
16121 Some(None),
16122 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16123 );
16124
16125 let mut candidate = Command::new("/nonexistent");
16126 apply_child_env(&mut candidate, &spec(Vec::new()));
16127 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16128 assert_eq!(
16129 role(&candidate),
16130 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16131 );
16132 }
16133
16134 #[test]
16140 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16141 let mut command = Command::new("/nonexistent");
16142 apply_child_env(
16143 &mut command,
16144 &spec(vec![
16145 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16146 ("KEPT".to_string(), "yes".to_string()),
16147 ]),
16148 );
16149 let keys: Vec<String> = command
16150 .as_std()
16151 .get_envs()
16152 .filter(|(_, value)| value.is_some())
16153 .map(|(key, _)| key.to_string_lossy().into_owned())
16154 .collect();
16155 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16156 assert!(
16157 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16158 "daemon-private capture key leaked to the child: {keys:?}"
16159 );
16160 }
16161}
16162
16163#[cfg(test)]
16164mod jitter_tests {
16165 use super::jittered_health_delay;
16166 use std::{collections::HashSet, time::Duration};
16167
16168 const FLEET: [&str; 14] = [
16177 "aft",
16178 "alfonso-core",
16179 "magic-context",
16180 "broca",
16181 "thalamus",
16182 "quota",
16183 "engram",
16184 "plexus",
16185 "cerebellum",
16186 "astrocyte",
16187 "synapse",
16188 "subc-mcp",
16189 "cortexkit-credentials",
16190 "subc-federation",
16191 ];
16192
16193 #[test]
16201 fn probe_delays_disperse_across_the_fleet() {
16202 let cadence = Duration::from_secs(30);
16203 let delays: HashSet<Duration> = FLEET
16204 .iter()
16205 .map(|id| jittered_health_delay(id, 0, cadence))
16206 .collect();
16207 assert_eq!(
16208 delays.len(),
16209 FLEET.len(),
16210 "every supervised module must land on its own probe offset"
16211 );
16212 }
16213
16214 #[test]
16220 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16221 let cadence = Duration::from_secs(30);
16222 let span = cadence / 10;
16223 for id in FLEET {
16224 for probe_index in 0..8 {
16225 let delay = jittered_health_delay(id, probe_index, cadence);
16226 assert!(
16227 delay >= cadence,
16228 "{id}#{probe_index}: jitter must not shorten the cadence"
16229 );
16230 assert!(
16231 delay < cadence + span,
16232 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16233 );
16234 }
16235 }
16236 }
16237
16238 #[test]
16244 fn a_module_offset_is_stable_across_restarts() {
16245 let cadence = Duration::from_secs(30);
16246 for id in FLEET {
16247 assert_eq!(
16248 jittered_health_delay(id, 0, cadence),
16249 jittered_health_delay(id, 0, cadence),
16250 "{id}: the same module and probe index must produce the same offset"
16251 );
16252 }
16253 }
16254
16255 #[test]
16257 fn zero_cadence_yields_zero_delay() {
16258 assert_eq!(
16259 jittered_health_delay("aft", 0, Duration::ZERO),
16260 Duration::ZERO
16261 );
16262 }
16263}
16264
16265#[cfg(all(test, target_os = "linux"))]
16266mod cgroup_placement_tests {
16267 use super::{
16268 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16269 SuperviseError, SupervisedChild,
16270 };
16271 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16272 use cortexkit_test_support::ScratchDir;
16273 use std::{
16274 fs, io,
16275 path::{Path, PathBuf},
16276 sync::{Arc, Mutex},
16277 };
16278 use tokio::process::Command;
16279
16280 #[tokio::test]
16281 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16282 use super::*;
16283 let dir = ScratchDir::new("unique-spawn-cgroups");
16284 let root = PathBuf::from(format!(
16285 "/sys/fs/cgroup/subc-unique-test-{}-{}",
16286 std::process::id(),
16287 unix_ms_now()
16288 ));
16289 if let Err(error) = fs::create_dir(&root) {
16290 assert!(
16291 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16292 "required cgroup test cannot execute: {error}"
16293 );
16294 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16295 return;
16296 }
16297 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16298 let group_count = || {
16299 fs::read_dir(root.join("subc-modules"))
16300 .unwrap()
16301 .map(|entry| entry.unwrap().file_type().unwrap())
16302 .filter(|kind| kind.is_dir())
16303 .count()
16304 };
16305 let supervisor =
16306 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16307 .with_cgroup_placement(Some(placement.clone()));
16308 let runtime = supervisor.runtime_config();
16309 let mut spec = ModuleSpec {
16310 module_id: "unique-spawn".into(),
16311 program: PathBuf::from("/bin/sleep"),
16312 args: vec!["60".into()],
16313 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16314 .into_iter()
16315 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
16316 .collect(),
16317 reserved: false,
16318 reserved_prefixes: vec![],
16319 protocol: ModuleProtocol::None,
16320 overlap: Default::default(),
16321 };
16322 let spawn = |spec: &ModuleSpec| {
16323 spawn_child(
16324 spec,
16325 None,
16326 None,
16327 &runtime.stderr_ring,
16328 None,
16329 &runtime.child_roster,
16330 Some(&placement),
16331 )
16332 .unwrap()
16333 };
16334 let mut live = spawn(&spec);
16335 for _ in 0..3 {
16336 let next = spawn(&spec);
16338 assert_ne!(live.module_id, next.module_id);
16339 live.start_kill().unwrap();
16340 live.wait().await.unwrap();
16341 live = next;
16342 assert!(
16343 live.child.try_wait().unwrap().is_none(),
16344 "retiring the old slot must not kill the replacement"
16345 );
16346 assert_eq!(
16347 group_count(),
16348 1,
16349 "only the live spawn's cgroup should remain"
16350 );
16351 }
16352 supervisor.begin_daemon_shutdown();
16353 let reap = tokio::spawn(async move {
16354 live.wait().await.unwrap();
16355 });
16356 supervisor
16357 .end_children_for_daemon_shutdown(false, std::future::pending())
16358 .await;
16359 reap.await.unwrap();
16360 assert_eq!(group_count(), 0);
16361 spec.program = PathBuf::from("/bin/true");
16363 spec.args.clear();
16364 let fresh_roster = ChildRoster::default();
16365 let mut short = spawn_child(
16366 &spec,
16367 None,
16368 None,
16369 &runtime.stderr_ring,
16370 None,
16371 &fresh_roster,
16372 Some(&placement),
16373 )
16374 .unwrap();
16375 short.wait().await.unwrap();
16376 assert_eq!(group_count(), 0);
16377 spec.module_id = "_".repeat(255);
16378 let mut long_id = spawn_child(
16379 &spec,
16380 None,
16381 None,
16382 &runtime.stderr_ring,
16383 None,
16384 &fresh_roster,
16385 Some(&placement),
16386 )
16387 .unwrap();
16388 long_id.wait().await.unwrap();
16389 assert_eq!(
16390 group_count(),
16391 0,
16392 "valid long module IDs must not exceed cgroup NAME_MAX"
16393 );
16394 fs::remove_dir(root.join("subc-modules")).unwrap();
16395 fs::remove_dir(root).unwrap();
16396 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
16397 }
16398
16399 #[test]
16400 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
16401 let path = Path::new("/definitely-missing-subc-cgroup");
16402 let mut command = Command::new("true");
16403 let error = apply_cgroup_placement(
16404 &mut command,
16405 &ModuleSpec {
16406 module_id: "broken-cgroup".to_string(),
16407 program: PathBuf::from("true"),
16408 args: Vec::new(),
16409 env: Vec::new(),
16410 reserved: false,
16411 reserved_prefixes: Vec::new(),
16412 protocol: ModuleProtocol::Subc,
16413 overlap: Default::default(),
16414 },
16415 path,
16416 )
16417 .expect_err("a parent cgroup open failure must reject the supervised spawn");
16418 let reason = error.to_string();
16419
16420 assert!(
16421 matches!(error, SuperviseError::Cgroup { .. }),
16422 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
16423 );
16424 assert!(
16425 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
16426 "parent cgroup open failure must name cgroup.procs: {reason}"
16427 );
16428 }
16429
16430 #[tokio::test]
16431 async fn reaping_a_child_removes_its_empty_module_cgroup() {
16432 let root = ScratchDir::new("supervisor-reap-cgroup");
16433 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16434 let placement = subc_cgroup::prepare_at(&root)
16435 .expect("prepare scratch cgroup root")
16436 .expect("scratch root has a cgroup.procs marker");
16437 let module_id = "reaped-module";
16438 let module = placement
16439 .module_path(module_id)
16440 .expect("create scratch module cgroup");
16441 let child = Command::new("true")
16442 .env("XDG_DATA_HOME", root.path())
16443 .env("XDG_RUNTIME_DIR", root.path())
16444 .env("XDG_CONFIG_HOME", root.path())
16445 .spawn()
16446 .expect("spawn short-lived child");
16447 let pid = child.id().expect("spawned child has pid");
16448 let mut child = SupervisedChild {
16449 child,
16450 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
16451 protocol: ModuleProtocol::Subc,
16452 module_id: module_id.to_string(),
16453 cgroup_placement: Some(placement),
16454 stdout_pump: None,
16455 stderr_pump: None,
16456 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
16457 spawned_at: tokio::time::Instant::now(),
16458 registration_deadline: None,
16459 registration_watch: None,
16460 spawned_at_ms: 0,
16461 spawned_from: PathBuf::from("true"),
16462 launch_spec_digest: LaunchSpecDigest([0; 32]),
16463 spawned_file_identity: None,
16464 process_start_time: None,
16465 process_identity: None,
16466 pid,
16467 roster_guard: None,
16468 #[cfg(target_os = "macos")]
16469 privacy_exec: None,
16470 spawn_failure: None,
16471 };
16472
16473 child.wait().await.expect("reap short-lived child");
16474
16475 assert!(
16476 !module.exists(),
16477 "reaping the supervised child must remove its empty cgroup"
16478 );
16479 }
16480
16481 #[test]
16482 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
16483 let root = ScratchDir::new("supervisor-non-empty-cgroup");
16484 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16485 let placement = subc_cgroup::prepare_at(&root)
16486 .expect("prepare scratch cgroup root")
16487 .expect("scratch root has a cgroup.procs marker");
16488 let module = placement
16489 .module_path("surviving-module")
16490 .expect("create scratch module cgroup");
16491 fs::write(module.join("surviving-process"), b"still present")
16492 .expect("make scratch cgroup non-empty");
16493 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
16494
16495 remove_module_cgroup(&placement, "surviving-module");
16496
16497 let logs = crate::router::test_log::captured_logs(&logs);
16498 assert!(
16499 module.exists(),
16500 "failed removal must leave the cgroup intact"
16501 );
16502 assert!(
16503 logs.contains("could not remove module cgroup after process exit; continuing teardown")
16504 && logs.contains("surviving-module"),
16505 "best-effort removal must report the failure without returning it: {logs}"
16506 );
16507 }
16508
16509 #[test]
16510 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
16511 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
16512 let reason = SuperviseError::Spawn {
16513 program: PathBuf::from("/bin/true"),
16514 source: io::Error::from_raw_os_error(13),
16515 cgroup_path: Some(cgroup_path.clone()),
16516 }
16517 .to_string();
16518
16519 assert!(
16520 reason.contains(&cgroup_path.display().to_string()),
16521 "a pre_exec spawn failure must name the cgroup path: {reason}"
16522 );
16523 }
16524}
16525
16526#[cfg(test)]
16527mod spawn_subscriber_lag_tests {
16528 use super::*;
16529
16530 #[tokio::test]
16535 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
16536 let feed = SpawnEventFeed::default();
16537 feed.configure_incarnation("lag-incarnation".to_string());
16538 let (tx, mut rx) = mpsc::channel(1);
16541 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
16542 .expect("subscribe");
16543 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
16544 for index in 0..emitted {
16545 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
16546 tokio::task::yield_now().await;
16549 }
16550 assert_eq!(
16551 feed.subscriber_count(),
16552 0,
16553 "the lagged subscriber must be removed"
16554 );
16555
16556 let mut data = Vec::new();
16557 let mut last = None;
16558 loop {
16559 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16560 .await
16561 .expect("the forwarder must finish once the subscriber is dropped");
16562 let Some(outbound) = next else { break };
16563 let frame = outbound.frame;
16564 if frame.header.ty == FrameType::StreamData {
16565 assert!(last.is_none(), "no data may follow the terminal frame");
16566 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
16567 data.push(event.cursor.seq);
16568 } else {
16569 assert!(last.is_none(), "exactly one terminal frame");
16570 last = Some(frame);
16571 }
16572 }
16573 assert!(!data.is_empty(), "queued frames drain before the terminal");
16574 for pair in data.windows(2) {
16575 assert_eq!(
16576 pair[1],
16577 pair[0] + 1,
16578 "queued frames arrive dense and in order"
16579 );
16580 }
16581 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
16582 assert_eq!(terminal.header.ty, FrameType::Error);
16583 assert_eq!(terminal.header.corr, 7);
16584 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
16585 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
16586 let detail = body.detail.expect("lagged error carries detail");
16587 assert_eq!(
16588 detail["first_undelivered_cursor"]["seq"],
16589 data.last().unwrap() + 1,
16590 "the named cursor is the first event the subscriber did not receive"
16591 );
16592 assert_eq!(
16593 detail["first_undelivered_cursor"]["daemon_incarnation"],
16594 "lag-incarnation"
16595 );
16596 }
16597}
16598
16599#[cfg(test)]
16600mod terminal_history_read_concurrency_tests {
16601 use super::*;
16602 use crate::terminal_journal::read_pause;
16603 use cortexkit_test_support::ScratchDir;
16604 use std::sync::mpsc as std_mpsc;
16605
16606 fn journaled_ring(
16607 journal: &Arc<crate::terminal_journal::TerminalJournal>,
16608 ) -> Arc<Mutex<TerminalRing>> {
16609 Arc::new(Mutex::new(
16610 TerminalRing::new(TerminalRingConfig::default(), 1)
16611 .with_journal(Some(Arc::clone(journal))),
16612 ))
16613 }
16614
16615 fn crash(at_ms: u64) -> ExitReport {
16616 ExitReport {
16617 kind: ExitKind::Crash,
16618 code: Some(1),
16619 signal: None,
16620 at_ms,
16621 }
16622 }
16623
16624 fn record_within(
16627 module_id: &'static str,
16628 ring: &Arc<Mutex<TerminalRing>>,
16629 at_ms: u64,
16630 bound: Duration,
16631 ) -> bool {
16632 let ring = Arc::clone(ring);
16633 let (done, done_rx) = std_mpsc::channel();
16634 std::thread::spawn(move || {
16635 record_terminal(
16636 module_id,
16637 &ring,
16638 &SpawnEventFeed::default(),
16639 &crash(at_ms),
16640 TerminalDisposition::Restarting,
16641 );
16642 let _ = done.send(());
16643 });
16644 done_rx.recv_timeout(bound).is_ok()
16645 }
16646
16647 #[test]
16652 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
16653 let dir = ScratchDir::new("terminal-history-concurrent-read");
16654 let path = dir.join("terminals.jsonl");
16655 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
16656 path.clone(),
16657 "daemon".into(),
16658 ));
16659 let reader_ring = journaled_ring(&journal);
16660 let other_ring = journaled_ring(&journal);
16661 assert!(record_within(
16662 "reader-module",
16663 &reader_ring,
16664 10,
16665 Duration::from_secs(5)
16666 ));
16667
16668 let (started, release) = read_pause::install(&path);
16669 let reading = {
16670 let ring = Arc::clone(&reader_ring);
16671 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
16672 };
16673 started
16674 .recv_timeout(Duration::from_secs(5))
16675 .expect("the history read reached its pause");
16676
16677 let bound = Duration::from_secs(1);
16678 assert!(
16679 record_within("other-module", &other_ring, 20, bound),
16680 "another module's exit waited on a history read (journal writer held)"
16681 );
16682 assert!(
16683 record_within("reader-module", &reader_ring, 30, bound),
16684 "the read module's own exit waited on its history read (ring held)"
16685 );
16686
16687 drop(release);
16688 let paused = reading.join().unwrap();
16689 assert_eq!(
16690 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16691 vec![10],
16692 "an exit recorded after the read began lands in neither half of it"
16693 );
16694 assert_eq!(paused.journal_skipped_lines, 0);
16695 assert_eq!(paused.journal_read_errors, 0);
16696
16697 let after = durable_terminal_history_of(&reader_ring, "reader-module");
16698 assert_eq!(
16699 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16700 vec![10, 30],
16701 "the next read merges ring and journal with no duplicate"
16702 );
16703 assert_eq!(after.journal_skipped_lines, 0);
16704 }
16705}
16706
16707#[cfg(test)]
16712mod stderr_settle_tests {
16713 use std::{
16714 future::Future,
16715 io,
16716 pin::Pin,
16717 sync::{Arc, Mutex},
16718 task::{Context, Poll},
16719 time::Duration,
16720 };
16721
16722 use tokio::{
16723 io::{AsyncRead, ReadBuf},
16724 sync::oneshot,
16725 time::Instant,
16726 };
16727
16728 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
16729 use crate::stderr_tail::{
16730 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
16731 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
16732 };
16733
16734 const BOUND: Duration = Duration::from_millis(250);
16735
16736 struct HeldReader {
16740 before: Option<Vec<u8>>,
16741 gate: Option<oneshot::Receiver<()>>,
16742 after: io::Cursor<Vec<u8>>,
16743 }
16744
16745 impl AsyncRead for HeldReader {
16746 fn poll_read(
16747 mut self: Pin<&mut Self>,
16748 cx: &mut Context<'_>,
16749 buf: &mut ReadBuf<'_>,
16750 ) -> Poll<io::Result<()>> {
16751 if let Some(bytes) = self.before.take() {
16752 buf.put_slice(&bytes);
16753 return Poll::Ready(Ok(()));
16754 }
16755 if let Some(gate) = self.gate.as_mut() {
16756 match Pin::new(gate).poll(cx) {
16757 Poll::Pending => return Poll::Pending,
16758 Poll::Ready(_) => self.gate = None,
16759 }
16760 }
16761 Pin::new(&mut self.after).poll_read(cx, buf)
16762 }
16763 }
16764
16765 struct DiscardSink;
16766
16767 impl OutputSink for DiscardSink {
16768 fn write_line(&mut self, _line: &[u8]) {}
16769 }
16770
16771 fn line(text: &str) -> TailEntry {
16772 TailEntry::Line {
16773 text: text.to_string(),
16774 truncated: false,
16775 at_ms: None,
16776 }
16777 }
16778
16779 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16780 ring.lock().unwrap()
16781 }
16782
16783 fn held_pump(
16787 ring: &Arc<Mutex<StderrRing>>,
16788 before: &str,
16789 after: &str,
16790 ) -> (StderrPump, oneshot::Sender<()>) {
16791 let generation = lock(ring).begin_process();
16792 let (release, gate) = oneshot::channel();
16793 let reader = HeldReader {
16794 before: Some(before.as_bytes().to_vec()),
16795 gate: Some(gate),
16796 after: io::Cursor::new(after.as_bytes().to_vec()),
16797 };
16798 let task = tokio::spawn(pump_stderr_to(
16799 reader,
16800 Arc::clone(ring),
16801 generation,
16802 DiscardSink,
16803 ));
16804 (StderrPump { task, generation }, release)
16805 }
16806
16807 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16808 for _ in 0..1000 {
16809 if done(&lock(ring)) {
16810 return;
16811 }
16812 tokio::time::sleep(Duration::from_millis(1)).await;
16813 }
16814 panic!(
16815 "ring never reached the expected state: {:?}",
16816 lock(ring).snapshot(None, None)
16817 );
16818 }
16819
16820 #[tokio::test(start_paused = true)]
16821 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16822 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16823 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16824
16825 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16826 let before_release = lock(&ring).snapshot(None, None);
16827 assert!(
16828 matches!(before_release.capture, CaptureState::Incomplete { .. }),
16829 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16830 );
16831
16832 let next = lock(&ring).begin_process();
16835 lock(&ring).push_line_from(next, "next process booting");
16836 release.send(()).unwrap();
16837 wait_until(&ring, |ring| {
16838 ring.snapshot(None, None).capture == CaptureState::Captured
16839 })
16840 .await;
16841
16842 assert_eq!(
16843 untimed(lock(&ring).snapshot(None, None).entries),
16844 vec![
16845 line("booting"),
16846 line("config error: missing storage"),
16847 TailEntry::ProcessStart,
16848 line("next process booting"),
16849 ],
16850 "the crash's last line must survive a slow reader and stay in the crashed process's section"
16851 );
16852 }
16853
16854 #[tokio::test(start_paused = true)]
16855 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16856 ) {
16857 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16858 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16861
16862 let started = Instant::now();
16863 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16864 assert_eq!(
16865 started.elapsed(),
16866 BOUND,
16867 "the restart must wait exactly the bound for a pipe that stays open, no longer"
16868 );
16869
16870 let next = lock(&ring).begin_process();
16871 lock(&ring).push_line_from(next, "next process booting");
16872 tokio::time::sleep(Duration::from_secs(60)).await;
16873
16874 let snapshot = lock(&ring).snapshot(None, None);
16875 match &snapshot.capture {
16876 CaptureState::Incomplete { reason } => assert!(
16877 reason.contains("had not reached EOF") && reason.contains("250ms"),
16878 "the reason must say what is missing and after how long: {reason}"
16879 ),
16880 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16881 }
16882 assert_eq!(
16883 untimed(snapshot.entries),
16884 vec![
16885 line("parent exiting"),
16886 TailEntry::ProcessStart,
16887 line("next process booting"),
16888 ]
16889 );
16890 }
16891
16892 #[tokio::test(start_paused = true)]
16893 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16894 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16895 let (pump, release) = held_pump(&ring, "one\n", "two\n");
16896 release.send(()).unwrap();
16897
16898 settle_stderr_pump("clean", &ring, pump, BOUND).await;
16899
16900 let snapshot = lock(&ring).snapshot(None, None);
16901 assert_eq!(snapshot.capture, CaptureState::Captured);
16902 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16903 }
16904
16905 #[tokio::test(start_paused = true)]
16906 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16907 const BURST: usize = 150;
16908 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16909 let path = capture.join("two-pipe.stderr.log");
16910 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16911 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16912 let generation = lock(&ring).begin_process();
16913 let stdout = (0..BURST)
16914 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16915 .collect::<String>();
16916 let stderr = (0..BURST)
16917 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16918 .collect::<String>();
16919 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
16924 let (release, gate) = oneshot::channel();
16925 let reader = HeldReader {
16926 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
16927 gate: Some(gate),
16928 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
16929 };
16930 let (finished, completion) = oneshot::channel();
16931 let stdout_sink = sink.clone();
16932 let pump = tokio::spawn(async move {
16933 pump_stdout_to(reader, stdout_sink).await;
16934 finished.send(()).unwrap();
16935 });
16936 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
16937
16938 let started = Instant::now();
16939 settle_stdout_pump("two-pipe", pump, BOUND).await;
16940 assert_eq!(
16941 started.elapsed(),
16942 BOUND,
16943 "a late reader must not hold up restart"
16944 );
16945 assert_eq!(
16946 std::fs::read_to_string(&path).unwrap().lines().count(),
16947 BURST + 1
16948 );
16949
16950 release
16951 .send(())
16952 .expect("stdout reader must survive the exit drain bound");
16953 tokio::time::timeout(Duration::from_secs(1), completion)
16954 .await
16955 .expect("released stdout reader must reach EOF")
16956 .expect("stdout capture must finish at EOF, not be cancelled");
16957 let contents = std::fs::read_to_string(&path).unwrap();
16958 let mut lines = contents
16959 .lines()
16960 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
16961 .collect::<Vec<_>>();
16962 let mut expected = (0..BURST)
16963 .flat_map(|i| {
16964 [
16965 format!("out-{i:04}-{}", "o".repeat(64)),
16966 format!("err-{i:04}-{}", "e".repeat(64)),
16967 ]
16968 })
16969 .collect::<Vec<_>>();
16970 lines.sort_unstable();
16971 expected.sort_unstable();
16972 assert_eq!(
16973 lines, expected,
16974 "both pipes must retain every complete line exactly once"
16975 );
16976 }
16977}
16978
16979#[cfg(all(test, windows))]
16993mod job_containment_tests {
16994 use super::*;
16995 use cortexkit_test_support::ScratchDir;
16996 use std::{
16997 path::{Path, PathBuf},
16998 sync::{Arc, Mutex},
16999 time::{Duration, Instant},
17000 };
17001
17002 fn stub_path() -> PathBuf {
17008 let mut path = std::env::current_exe().expect("current_exe available in tests");
17009 path.pop();
17010 path.pop();
17011 path.push("fake-aft-stub.exe");
17012 assert!(
17013 path.exists(),
17014 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17015 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17016 path.display()
17017 );
17018 path
17019 }
17020
17021 fn read_grandchild_pid(path: &Path) -> u32 {
17023 let deadline = Instant::now() + Duration::from_secs(10);
17024 loop {
17025 if let Ok(contents) = std::fs::read_to_string(path) {
17026 if let Ok(pid) = contents.trim().parse() {
17027 return pid;
17028 }
17029 }
17030 assert!(
17031 Instant::now() < deadline,
17032 "the stub never recorded a grandchild pid at {}",
17033 path.display()
17034 );
17035 std::thread::sleep(Duration::from_millis(10));
17036 }
17037 }
17038
17039 struct Fixture {
17042 _dir: ScratchDir,
17043 module_id: String,
17044 grandchild: u32,
17045 child: Option<SupervisedChild>,
17046 registry: Arc<Registry>,
17047 snapshot: Arc<Mutex<SupervisorSnapshot>>,
17048 terminal_ring: Arc<Mutex<TerminalRing>>,
17049 spawn_events: SpawnEventFeed,
17050 }
17051
17052 fn fixture(label: &str, module_id: &str) -> Fixture {
17053 let dir = ScratchDir::new(label);
17054 let pid_file = dir.join("grandchild.pid");
17055 let supervisor = Supervisor::new_for_test(
17056 Arc::new(Registry::default()),
17057 RestartPolicy::new(3, Duration::ZERO),
17058 );
17059 let runtime = supervisor.runtime_config();
17060 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17061 let spec = ModuleSpec {
17062 module_id: module_id.to_string(),
17063 program: stub_path(),
17064 args: Vec::new(),
17068 env: vec![
17069 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17070 (
17071 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17072 pid_file.display().to_string(),
17073 ),
17074 ],
17075 reserved: false,
17076 reserved_prefixes: Vec::new(),
17077 protocol: ModuleProtocol::Subc,
17078 overlap: Default::default(),
17079 };
17080 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17081 .expect("spawn the supervised fixture");
17082 let grandchild = read_grandchild_pid(&pid_file);
17083 Fixture {
17084 _dir: dir,
17085 module_id: module_id.to_string(),
17086 grandchild,
17087 child: Some(child),
17088 registry: Arc::new(Registry::default()),
17089 snapshot,
17090 terminal_ring: Arc::clone(&runtime.terminal_ring),
17091 spawn_events: SpawnEventFeed::default(),
17092 }
17093 }
17094
17095 impl Fixture {
17096 async fn drain(&mut self) {
17098 let child = self
17099 .child
17100 .take()
17101 .expect("the fixture child is still present");
17102 drain_child_to_state(
17103 &self.module_id,
17104 ModuleProtocol::Subc,
17105 StopNotice::NotSent,
17108 &self.registry,
17109 None,
17110 &self.snapshot,
17111 &self.terminal_ring,
17112 &self.spawn_events,
17113 child,
17114 Duration::from_millis(500),
17115 ModuleState::Stopped,
17116 Some(false),
17117 &FailureContext::operator(),
17118 )
17119 .await
17120 .expect("drain the supervised fixture");
17121 }
17122 }
17123
17124 #[tokio::test]
17130 async fn teardown_reaps_the_grandchild() {
17131 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17132 let grandchild = fixture.grandchild;
17133
17134 assert!(
17135 subc_jobobject::process_exists(grandchild),
17136 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17137 );
17138
17139 fixture.drain().await;
17140
17141 assert!(
17142 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17143 "grandchild {grandchild} outlived module teardown: the tree was not contained"
17144 );
17145 }
17146
17147 #[test]
17160 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17161 let dir = ScratchDir::new("teardown-uncontained");
17162 let pid_file = dir.join("grandchild.pid");
17163 let mut child = std::process::Command::new(stub_path())
17164 .env("FAKE_AFT_NEVER_CONNECT", "1")
17165 .env(
17166 "FAKE_AFT_GRANDCHILD_PID_FILE",
17167 pid_file.display().to_string(),
17168 )
17169 .stdin(std::process::Stdio::null())
17170 .stdout(std::process::Stdio::null())
17171 .stderr(std::process::Stdio::null())
17172 .spawn()
17173 .expect("spawn the uncontained fixture");
17174 let grandchild = read_grandchild_pid(&pid_file);
17175
17176 child.kill().expect("kill the direct child");
17178 let _ = child.wait();
17179
17180 assert!(
17181 subc_jobobject::process_exists(grandchild),
17182 "grandchild {grandchild} died with the direct child, so this control no longer \
17183 distinguishes contained from uncontained teardown and the regression test is \
17184 passing vacuously"
17185 );
17186
17187 kill_tree(grandchild);
17190 }
17191
17192 #[tokio::test]
17201 async fn dropping_containment_reaps_the_grandchild() {
17202 let mut fixture = fixture("drop-containment", "tree-drop");
17203 let grandchild = fixture.grandchild;
17204
17205 assert!(subc_jobobject::process_exists(grandchild));
17206
17207 fixture.child.as_mut().expect("child present").job = None;
17209
17210 assert!(
17211 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17212 "grandchild {grandchild} survived the containment handle closing, so a daemon \
17213 crash would leave the tree behind"
17214 );
17215 }
17216
17217 fn kill_tree(pid: u32) {
17219 let _ = std::process::Command::new("taskkill.exe")
17220 .args(["/PID", &pid.to_string(), "/T", "/F"])
17221 .stdin(std::process::Stdio::null())
17222 .stdout(std::process::Stdio::null())
17223 .stderr(std::process::Stdio::null())
17224 .status();
17225 assert!(
17226 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17227 "could not clean up grandchild {pid}"
17228 );
17229 }
17230}
17231
17232#[cfg(test)]
17233mod privacy_trampoline_configuration_tests {
17234 #[tokio::test]
17235 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17236 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17237 #[cfg(target_os = "macos")]
17238 {
17239 let supervisor = super::Supervisor::new(
17240 std::sync::Arc::new(crate::Registry::default()),
17241 super::RestartPolicy::default(),
17242 );
17243 let error = supervisor.spawn(spec()).unwrap_err();
17244 assert!(
17245 error
17246 .to_string()
17247 .contains("no privacy trampoline configured"),
17248 "{error}"
17249 );
17250 }
17251 }
17252
17253 #[tokio::test]
17254 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17255 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17256 #[cfg(target_os = "macos")]
17257 {
17258 let supervisor = super::Supervisor::new(
17259 std::sync::Arc::new(crate::Registry::default()),
17260 super::RestartPolicy::default(),
17261 )
17262 .with_privacy_trampoline(std::env::current_exe().unwrap());
17263 let error = supervisor.spawn(spec()).unwrap_err();
17264 assert!(
17265 error
17266 .to_string()
17267 .contains("binary does not implement the privacy trampoline protocol"),
17268 "{error}"
17269 );
17270 }
17271 }
17272
17273 #[cfg(target_os = "macos")]
17274 fn spec() -> super::ModuleSpec {
17275 super::ModuleSpec {
17276 module_id: "privacy-configuration".into(),
17277 program: "/bin/sleep".into(),
17278 args: vec!["30".into()],
17279 env: vec![],
17280 reserved: false,
17281 reserved_prefixes: vec![],
17282 protocol: subc_control::ModuleProtocol::None,
17283 overlap: super::ModuleOverlap::Exclusive,
17284 }
17285 }
17286}
17287
17288#[cfg(test)]
17289mod privacy_exec_boundary_tests {
17290 #[cfg(target_os = "macos")]
17291 use super::*;
17292 #[cfg(target_os = "macos")]
17293 use std::{
17294 io::{Read, Write},
17295 net::{TcpListener, TcpStream},
17296 };
17297
17298 #[cfg(target_os = "macos")]
17301 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
17302 if let Some((_, path)) = spec
17303 .env
17304 .iter()
17305 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
17306 {
17307 let mut barrier = TcpStream::connect(path).unwrap();
17308 barrier
17309 .set_read_timeout(Some(Duration::from_secs(30)))
17310 .unwrap();
17311 barrier.write_all(&pid.to_ne_bytes()).unwrap();
17312 let mut release = [0];
17313 barrier.read_exact(&mut release).unwrap();
17314 assert_eq!(&release, b"X");
17315 }
17316 }
17317
17318 #[cfg(target_os = "macos")]
17319 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
17320 ModuleSpec {
17321 module_id: "privacy-boundary".into(),
17322 program: program.into(),
17323 args: args.iter().map(|arg| (*arg).into()).collect(),
17324 env: vec![],
17325 reserved: false,
17326 reserved_prefixes: vec![],
17327 protocol: ModuleProtocol::None,
17328 overlap: ModuleOverlap::Exclusive,
17329 }
17330 }
17331
17332 #[cfg(target_os = "macos")]
17333 async fn accept(listener: TcpListener) -> TcpStream {
17334 let listener = tokio::net::TcpListener::from_std({
17336 listener.set_nonblocking(true).unwrap();
17337 listener
17338 })
17339 .unwrap();
17340 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
17341 .await
17342 .unwrap()
17343 .unwrap();
17344 let stream = stream.into_std().unwrap();
17345 stream.set_nonblocking(false).unwrap();
17346 stream
17347 .set_read_timeout(Some(Duration::from_secs(30)))
17348 .unwrap();
17349 stream
17350 }
17351
17352 #[tokio::test]
17353 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17354 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
17355 #[cfg(target_os = "macos")]
17356 {
17357 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
17358 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17361 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17362 let record = root.join("live-children.json");
17363 let supervisor =
17364 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17365 .with_live_children_record(&record);
17366 let runtime = supervisor.runtime_config();
17367 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17368 let mut spec = spec("/bin/sleep", &["30"]);
17369 spec.env = vec![
17370 (
17371 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
17372 exec_listener.local_addr().unwrap().to_string(),
17373 ),
17374 (
17375 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
17376 sample_listener.local_addr().unwrap().to_string(),
17377 ),
17378 ];
17379 let spawn = tokio::task::spawn_blocking(move || {
17380 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
17381 });
17382 let mut sample = accept(sample_listener).await;
17383 let mut pid = [0; 4];
17384 sample.read_exact(&mut pid).unwrap();
17385 let pid = u32::from_ne_bytes(pid);
17386 let mut exec = accept(exec_listener).await;
17387 let mut ready = [0];
17388 exec.read_exact(&mut ready).unwrap();
17389 assert_eq!(&ready, b"R");
17390 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
17393 assert_eq!(
17394 observe_spawned_image(pid).unwrap().executable,
17395 Some(trampoline)
17396 );
17397 sample.write_all(b"X").unwrap();
17398 let mut child = spawn.await.unwrap();
17399 let early = crate::live_children::read_record(&record).unwrap();
17400 assert_eq!(early.len(), 1);
17401 assert_eq!(early[0].pid, pid);
17402 assert_eq!(
17403 early[0].executable, None,
17404 "unconfirmed trampoline image entered the roster"
17405 );
17406 assert!(child.report_ready.get().is_none());
17407 child.privacy_exec.as_mut().unwrap().deadline =
17410 tokio::time::Instant::now() + Duration::from_secs(30);
17411 exec.write_all(b"X").unwrap();
17412 child.confirm_privacy_exec().await;
17413 assert_eq!(child.spawn_failure, None);
17414 assert!(child.report_ready.get().is_some());
17415 let confirmed = crate::live_children::read_record(&record).unwrap();
17416 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
17417 assert_ne!(module, trampoline);
17418 assert_eq!(confirmed[0].executable, Some(module.into()));
17419 child.start_kill().unwrap();
17420 child.wait().await.unwrap();
17421 child.release_roster();
17422 }
17423 }
17424
17425 #[tokio::test]
17426 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17427 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
17428 #[cfg(target_os = "macos")]
17429 {
17430 let registry = Arc::new(Registry::default());
17431 let policy = RestartPolicy::new(0, Duration::ZERO);
17432 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
17433 let runtime = supervisor.runtime_config();
17434 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17435 let spec = spec("/bin/sh", &["-c", "exit 121"]);
17436 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17440 let pid = child.pid;
17441 tokio::task::spawn_blocking(move || {
17442 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
17443 })
17444 .await
17445 .unwrap()
17446 .unwrap();
17447 child.privacy_exec.as_mut().unwrap().deadline =
17448 tokio::time::Instant::now() + Duration::from_secs(30);
17449 let status = child.wait().await.unwrap();
17450 assert_eq!(status.code(), Some(121));
17451 assert!(child.privacy_exec.is_none());
17452 assert!(
17453 child.report_ready.get().is_none(),
17454 "an exited module must not publish a live pid"
17455 );
17456 let report = classify_reaped_child_exit(&snapshot, &child, &status);
17457 on_child_exit(
17458 &spec,
17459 policy,
17460 ®istry,
17461 &snapshot,
17462 &runtime.terminal_ring,
17463 &runtime.spawn_events,
17464 &runtime.child_roster,
17465 report,
17466 true,
17467 )
17468 .await;
17469 let state = lock_snapshot(&snapshot).unwrap();
17470 assert_eq!(state.state, ModuleState::Failed);
17471 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
17472 assert_eq!(state.reported_pid(), None);
17473 drop(state);
17474 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17475 assert_eq!(history.entries.len(), 1);
17476 let terminal = &history.entries[0];
17477 assert_eq!(terminal.exit_code, Some(121));
17478 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
17479 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
17480 assert_eq!(
17481 terminal.disposition_detail.as_deref(),
17482 Some(policy.budget_exhausted_detail().as_str()),
17483 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
17484 );
17485 assert_eq!(child.spawn_failure, None);
17486 child.release_roster();
17487 }
17488 }
17489}
17490
17491#[cfg(all(test, unix))]
17495mod launch_nonce_descriptor_tests {
17496 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
17497 use crate::stderr_tail::{StderrRing, StderrTailConfig};
17498 use cortexkit_test_support::ScratchDir;
17499 use std::{
17500 path::PathBuf,
17501 sync::{Arc, Mutex},
17502 time::{Duration, Instant},
17503 };
17504
17505 async fn probe(role: super::SpawnRole) {
17506 let scratch = ScratchDir::new("launch-nonce-descriptor");
17507 let fd_copy = scratch.join("from-descriptor");
17508 let env_copy = scratch.join("environment");
17509 let script = format!(
17510 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
17511 fd = fd_copy.display(), env = env_copy.display(),
17512 );
17513 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
17514 let spec = ModuleSpec {
17515 module_id: "nonce-descriptor-probe".to_string(),
17516 program: PathBuf::from("/bin/sh"),
17517 args: vec!["-c".to_string(), script],
17518 env: vec![
17519 xdg("XDG_DATA_HOME"),
17520 xdg("XDG_RUNTIME_DIR"),
17521 xdg("XDG_CONFIG_HOME"),
17522 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
17523 ],
17524 reserved: true,
17525 reserved_prefixes: Vec::new(),
17526 protocol: ModuleProtocol::Subc,
17527 overlap: Default::default(),
17528 };
17529 let handle = SupervisorHandle::new();
17530 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17531 let roster = ChildRoster::default();
17532 #[cfg(target_os = "macos")]
17533 {
17534 let path = super::test_privacy_trampoline();
17535 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
17536 }
17537 let child = super::spawn_child_in_slot(
17538 &spec,
17539 None,
17540 Some(&handle),
17541 &ring,
17542 None,
17543 &roster,
17544 #[cfg(target_os = "linux")]
17545 None,
17546 role,
17547 matches!(role, super::SpawnRole::SwapCandidate),
17548 )
17549 .expect("spawn probe");
17550 let deadline = Instant::now() + Duration::from_secs(10);
17551 while !(fd_copy.exists() && env_copy.exists()) {
17552 assert!(Instant::now() < deadline, "probe never wrote its copies");
17553 tokio::time::sleep(Duration::from_millis(20)).await;
17554 }
17555 let nonce = std::fs::read_to_string(fd_copy).unwrap();
17556 assert!(!nonce.is_empty());
17557 let environment = std::fs::read_to_string(env_copy).unwrap();
17558 assert!(environment
17559 .lines()
17560 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
17561 let copy = environment
17562 .lines()
17563 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
17564 assert_eq!(
17565 copy, None,
17566 "Unix children must never receive the environment nonce"
17567 );
17568 if matches!(role, super::SpawnRole::Plain) {
17569 assert_eq!(
17570 handle.spawn_nonce(&spec.module_id).as_deref(),
17571 Some(nonce.as_str())
17572 );
17573 }
17574 drop(child);
17575 }
17576
17577 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17578 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
17579 probe(super::SpawnRole::Plain).await;
17580 }
17581
17582 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17583 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
17584 probe(super::SpawnRole::SwapCandidate).await;
17585 }
17586}
17587
17588#[cfg(all(test, target_os = "linux"))]
17589mod cgroup_containment_tests {
17590 use super::*;
17591 use cortexkit_test_support::ScratchDir;
17592
17593 fn running(pid: u32) -> bool {
17594 std::fs::read_to_string(format!("/proc/{pid}/stat"))
17596 .ok()
17597 .and_then(|stat| {
17598 stat.rsplit_once(") ")
17599 .map(|(_, rest)| rest.starts_with('Z'))
17600 })
17601 .is_some_and(|zombie| !zombie)
17602 }
17603
17604 #[tokio::test]
17605 async fn linux_teardown_reaps_the_grandchild() {
17606 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
17607 }
17608
17609 #[tokio::test]
17610 async fn linux_shutdown_straggler_reaps_the_grandchild() {
17611 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
17612 }
17613
17614 async fn teardown_tree(test_name: &str, shutdown: bool) {
17615 let dir = ScratchDir::new(test_name);
17616 let root = PathBuf::from(format!(
17617 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
17618 std::process::id(),
17619 unix_ms_now()
17620 ));
17621 if let Err(error) = std::fs::create_dir(&root) {
17622 assert!(
17623 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17624 "required cgroup test cannot execute: {error}"
17625 );
17626 eprintln!(
17627 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
17628 root.display()
17629 );
17630 return;
17631 }
17632 let placement = subc_cgroup::prepare_at(&root)
17633 .expect("prepare isolated kernel cgroup")
17634 .expect("isolated cgroup is delegated");
17635 let module_id = "tree-teardown";
17636 let module = placement
17637 .module_path(module_id)
17638 .expect("create isolated module cgroup");
17639 if !module.join("cgroup.kill").exists() {
17640 std::fs::remove_dir(&module).unwrap();
17641 std::fs::remove_dir(root.join("subc-modules")).unwrap();
17642 std::fs::remove_dir(&root).unwrap();
17643 assert!(
17644 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17645 "required cgroup.kill interface unavailable"
17646 );
17647 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
17648 return;
17649 }
17650 let supervisor = Supervisor::new_for_test(
17651 Arc::new(Registry::default()),
17652 RestartPolicy::new(3, Duration::ZERO),
17653 )
17654 .with_cgroup_placement(Some(placement));
17655 let mut runtime = supervisor.runtime_config();
17656 runtime.child_roster = runtime
17657 .child_roster
17658 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
17659 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17660 let pid_file = dir.join("grandchild.pid");
17661 let spec = ModuleSpec {
17662 module_id: module_id.to_string(),
17663 program: PathBuf::from("/bin/sh"),
17664 args: vec![
17665 "-c".into(),
17666 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
17667 "fixture".into(),
17668 pid_file.display().to_string(),
17669 ],
17670 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17671 .into_iter()
17672 .map(|key| (key.to_string(), dir.display().to_string()))
17673 .collect(),
17674 reserved: false,
17675 reserved_prefixes: Vec::new(),
17676 protocol: ModuleProtocol::None,
17677 overlap: Default::default(),
17678 };
17679 let child =
17680 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
17681 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17682 let grandchild: u32 = loop {
17683 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
17684 if let Ok(pid) = contents.trim().parse() {
17685 break pid;
17686 }
17687 }
17688 assert!(
17689 tokio::time::Instant::now() < deadline,
17690 "grandchild pid was not recorded"
17691 );
17692 tokio::time::sleep(Duration::from_millis(10)).await;
17693 };
17694 assert!(
17695 running(grandchild),
17696 "grandchild must be alive before teardown"
17697 );
17698 if shutdown {
17699 let mut child = child;
17700 crate::child_roster::end_children_for_daemon_shutdown(
17701 &runtime.child_roster,
17702 false,
17703 std::future::pending(),
17704 )
17705 .await;
17706 child.wait().await.expect("reap shutdown straggler");
17707 } else {
17708 drain_child_to_state(
17709 module_id,
17710 ModuleProtocol::None,
17711 StopNotice::NotSent,
17712 &Registry::default(),
17713 None,
17714 &snapshot,
17715 &runtime.terminal_ring,
17716 &SpawnEventFeed::default(),
17717 child,
17718 Duration::from_millis(100),
17719 ModuleState::Stopped,
17720 Some(false),
17721 &FailureContext::operator(),
17722 )
17723 .await
17724 .expect("real supervisor teardown");
17725 }
17726 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
17727 while running(grandchild) && tokio::time::Instant::now() < deadline {
17728 tokio::time::sleep(Duration::from_millis(10)).await;
17729 }
17730 let survived = running(grandchild);
17731 if survived {
17733 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
17734 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
17735 tokio::time::sleep(Duration::from_millis(100)).await;
17736 }
17737 if module.exists() {
17738 std::fs::remove_dir(&module).expect("remove empty module cgroup");
17739 }
17740 std::fs::remove_dir(root.join("subc-modules")).unwrap();
17741 std::fs::remove_dir(&root).unwrap();
17742 assert!(
17743 !survived,
17744 "grandchild {grandchild} outlived module teardown"
17745 );
17746 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
17747 }
17748}
17749
17750#[cfg(all(test, unix))]
17753mod record_write_stall_tests {
17754 use super::*;
17755 use std::sync::atomic::{AtomicBool, Ordering};
17756 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
17757
17758 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
17761 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
17765
17766 fn sleeping_spec() -> ModuleSpec {
17778 ModuleSpec {
17779 module_id: "record-write-stall".into(),
17780 program: "/bin/sleep".into(),
17781 args: vec!["600".into()],
17782 env: vec![],
17783 reserved: false,
17784 reserved_prefixes: vec![],
17785 protocol: ModuleProtocol::None,
17786 overlap: Default::default(),
17787 }
17788 }
17789
17790 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17791 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17792 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17793 let supervisor =
17794 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17795 .with_live_children_record(home.join("live-children.json"));
17796 let module = supervisor.spawn(sleeping_spec()).unwrap();
17797 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17802 loop {
17803 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17804 .unwrap()
17805 .reported_pid()
17806 {
17807 return pid;
17808 }
17809 tokio::time::sleep(Duration::from_millis(10)).await;
17810 }
17811 })
17812 .await
17813 .expect("the first process was admitted as the module");
17814
17815 let key = vec![0x42; 32];
17818 let daemon_id = [0x24; 16];
17819 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17820 let port = listener.local_addr().unwrap().port();
17821 let server = tokio::spawn(crate::server::serve_listener(
17822 listener,
17823 Arc::new(crate::router::Router::with_default_self_handler()),
17824 crate::server::ServerAuth::with_limits(
17825 key.clone(),
17826 daemon_id,
17827 "test",
17828 crate::server::DEFAULT_AUTH_DEADLINE,
17829 4,
17830 ),
17831 ));
17832
17833 let (held_tx, held_rx) = std::sync::mpsc::channel();
17836 let armed = AtomicBool::new(true);
17837 supervisor
17838 .child_roster
17839 .set_record_write_hook_for_test(move |live| {
17840 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17841 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17842 let _ = held_tx.send(());
17843 std::thread::sleep(HELD_RECORD_WRITE);
17844 }
17845 });
17846
17847 let module_process = std::thread::spawn(move || {
17850 held_rx
17851 .recv_timeout(Duration::from_secs(30))
17852 .expect("the restart reached the replacement's record write");
17853 let conn = ConnectionInfo {
17854 schema: SCHEMA_VERSION,
17855 wire_version: None,
17856 endpoints: vec![Endpoint {
17857 host: "127.0.0.1".to_owned(),
17858 port,
17859 }],
17860 key,
17861 daemon_id,
17862 pid: std::process::id(),
17863 daemon_ver: "test".to_owned(),
17864 };
17865 tokio::runtime::Builder::new_current_thread()
17866 .enable_all()
17867 .build()
17868 .unwrap()
17869 .block_on(async {
17870 let started = Instant::now();
17871 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17872 .await
17873 .unwrap();
17874 let result =
17875 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17876 (result, started.elapsed())
17877 })
17878 });
17879
17880 module.restart(Some(0)).await.unwrap();
17881 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17882 .await
17883 .unwrap();
17884 assert!(
17885 auth.is_ok(),
17886 "the daemon must answer a module's handshake while a restart waits on the \
17887 live-children record write; got {auth:?} after {elapsed:?}"
17888 );
17889
17890 tokio::time::timeout(Duration::from_secs(20), async {
17891 while lock_snapshot(&module.inner.snapshot)
17892 .unwrap()
17893 .reported_pid()
17894 .is_none_or(|pid| pid == old_pid)
17895 {
17896 tokio::time::sleep(Duration::from_millis(10)).await;
17897 }
17898 })
17899 .await
17900 .expect("the restart published its replacement");
17901 module.stop().await.unwrap();
17902 server.abort();
17903 }
17904}