1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62mod slow_retry;
63use slow_retry::{FailedRetry, FailureContext};
64
65#[cfg(all(test, unix))]
66mod slow_retry_tests;
67
68pub const SUBC_ARG: &str = "--subc";
74
75const DEFAULT_MAX_RESTARTS: u32 = 3;
76const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
77const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
78const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
82pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
93pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
94const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
95const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
96const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
108const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
131pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
133const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
134pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
139
140#[cfg(target_os = "macos")]
141struct PrivacyExec {
142 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
143 deadline: tokio::time::Instant,
144 expected: Option<subc_os::FileIdentity>,
145 trampoline: Option<subc_os::FileIdentity>,
146 script: bool,
147 module_id: String,
148 #[cfg(test)]
149 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
150}
151
152#[cfg(all(test, target_os = "macos"))]
153pub(crate) fn test_privacy_trampoline() -> PathBuf {
154 let path = std::env::current_exe()
157 .unwrap()
158 .parent()
159 .unwrap()
160 .parent()
161 .unwrap()
162 .join("privacy-trampoline-fixture");
163 assert!(
167 path.exists(),
168 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
169 --bins --features test-support` or `cargo test -p subc-daemon` first",
170 path.display()
171 );
172 path
173}
174
175#[cfg(target_os = "macos")]
176fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
177 use std::io::Read;
178 let mut probe = std::process::Command::new(path)
179 .args(["__disclaim-exec", "--probe"])
180 .stdin(Stdio::null())
181 .stdout(Stdio::piped())
182 .stderr(Stdio::piped())
183 .spawn()
184 .map_err(|error| {
185 format!(
186 "privacy trampoline probe failed for {}: {error}",
187 path.display()
188 )
189 })?;
190 let deadline = std::time::Instant::now() + Duration::from_secs(5);
191 let status = loop {
192 match probe.try_wait() {
193 Ok(Some(status)) => break status,
194 Ok(None) if std::time::Instant::now() < deadline => {
195 std::thread::sleep(Duration::from_millis(5))
196 }
197 result => {
198 let _ = probe.kill();
199 let _ = probe.wait();
200 return Err(format!(
201 "privacy trampoline probe failed or timed out for {}: {result:?}",
202 path.display()
203 ));
204 }
205 }
206 };
207 let mut answer = String::new();
208 if let Some(stdout) = probe.stdout.take() {
209 let _ = stdout.take(256).read_to_string(&mut answer);
210 }
211 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
212 return Ok(());
213 }
214 let mut diagnostic = String::new();
215 if let Some(stderr) = probe.stderr.take() {
216 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
217 }
218 let cause = diagnostic
219 .trim()
220 .strip_prefix("ck-subc: own privacy identity refused: ")
221 .unwrap_or("binary does not implement the privacy trampoline protocol");
222 Err(format!(
223 "{cause}: probe of {} exited {status}",
224 path.display()
225 ))
226}
227
228#[cfg(target_os = "macos")]
229fn privacy_command(
230 spec: &ModuleSpec,
231 roster: &ChildRoster,
232) -> Result<
233 (
234 Command,
235 Option<PrivacyExec>,
236 subc_os::privacy_identity::ExecAcknowledgement,
237 ),
238 SuperviseError,
239> {
240 let failure = |cause: String| {
241 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
242 SuperviseError::Spawn {
243 program: spec.program.clone(),
244 source: io::Error::other(cause),
245 cgroup_path: None,
246 }
247 };
248 let trampoline = roster.privacy_trampoline().map_err(failure)?;
249 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
254 let path = spec
255 .env
256 .iter()
257 .find(|(key, _)| key == "PATH")
258 .map(|(_, value)| std::ffi::OsString::from(value))
259 .or_else(|| std::env::var_os("PATH"))
260 .unwrap_or_else(|| "/usr/bin:/bin".into());
261 std::env::split_paths(&path)
262 .map(|dir| dir.join(&spec.program))
263 .find(|path| path.is_file())
264 .unwrap_or_else(|| spec.program.clone())
265 } else {
266 spec.program.clone()
267 };
268 let expected = subc_os::file_identity(&program);
269 let trampoline_image = subc_os::file_identity(&trampoline);
270 let script = {
271 use std::io::Read;
272 let mut prefix = [0u8; 2];
273 std::fs::File::open(&program)
274 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
275 };
276 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
277 return Err(failure(
278 "privacy identity module executable is missing or is the trampoline itself".to_string(),
279 ));
280 }
281 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
282 .map_err(|error| failure(error.to_string()))?;
283 let reader =
284 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
285 let mut command = Command::new(&trampoline);
286 command
287 .arg("__disclaim-exec")
288 .arg(ack.fd().to_string())
289 .arg(&program);
290 ack.install(command.as_std_mut());
291 Ok((
292 command,
293 Some(PrivacyExec {
294 reader,
295 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
296 expected,
297 trampoline: trampoline_image,
298 script,
299 module_id: spec.module_id.clone(),
300 #[cfg(test)]
301 confirmation_pause: spec
302 .env
303 .iter()
304 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
305 .map(|(_, address)| (address.clone(), None)),
306 }),
307 ack,
308 ))
309}
310
311struct SupervisedChild {
312 child: Child,
313 exec: crate::activity::ExecOwner,
314 #[cfg(windows)]
315 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
316 #[cfg(target_os = "macos")]
317 privacy_exec: Option<PrivacyExec>,
318 #[cfg(target_os = "macos")]
325 report_ready: Arc<OnceLock<()>>,
326 spawn_failure: Option<String>,
329 protocol: ModuleProtocol,
334 #[cfg(target_os = "linux")]
340 module_id: String,
341 #[cfg(target_os = "linux")]
342 cgroup_placement: Option<subc_cgroup::Placement>,
343 #[cfg(windows)]
365 job: Option<subc_jobobject::JobObject>,
366 stdout_pump: Option<JoinHandle<()>>,
367 stderr_pump: Option<StderrPump>,
368 stderr_ring: Arc<Mutex<StderrRing>>,
369 spawned_at: Instant,
372 registration_deadline: Option<Instant>,
375 registration_watch: Option<RegistrationWatch>,
376 spawned_at_ms: u64,
377 spawned_from: PathBuf,
378 launch_spec_digest: LaunchSpecDigest,
379 spawned_file_identity: Option<SpawnedFileIdentity>,
380 process_start_time: Option<u64>,
381 process_identity: Option<ProcessIdentity>,
382 pid: u32,
383 roster_guard: Option<crate::child_roster::RosterGuard>,
386}
387
388impl SupervisedChild {
389 fn registered_since_spawn(&self) -> bool {
390 self.registration_watch
391 .as_ref()
392 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
393 }
394
395 fn id(&self) -> Option<u32> {
396 Some(self.pid)
397 }
398
399 fn process_identity(&self) -> Option<ProcessIdentity> {
400 self.process_identity
401 }
402
403 async fn wait(&mut self) -> io::Result<ExitStatus> {
404 #[cfg(target_os = "macos")]
405 self.confirm_privacy_exec().await;
406 let result = self.child.wait().await;
414 #[cfg(target_os = "linux")]
415 if result.is_ok() {
416 if let Some(placement) = self.cgroup_placement.as_ref() {
417 cleanup_reaped_cgroup(placement, &self.module_id).await;
418 self.cgroup_placement = None;
421 }
422 }
423 result
424 }
425
426 #[cfg(target_os = "macos")]
427 async fn confirm_privacy_exec(&mut self) {
428 let Some(pending) = &mut self.privacy_exec else {
429 return;
430 };
431 let result = tokio::time::timeout_at(pending.deadline, async {
432 let mut record = Vec::new();
433 loop {
434 let mut ready = pending.reader.readable().await?;
435 let read = ready.try_io(|reader| {
436 use std::io::Read;
437 let mut reader = reader.get_ref();
438 let mut buffer = [0u8; 256];
439 reader.read(&mut buffer).map(|count| (count, buffer))
440 });
441 match read {
442 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
443 Ok(Ok((count, buffer))) => {
444 if record.len() + count > 1024 {
445 return Err(io::Error::other(
446 "privacy exec refusal record is too long",
447 ));
448 }
449 record.extend_from_slice(&buffer[..count]);
450 }
451 Ok(Err(error)) => return Err(error),
452 Err(_) => continue,
453 }
454 }
455 })
456 .await;
457 #[cfg(test)]
458 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
459 use tokio::io::{AsyncReadExt, AsyncWriteExt};
460 if stream.is_none() {
461 *stream = Some(
462 tokio::net::TcpStream::connect(address.as_str())
463 .await
464 .unwrap(),
465 );
466 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
467 stream
468 .as_mut()
469 .unwrap()
470 .write_all(&self.pid.to_ne_bytes())
471 .await
472 .unwrap();
473 }
474 let mut reply = [0];
475 stream
476 .as_mut()
477 .unwrap()
478 .read_exact(&mut reply)
479 .await
480 .unwrap();
481 pending.confirmation_pause = None;
482 if &reply == b"F" {
483 Ok(Err(io::Error::other("injected late exec refusal")))
484 } else {
485 assert_eq!(&reply, b"X");
486 result
487 }
488 } else {
489 result
490 };
491 let pending = self.privacy_exec.as_ref().expect("pending exec");
494 let cause = match result {
495 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
496 Ok(Err(error)) => Some(format!(
497 "privacy identity exec acknowledgement failed: {error}"
498 )),
499 Ok(Ok(record)) if !record.is_empty() => Some(
500 std::str::from_utf8(&record)
501 .ok()
502 .and_then(|record| {
503 record
504 .trim()
505 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
506 })
507 .filter(|cause| !cause.is_empty())
508 .unwrap_or("invalid privacy exec refusal record")
509 .to_string(),
510 ),
511 Ok(Ok(_)) => match self.child.try_wait() {
512 Ok(Some(_status)) => None,
516 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
517 Ok(None) => {
518 let image = observe_spawned_image(self.pid);
519 if let Some(image) = image.filter(|image| {
520 image.executable.is_some()
521 && image.executable != pending.trampoline
522 && (image.executable == pending.expected || pending.script)
523 }) {
524 if let Err(error) = self
525 .roster_guard
526 .as_ref()
527 .map(|guard| guard.confirm_executable(image))
528 .transpose()
529 {
530 Some(format!(
531 "could not durably record confirmed module image: {error}"
532 ))
533 } else {
534 let _ = self.report_ready.set(());
535 info!(module_id = %pending.module_id, pid = self.pid,
536 "module spawned with own privacy identity (responsibility disclaimed)");
537 None
538 }
539 } else if image.is_none()
540 || image.is_some_and(|image| image.executable.is_none())
541 {
542 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
549 Ok(Ok(_status)) => None,
550 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
551 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
552 }
553 } else {
554 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
555 }
556 }
557 },
558 };
559 let pending = self.privacy_exec.take().expect("pending exec");
560 self.exec.0.finish(if cause.is_some() {
563 crate::activity::ExecState::Refused
564 } else {
565 crate::activity::ExecState::CommittedExec
566 });
567 if let Some(cause) = cause {
568 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
569 self.spawn_failure = Some(cause);
570 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
573 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
574 }
575 let _ = self.child.start_kill();
576 }
577 }
578
579 fn release_roster(&mut self) {
583 self.roster_guard = None;
584 }
585
586 fn start_kill(&mut self) -> io::Result<()> {
599 #[cfg(windows)]
600 if let Some(job) = &self.job {
601 if let Err(error) = job.terminate() {
602 debug!(
603 error = %error,
604 "job termination failed; the direct-child kill still owns the outcome"
605 );
606 }
607 }
608 #[cfg(target_os = "linux")]
609 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
610 self.child.start_kill()
611 }
612
613 async fn drain_stderr(&mut self, module_id: &str) {
614 if let Some(pump) = self.stdout_pump.take() {
615 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
616 }
617
618 let Some(pump) = self.stderr_pump.take() else {
619 return;
620 };
621 settle_stderr_pump(
622 module_id,
623 &self.stderr_ring,
624 pump,
625 STDERR_PUMP_DRAIN_TIMEOUT,
626 )
627 .await;
628 }
629}
630
631async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
635 match timeout(bound, &mut pump).await {
636 Ok(Ok(())) => {}
637 Ok(Err(error)) => {
638 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
639 }
640 Err(_) => {
641 drop(pump);
642 warn!(
643 module_id,
644 waited = ?bound,
645 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
646 );
647 }
648 }
649}
650
651struct StderrPump {
654 task: JoinHandle<()>,
655 generation: u64,
656}
657
658async fn settle_stderr_pump(
664 module_id: &str,
665 ring: &Arc<Mutex<StderrRing>>,
666 pump: StderrPump,
667 bound: Duration,
668) {
669 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
670 let StderrPump {
671 mut task,
672 generation,
673 } = pump;
674 lock().retire_pump(generation);
675 match timeout(bound, &mut task).await {
676 Ok(Ok(())) => {}
677 Ok(Err(err)) => {
678 let mut ring = lock();
679 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
680 ring.finish_pump(generation);
681 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
682 }
683 Err(_) => {
684 drop(task);
686 lock().mark_pump_late(
687 generation,
688 format!(
689 "stderr of the exited process had not reached EOF {bound:?} after it was \
690 retired (a descendant may still hold the pipe open); lines it still \
691 writes are kept in that process's section"
692 ),
693 );
694 warn!(
695 module_id,
696 waited = ?bound,
697 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
698 );
699 }
700 }
701}
702
703fn registration_release_events() -> &'static watch::Sender<u64> {
704 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
705 EVENTS.get_or_init(|| {
706 let (sender, _receiver) = watch::channel(0);
707 sender
708 })
709}
710
711pub(crate) fn notify_registration_release() {
712 let events = registration_release_events();
713 let next_generation = (*events.borrow()).wrapping_add(1);
714 events.send_replace(next_generation);
715}
716
717#[derive(Debug, Clone, PartialEq, Eq)]
719pub struct ModuleSpec {
720 pub module_id: String,
721 pub program: PathBuf,
722 pub args: Vec<String>,
723 pub env: Vec<(String, String)>,
724 pub reserved: bool,
729 pub reserved_prefixes: Vec<String>,
734 pub protocol: ModuleProtocol,
753 pub overlap: ModuleOverlap,
758}
759
760#[derive(Clone, PartialEq, Eq)]
763pub(crate) struct LaunchSpecDigest([u8; 32]);
764
765impl fmt::Debug for LaunchSpecDigest {
766 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
767 f.write_str("LaunchSpecDigest([redacted])")
768 }
769}
770
771impl LaunchSpecDigest {
772 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
773 fn field(hash: &mut Sha256, bytes: &[u8]) {
774 hash.update((bytes.len() as u64).to_le_bytes());
775 hash.update(bytes);
776 }
777 let mut hash = Sha256::new();
778 if let Some(program) = spec.program.to_str() {
779 field(&mut hash, b"utf8-program");
780 field(&mut hash, program.as_bytes());
781 } else {
782 #[cfg(unix)]
785 {
786 use std::os::unix::ffi::OsStrExt;
787 field(&mut hash, b"unix-program");
788 field(&mut hash, spec.program.as_os_str().as_bytes());
789 }
790 #[cfg(windows)]
791 {
792 use std::os::windows::ffi::OsStrExt;
793 field(&mut hash, b"windows-program");
794 let bytes = spec
795 .program
796 .as_os_str()
797 .encode_wide()
798 .flat_map(u16::to_le_bytes)
799 .collect::<Vec<_>>();
800 field(&mut hash, &bytes);
801 }
802 #[cfg(not(any(unix, windows)))]
803 field(&mut hash, spec.program.to_string_lossy().as_bytes());
804 }
805 hash.update((spec.args.len() as u64).to_le_bytes());
806 for arg in &spec.args {
807 field(&mut hash, arg.as_bytes());
808 }
809 let mut env = spec.env.iter().collect::<Vec<_>>();
810 env.sort_unstable();
811 hash.update((env.len() as u64).to_le_bytes());
812 for (key, value) in env {
813 field(&mut hash, key.as_bytes());
814 field(&mut hash, value.as_bytes());
815 }
816 field(
817 &mut hash,
818 match spec.protocol {
819 ModuleProtocol::Subc => b"subc",
820 ModuleProtocol::None => b"none",
821 },
822 );
823 Self(hash.finalize().into())
824 }
825}
826
827#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
834pub enum ModuleOverlap {
835 #[default]
837 Exclusive,
838 Safe,
850}
851
852impl ModuleOverlap {
853 pub fn as_str(self) -> &'static str {
854 match self {
855 Self::Exclusive => "exclusive",
856 Self::Safe => "safe",
857 }
858 }
859}
860
861pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
871pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
873pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
878
879#[derive(Debug, Clone, Copy, PartialEq, Eq)]
897pub struct RestartPolicy {
898 pub max_restarts: u32,
899 pub backoff: Duration,
902 pub max_backoff: Duration,
904 pub window: Duration,
908}
909
910impl RestartPolicy {
911 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
915 Self {
916 max_restarts,
917 backoff,
918 max_backoff: DEFAULT_MAX_BACKOFF,
919 window: DEFAULT_RESTART_WINDOW,
920 }
921 }
922
923 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
924 self.max_backoff = max_backoff;
925 self
926 }
927
928 pub fn with_window(mut self, window: Duration) -> Self {
929 self.window = window;
930 self
931 }
932
933 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
938 if self.backoff.is_zero() || self.max_backoff.is_zero() {
939 return Duration::ZERO;
940 }
941
942 let mut delay = self.backoff;
943 for _ in 0..restart_in_window {
944 if delay >= self.max_backoff {
945 return self.max_backoff;
946 }
947 delay = delay
948 .checked_mul(10)
949 .unwrap_or(self.max_backoff)
950 .min(self.max_backoff);
951 }
952 delay.min(self.max_backoff)
953 }
954
955 fn budget_exhausted_detail(&self) -> String {
960 format!(
961 "crash budget exhausted: max_restarts={} within window_secs={}",
962 self.max_restarts,
963 self.window.as_secs()
964 )
965 }
966}
967
968impl Default for RestartPolicy {
969 fn default() -> Self {
970 Self {
971 max_restarts: DEFAULT_MAX_RESTARTS,
972 backoff: DEFAULT_BACKOFF,
973 max_backoff: DEFAULT_MAX_BACKOFF,
974 window: DEFAULT_RESTART_WINDOW,
975 }
976 }
977}
978
979#[derive(Debug, Clone, Copy, PartialEq, Eq)]
980struct CrashRestartSchedule {
981 restart_in_window: u32,
982 delay: Duration,
983}
984
985fn daemon_will_restart(
992 state: &mut SupervisorSnapshot,
993 policy: &RestartPolicy,
994 now: Instant,
995) -> bool {
996 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
997}
998
999const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1000const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1001const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1002const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1003
1004#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1005pub enum HealthAction {
1006 Report,
1007 Restart,
1008 Alert,
1009}
1010
1011impl fmt::Display for HealthAction {
1012 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1013 f.write_str(match self {
1014 Self::Report => "report",
1015 Self::Restart => "restart",
1016 Self::Alert => "alert",
1017 })
1018 }
1019}
1020
1021#[derive(Debug, Clone, PartialEq, Eq)]
1022pub struct HealthConfig {
1023 pub http: Option<String>,
1027 pub cadence: Duration,
1028 pub deadline: Duration,
1029 pub failure_threshold: u32,
1030 pub on_degraded: HealthAction,
1031 pub on_failing: HealthAction,
1032 pub critical: bool,
1033}
1034
1035impl Default for HealthConfig {
1036 fn default() -> Self {
1037 Self {
1038 http: None,
1039 cadence: DEFAULT_HEALTH_CADENCE,
1040 deadline: DEFAULT_HEALTH_DEADLINE,
1041 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1042 on_degraded: HealthAction::Report,
1043 on_failing: HealthAction::Report,
1044 critical: false,
1045 }
1046 }
1047}
1048
1049#[derive(Debug, Clone, PartialEq)]
1067pub struct ModuleHealthStatus {
1068 pub status: SupervisorHealthStatus,
1069 pub last_probe_ms: Option<u64>,
1070 pub detail: Option<String>,
1071 pub metrics: Option<Value>,
1072 pub consecutive_failures: u32,
1073 pub late_answer_count: u64,
1076 pub last_late_answer_latency_ms: Option<u64>,
1078 pub last_action: Option<String>,
1079 pub last_action_ms: Option<u64>,
1083}
1084
1085impl Default for ModuleHealthStatus {
1086 fn default() -> Self {
1087 Self {
1088 status: SupervisorHealthStatus::Unknown,
1089 last_probe_ms: None,
1090 detail: None,
1091 metrics: None,
1092 consecutive_failures: 0,
1093 late_answer_count: 0,
1094 last_late_answer_latency_ms: None,
1095 last_action: None,
1096 last_action_ms: None,
1097 }
1098 }
1099}
1100
1101#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1103pub enum ModuleState {
1104 Starting,
1105 Running,
1106 Unresponsive,
1107 Restarting,
1108 Draining,
1109 Stopped,
1110 Failed,
1111 Disabled,
1112}
1113
1114impl fmt::Display for ModuleState {
1115 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1116 f.write_str(match self {
1117 Self::Starting => "starting",
1118 Self::Running => "running",
1119 Self::Unresponsive => "unresponsive",
1120 Self::Restarting => "restarting",
1121 Self::Draining => "draining",
1122 Self::Stopped => "stopped",
1123 Self::Failed => "failed",
1124 Self::Disabled => "disabled",
1125 })
1126 }
1127}
1128
1129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1131pub enum ExitKind {
1132 Clean,
1133 Crash,
1134 DeliberateSeverance,
1135}
1136
1137impl From<ExitKind> for TerminalExitKind {
1138 fn from(kind: ExitKind) -> Self {
1139 match kind {
1140 ExitKind::Clean => Self::Clean,
1141 ExitKind::Crash => Self::Crash,
1142 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1143 }
1144 }
1145}
1146
1147#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1150pub(crate) struct ProcessIdentity {
1151 pub(crate) pid: u32,
1152 pub(crate) start_time: u64,
1153}
1154
1155#[derive(Debug, Clone, PartialEq, Eq)]
1157pub struct ExitReport {
1158 pub kind: ExitKind,
1159 pub code: Option<i32>,
1160 pub signal: Option<i32>,
1161 pub at_ms: u64,
1162}
1163
1164#[derive(Debug, Clone, PartialEq)]
1167pub struct ModuleStatus {
1168 pub module_id: String,
1169 pub waiting_for_previous_pid: Option<u32>,
1171 pub waiting_for_previous_detail: Option<String>,
1173 pub launch_nonce_source: Option<String>,
1178 pub state: ModuleState,
1179 pub enabled: bool,
1180 pub process_alive: bool,
1181 pub registration_active: bool,
1182 pub protocol: ModuleProtocol,
1187 pub live: bool,
1198 pub restart_count: u32,
1202 pub lifetime_restarts: u32,
1206 pub spawn_generation: u64,
1207 pub max_restarts: u32,
1212 pub restart_window: Duration,
1216 pub drain_timeout: Duration,
1220 pub restart_backoff: Duration,
1221 pub restart_max_backoff: Duration,
1222 pub pid: Option<u32>,
1227 pub spawned_at_ms: Option<u64>,
1228 pub spawned_from: Option<PathBuf>,
1229 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1230 pub process_start_time: Option<u64>,
1231 pub last_exit: Option<ExitReport>,
1232 pub health: ModuleHealthStatus,
1233}
1234
1235#[cfg(test)]
1236#[derive(Debug, Clone, Copy, PartialEq)]
1237struct ActorSelectCheckpoint {
1238 generation: u64,
1239 turn: u64,
1240 registered_connection: Option<ConnectionId>,
1241 next_probe_at: Option<Instant>,
1242 wake_after: Option<Duration>,
1243}
1244
1245#[derive(Debug, Clone, PartialEq)]
1246struct SupervisorSnapshot {
1247 failed_retry: FailedRetry,
1248 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1249 #[cfg(windows)]
1250 pipe_consumed: Option<Arc<OnceLock<()>>>,
1251 #[cfg(test)]
1253 actor_turns: u64,
1254 #[cfg(test)]
1257 actor_select: Option<ActorSelectCheckpoint>,
1258 state: ModuleState,
1259 enabled: bool,
1260 process_alive: bool,
1261 spawned_protocol: Option<ModuleProtocol>,
1262 spawn_failure: Option<String>,
1263 waiting_for_previous_pid: Option<u32>,
1264 waiting_for_previous_detail: Option<String>,
1265 crash_restarts: VecDeque<Instant>,
1271 lifetime_restarts: u32,
1272 spawn_generation: u64,
1281 pid: Option<u32>,
1282 #[cfg(target_os = "macos")]
1283 report_ready: Option<Arc<OnceLock<()>>>,
1284 reaped_pid: Option<u32>,
1286 respawn_pending: bool,
1288 coalesced_restart_pending: bool,
1290 spawned_at_ms: Option<u64>,
1291 spawned_from: Option<PathBuf>,
1292 launch_spec_digest: Option<LaunchSpecDigest>,
1293 spawned_file_identity: Option<SpawnedFileIdentity>,
1294 process_start_time: Option<u64>,
1295 deliberate_severance: Option<ProcessIdentity>,
1296 last_exit: Option<ExitReport>,
1297 drain_disposition_detail: Option<String>,
1299 health: ModuleHealthStatus,
1300 in_alternate_slot: bool,
1305 draining_to_replace: bool,
1312 configuration_updated_since_spawn: bool,
1318}
1319
1320impl SupervisorSnapshot {
1321 fn reported_pid(&self) -> Option<u32> {
1326 #[cfg(target_os = "macos")]
1327 if self
1328 .report_ready
1329 .as_ref()
1330 .is_some_and(|ready| ready.get().is_none())
1331 {
1332 return None;
1333 }
1334 self.pid
1335 }
1336
1337 fn starting() -> Self {
1338 Self::new(ModuleState::Starting, true)
1339 }
1340
1341 fn disabled() -> Self {
1342 Self::new(ModuleState::Disabled, false)
1343 }
1344
1345 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1349 while let Some(oldest) = self.crash_restarts.front() {
1350 if now.duration_since(*oldest) > window {
1351 self.crash_restarts.pop_front();
1352 } else {
1353 break;
1354 }
1355 }
1356 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1357 }
1358
1359 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1365 self.crash_restarts.push_back(now);
1366 while self.crash_restarts.len() > policy.max_restarts as usize {
1367 self.crash_restarts.pop_front();
1368 }
1369 self.lifetime_restarts += 1;
1370 }
1371
1372 fn next_crash_restart(
1376 &mut self,
1377 policy: &RestartPolicy,
1378 now: Instant,
1379 ) -> Option<CrashRestartSchedule> {
1380 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1381 if restart_in_window >= policy.max_restarts {
1382 return None;
1383 }
1384 self.record_crash_restart(policy, now);
1385 Some(CrashRestartSchedule {
1386 restart_in_window,
1387 delay: policy.delay_for_restart(restart_in_window),
1388 })
1389 }
1390
1391 fn clear_crash_restarts(&mut self) {
1396 self.crash_restarts.clear();
1397 }
1398
1399 fn new(state: ModuleState, enabled: bool) -> Self {
1400 Self {
1401 failed_retry: FailedRetry::default(),
1402 exec_outcome: None,
1403 #[cfg(test)]
1404 actor_turns: 0,
1405 #[cfg(test)]
1406 actor_select: None,
1407 state,
1408 enabled,
1409 process_alive: false,
1410 #[cfg(windows)]
1411 pipe_consumed: None,
1412 spawned_protocol: None,
1413 spawn_failure: None,
1414 waiting_for_previous_pid: None,
1415 waiting_for_previous_detail: None,
1416 crash_restarts: VecDeque::new(),
1417 lifetime_restarts: 0,
1418 spawn_generation: 0,
1419 pid: None,
1420 #[cfg(target_os = "macos")]
1421 report_ready: None,
1422 reaped_pid: None,
1423 respawn_pending: false,
1424 coalesced_restart_pending: false,
1425 spawned_at_ms: None,
1426 spawned_from: None,
1427 launch_spec_digest: None,
1428 spawned_file_identity: None,
1429 process_start_time: None,
1430 deliberate_severance: None,
1431 last_exit: None,
1432 drain_disposition_detail: None,
1433 health: ModuleHealthStatus::default(),
1434 in_alternate_slot: false,
1435 draining_to_replace: false,
1436 configuration_updated_since_spawn: false,
1437 }
1438 }
1439}
1440
1441type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1442
1443type SpawnSubscriberKey = (ConnectionId, u64);
1444
1445#[derive(Debug)]
1446struct SpawnSubscriber {
1447 version: u8,
1448 frames: mpsc::Sender<Frame>,
1449 lagged: Option<oneshot::Sender<SpawnCursor>>,
1453}
1454
1455#[derive(Debug)]
1456struct SpawnEventState {
1457 daemon_incarnation: String,
1458 seq: u64,
1459 capacity: usize,
1460 live: HashMap<String, LiveSpawn>,
1461 generations: HashMap<String, u64>,
1462 events: VecDeque<SpawnEvent>,
1463 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1464}
1465
1466impl Default for SpawnEventState {
1467 fn default() -> Self {
1468 Self {
1469 daemon_incarnation: "unconfigured".to_string(),
1470 seq: 0,
1471 capacity: SPAWN_EVENT_RING_CAPACITY,
1472 live: HashMap::new(),
1473 generations: HashMap::new(),
1474 events: VecDeque::new(),
1475 subscribers: HashMap::new(),
1476 }
1477 }
1478}
1479
1480#[derive(Debug, Clone, Default)]
1481struct SpawnEventFeed(
1482 Arc<Mutex<SpawnEventState>>,
1483 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1484);
1485
1486#[derive(Debug, Clone, PartialEq, Eq)]
1487pub(crate) enum SpawnSubscribeRefusal {
1488 ForeignIncarnation { current: String },
1489 TooOld { oldest: SpawnCursor },
1490 Frame(String),
1491}
1492
1493impl SpawnEventFeed {
1494 fn configure_incarnation(&self, daemon_incarnation: String) {
1495 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1496 state.daemon_incarnation = daemon_incarnation;
1497 state.seq = 0;
1498 state.live.clear();
1499 state.generations.clear();
1500 state.events.clear();
1501 state.subscribers.clear();
1502 }
1503
1504 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1505 SpawnCursor {
1506 daemon_incarnation: state.daemon_incarnation.clone(),
1507 seq: state.seq,
1508 }
1509 }
1510
1511 fn snapshot(&self) -> SpawnSnapshot {
1512 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1513 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1514 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1515 SpawnSnapshot {
1516 cursor: Self::cursor(&state),
1517 ring_bound: state.capacity as u64,
1518 live,
1519 }
1520 }
1521
1522 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1523 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1524 let generation = state
1525 .generations
1526 .get(module_id)
1527 .copied()
1528 .unwrap_or(0)
1529 .checked_add(1)
1530 .expect("spawn generation exhausted");
1531 state.generations.insert(module_id.to_string(), generation);
1532 let live = LiveSpawn {
1533 module_id: module_id.to_string(),
1534 spawn_generation: generation,
1535 pid,
1536 spawned_at_ms,
1537 };
1538 state.live.insert(module_id.to_string(), live);
1539 Self::emit_locked(
1540 &mut state,
1541 SpawnEventKind::Spawned,
1542 module_id.to_string(),
1543 generation,
1544 pid,
1545 None,
1546 None,
1547 );
1548 generation
1549 }
1550
1551 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1552 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1553 let Some(live) = state.live.remove(module_id) else {
1554 warn!(
1555 module_id,
1556 "terminal record had no live spawn event identity"
1557 );
1558 return;
1559 };
1560 Self::emit_locked(
1561 &mut state,
1562 SpawnEventKind::Exited,
1563 module_id.to_string(),
1564 live.spawn_generation,
1565 live.pid,
1566 exit_code,
1567 exit_signal,
1568 );
1569 }
1570
1571 fn emit_superseded_exited(
1578 &self,
1579 module_id: &str,
1580 spawn_generation: u64,
1581 pid: u32,
1582 exit_code: Option<i32>,
1583 exit_signal: Option<i32>,
1584 ) {
1585 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1586 if state
1587 .live
1588 .get(module_id)
1589 .is_some_and(|live| live.spawn_generation == spawn_generation)
1590 {
1591 state.live.remove(module_id);
1592 }
1593 Self::emit_locked(
1594 &mut state,
1595 SpawnEventKind::Exited,
1596 module_id.to_string(),
1597 spawn_generation,
1598 pid,
1599 exit_code,
1600 exit_signal,
1601 );
1602 }
1603
1604 #[allow(clippy::too_many_arguments)]
1605 fn emit_locked(
1606 state: &mut SpawnEventState,
1607 kind: SpawnEventKind,
1608 module_id: String,
1609 spawn_generation: u64,
1610 pid: u32,
1611 exit_code: Option<i32>,
1612 exit_signal: Option<i32>,
1613 ) {
1614 state.seq = state
1615 .seq
1616 .checked_add(1)
1617 .expect("spawn event sequence exhausted");
1618 let event = SpawnEvent {
1619 cursor: Self::cursor(state),
1620 kind,
1621 module_id,
1622 spawn_generation,
1623 pid,
1624 exit_code,
1625 exit_signal,
1626 };
1627 state.events.push_back(event.clone());
1628 while state.events.len() > state.capacity {
1629 state.events.pop_front();
1630 }
1631 let body = match serde_json::to_vec(&event) {
1632 Ok(body) => body,
1633 Err(error) => {
1634 error!(%error, "failed to serialize supervisor spawn event");
1635 return;
1636 }
1637 };
1638 state.subscribers.retain(|(connection_id, corr), subscriber| {
1639 let frame = Frame::build_with_version(
1640 subscriber.version,
1641 FrameType::StreamData,
1642 control_flags(),
1643 0,
1644 0,
1645 *corr,
1646 body.clone(),
1647 );
1648 match frame {
1649 Ok(frame) => {
1650 if subscriber.frames.try_send(frame).is_ok() {
1651 true
1652 } else {
1653 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1654 if let Some(lagged) = subscriber.lagged.take() {
1655 let _ = lagged.send(event.cursor.clone());
1656 }
1657 false
1658 }
1659 }
1660 Err(error) => {
1661 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1662 false
1663 }
1664 }
1665 });
1666 }
1667
1668 fn subscribe(
1669 &self,
1670 connection_id: ConnectionId,
1671 corr: u64,
1672 version: u8,
1673 since: Option<SpawnCursor>,
1674 sink: FrameSink,
1675 ) -> Result<(), SpawnSubscribeRefusal> {
1676 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1677 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1678 {
1679 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1680 let replay = if let Some(since) = since {
1681 if since.daemon_incarnation != state.daemon_incarnation {
1682 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1683 current: state.daemon_incarnation.clone(),
1684 });
1685 }
1686 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1687 if since.seq < oldest.seq.saturating_sub(1) {
1688 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1689 }
1690 }
1691 state
1692 .events
1693 .iter()
1694 .filter(|event| event.cursor.seq > since.seq)
1695 .cloned()
1696 .collect::<Vec<_>>()
1697 } else {
1698 Vec::new()
1699 };
1700 for event in replay {
1701 let body = serde_json::to_vec(&event)
1702 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1703 let frame = Frame::build_with_version(
1704 version,
1705 FrameType::StreamData,
1706 control_flags(),
1707 0,
1708 0,
1709 corr,
1710 body,
1711 )
1712 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1713 frames
1714 .try_send(frame)
1715 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1716 }
1717 state.subscribers.insert(
1718 (connection_id, corr),
1719 SpawnSubscriber {
1720 version,
1721 frames,
1722 lagged: Some(lagged),
1723 },
1724 );
1725 }
1726 tokio::spawn(async move {
1737 while let Some(frame) = receiver.recv().await {
1738 if sink.send(frame).await.is_err() {
1739 return;
1740 }
1741 }
1742 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1743 return;
1744 };
1745 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1746 Ok(frame) => {
1747 let _ = sink.send(frame).await;
1748 }
1749 Err(error) => {
1750 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1751 }
1752 }
1753 });
1754 Ok(())
1755 }
1756
1757 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1758 let Some(subscriber) = self
1759 .0
1760 .lock()
1761 .unwrap_or_else(|p| p.into_inner())
1762 .subscribers
1763 .remove(&(connection_id, corr))
1764 else {
1765 return false;
1766 };
1767 if let Ok(frame) = Frame::build_with_version(
1768 subscriber.version,
1769 FrameType::StreamEnd,
1770 control_flags(),
1771 0,
1772 0,
1773 corr,
1774 Vec::new(),
1775 ) {
1776 tokio::spawn(async move {
1777 let _ = subscriber.frames.send(frame).await;
1778 });
1779 }
1780 true
1781 }
1782
1783 fn remove_connection(&self, connection_id: ConnectionId) {
1784 self.0
1785 .lock()
1786 .unwrap_or_else(|p| p.into_inner())
1787 .subscribers
1788 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1789 }
1790
1791 #[cfg(any(test, feature = "test-support"))]
1792 fn set_capacity(&self, capacity: usize) {
1793 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1794 }
1795
1796 #[cfg(any(test, feature = "test-support"))]
1797 fn subscriber_count(&self) -> usize {
1798 self.0
1799 .lock()
1800 .unwrap_or_else(|p| p.into_inner())
1801 .subscribers
1802 .len()
1803 }
1804}
1805
1806fn spawn_subscriber_lagged_frame(
1809 version: u8,
1810 corr: u64,
1811 first_undelivered: SpawnCursor,
1812) -> Result<Frame, String> {
1813 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1814 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1815 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1816 .to_string(),
1817 detail: Some(serde_json::json!({
1818 "first_undelivered_cursor": first_undelivered
1819 })),
1820 })
1821 .map_err(|error| error.to_string())?;
1822 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1823 .map_err(|error| error.to_string())
1824}
1825
1826pub trait ModuleProcessLiveness: Send + Sync {
1827 fn process_live(&self, module_id: &str) -> Option<bool>;
1828
1829 fn process_replacing(&self, _module_id: &str) -> bool {
1835 false
1836 }
1837}
1838
1839#[derive(Debug, Clone, Default)]
1841pub struct SupervisorProcessLiveness {
1842 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1843}
1844
1845impl SupervisorProcessLiveness {
1846 pub fn new() -> Self {
1847 Self::default()
1848 }
1849
1850 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1851 let mut snapshots = self
1852 .snapshots
1853 .lock()
1854 .unwrap_or_else(|poisoned| poisoned.into_inner());
1855 snapshots.insert(module_id, snapshot);
1856 }
1857
1858 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1859 let mut snapshots = self
1860 .snapshots
1861 .lock()
1862 .unwrap_or_else(|poisoned| poisoned.into_inner());
1863 let is_current = snapshots
1864 .get(module_id)
1865 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1866 .unwrap_or(false);
1867 if is_current {
1868 snapshots.remove(module_id);
1869 }
1870 }
1871}
1872
1873impl ModuleProcessLiveness for SupervisorProcessLiveness {
1874 fn process_live(&self, module_id: &str) -> Option<bool> {
1875 let snapshot = {
1876 let snapshots = self
1877 .snapshots
1878 .lock()
1879 .unwrap_or_else(|poisoned| poisoned.into_inner());
1880 snapshots.get(module_id).cloned()
1881 }?;
1882 let snapshot = snapshot
1883 .lock()
1884 .unwrap_or_else(|poisoned| poisoned.into_inner());
1885 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1886 }
1887
1888 fn process_replacing(&self, module_id: &str) -> bool {
1889 let Some(snapshot) = self
1890 .snapshots
1891 .lock()
1892 .unwrap_or_else(|poisoned| poisoned.into_inner())
1893 .get(module_id)
1894 .cloned()
1895 else {
1896 return false;
1897 };
1898 let snapshot = snapshot
1899 .lock()
1900 .unwrap_or_else(|poisoned| poisoned.into_inner());
1901 snapshot.enabled
1902 && match snapshot.state {
1903 ModuleState::Restarting => true,
1904 ModuleState::Draining => snapshot.draining_to_replace,
1905 ModuleState::Starting
1906 | ModuleState::Running
1907 | ModuleState::Unresponsive
1908 | ModuleState::Stopped
1909 | ModuleState::Failed
1910 | ModuleState::Disabled => false,
1911 }
1912 }
1913}
1914
1915#[cfg(test)]
1916#[derive(Debug, Default)]
1917struct ReloadExitRecordGate {
1918 reached: tokio::sync::Notify,
1919 resume: tokio::sync::Notify,
1920}
1921
1922#[derive(Debug, Clone, Copy)]
1923enum RespawnKind {
1924 Spawn,
1925 Reload,
1926}
1927
1928#[derive(Debug, Clone, Copy)]
1929struct PendingRespawn {
1930 deadline: Instant,
1931 kind: RespawnKind,
1932 daemon_initiated: bool,
1933}
1934
1935type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1936
1937#[derive(Debug, Clone)]
1938struct SupervisorRuntimeConfig {
1939 registry: Arc<Registry>,
1940 recovery_is_daemon: bool,
1942 start_requester: Option<Principal>,
1943 pending_activity_restart: bool,
1944 provenance_probe: ExecutableIdentityProbe,
1945 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1947 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1949 restart_policy: RestartPolicy,
1950 register_timeout: Duration,
1951 drain_timeout: Duration,
1954 effective_drain_timeout: Arc<Mutex<Duration>>,
1957 default_drain_timeout: Duration,
1960 health: HealthConfig,
1961 connection_file_path: Option<PathBuf>,
1962 capture_logs_dir: Option<PathBuf>,
1963 forwarding: Option<Arc<ForwardingTable>>,
1964 supervisor_handle: Option<SupervisorHandle>,
1967 stderr_ring: Arc<Mutex<StderrRing>>,
1974 terminal_ring: Arc<Mutex<TerminalRing>>,
1975 spawn_events: SpawnEventFeed,
1976 child_roster: ChildRoster,
1977 #[cfg(target_os = "linux")]
1978 cgroup_placement: Option<subc_cgroup::Placement>,
1979 #[cfg(test)]
1980 test_seed_stale_facts_before_enable_spawn: bool,
1981 #[cfg(test)]
1982 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1983}
1984
1985impl SupervisorRuntimeConfig {
1986 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
1987 FailureContext {
1988 daemon_initiated,
1989 shutdown: self.child_roster.shutdown_flag(),
1990 }
1991 }
1992}
1993
1994#[derive(Debug, Clone, PartialEq, Eq)]
1995struct SupervisedConfiguration {
1996 spec: ModuleSpec,
1997 health: HealthConfig,
1998}
1999
2000#[derive(Debug, Clone, Default)]
2006pub struct SupervisorHandle {
2007 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2008 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2009 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2010 configured_ids: Arc<Mutex<HashSet<String>>>,
2022 spawn_events: SpawnEventFeed,
2023 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2034 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2040 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2044 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2052 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2058 promotion_observer: PromotionObserverSlot,
2060 operation_lock: Arc<AsyncMutex<()>>,
2064}
2065
2066pub(crate) trait SwapPromotionObserver: Send + Sync {
2075 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2076}
2077
2078#[derive(Clone, Default)]
2082struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2083
2084impl fmt::Debug for PromotionObserverSlot {
2085 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2086 f.write_str("PromotionObserverSlot")
2087 }
2088}
2089
2090#[derive(Debug, Clone)]
2092struct OpenSwap {
2093 candidate_nonce: String,
2096 incumbent_nonce: Option<String>,
2101 candidate_admitted: bool,
2105}
2106
2107#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2110pub(crate) enum SwapHelloAdmission {
2111 NotSwapping,
2114 Candidate,
2116 Refused,
2119}
2120
2121#[derive(Debug, Clone, PartialEq, Eq)]
2122pub(crate) enum ReservedHelloRejection {
2123 Exact {
2124 module_id: String,
2125 },
2126 Prefix {
2127 prefix: String,
2128 owner_module_id: String,
2129 },
2130}
2131
2132impl SupervisorHandle {
2133 pub fn new() -> Self {
2134 Self::default()
2135 }
2136
2137 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2138 self.spawn_events.snapshot()
2139 }
2140
2141 pub(crate) fn subscribe_spawns(
2142 &self,
2143 connection_id: ConnectionId,
2144 corr: u64,
2145 version: u8,
2146 since: Option<SpawnCursor>,
2147 sink: FrameSink,
2148 ) -> Result<(), SpawnSubscribeRefusal> {
2149 self.spawn_events
2150 .subscribe(connection_id, corr, version, since, sink)
2151 }
2152
2153 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2154 self.spawn_events.cancel(connection_id, corr)
2155 }
2156
2157 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2158 self.spawn_events.remove_connection(connection_id);
2159 }
2160
2161 #[cfg(any(test, feature = "test-support"))]
2162 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2163 assert!(capacity > 0, "spawn event capacity must be non-zero");
2164 self.spawn_events.set_capacity(capacity);
2165 }
2166
2167 #[cfg(any(test, feature = "test-support"))]
2168 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2169 self.spawn_events.subscriber_count()
2170 }
2171
2172 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2175 self.spawn_nonces
2176 .lock()
2177 .unwrap_or_else(|poisoned| poisoned.into_inner())
2178 .insert(module_id.to_string(), nonce);
2179 }
2180
2181 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2184 self.reserved_nonces
2185 .lock()
2186 .unwrap_or_else(|poisoned| poisoned.into_inner())
2187 .insert(module_id.to_string(), Some(nonce));
2188 }
2189
2190 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2192 let mut owners = self
2193 .reserved_prefix_owners
2194 .lock()
2195 .unwrap_or_else(|poisoned| poisoned.into_inner());
2196 owners.retain(|_, owner| owner != owner_module_id);
2197 for prefix in prefixes {
2198 owners.insert(prefix.clone(), owner_module_id.to_string());
2199 }
2200 }
2201
2202 #[cfg(test)]
2204 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2205 self.spawn_nonces
2206 .lock()
2207 .unwrap_or_else(|poisoned| poisoned.into_inner())
2208 .get(module_id)
2209 .cloned()
2210 }
2211
2212 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2213 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2214 let spawn_nonce = self
2215 .spawn_nonces
2216 .lock()
2217 .unwrap_or_else(|poisoned| poisoned.into_inner())
2218 .get(&spec.module_id)
2219 .cloned();
2220 let mut reserved_nonces = self
2221 .reserved_nonces
2222 .lock()
2223 .unwrap_or_else(|poisoned| poisoned.into_inner());
2224 if spec.reserved {
2225 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2230 }
2231 drop(reserved_nonces);
2232 self.removal_tombstones
2236 .lock()
2237 .unwrap_or_else(|poisoned| poisoned.into_inner())
2238 .remove(&spec.module_id);
2239 }
2240
2241 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2246 self.reserved_hello_verification(module_id, presented)
2247 .is_ok()
2248 }
2249
2250 pub(crate) fn reserved_hello_verification(
2253 &self,
2254 module_id: &str,
2255 presented: Option<&str>,
2256 ) -> Result<bool, ReservedHelloRejection> {
2257 let nonces = self
2258 .reserved_nonces
2259 .lock()
2260 .unwrap_or_else(|poisoned| poisoned.into_inner());
2261 if let Some(expected) = nonces.get(module_id) {
2262 let authorized = match expected {
2266 Some(expected) => {
2267 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2268 }
2269 None => false,
2270 };
2271 if authorized {
2272 return Ok(true);
2273 }
2274 return Err(ReservedHelloRejection::Exact {
2275 module_id: module_id.to_string(),
2276 });
2277 }
2278 drop(nonces);
2279
2280 let matched_prefix = self
2281 .reserved_prefix_owners
2282 .lock()
2283 .unwrap_or_else(|poisoned| poisoned.into_inner())
2284 .iter()
2285 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2286 .max_by_key(|(prefix, _)| prefix.len())
2287 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2288 let Some((prefix, owner_module_id)) = matched_prefix else {
2289 return Ok(false);
2290 };
2291
2292 let authorized = presented.is_some_and(|presented| {
2293 self.spawn_nonces
2294 .lock()
2295 .unwrap_or_else(|poisoned| poisoned.into_inner())
2296 .get(&owner_module_id)
2297 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2298 || self.swap_nonce_matches(&owner_module_id, presented)
2301 });
2302 if authorized {
2303 Ok(true)
2304 } else {
2305 Err(ReservedHelloRejection::Prefix {
2306 prefix,
2307 owner_module_id,
2308 })
2309 }
2310 }
2311
2312 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2317 if presented.is_empty() {
2318 return false;
2319 }
2320 let nonces = self
2321 .spawn_nonces
2322 .lock()
2323 .unwrap_or_else(|poisoned| poisoned.into_inner());
2324 let current = nonces
2325 .get(module_id)
2326 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2327 drop(nonces);
2328 current || self.swap_nonce_matches(module_id, presented)
2333 }
2334
2335 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2337 let swaps = self
2338 .swaps
2339 .lock()
2340 .unwrap_or_else(|poisoned| poisoned.into_inner());
2341 swaps.get(module_id).is_some_and(|swap| {
2342 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2343 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2344 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2345 })
2346 })
2347 }
2348
2349 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2352 let incumbent_nonce = self
2353 .spawn_nonces
2354 .lock()
2355 .unwrap_or_else(|poisoned| poisoned.into_inner())
2356 .get(module_id)
2357 .cloned();
2358 self.swaps
2359 .lock()
2360 .unwrap_or_else(|poisoned| poisoned.into_inner())
2361 .insert(
2362 module_id.to_string(),
2363 OpenSwap {
2364 candidate_nonce,
2365 incumbent_nonce,
2366 candidate_admitted: false,
2367 },
2368 );
2369 }
2370
2371 pub(crate) fn close_swap(&self, module_id: &str) {
2374 self.swaps
2375 .lock()
2376 .unwrap_or_else(|poisoned| poisoned.into_inner())
2377 .remove(module_id);
2378 }
2379
2380 pub(crate) fn set_swap_promotion_observer(
2383 &self,
2384 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2385 ) {
2386 *self
2387 .promotion_observer
2388 .0
2389 .lock()
2390 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2391 }
2392
2393 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2396 let observer = self
2397 .promotion_observer
2398 .0
2399 .lock()
2400 .unwrap_or_else(|poisoned| poisoned.into_inner())
2401 .as_ref()
2402 .and_then(std::sync::Weak::upgrade);
2403 if let Some(observer) = observer {
2404 observer.swap_promoted(registration);
2405 }
2406 }
2407
2408 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2410 self.swaps
2411 .lock()
2412 .unwrap_or_else(|poisoned| poisoned.into_inner())
2413 .contains_key(module_id)
2414 }
2415
2416 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2421 let candidate_nonce = self
2422 .swaps
2423 .lock()
2424 .unwrap_or_else(|poisoned| poisoned.into_inner())
2425 .get(module_id)
2426 .map(|swap| swap.candidate_nonce.clone());
2427 let Some(nonce) = candidate_nonce else {
2428 return;
2429 };
2430 self.set_spawn_nonce(module_id, nonce.clone());
2431 if reserved {
2432 self.set_reserved_nonce(module_id, nonce);
2433 }
2434 }
2435
2436 pub(crate) fn swap_hello_admission(
2451 &self,
2452 module_id: &str,
2453 presented: Option<&str>,
2454 ) -> SwapHelloAdmission {
2455 let swaps = self
2456 .swaps
2457 .lock()
2458 .unwrap_or_else(|poisoned| poisoned.into_inner());
2459 let Some(swap) = swaps.get(module_id) else {
2460 return SwapHelloAdmission::NotSwapping;
2461 };
2462 let Some(presented) = presented else {
2463 return SwapHelloAdmission::Refused;
2464 };
2465 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2466 return if swap.candidate_admitted {
2467 SwapHelloAdmission::Refused
2468 } else {
2469 SwapHelloAdmission::Candidate
2470 };
2471 }
2472 if swap
2473 .incumbent_nonce
2474 .as_deref()
2475 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2476 {
2477 return SwapHelloAdmission::NotSwapping;
2478 }
2479 SwapHelloAdmission::Refused
2480 }
2481
2482 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2485 if let Some(swap) = self
2486 .swaps
2487 .lock()
2488 .unwrap_or_else(|poisoned| poisoned.into_inner())
2489 .get_mut(module_id)
2490 {
2491 swap.candidate_admitted = true;
2492 }
2493 }
2494
2495 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2497 self.spawn_nonces
2498 .lock()
2499 .unwrap_or_else(|poisoned| poisoned.into_inner())
2500 .get(module_id)
2501 .cloned()
2502 }
2503
2504 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2506 self.reserved_nonces
2507 .lock()
2508 .unwrap_or_else(|poisoned| poisoned.into_inner())
2509 .get(module_id)
2510 .cloned()
2511 .flatten()
2512 }
2513
2514 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2515 self.mark_configured(module.module_id());
2519 let mut modules = self
2520 .modules
2521 .lock()
2522 .unwrap_or_else(|poisoned| poisoned.into_inner());
2523 modules.insert(module.module_id().to_string(), module)
2524 }
2525
2526 fn mark_configured(&self, module_id: &str) {
2530 self.configured_ids
2531 .lock()
2532 .unwrap_or_else(|poisoned| poisoned.into_inner())
2533 .insert(module_id.to_string());
2534 }
2535
2536 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2540 let modules = self
2541 .modules
2542 .lock()
2543 .unwrap_or_else(|poisoned| poisoned.into_inner());
2544 if !modules.contains_key(module_id) {
2545 self.configured_ids
2546 .lock()
2547 .unwrap_or_else(|poisoned| poisoned.into_inner())
2548 .remove(module_id);
2549 }
2550 }
2551
2552 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2560 self.configured_ids
2561 .lock()
2562 .unwrap_or_else(|poisoned| poisoned.into_inner())
2563 .contains(module_id)
2564 }
2565
2566 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2567 let modules = self
2568 .modules
2569 .lock()
2570 .unwrap_or_else(|poisoned| poisoned.into_inner());
2571 modules.get(module_id).cloned()
2572 }
2573
2574 pub(crate) fn record_late_health_answer(
2575 &self,
2576 module_id: &str,
2577 latency_ms: u64,
2578 ) -> Result<bool, SuperviseError> {
2579 let Some(module) = self.get(module_id) else {
2580 return Ok(false);
2581 };
2582 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2583 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2584 state.health.last_late_answer_latency_ms = Some(latency_ms);
2585 state.health.consecutive_failures = 0;
2593 })?;
2594 Ok(true)
2595 }
2596
2597 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2603 let Some(module) = self.get(module_id) else {
2604 return Ok(false);
2605 };
2606 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2607 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2608 return Ok(false);
2609 };
2610 drop(snapshot);
2611 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2612 }
2613
2614 pub fn list(&self) -> Vec<SupervisedModule> {
2615 let modules = self
2616 .modules
2617 .lock()
2618 .unwrap_or_else(|poisoned| poisoned.into_inner());
2619 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2620 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2621 modules
2622 }
2623
2624 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2625 self.activity_images
2626 .lock()
2627 .unwrap_or_else(|p| p.into_inner())
2628 .remove(module_id);
2629 self.spawn_nonces
2630 .lock()
2631 .unwrap_or_else(|poisoned| poisoned.into_inner())
2632 .remove(module_id);
2633 self.close_swap(module_id);
2634 let mut reserved_nonces = self
2635 .reserved_nonces
2636 .lock()
2637 .unwrap_or_else(|poisoned| poisoned.into_inner());
2638 if reserved_nonces.contains_key(module_id) {
2639 reserved_nonces.insert(module_id.to_string(), None);
2642 }
2643 drop(reserved_nonces);
2644 self.reserved_prefix_owners
2645 .lock()
2646 .unwrap_or_else(|poisoned| poisoned.into_inner())
2647 .retain(|_, owner| owner != module_id);
2648 let removed = self
2649 .modules
2650 .lock()
2651 .unwrap_or_else(|poisoned| poisoned.into_inner())
2652 .remove(module_id);
2653 self.configured_ids
2654 .lock()
2655 .unwrap_or_else(|poisoned| poisoned.into_inner())
2656 .remove(module_id);
2657 removed
2658 }
2659
2660 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2663 self.removal_tombstones
2664 .lock()
2665 .unwrap_or_else(|poisoned| poisoned.into_inner())
2666 .insert(module_id.to_string(), unix_ms_now());
2667 }
2668
2669 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2671 self.removal_tombstones
2672 .lock()
2673 .unwrap_or_else(|poisoned| poisoned.into_inner())
2674 .get(module_id)
2675 .copied()
2676 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2677 }
2678
2679 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2684 if self.get(module_id).is_some() {
2685 return false;
2686 }
2687 let mut reserved_nonces = self
2688 .reserved_nonces
2689 .lock()
2690 .unwrap_or_else(|poisoned| poisoned.into_inner());
2691 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2692 return false;
2693 }
2694 reserved_nonces.remove(module_id);
2695 true
2696 }
2697
2698 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2699 Arc::clone(&self.operation_lock)
2700 }
2701}
2702
2703#[derive(Debug, Clone)]
2705pub struct Supervisor {
2706 registry: Arc<Registry>,
2707 restart_policy: RestartPolicy,
2708 drain_timeout: Duration,
2709 connection_file_path: Option<PathBuf>,
2710 capture_logs_dir: Option<PathBuf>,
2711 forwarding: Option<Arc<ForwardingTable>>,
2712 process_liveness: Arc<SupervisorProcessLiveness>,
2713 supervisor_handle: Option<SupervisorHandle>,
2714 health: HealthConfig,
2715 daemon_start_clock: crate::clock::StartClock,
2716 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2717 spawn_events: SpawnEventFeed,
2718 provenance_probe: ExecutableIdentityProbe,
2719 child_roster: ChildRoster,
2722 #[cfg(target_os = "linux")]
2723 cgroup_placement: Option<subc_cgroup::Placement>,
2724 #[cfg(test)]
2725 test_after_first_spawn: AfterFirstSpawnHook,
2726}
2727
2728pub(crate) struct RegisteredConfiguredModule {
2732 spec: ModuleSpec,
2733 runtime: SupervisorRuntimeConfig,
2734 module: SupervisedModule,
2735 commands: mpsc::Receiver<SupervisorCommand>,
2736}
2737
2738#[cfg(test)]
2744#[derive(Clone, Default)]
2745struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2746
2747#[cfg(test)]
2748type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2749
2750#[cfg(test)]
2751impl fmt::Debug for AfterFirstSpawnHook {
2752 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2753 f.write_str("AfterFirstSpawnHook")
2754 }
2755}
2756
2757#[cfg(test)]
2758impl AfterFirstSpawnHook {
2759 fn run(&self, module_id: &str) {
2760 if let Some(hook) = &self.0 {
2761 hook(module_id);
2762 }
2763 }
2764}
2765
2766impl Supervisor {
2767 #[cfg(test)]
2768 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2769 let supervisor = Self::new(registry, policy);
2770 #[cfg(target_os = "macos")]
2771 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2772 supervisor
2773 }
2774 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2779 let path = path.into();
2780 #[cfg(target_os = "macos")]
2781 {
2782 let result = probe_privacy_trampoline(&path).map(|()| path);
2783 if let Err(cause) = &result {
2784 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2785 }
2786 self.child_roster.set_privacy_trampoline(result);
2787 }
2788 #[cfg(not(target_os = "macos"))]
2789 let _ = path;
2790 self
2791 }
2792 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2795 self.child_roster.shutdown_flag()
2796 }
2797
2798 #[cfg(unix)]
2809 pub(crate) fn begin_daemon_shutdown(&self) {
2810 self.child_roster.close();
2811 if let Some(journal) = &self.terminal_journal {
2812 journal.stamp_shutdown();
2813 }
2814 }
2815
2816 #[cfg(unix)]
2820 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2821 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2822 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2823 let Some(forwarding) = &self.forwarding else {
2824 return Ok(());
2825 };
2826 let notice_started = Instant::now();
2827 let modules = forwarding
2828 .begin_daemon_drain()
2829 .map_err(SuperviseError::Forwarding)?;
2830 let deadline_ms =
2831 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2832 let mut notices = tokio::task::JoinSet::new();
2833 let mut drains = Vec::new();
2834 for (module_id, endpoint) in modules {
2835 let Some(target) = forwarding
2836 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2837 .map_err(SuperviseError::Forwarding)?
2838 else {
2839 continue;
2840 };
2841 let routes = forwarding
2842 .endpoint_routes(target.endpoint)
2843 .map_err(SuperviseError::Forwarding)?;
2844 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2850 reason: RouteCloseReason::Restart,
2851 deadline_ms,
2852 })
2853 .expect("module draining serializes");
2854 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2855 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2856 for route in routes {
2857 let client = route.goodbye_target;
2858 if let Some((_, channels)) = clients
2859 .iter_mut()
2860 .find(|(existing, _)| existing.connection_id == client.connection_id)
2861 {
2862 channels.push(client.channel);
2863 } else {
2864 let channel = client.channel;
2865 clients.push((client, vec![channel]));
2866 }
2867 }
2868 for (client, mut channels) in clients {
2869 channels.sort_unstable();
2870 channels.dedup();
2871 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2872 module_id: module_id.clone(),
2873 channels,
2874 reason: RouteCloseReason::Restart,
2875 })
2876 .expect("route closing serializes");
2877 recipients.push((client.sink, client.negotiated_ver, closing));
2878 }
2879 for (sink, version, body) in recipients {
2880 notices.spawn(async move {
2881 let frame = Frame::build_with_version(
2882 version,
2883 FrameType::Push,
2884 control_flags(),
2885 0,
2886 0,
2887 0,
2888 body,
2889 )
2890 .expect("bounded lifecycle notice frame builds");
2891 sink.send_flushed(frame).await
2892 });
2893 }
2894 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2895 drains.push((module_id, target.endpoint, gauges));
2896 }
2897 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2900 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2901 if !matches!(result, Ok(Ok(()))) {
2902 warn!(?result, "daemon shutdown notice delivery failed");
2903 }
2904 }
2905 notices.abort_all();
2906 let notice_elapsed = notice_started.elapsed();
2907 let drain_started = Instant::now();
2908 let deadline = drain_started + DRAIN_BUDGET;
2909 let mut waits = tokio::task::JoinSet::new();
2910 for (module_id, endpoint, gauges) in drains {
2911 let forwarding = Arc::clone(forwarding);
2912 let mut runtime = self.runtime_config();
2913 runtime.health.cadence = Duration::from_millis(100);
2914 waits.spawn(async move {
2915 let wait_result = wait_for_forwarding_quiescence(
2916 &forwarding,
2917 &module_id,
2918 &runtime,
2919 endpoint,
2920 deadline,
2921 &gauges,
2922 DrainScope::Endpoint(endpoint),
2923 &mut DrainGaugeTrace::new(&gauges),
2924 )
2925 .await;
2926 forwarding
2930 .record_daemon_drain_result(
2931 endpoint,
2932 drained_after_quiescence_wait(&wait_result),
2933 )
2934 .map_err(SuperviseError::Forwarding)?;
2935 wait_result
2936 });
2937 }
2938 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2939 if !matches!(result, Ok(Ok(true))) {
2940 warn!(?result, "daemon shutdown drain did not reach quiescence");
2941 }
2942 }
2943 info!(
2946 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
2947 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
2948 "daemon shutdown forwarding drain complete"
2949 );
2950 Ok(())
2951 }
2952
2953 #[cfg(unix)]
2965 pub(crate) async fn end_children_for_daemon_shutdown(
2966 &self,
2967 already_escalated: bool,
2968 escalate: impl std::future::Future<Output = ()>,
2969 ) {
2970 tokio::pin!(escalate);
2971 let mut escalated = already_escalated;
2972 if let Some(forwarding) = &self.forwarding {
2973 let reason = CloseReason::new(
2974 "daemon_shutdown",
2975 "the daemon is exiting after its shutdown notice and drain",
2976 );
2977 if escalated {
2978 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2981 } else {
2982 tokio::select! {
2983 biased;
2984 _ = escalate.as_mut() => {
2985 info!("second SIGTERM: abandoning module GOODBYE delivery");
2986 escalated = true;
2987 }
2988 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2989 }
2990 }
2991 let closed = forwarding.close_all_connections(&reason);
2992 debug!(closed, "closed established connections for daemon shutdown");
2993 }
2994 let escalated_here = escalated && !already_escalated;
2998 let remaining_escalate = async move {
2999 if escalated_here {
3000 std::future::pending::<()>().await;
3001 } else {
3002 escalate.await;
3003 }
3004 };
3005 crate::child_roster::end_children_for_daemon_shutdown(
3006 &self.child_roster,
3007 escalated,
3008 remaining_escalate,
3009 )
3010 .await;
3011 }
3012
3013 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3014 Self {
3015 registry,
3016 restart_policy,
3017 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3018 connection_file_path: None,
3019 capture_logs_dir: None,
3020 forwarding: None,
3021 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3022 supervisor_handle: None,
3023 health: HealthConfig::default(),
3024 daemon_start_clock: crate::clock::StartClock::capture(),
3025 terminal_journal: None,
3026 spawn_events: SpawnEventFeed::default(),
3027 provenance_probe: ExecutableIdentityProbe::default(),
3028 child_roster: ChildRoster::default(),
3029 #[cfg(target_os = "linux")]
3030 cgroup_placement: None,
3031 #[cfg(test)]
3032 test_after_first_spawn: AfterFirstSpawnHook::default(),
3033 }
3034 }
3035
3036 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3037 self.drain_timeout = drain_timeout;
3038 self
3039 }
3040
3041 pub(crate) fn default_drain_timeout(&self) -> Duration {
3043 self.drain_timeout
3044 }
3045
3046 pub fn with_process_liveness(
3047 mut self,
3048 process_liveness: Arc<SupervisorProcessLiveness>,
3049 ) -> Self {
3050 self.process_liveness = process_liveness;
3051 self
3052 }
3053
3054 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3055 self.connection_file_path = Some(connection_file_path.into());
3056 self
3057 }
3058
3059 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3061 self.capture_logs_dir = Some(logs_dir.into());
3062 self
3063 }
3064
3065 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3068 self.spawn_events.configure_incarnation(daemon_incarnation);
3072 self
3073 }
3074
3075 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3078 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3079 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3080 path,
3081 daemon_incarnation,
3082 )));
3083 this
3084 }
3085
3086 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3087 self.forwarding = Some(forwarding);
3088 self
3089 }
3090
3091 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3092 self.spawn_events = supervisor_handle.spawn_events.clone();
3093 self.supervisor_handle = Some(supervisor_handle);
3094 self
3095 }
3096
3097 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3098 self.health = health;
3099 self
3100 }
3101
3102 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3106 self.child_roster.record_to(path.into());
3107 self
3108 }
3109
3110 #[cfg(target_os = "linux")]
3111 pub fn with_cgroup_placement(
3112 mut self,
3113 cgroup_placement: Option<subc_cgroup::Placement>,
3114 ) -> Self {
3115 self.cgroup_placement = cgroup_placement;
3116 self
3117 }
3118
3119 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3125 validate_spec(&spec)?;
3126 self.establish_identity(&spec);
3127
3128 let runtime = self.runtime_config();
3129 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3130 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3131 let registration_watch = watch_registration(&self.registry, &spec);
3132 let spawned = spawn_child(
3133 &spec,
3134 runtime.connection_file_path.as_deref(),
3135 self.supervisor_handle.as_ref(),
3136 &runtime.stderr_ring,
3137 runtime.capture_logs_dir.as_deref(),
3138 &runtime.child_roster,
3139 #[cfg(target_os = "linux")]
3140 runtime.cgroup_placement.as_ref(),
3141 );
3142 #[cfg(test)]
3143 self.test_after_first_spawn.run(&spec.module_id);
3144 let mut child = match spawned {
3145 Ok(child) => child,
3146 Err(err) => {
3147 self.abandon_unrostered(&spec.module_id);
3150 return Err(err);
3151 }
3152 };
3153 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3154 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3155 if let Some(spawn) = activity_spawn {
3156 spawn.complete(&snapshot);
3157 }
3158
3159 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3160 }
3161
3162 fn establish_identity(&self, spec: &ModuleSpec) {
3175 if let Some(supervisor_handle) = &self.supervisor_handle {
3176 supervisor_handle.apply_identity_configuration(spec);
3177 supervisor_handle.mark_configured(&spec.module_id);
3178 }
3179 }
3180
3181 fn abandon_unrostered(&self, module_id: &str) {
3184 if let Some(supervisor_handle) = &self.supervisor_handle {
3185 supervisor_handle.unmark_configured_unless_rostered(module_id);
3186 }
3187 }
3188
3189 fn mark_first_process_running(
3192 &self,
3193 spec: &ModuleSpec,
3194 runtime: &SupervisorRuntimeConfig,
3195 snapshot: &SharedSnapshot,
3196 child: &mut SupervisedChild,
3197 ) -> Result<(), SuperviseError> {
3198 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3199 self.abandon_unrostered(&spec.module_id);
3200 return Err(err);
3201 }
3202 self.process_liveness
3203 .track(spec.module_id.clone(), Arc::clone(snapshot));
3204 Ok(())
3205 }
3206
3207 pub fn supervise_configured(
3213 &self,
3214 spec: ModuleSpec,
3215 enabled: bool,
3216 ) -> Result<SupervisedModule, SuperviseError> {
3217 self.supervise_configured_requested(
3218 spec,
3219 enabled,
3220 self.health.clone(),
3221 None,
3222 self.restart_policy,
3223 DEFAULT_REGISTER_TIMEOUT,
3224 None,
3225 )
3226 }
3227
3228 pub fn supervise_configured_with_health(
3234 &self,
3235 spec: ModuleSpec,
3236 enabled: bool,
3237 health: HealthConfig,
3238 drain_timeout_ms: Option<u64>,
3239 restart_policy: RestartPolicy,
3240 ) -> Result<SupervisedModule, SuperviseError> {
3241 self.supervise_configured_requested(
3242 spec,
3243 enabled,
3244 health,
3245 drain_timeout_ms,
3246 restart_policy,
3247 DEFAULT_REGISTER_TIMEOUT,
3248 None,
3249 )
3250 }
3251
3252 #[allow(clippy::too_many_arguments)]
3253 pub(crate) fn supervise_configured_requested(
3254 &self,
3255 spec: ModuleSpec,
3256 enabled: bool,
3257 health: HealthConfig,
3258 drain_timeout_ms: Option<u64>,
3259 restart_policy: RestartPolicy,
3260 register_timeout: Duration,
3261 requester: Option<Principal>,
3262 ) -> Result<SupervisedModule, SuperviseError> {
3263 let registered = self.register_configured_with_health(
3264 spec,
3265 enabled,
3266 health,
3267 drain_timeout_ms,
3268 restart_policy,
3269 register_timeout,
3270 )?;
3271 let daemon_initiated = requester.is_none();
3272 self.start_registered_configured(registered, requester, daemon_initiated)
3273 }
3274
3275 pub(crate) fn register_configured_with_health(
3280 &self,
3281 spec: ModuleSpec,
3282 enabled: bool,
3283 health: HealthConfig,
3284 drain_timeout_ms: Option<u64>,
3285 restart_policy: RestartPolicy,
3286 register_timeout: Duration,
3287 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3288 validate_spec(&spec)?;
3289 self.establish_identity(&spec);
3290
3291 let mut runtime = self.runtime_config();
3292 runtime.health = health;
3293 runtime.restart_policy = restart_policy;
3294 runtime.register_timeout = register_timeout;
3295 if let Some(ms) = drain_timeout_ms {
3296 runtime.drain_timeout = Duration::from_millis(ms);
3297 *runtime
3298 .effective_drain_timeout
3299 .lock()
3300 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3301 }
3302 let snapshot = if enabled {
3303 SupervisorSnapshot::starting()
3304 } else {
3305 SupervisorSnapshot::disabled()
3306 };
3307 let snapshot = {
3308 let mut snapshot = snapshot;
3309 if enabled {
3310 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3311 {
3312 snapshot.waiting_for_previous_pid = Some(pid);
3313 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3314 }
3315 }
3316 snapshot
3317 };
3318 let (module, commands) =
3319 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3320 Ok(RegisteredConfiguredModule {
3321 spec,
3322 runtime,
3323 module,
3324 commands,
3325 })
3326 }
3327
3328 pub(crate) fn start_registered_configured(
3331 &self,
3332 registered: RegisteredConfiguredModule,
3333 requester: Option<Principal>,
3334 daemon_initiated: bool,
3335 ) -> Result<SupervisedModule, SuperviseError> {
3336 let RegisteredConfiguredModule {
3337 spec,
3338 mut runtime,
3339 module,
3340 commands,
3341 } = registered;
3342 runtime.start_requester = requester;
3343 runtime.recovery_is_daemon = daemon_initiated;
3344 let snapshot = &module.inner.snapshot;
3345 let should_spawn = {
3346 let state = lock_snapshot(snapshot)?;
3347 state.enabled && state.waiting_for_previous_pid.is_none()
3348 };
3349 let child = if should_spawn {
3350 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3351 let registration_watch = watch_registration(&self.registry, &spec);
3352 let spawned = spawn_child(
3353 &spec,
3354 runtime.connection_file_path.as_deref(),
3355 self.supervisor_handle.as_ref(),
3356 &runtime.stderr_ring,
3357 runtime.capture_logs_dir.as_deref(),
3358 &runtime.child_roster,
3359 #[cfg(target_os = "linux")]
3360 runtime.cgroup_placement.as_ref(),
3361 );
3362 #[cfg(test)]
3363 self.test_after_first_spawn.run(&spec.module_id);
3364 match spawned {
3365 Ok(mut child) => {
3366 arm_registration_deadline(
3367 &mut child,
3368 runtime.register_timeout,
3369 registration_watch,
3370 );
3371 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3372 if let Some(spawn) = activity_spawn {
3373 spawn.complete(snapshot);
3374 }
3375 Some(child)
3376 }
3377 Err(err) => {
3378 if runtime.health.critical {
3379 error!(
3380 module_id = %spec.module_id,
3381 program = %spec.program.display(),
3382 error = %err,
3383 "critical configured module failed to spawn; marking failed and alerting"
3384 );
3385 } else {
3386 error!(
3387 module_id = %spec.module_id,
3388 program = %spec.program.display(),
3389 error = %err,
3390 "configured module failed to spawn; marking failed and continuing"
3391 );
3392 }
3393 fail_snapshot(
3394 snapshot,
3395 Some(&spec.module_id),
3396 None,
3397 &runtime.failure_context(daemon_initiated),
3398 );
3399 None
3400 }
3401 }
3402 } else {
3403 None
3404 };
3405 self.start_monitor(spec, runtime, &module, child, commands);
3406 Ok(module)
3407 }
3408
3409 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3410 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3411 SupervisorRuntimeConfig {
3412 registry: self.registry.clone(),
3413 recovery_is_daemon: true,
3414 start_requester: None,
3415 pending_activity_restart: false,
3416 provenance_probe: self.provenance_probe.clone(),
3417 scheduled_respawn: Arc::default(),
3418 deferred_reload_reply: Arc::default(),
3419 restart_policy: self.restart_policy,
3420 register_timeout: DEFAULT_REGISTER_TIMEOUT,
3421 drain_timeout: self.drain_timeout,
3422 child_roster: self
3425 .child_roster
3426 .for_module(Arc::clone(&effective_drain_timeout)),
3427 effective_drain_timeout,
3428 default_drain_timeout: self.drain_timeout,
3429 health: self.health.clone(),
3430 connection_file_path: self.connection_file_path.clone(),
3431 capture_logs_dir: self.capture_logs_dir.clone(),
3432 forwarding: self.forwarding.clone(),
3433 supervisor_handle: self.supervisor_handle.clone(),
3434 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3435 terminal_ring: Arc::new(Mutex::new(
3436 TerminalRing::new(
3437 TerminalRingConfig::default(),
3438 self.daemon_start_clock.started_at_ms(),
3439 )
3440 .with_start_clock(self.daemon_start_clock)
3441 .with_journal(self.terminal_journal.clone())
3442 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3443 )),
3444 spawn_events: self.spawn_events.clone(),
3445 #[cfg(target_os = "linux")]
3446 cgroup_placement: self.cgroup_placement.clone(),
3447 #[cfg(test)]
3448 test_seed_stale_facts_before_enable_spawn: false,
3449 #[cfg(test)]
3450 test_reload_exit_record_gate: None,
3451 }
3452 }
3453
3454 fn supervised_module(
3455 &self,
3456 spec: ModuleSpec,
3457 runtime: SupervisorRuntimeConfig,
3458 snapshot: SharedSnapshot,
3459 child: Option<SupervisedChild>,
3460 ) -> SupervisedModule {
3461 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3462 self.start_monitor(spec, runtime, &module, child, commands);
3463 module
3464 }
3465
3466 fn register_module(
3467 &self,
3468 spec: &ModuleSpec,
3469 runtime: &SupervisorRuntimeConfig,
3470 snapshot: SharedSnapshot,
3471 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3472 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3473 spec: spec.clone(),
3474 health: runtime.health.clone(),
3475 }));
3476 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3477 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3478 let restart_policy = runtime.restart_policy;
3482 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3483 let (tx, rx) = mpsc::channel(4);
3484 let module_id = spec.module_id.clone();
3485 let module = SupervisedModule {
3486 inner: Arc::new(SupervisedModuleInner {
3487 module_id: module_id.clone(),
3488 registry: Arc::clone(&self.registry),
3489 snapshot,
3490 configuration,
3491 stderr_ring,
3492 terminal_ring,
3493 commands: tx,
3494 monitor: Mutex::new(None),
3495 restart_policy,
3496 effective_drain_timeout,
3497 provenance_probe: self.provenance_probe.clone(),
3498 }),
3499 };
3500 if let Some(supervisor_handle) = &self.supervisor_handle {
3503 supervisor_handle.insert(module.clone());
3504 }
3505 (module, rx)
3506 }
3507
3508 fn start_monitor(
3509 &self,
3510 spec: ModuleSpec,
3511 runtime: SupervisorRuntimeConfig,
3512 module: &SupervisedModule,
3513 child: Option<SupervisedChild>,
3514 commands: mpsc::Receiver<SupervisorCommand>,
3515 ) {
3516 let monitor = tokio::spawn(supervise_loop(
3517 spec,
3518 runtime,
3519 Arc::clone(&self.registry),
3520 Arc::clone(&self.process_liveness),
3521 Arc::clone(&module.inner.snapshot),
3522 child,
3523 commands,
3524 ));
3525 *module
3526 .inner
3527 .monitor
3528 .lock()
3529 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3530 }
3531}
3532
3533impl Default for Supervisor {
3534 fn default() -> Self {
3535 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3536 }
3537}
3538
3539#[derive(Clone)]
3541pub struct SupervisedModule {
3542 inner: Arc<SupervisedModuleInner>,
3543}
3544
3545struct SupervisedModuleInner {
3546 module_id: String,
3547 registry: Arc<Registry>,
3548 snapshot: SharedSnapshot,
3549 configuration: Arc<Mutex<SupervisedConfiguration>>,
3550 stderr_ring: Arc<Mutex<StderrRing>>,
3551 terminal_ring: Arc<Mutex<TerminalRing>>,
3552 commands: mpsc::Sender<SupervisorCommand>,
3553 monitor: Mutex<Option<JoinHandle<()>>>,
3554 restart_policy: RestartPolicy,
3558 effective_drain_timeout: Arc<Mutex<Duration>>,
3559 provenance_probe: ExecutableIdentityProbe,
3560}
3561
3562impl fmt::Debug for SupervisedModule {
3563 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3564 f.debug_struct("SupervisedModule")
3565 .field("module_id", &self.inner.module_id)
3566 .field("status", &self.status())
3567 .finish_non_exhaustive()
3568 }
3569}
3570
3571impl SupervisedModule {
3572 pub fn module_id(&self) -> &str {
3573 &self.inner.module_id
3574 }
3575
3576 #[cfg(test)]
3580 pub(crate) fn record_health_probe_failure_for_test(
3581 &self,
3582 detail: &str,
3583 ) -> Result<(), SuperviseError> {
3584 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3585 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3586 state.health.detail = Some(detail.to_string());
3587 })
3588 }
3589
3590 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3591 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3592 }
3593
3594 pub(crate) fn registration_end_reason(
3595 &self,
3596 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3597 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3598 Ok(match snapshot.state {
3599 ModuleState::Draining if snapshot.draining_to_replace => {
3600 Some(RegistrationEndReason::SupervisorRestart)
3601 }
3602 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3603 ModuleState::Restarting if snapshot.draining_to_replace => {
3604 Some(RegistrationEndReason::SupervisorRestart)
3605 }
3606 _ => None,
3607 })
3608 }
3609
3610 pub fn stderr_tail(
3617 &self,
3618 max_lines: Option<usize>,
3619 max_bytes: Option<usize>,
3620 ) -> StderrTailSnapshot {
3621 self.inner
3622 .stderr_ring
3623 .lock()
3624 .unwrap_or_else(|poisoned| poisoned.into_inner())
3625 .snapshot(max_lines, max_bytes)
3626 }
3627
3628 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3633 self.inner
3634 .terminal_ring
3635 .lock()
3636 .unwrap_or_else(|poisoned| poisoned.into_inner())
3637 .snapshot()
3638 }
3639
3640 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3645 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3646 }
3647
3648 pub(crate) async fn read_durable_terminal_history(
3653 &self,
3654 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3655 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3656 let module_id = self.inner.module_id.clone();
3657 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3658 .await
3659 }
3660
3661 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3662 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3663 .map(|(status, _)| status)
3664 }
3665
3666 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3667 let state = lock_snapshot(&self.inner.snapshot)?;
3668 Ok(state
3669 .failed_retry
3670 .pending
3671 .filter(|_| state.state == ModuleState::Failed && state.enabled)
3672 .map(|retry| {
3673 unix_ms_now().saturating_add(
3674 u64::try_from(
3675 retry
3676 .deadline
3677 .saturating_duration_since(Instant::now())
3678 .as_millis(),
3679 )
3680 .unwrap_or(u64::MAX),
3681 )
3682 }))
3683 }
3684
3685 pub(crate) fn record_deliberate_severance(
3686 &self,
3687 identity: ProcessIdentity,
3688 ) -> Result<bool, SuperviseError> {
3689 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3690 if snapshot.pid != Some(identity.pid)
3691 || snapshot.process_start_time != Some(identity.start_time)
3692 {
3693 return Ok(false);
3694 }
3695 snapshot.deliberate_severance = Some(identity);
3696 Ok(true)
3697 }
3698
3699 pub(crate) fn status_for_control(
3704 &self,
3705 caller: &'static str,
3706 ) -> Result<ModuleStatus, SuperviseError> {
3707 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3708 .map(|(status, _)| status)
3709 }
3710
3711 fn status_with_snapshot_lock(
3712 &self,
3713 snapshot: &SharedSnapshot,
3714 caller: Option<&'static str>,
3715 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3716 let mut guard = match caller {
3717 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3718 None => lock_snapshot(snapshot)?,
3719 };
3720 let restart_count =
3723 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3724 let snapshot = guard.clone();
3725 drop(guard);
3726 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3727 SuperviseError::StatePoisoned {
3728 module_id: Some(self.inner.module_id.clone()),
3729 }
3730 })?;
3731 let registration_active = self
3732 .inner
3733 .registry
3734 .get_module(&self.inner.module_id)
3735 .map_err(SuperviseError::Registry)?
3736 .is_some();
3737 let protocol = snapshot
3738 .spawned_protocol
3739 .unwrap_or(self.declared_protocol()?);
3740 let running_process =
3741 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3742 let live = match protocol {
3748 ModuleProtocol::Subc => running_process && registration_active,
3749 ModuleProtocol::None => running_process,
3750 };
3751
3752 Ok((
3753 ModuleStatus {
3754 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3755 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3756 module_id: self.inner.module_id.clone(),
3757 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3758 state: snapshot.state,
3759 enabled: snapshot.enabled,
3760 process_alive: snapshot.process_alive,
3761 registration_active,
3762 protocol,
3763 live,
3764 restart_count,
3765 lifetime_restarts: snapshot.lifetime_restarts,
3766 spawn_generation: snapshot.spawn_generation,
3767 max_restarts: self.inner.restart_policy.max_restarts,
3768 restart_window: self.inner.restart_policy.window,
3769 drain_timeout,
3770 restart_backoff: self.inner.restart_policy.backoff,
3771 restart_max_backoff: self.inner.restart_policy.max_backoff,
3772 pid: snapshot.reported_pid(),
3773 spawned_at_ms: snapshot.spawned_at_ms,
3774 spawned_from: snapshot.spawned_from,
3775 launch_spec_digest: snapshot.launch_spec_digest,
3776 process_start_time: snapshot.process_start_time,
3777 last_exit: snapshot.last_exit,
3778 health: snapshot.health,
3779 },
3780 snapshot.spawned_file_identity.clone(),
3781 ))
3782 }
3783
3784 #[cfg(test)]
3785 pub(crate) fn hold_snapshot_for_test(
3786 &self,
3787 acquired: std::sync::mpsc::Sender<()>,
3788 hold: Duration,
3789 ) -> std::thread::JoinHandle<()> {
3790 let snapshot = Arc::clone(&self.inner.snapshot);
3791 std::thread::spawn(move || {
3792 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3793 acquired
3794 .send(())
3795 .expect("test receiver waits for snapshot lock");
3796 std::thread::sleep(hold);
3797 })
3798 }
3799
3800 pub(crate) async fn status_and_running_image_agreement(
3806 &self,
3807 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3808 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3809 let image = self
3810 .inner
3811 .provenance_probe
3812 .observe(
3813 status.pid,
3814 status.spawned_from.as_deref(),
3815 identity,
3816 status.process_start_time,
3817 )
3818 .await;
3819 Ok((status, image))
3820 }
3821
3822 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3823 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3824 Ok(snapshot) => snapshot.clone(),
3825 Err(_) => {
3826 return subc_control::RunningImageAgreement::Unavailable {
3827 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3828 };
3829 }
3830 };
3831 self.inner
3832 .provenance_probe
3833 .observe(
3834 snapshot.reported_pid(),
3835 snapshot.spawned_from.as_deref(),
3836 snapshot.spawned_file_identity.clone(),
3837 snapshot.process_start_time,
3838 )
3839 .await
3840 }
3841
3842 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3845 #[cfg(windows)]
3846 {
3847 let image = lock_snapshot(&self.inner.snapshot)
3848 .ok()
3849 .and_then(|snapshot| {
3850 snapshot
3851 .spawned_file_identity
3852 .as_ref()
3853 .and_then(|identity| identity.windows_image.clone())
3854 });
3855 match image {
3856 Some(image) => {
3857 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3858 }
3859 None => subc_control::ChildResourceUsage::Unavailable {
3860 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3861 },
3862 }
3863 }
3864 #[cfg(not(windows))]
3865 {
3866 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3867 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3868 Err(_) => {
3869 return subc_control::ChildResourceUsage::Unavailable {
3870 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3871 }
3872 }
3873 };
3874 crate::child_resources::read(pid, start_time)
3875 }
3876 }
3877
3878 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3879 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3880 Ok(match snapshot.state {
3881 ModuleState::Restarting => true,
3882 ModuleState::Failed | ModuleState::Disabled => false,
3883 _ if snapshot.enabled && snapshot.respawn_pending => true,
3884 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3885 })
3886 }
3887
3888 #[cfg(test)]
3889 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3890 self.is_warming_with_snapshot_lock(None)
3891 }
3892
3893 pub(crate) fn is_warming_for_control(
3894 &self,
3895 caller: &'static str,
3896 ) -> Result<bool, SuperviseError> {
3897 self.is_warming_with_snapshot_lock(Some(caller))
3898 }
3899
3900 fn is_warming_with_snapshot_lock(
3901 &self,
3902 caller: Option<&'static str>,
3903 ) -> Result<bool, SuperviseError> {
3904 let snapshot = match caller {
3905 Some(caller) => {
3906 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3907 }
3908 None => lock_snapshot(&self.inner.snapshot)?,
3909 }
3910 .clone();
3911 Ok(matches!(
3912 snapshot.state,
3913 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3914 ))
3915 }
3916
3917 pub async fn drain(&self) -> Result<(), SuperviseError> {
3919 self.stop().await
3920 }
3921
3922 pub(crate) async fn retire_requested(
3923 &self,
3924 requester: Option<Principal>,
3925 ) -> Result<(), SuperviseError> {
3926 match self.state()? {
3927 ModuleState::Stopped => return Ok(()),
3928 ModuleState::Failed
3929 if lock_snapshot(&self.inner.snapshot)?
3930 .failed_retry
3931 .pending
3932 .is_none() =>
3933 {
3934 return Ok(())
3935 }
3936 ModuleState::Failed
3937 | ModuleState::Starting
3938 | ModuleState::Running
3939 | ModuleState::Unresponsive
3940 | ModuleState::Restarting
3941 | ModuleState::Draining
3942 | ModuleState::Disabled => {}
3943 }
3944
3945 let (reply_tx, reply_rx) = oneshot::channel();
3946 self.inner
3947 .commands
3948 .send(SupervisorCommand::Retire {
3949 requester,
3950 reply: reply_tx,
3951 })
3952 .await
3953 .map_err(|_| SuperviseError::CommandClosed {
3954 module_id: self.inner.module_id.clone(),
3955 })?;
3956 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3957 module_id: self.inner.module_id.clone(),
3958 })?
3959 }
3960
3961 pub async fn stop(&self) -> Result<(), SuperviseError> {
3962 self.stop_requested(None).await
3963 }
3964
3965 pub(crate) async fn stop_requested(
3966 &self,
3967 requester: Option<Principal>,
3968 ) -> Result<(), SuperviseError> {
3969 match self.state()? {
3970 ModuleState::Stopped => return Ok(()),
3971 ModuleState::Failed
3972 if lock_snapshot(&self.inner.snapshot)?
3973 .failed_retry
3974 .pending
3975 .is_none() =>
3976 {
3977 return Ok(())
3978 }
3979 ModuleState::Failed
3980 | ModuleState::Starting
3981 | ModuleState::Running
3982 | ModuleState::Unresponsive
3983 | ModuleState::Restarting
3984 | ModuleState::Draining
3985 | ModuleState::Disabled => {}
3986 }
3987
3988 let (reply_tx, reply_rx) = oneshot::channel();
3989 self.inner
3990 .commands
3991 .send(SupervisorCommand::Drain {
3992 requester,
3993 reply: reply_tx,
3994 })
3995 .await
3996 .map_err(|_| SuperviseError::CommandClosed {
3997 module_id: self.inner.module_id.clone(),
3998 })?;
3999 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4000 module_id: self.inner.module_id.clone(),
4001 })?
4002 }
4003
4004 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4005 self.restart_requested(drain_timeout_ms, None).await
4006 }
4007
4008 pub(crate) async fn restart_requested(
4009 &self,
4010 drain_timeout_ms: Option<u64>,
4011 requester: Option<Principal>,
4012 ) -> Result<(), SuperviseError> {
4013 let queued_at = Instant::now();
4014 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4015 let (reply_tx, reply_rx) = oneshot::channel();
4016 self.inner
4017 .commands
4018 .send(SupervisorCommand::Restart {
4019 requester,
4020 drain_timeout_ms,
4021 received_at_generation,
4022 queued_at,
4023 reply: reply_tx,
4024 })
4025 .await
4026 .map_err(|_| SuperviseError::CommandClosed {
4027 module_id: self.inner.module_id.clone(),
4028 })?;
4029 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4030 module_id: self.inner.module_id.clone(),
4031 })?
4032 }
4033
4034 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4039 self.swap_requested(ready_timeout, None).await
4040 }
4041
4042 pub(crate) async fn swap_requested(
4043 &self,
4044 ready_timeout: Option<Duration>,
4045 requester: Option<Principal>,
4046 ) -> Result<(), SuperviseError> {
4047 let (reply_tx, reply_rx) = oneshot::channel();
4048 self.inner
4049 .commands
4050 .send(SupervisorCommand::Swap {
4051 requester,
4052 ready_timeout,
4053 reply: reply_tx,
4054 })
4055 .await
4056 .map_err(|_| SuperviseError::CommandClosed {
4057 module_id: self.inner.module_id.clone(),
4058 })?;
4059 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4060 module_id: self.inner.module_id.clone(),
4061 })?
4062 }
4063
4064 pub async fn reload(&self) -> Result<(), SuperviseError> {
4065 self.reload_requested(None).await
4066 }
4067
4068 pub(crate) async fn reload_requested(
4069 &self,
4070 requester: Option<Principal>,
4071 ) -> Result<(), SuperviseError> {
4072 let (reply_tx, reply_rx) = oneshot::channel();
4073 self.inner
4074 .commands
4075 .send(SupervisorCommand::Reload {
4076 requester,
4077 reply: reply_tx,
4078 })
4079 .await
4080 .map_err(|_| SuperviseError::CommandClosed {
4081 module_id: self.inner.module_id.clone(),
4082 })?;
4083 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4084 module_id: self.inner.module_id.clone(),
4085 })?
4086 }
4087
4088 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4089 self.set_enabled_requested(enabled, None).await
4090 }
4091
4092 pub(crate) async fn set_enabled_requested(
4093 &self,
4094 enabled: bool,
4095 requester: Option<Principal>,
4096 ) -> Result<bool, SuperviseError> {
4097 let (reply_tx, reply_rx) = oneshot::channel();
4098 self.inner
4099 .commands
4100 .send(SupervisorCommand::SetEnabled {
4101 requester,
4102 enabled,
4103 reply: reply_tx,
4104 })
4105 .await
4106 .map_err(|_| SuperviseError::CommandClosed {
4107 module_id: self.inner.module_id.clone(),
4108 })?;
4109 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4110 module_id: self.inner.module_id.clone(),
4111 })?
4112 }
4113
4114 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4118 let configured = self
4119 .inner
4120 .configuration
4121 .lock()
4122 .map_err(|_| SuperviseError::StatePoisoned {
4123 module_id: Some(self.inner.module_id.clone()),
4124 })?
4125 .spec
4126 .protocol;
4127 let state = lock_snapshot(&self.inner.snapshot)?;
4128 Ok(state.spawned_protocol.unwrap_or(configured))
4129 }
4130
4131 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4132 let configuration =
4133 self.inner
4134 .configuration
4135 .lock()
4136 .map_err(|_| SuperviseError::StatePoisoned {
4137 module_id: Some(self.inner.module_id.clone()),
4138 })?;
4139 Ok((configuration.spec.clone(), configuration.health.clone()))
4140 }
4141
4142 #[cfg(any(test, feature = "test-support"))]
4146 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4147 let (_, health) = self.configuration()?;
4148 let drain_timeout_ms = u64::try_from(
4149 self.inner
4150 .effective_drain_timeout
4151 .lock()
4152 .unwrap_or_else(|poisoned| poisoned.into_inner())
4153 .as_millis(),
4154 )
4155 .ok();
4156 self.update_configuration(spec, health, drain_timeout_ms)
4157 .await
4158 }
4159
4160 pub(crate) async fn update_configuration(
4161 &self,
4162 spec: ModuleSpec,
4163 health: HealthConfig,
4164 drain_timeout_ms: Option<u64>,
4165 ) -> Result<(), SuperviseError> {
4166 if spec.module_id != self.inner.module_id {
4167 return Err(SuperviseError::InvalidSpec {
4168 reason: "a supervised module's module_id cannot be changed".to_string(),
4169 });
4170 }
4171 validate_spec(&spec)?;
4172 let (reply_tx, reply_rx) = oneshot::channel();
4173 self.inner
4174 .commands
4175 .send(SupervisorCommand::UpdateConfiguration {
4176 spec: spec.clone(),
4177 health: health.clone(),
4178 drain_timeout_ms,
4179 reply: reply_tx,
4180 })
4181 .await
4182 .map_err(|_| SuperviseError::CommandClosed {
4183 module_id: self.inner.module_id.clone(),
4184 })?;
4185 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4186 module_id: self.inner.module_id.clone(),
4187 })?;
4188 let mut configuration =
4189 self.inner
4190 .configuration
4191 .lock()
4192 .map_err(|_| SuperviseError::StatePoisoned {
4193 module_id: Some(self.inner.module_id.clone()),
4194 })?;
4195 configuration.spec = spec;
4196 configuration.health = health;
4197 Ok(())
4198 }
4199}
4200
4201impl Drop for SupervisedModuleInner {
4202 fn drop(&mut self) {
4203 let Ok(mut monitor) = self.monitor.lock() else {
4204 return;
4205 };
4206 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4207 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4208 state.state = ModuleState::Stopped;
4209 clear_current_process_facts(state);
4210 });
4211 monitor.abort();
4212 }
4213 let _ = monitor.take();
4214 }
4215}
4216
4217#[derive(Debug)]
4218enum SupervisorCommand {
4219 Drain {
4220 requester: Option<Principal>,
4221 reply: oneshot::Sender<Result<(), SuperviseError>>,
4222 },
4223 Retire {
4224 requester: Option<Principal>,
4225 reply: oneshot::Sender<Result<(), SuperviseError>>,
4226 },
4227 Restart {
4228 requester: Option<Principal>,
4229 drain_timeout_ms: Option<u64>,
4234 received_at_generation: u64,
4238 queued_at: Instant,
4241 reply: oneshot::Sender<Result<(), SuperviseError>>,
4242 },
4243 Reload {
4244 requester: Option<Principal>,
4245 reply: oneshot::Sender<Result<(), SuperviseError>>,
4246 },
4247 SetEnabled {
4248 requester: Option<Principal>,
4249 enabled: bool,
4250 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4251 },
4252 UpdateConfiguration {
4253 spec: ModuleSpec,
4254 health: HealthConfig,
4255 drain_timeout_ms: Option<u64>,
4258 reply: oneshot::Sender<()>,
4259 },
4260 Swap {
4261 requester: Option<Principal>,
4262 ready_timeout: Option<Duration>,
4265 reply: oneshot::Sender<Result<(), SuperviseError>>,
4267 },
4268}
4269
4270#[derive(Debug)]
4271pub enum SuperviseError {
4272 InvalidSpec {
4273 reason: String,
4274 },
4275 Spawn {
4276 program: PathBuf,
4277 source: io::Error,
4278 cgroup_path: Option<PathBuf>,
4279 },
4280 Cgroup {
4281 module_id: String,
4282 source: io::Error,
4283 },
4284 LaunchNonce {
4287 reason: String,
4288 },
4289 Wait {
4290 module_id: String,
4291 source: io::Error,
4292 },
4293 Kill {
4294 module_id: String,
4295 source: io::Error,
4296 },
4297 Forwarding(ForwardingError),
4298 Registry(RegistryError),
4299 ReloadUnavailable {
4300 module_id: String,
4301 reason: String,
4302 },
4303 Disabled {
4308 module_id: String,
4309 },
4310 ReloadFailed {
4311 module_id: String,
4312 reason: String,
4313 },
4314 RegistrationStillActive {
4315 module_id: String,
4316 waited: Duration,
4317 },
4318 StatePoisoned {
4319 module_id: Option<String>,
4320 },
4321 CommandClosed {
4322 module_id: String,
4323 },
4324 SwapInProgress {
4328 module_id: String,
4329 },
4330 SwapRefused {
4332 module_id: String,
4333 reason: SwapRefusal,
4334 },
4335 SwapFailed {
4339 module_id: String,
4340 arm: SwapFailureArm,
4341 detail: String,
4342 candidate_exit: Option<ExitReport>,
4345 },
4346}
4347
4348#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4350pub enum SwapRefusal {
4351 OverlapExclusive,
4353 NotRegistered,
4356 ProtocolNone,
4359 NotConfigured,
4362 AlreadySwapping,
4364}
4365
4366impl SwapRefusal {
4367 pub fn as_str(self) -> &'static str {
4368 match self {
4369 Self::OverlapExclusive => "overlap_exclusive",
4370 Self::NotRegistered => "not_registered",
4371 Self::ProtocolNone => "protocol_none",
4372 Self::NotConfigured => "not_configured",
4373 Self::AlreadySwapping => "already_swapping",
4374 }
4375 }
4376}
4377
4378#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4381pub enum SwapFailureArm {
4382 SpawnFailed,
4384 NeverRegistered,
4386 NeverReady,
4388 CandidateExited,
4390 CandidateUnhealthy,
4392 Interrupted,
4396 CutoverLost,
4401}
4402
4403impl SwapFailureArm {
4404 pub fn as_str(self) -> &'static str {
4405 match self {
4406 Self::SpawnFailed => "spawn_failed",
4407 Self::NeverRegistered => "never_registered",
4408 Self::NeverReady => "never_ready",
4409 Self::CandidateExited => "candidate_exited",
4410 Self::CandidateUnhealthy => "candidate_unhealthy",
4411 Self::Interrupted => "interrupted",
4412 Self::CutoverLost => "cutover_lost",
4413 }
4414 }
4415}
4416
4417impl fmt::Display for SuperviseError {
4418 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4419 match self {
4420 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4421 Self::Spawn {
4422 program,
4423 source,
4424 cgroup_path: Some(cgroup_path),
4425 } => write!(
4426 f,
4427 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4428 cgroup_path.display(),
4429 program.display()
4430 ),
4431 Self::Spawn {
4432 program,
4433 source,
4434 cgroup_path: None,
4435 } => write!(
4436 f,
4437 "failed to spawn module '{}': {source}",
4438 program.display()
4439 ),
4440 Self::Cgroup { module_id, source } => {
4441 write!(
4442 f,
4443 "failed to prepare cgroup for module '{module_id}': {source}"
4444 )
4445 }
4446 Self::LaunchNonce { reason } => {
4447 write!(
4448 f,
4449 "failed to generate reserved-module launch nonce: {reason}"
4450 )
4451 }
4452 Self::Wait { module_id, source } => {
4453 write!(f, "failed to wait for module '{module_id}': {source}")
4454 }
4455 Self::Kill { module_id, source } => {
4456 write!(f, "failed to kill module '{module_id}': {source}")
4457 }
4458 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4459 Self::Registry(err) => write!(f, "registry error: {err}"),
4460 Self::ReloadUnavailable { module_id, reason } => {
4461 write!(f, "reload unavailable for module '{module_id}': {reason}")
4462 }
4463 Self::Disabled { module_id } => {
4464 write!(
4465 f,
4466 "module '{module_id}' is disabled; enable it before restart or reload"
4467 )
4468 }
4469 Self::ReloadFailed { module_id, reason } => {
4470 write!(f, "reload failed for module '{module_id}': {reason}")
4471 }
4472 Self::RegistrationStillActive { module_id, waited } => write!(
4473 f,
4474 "module '{module_id}' registration remained active after waiting {waited:?}"
4475 ),
4476 Self::StatePoisoned { module_id } => match module_id {
4477 Some(module_id) => {
4478 write!(f, "supervisor state for module '{module_id}' was poisoned")
4479 }
4480 None => write!(f, "supervisor state was poisoned"),
4481 },
4482 Self::CommandClosed { module_id } => {
4483 write!(
4484 f,
4485 "supervisor command channel for module '{module_id}' is closed"
4486 )
4487 }
4488 Self::SwapInProgress { module_id } => write!(
4489 f,
4490 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4491 ),
4492 Self::SwapRefused { module_id, reason } => match reason {
4493 SwapRefusal::OverlapExclusive => write!(
4494 f,
4495 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4496 ),
4497 SwapRefusal::NotRegistered => write!(
4498 f,
4499 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4500 ),
4501 SwapRefusal::ProtocolNone => write!(
4502 f,
4503 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4504 ),
4505 SwapRefusal::NotConfigured => write!(
4506 f,
4507 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4508 ),
4509 SwapRefusal::AlreadySwapping => {
4510 write!(f, "module '{module_id}' is already being swapped")
4511 }
4512 },
4513 Self::SwapFailed {
4514 module_id,
4515 arm,
4516 detail,
4517 ..
4518 } => write!(
4519 f,
4520 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4521 arm.as_str()
4522 ),
4523 }
4524 }
4525}
4526
4527impl Error for SuperviseError {
4528 fn source(&self) -> Option<&(dyn Error + 'static)> {
4529 match self {
4530 Self::Spawn { source, .. }
4531 | Self::Cgroup { source, .. }
4532 | Self::Wait { source, .. }
4533 | Self::Kill { source, .. } => Some(source),
4534 Self::Forwarding(err) => Some(err),
4535 Self::Registry(err) => Some(err),
4536 Self::LaunchNonce { .. }
4537 | Self::InvalidSpec { .. }
4538 | Self::ReloadUnavailable { .. }
4539 | Self::Disabled { .. }
4540 | Self::ReloadFailed { .. }
4541 | Self::RegistrationStillActive { .. }
4542 | Self::StatePoisoned { .. }
4543 | Self::CommandClosed { .. }
4544 | Self::SwapInProgress { .. }
4545 | Self::SwapRefused { .. }
4546 | Self::SwapFailed { .. } => None,
4547 }
4548 }
4549}
4550
4551pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4552 if spec.module_id.trim().is_empty() {
4553 return Err(SuperviseError::InvalidSpec {
4554 reason: "module_id must not be empty".to_string(),
4555 });
4556 }
4557
4558 Ok(())
4559}
4560
4561#[derive(Debug, Default)]
4562struct HealthProbeRuntime {
4563 spawn_generation: Option<u64>,
4564 configured_health: Option<HealthConfig>,
4565 registered_connection: Option<crate::ConnectionId>,
4566 advertised: bool,
4567 next_probe_at: Option<Instant>,
4568 probe_index: u64,
4569}
4570
4571fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4572 lock_snapshot(snapshot)
4573 .ok()
4574 .and_then(|state| state.spawned_protocol)
4575 .unwrap_or(spec.protocol)
4576}
4577
4578impl HealthProbeRuntime {
4579 fn refresh_registration(
4580 &mut self,
4581 spec: &ModuleSpec,
4582 runtime: &SupervisorRuntimeConfig,
4583 registry: &Registry,
4584 snapshot: &SharedSnapshot,
4585 ) {
4586 let generation = lock_snapshot(snapshot)
4587 .ok()
4588 .map(|state| state.spawn_generation);
4589 if self.spawn_generation != generation {
4590 *self = Self {
4594 spawn_generation: generation,
4595 ..Self::default()
4596 };
4597 }
4598 if self.configured_health.as_ref() != Some(&runtime.health) {
4599 self.configured_health = Some(runtime.health.clone());
4600 self.next_probe_at = None;
4601 self.registered_connection = None;
4602 self.probe_index = 0;
4603 }
4604 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4608 self.registered_connection = None;
4609 self.advertised = runtime.health.http.is_some();
4610 if !self.advertised {
4611 self.next_probe_at = None;
4612 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4613 let unknown = ModuleHealthStatus::default();
4614 if state.health != unknown {
4615 state.health = unknown;
4616 }
4617 });
4618 } else if self.next_probe_at.is_none() {
4619 self.next_probe_at = Some(
4620 Instant::now()
4621 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4622 );
4623 }
4624 return;
4625 }
4626
4627 let registration = match registry.get_module(&spec.module_id) {
4628 Ok(registration) => registration,
4629 Err(err) => {
4630 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4631 self.advertised = false;
4632 self.next_probe_at = None;
4633 return;
4634 }
4635 };
4636
4637 let Some(registration) = registration else {
4638 self.registered_connection = None;
4639 self.advertised = false;
4640 self.next_probe_at = None;
4641 return;
4642 };
4643
4644 let advertised = registration
4645 .control_ops
4646 .iter()
4647 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4648 if !advertised {
4649 self.registered_connection = Some(registration.connection_id);
4650 self.advertised = false;
4651 self.next_probe_at = None;
4652 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4653 if state.health.status != SupervisorHealthStatus::Unknown
4654 || state.health.consecutive_failures != 0
4655 || state.health.last_probe_ms.is_some()
4656 || state.health.detail.is_some()
4657 || state.health.metrics.is_some()
4658 {
4659 state.health.status = SupervisorHealthStatus::Unknown;
4660 state.health.consecutive_failures = 0;
4661 state.health.last_probe_ms = None;
4662 state.health.detail = None;
4663 state.health.metrics = None;
4664 }
4665 });
4666 return;
4667 }
4668
4669 let reregistered = self.registered_connection != Some(registration.connection_id);
4670 self.registered_connection = Some(registration.connection_id);
4671 self.advertised = true;
4672 if reregistered || self.next_probe_at.is_none() {
4673 self.probe_index = 0;
4674 self.next_probe_at = Some(
4675 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4676 );
4677 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4678 state.health.status = SupervisorHealthStatus::Unknown;
4679 state.health.consecutive_failures = 0;
4680 state.health.detail = None;
4681 state.health.metrics = None;
4682 });
4683 }
4684 }
4685
4686 fn wake_at(&self) -> Option<Instant> {
4687 if !self.advertised {
4688 return None;
4689 }
4690 self.next_probe_at
4691 }
4692
4693 fn due(&self) -> bool {
4694 self.advertised
4695 && self
4696 .next_probe_at
4697 .is_some_and(|next| Instant::now() >= next)
4698 }
4699
4700 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4701 self.probe_index = self.probe_index.wrapping_add(1);
4702 self.next_probe_at = Some(
4703 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4704 );
4705 }
4706}
4707
4708#[derive(Debug)]
4743enum HealthProbeEvidence {
4744 LaneDead,
4746 NoAnswer,
4748 BadAnswer,
4750 Misconfigured,
4752}
4753
4754#[derive(Debug)]
4755struct HealthProbeError {
4756 evidence: HealthProbeEvidence,
4757 message: String,
4758}
4759
4760impl HealthProbeError {
4761 fn lane_dead(message: impl Into<String>) -> Self {
4762 Self::with(HealthProbeEvidence::LaneDead, message)
4763 }
4764
4765 fn no_answer(message: impl Into<String>) -> Self {
4766 Self::with(HealthProbeEvidence::NoAnswer, message)
4767 }
4768
4769 fn bad_answer(message: impl Into<String>) -> Self {
4770 Self::with(HealthProbeEvidence::BadAnswer, message)
4771 }
4772
4773 fn misconfigured(message: impl Into<String>) -> Self {
4774 Self::with(HealthProbeEvidence::Misconfigured, message)
4775 }
4776
4777 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4778 Self {
4779 evidence,
4780 message: message.into(),
4781 }
4782 }
4783
4784 #[allow(dead_code)]
4798 fn is_proof_of_death(&self) -> bool {
4799 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4800 }
4801
4802 fn label(&self) -> &'static str {
4810 match self.evidence {
4811 HealthProbeEvidence::LaneDead => "lane-dead",
4812 HealthProbeEvidence::NoAnswer => "no-answer",
4813 HealthProbeEvidence::BadAnswer => "bad-answer",
4814 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4815 }
4816 }
4817}
4818
4819impl fmt::Display for HealthProbeError {
4820 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4821 f.write_str(&self.message)
4822 }
4823}
4824
4825async fn run_health_probe_cycle(
4826 spec: &ModuleSpec,
4827 runtime: &SupervisorRuntimeConfig,
4828 registry: &Registry,
4829 process_liveness: &SupervisorProcessLiveness,
4830 snapshot: &SharedSnapshot,
4831 child: &mut Option<SupervisedChild>,
4832) {
4833 let now_ms = unix_ms_now();
4834 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4835 .then_some(runtime.health.http.as_deref())
4836 .flatten();
4837 let result = match http {
4838 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4839 None => probe_module_health(&spec.module_id, runtime, None).await,
4840 };
4841 match result {
4842 Ok(report) => {
4843 handle_health_report(
4844 spec,
4845 runtime,
4846 registry,
4847 process_liveness,
4848 snapshot,
4849 child,
4850 report,
4851 now_ms,
4852 )
4853 .await;
4854 }
4855 Err(err) => {
4856 if http.is_some() {
4857 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4858 state.health.status = SupervisorHealthStatus::Failing;
4859 });
4860 }
4861 handle_health_probe_failure(
4862 spec,
4863 runtime,
4864 registry,
4865 process_liveness,
4866 snapshot,
4867 child,
4868 err,
4869 now_ms,
4870 )
4871 .await;
4872 }
4873 }
4874}
4875
4876pub(crate) struct HttpProbeTarget<'a> {
4877 address: std::net::SocketAddr,
4878 localhost: bool,
4879 authority: &'a str,
4880 path: String,
4881}
4882
4883pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4886 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4887 return Err("must not contain whitespace, controls, or a fragment".into());
4888 }
4889 let rest = url
4890 .strip_prefix("http://")
4891 .ok_or("must use plain http://")?;
4892 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4893 let (authority, suffix) = rest.split_at(split);
4894 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4895 ("::1", rest)
4896 } else {
4897 let split = authority.find(':').unwrap_or(authority.len());
4898 authority.split_at(split)
4899 };
4900 let ip: std::net::IpAddr = match host {
4901 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4902 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4903 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4904 };
4905 let port = if port.is_empty() {
4906 80
4907 } else {
4908 port.strip_prefix(':')
4909 .and_then(|p| p.parse::<u16>().ok())
4910 .filter(|p| *p > 0)
4911 .ok_or("must have a valid nonzero TCP port")?
4912 };
4913 let path = if suffix.is_empty() {
4914 "/".into()
4915 } else if suffix.starts_with('?') {
4916 format!("/{suffix}")
4917 } else {
4918 suffix.into()
4919 };
4920 Ok(HttpProbeTarget {
4921 address: std::net::SocketAddr::new(ip, port),
4922 localhost: host == "localhost",
4923 authority,
4924 path,
4925 })
4926}
4927
4928async fn probe_http_health(
4929 url: &str,
4930 deadline: Duration,
4931) -> Result<HealthReport, HealthProbeError> {
4932 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4933 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4934 let mut response_status = String::new();
4937 let mut body = Vec::new();
4938 let probe = async {
4939 let connection = match tokio::net::TcpStream::connect(target.address).await {
4942 Err(_) if target.localhost => {
4943 tokio::net::TcpStream::connect((
4944 std::net::Ipv6Addr::LOCALHOST,
4945 target.address.port(),
4946 ))
4947 .await
4948 }
4949 result => result,
4950 };
4951 let mut stream = connection.map_err(|error| {
4952 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4953 })?;
4954 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4955 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4956 let mut reader = BufReader::new(stream);
4957 let mut budget = 16 * 1024;
4958 let status = http_line(&mut reader, &mut budget).await?;
4959 let mut words = status.split_ascii_whitespace();
4960 let version = words.next();
4961 let code = words
4962 .next()
4963 .filter(|word| word.len() == 3)
4964 .and_then(|word| word.parse::<u16>().ok());
4965 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4966 || !code.is_some_and(|code| (100..600).contains(&code))
4967 {
4968 return Err(HealthProbeError::bad_answer(format!(
4969 "invalid HTTP status: {status}"
4970 )));
4971 }
4972 let code = code.expect("validated status code");
4973 response_status = status.clone();
4974 let mut length = None;
4975 let mut chunked = false;
4976 loop {
4977 let line = http_line(&mut reader, &mut budget).await?;
4978 if line.is_empty() {
4979 break;
4980 }
4981 if let Some((name, value)) = line.split_once(':') {
4982 if name.eq_ignore_ascii_case("content-length") {
4983 length = Some(value.trim().parse::<u64>().map_err(|_| {
4984 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4985 })?);
4986 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4987 chunked = value.trim().eq_ignore_ascii_case("chunked");
4988 }
4989 }
4990 }
4991 if chunked {
4992 while body.len() < 200 {
4993 let line = http_line(&mut reader, &mut budget).await?;
4994 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4995 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4996 if size == 0 {
4997 break;
4998 }
4999 let count = size.min((200 - body.len()) as u64) as usize;
5000 let start = body.len();
5001 (&mut reader)
5002 .take(count as u64)
5003 .read_to_end(&mut body)
5004 .await
5005 .map_err(|error| {
5006 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5007 })?;
5008 if body.len() - start != count {
5009 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5010 }
5011 if size > count as u64 || body.len() == 200 {
5012 break;
5013 }
5014 if !http_line(&mut reader, &mut budget).await?.is_empty() {
5015 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5016 }
5017 }
5018 } else {
5019 reader
5020 .take(length.unwrap_or(200).min(200))
5021 .read_to_end(&mut body)
5022 .await
5023 .map_err(|error| {
5024 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5025 })?;
5026 }
5027 if (200..300).contains(&code) {
5028 Ok(HealthReport::ok())
5029 } else {
5030 Err(HealthProbeError::bad_answer(
5031 "HTTP health endpoint returned non-2xx",
5032 ))
5033 }
5034 };
5035 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5036 Err(HealthProbeError::no_answer(format!(
5037 "HTTP probe timed out after {deadline:?}"
5038 )))
5039 });
5040 if let Err(error) = &mut result {
5041 if !response_status.is_empty() {
5042 error.message = format!(
5043 "{}; {response_status}: {}",
5044 error.message,
5045 String::from_utf8_lossy(&body)
5046 );
5047 }
5048 }
5049 result
5050}
5051
5052async fn http_line(
5053 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5054 remaining: &mut usize,
5055) -> Result<String, HealthProbeError> {
5056 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5057 let mut line = Vec::new();
5058 (&mut *reader)
5059 .take(*remaining as u64)
5060 .read_until(b'\n', &mut line)
5061 .await
5062 .map_err(|error| {
5063 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5064 })?;
5065 *remaining -= line.len();
5066 if !line.ends_with(b"\r\n") {
5067 return Err(HealthProbeError::bad_answer(
5068 "HTTP headers are incomplete or exceed 16 KiB",
5069 ));
5070 }
5071 line.truncate(line.len() - 2);
5072 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5073}
5074
5075async fn probe_module_health(
5076 module_id: &str,
5077 runtime: &SupervisorRuntimeConfig,
5078 drain_deadline: Option<Instant>,
5079) -> Result<HealthReport, HealthProbeError> {
5080 let Some(forwarding) = runtime.forwarding.as_ref() else {
5081 return Err(HealthProbeError::misconfigured(
5082 "supervisor was not configured with a forwarding table",
5083 ));
5084 };
5085 let probe_started_at = Instant::now();
5086 let mut deadline = probe_started_at + runtime.health.deadline;
5087 if let Some(drain_deadline) = drain_deadline {
5088 deadline = deadline.min(drain_deadline);
5089 }
5090 let pending = if drain_deadline.is_some() {
5091 forwarding.begin_drain_health_probe_rpc_for(
5092 module_id,
5093 MODULE_CONTROL_OP_HEALTH_CHECK,
5094 probe_started_at,
5095 deadline,
5096 )
5097 } else {
5098 forwarding.begin_health_probe_rpc_for(
5099 module_id,
5100 MODULE_CONTROL_OP_HEALTH_CHECK,
5101 probe_started_at,
5102 deadline,
5103 )
5104 }
5105 .map_err(|err| {
5106 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5109 })?;
5110 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5111}
5112
5113async fn probe_endpoint_health(
5120 endpoint: crate::ModuleEndpointId,
5121 runtime: &SupervisorRuntimeConfig,
5122 deadline_cap: Option<Instant>,
5123) -> Result<HealthReport, HealthProbeError> {
5124 let Some(forwarding) = runtime.forwarding.as_ref() else {
5125 return Err(HealthProbeError::misconfigured(
5126 "supervisor was not configured with a forwarding table",
5127 ));
5128 };
5129 let probe_started_at = Instant::now();
5130 let mut deadline = probe_started_at + runtime.health.deadline;
5131 if let Some(cap) = deadline_cap {
5132 deadline = deadline.min(cap);
5133 }
5134 let pending = forwarding
5135 .begin_endpoint_health_probe_rpc_for(
5136 endpoint,
5137 MODULE_CONTROL_OP_HEALTH_CHECK,
5138 probe_started_at,
5139 deadline,
5140 )
5141 .map_err(|err| {
5142 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5143 })?;
5144 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5145}
5146
5147async fn await_health_probe(
5149 forwarding: &ForwardingTable,
5150 pending: PendingModuleControlRpc,
5151 deadline: Instant,
5152 probe_budget: Duration,
5153) -> Result<HealthReport, HealthProbeError> {
5154 let PendingModuleControlRpc {
5155 endpoint,
5156 module_sink,
5157 negotiated_ver,
5158 corr,
5159 receiver,
5160 } = pending;
5161 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5162 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5163 })?;
5164 let frame = Frame::build_with_version(
5165 negotiated_ver,
5166 FrameType::Request,
5167 control_flags(),
5168 0,
5169 0,
5170 corr,
5171 body,
5172 )
5173 .map_err(|err| {
5174 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5175 })?;
5176
5177 match timeout_at(deadline, module_sink.send(frame)).await {
5183 Ok(Ok(())) => {}
5184 Ok(Err(err)) => {
5185 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5186 return Err(HealthProbeError::lane_dead(format!(
5189 "failed to send health.check: {err}"
5190 )));
5191 }
5192 Err(_elapsed) => {
5193 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5194 return Err(HealthProbeError::no_answer(
5198 "health.check send timed out before enqueue (module egress full)",
5199 ));
5200 }
5201 }
5202
5203 match timeout_at(deadline, receiver).await {
5204 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5208 response.health_report().ok_or_else(|| {
5209 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5210 })
5211 }
5212 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5213 format!("health.check rejected: {}", body.message),
5214 )),
5215 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5216 Err(HealthProbeError::lane_dead(message))
5217 }
5218 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5219 Err(HealthProbeError::bad_answer(message))
5220 }
5221 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5222 Err(HealthProbeError::bad_answer(format!(
5223 "expected module-control op '{expected}', got '{actual}'"
5224 )))
5225 }
5226 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5230 "module answered health.check after its daemon deadline",
5231 )),
5232 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5233 "health.check waiter was canceled before the module responded",
5234 )),
5235 Err(_) => {
5236 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5237 Err(HealthProbeError::no_answer(format!(
5238 "module did not answer health.check within {probe_budget:?}"
5239 )))
5240 }
5241 }
5242}
5243
5244#[allow(clippy::too_many_arguments)]
5245async fn handle_health_report(
5246 spec: &ModuleSpec,
5247 runtime: &SupervisorRuntimeConfig,
5248 registry: &Registry,
5249 process_liveness: &SupervisorProcessLiveness,
5250 snapshot: &SharedSnapshot,
5251 child: &mut Option<SupervisedChild>,
5252 report: HealthReport,
5253 now_ms: u64,
5254) {
5255 let status = supervisor_health_status(report.status);
5256 let detail = report.detail.clone();
5257 let metrics = truncate_health_metrics(report.metrics);
5258 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5259 state.health.status = status;
5260 state.health.last_probe_ms = Some(now_ms);
5261 state.health.detail = detail.clone();
5262 state.health.metrics = metrics.clone();
5263 state.health.consecutive_failures = 0;
5264 state.failed_retry.observe_health(
5265 report.status == HealthStatus::Ok,
5266 runtime.restart_policy.window,
5267 );
5268 });
5269
5270 let action = match report.status {
5271 HealthStatus::Ok => return,
5272 HealthStatus::Degraded => runtime.health.on_degraded,
5273 HealthStatus::Failing => runtime.health.on_failing,
5274 };
5275 apply_l3_health_action(
5276 spec,
5277 runtime,
5278 registry,
5279 process_liveness,
5280 snapshot,
5281 child,
5282 status,
5283 detail.as_deref(),
5284 action,
5285 now_ms,
5286 )
5287 .await;
5288}
5289
5290#[allow(clippy::too_many_arguments)]
5291async fn handle_health_probe_failure(
5292 spec: &ModuleSpec,
5293 runtime: &SupervisorRuntimeConfig,
5294 registry: &Registry,
5295 process_liveness: &SupervisorProcessLiveness,
5296 snapshot: &SharedSnapshot,
5297 child: &mut Option<SupervisedChild>,
5298 err: HealthProbeError,
5299 now_ms: u64,
5300) {
5301 let threshold = runtime.health.failure_threshold.max(1);
5302 let mut failures = 0;
5303 let detail = format!("[{}] {err}", err.label());
5308 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5309 state
5310 .failed_retry
5311 .observe_health(false, runtime.restart_policy.window);
5312 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5315 state.health.status = SupervisorHealthStatus::Unknown;
5316 }
5317 state.health.last_probe_ms = Some(now_ms);
5318 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5319 state.health.detail = Some(detail.clone());
5320 state.health.metrics = None;
5321 failures = state.health.consecutive_failures;
5322 });
5323
5324 if failures < threshold {
5325 warn!(
5326 module_id = %spec.module_id,
5327 consecutive_failures = failures,
5328 threshold,
5329 evidence = err.label(),
5330 detail = %detail,
5331 "health.check probe failed"
5332 );
5333 return;
5334 }
5335
5336 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5337 state.state = ModuleState::Unresponsive;
5338 state.health.status = SupervisorHealthStatus::Unresponsive;
5339 });
5340 if runtime.health.critical {
5344 error!(
5345 module_id = %spec.module_id,
5346 status = "unresponsive",
5347 evidence = err.label(),
5348 detail = %detail,
5349 "critical module health alert"
5350 );
5351 } else {
5352 warn!(
5353 module_id = %spec.module_id,
5354 status = "unresponsive",
5355 evidence = err.label(),
5356 detail = %detail,
5357 "module health threshold breached"
5358 );
5359 }
5360 if let Err(err) = health_restart_child(
5361 spec,
5362 runtime,
5363 registry,
5364 process_liveness,
5365 snapshot,
5366 child,
5367 SupervisorHealthStatus::Unresponsive,
5368 Some(&detail),
5369 now_ms,
5370 )
5371 .await
5372 {
5373 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5374 }
5375}
5376
5377#[allow(clippy::too_many_arguments)]
5378async fn apply_l3_health_action(
5379 spec: &ModuleSpec,
5380 runtime: &SupervisorRuntimeConfig,
5381 registry: &Registry,
5382 process_liveness: &SupervisorProcessLiveness,
5383 snapshot: &SharedSnapshot,
5384 child: &mut Option<SupervisedChild>,
5385 status: SupervisorHealthStatus,
5386 detail: Option<&str>,
5387 action: HealthAction,
5388 now_ms: u64,
5389) {
5390 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5391 match action {
5392 HealthAction::Report => {
5393 info!(
5394 module_id = %spec.module_id,
5395 status = ?status,
5396 detail,
5397 "module reported non-ok health"
5398 );
5399 }
5400 HealthAction::Alert => {
5401 error!(
5402 module_id = %spec.module_id,
5403 status = ?status,
5404 detail,
5405 "module health alert"
5406 );
5407 }
5408 HealthAction::Restart => {
5409 if let Err(err) = health_restart_child(
5410 spec,
5411 runtime,
5412 registry,
5413 process_liveness,
5414 snapshot,
5415 child,
5416 status,
5417 detail,
5418 now_ms,
5419 )
5420 .await
5421 {
5422 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5423 }
5424 }
5425 }
5426}
5427
5428#[allow(clippy::too_many_arguments)]
5429async fn health_restart_child(
5430 spec: &ModuleSpec,
5431 runtime: &SupervisorRuntimeConfig,
5432 registry: &Registry,
5433 process_liveness: &SupervisorProcessLiveness,
5434 snapshot: &SharedSnapshot,
5435 child: &mut Option<SupervisedChild>,
5436 status: SupervisorHealthStatus,
5437 detail: Option<&str>,
5438 now_ms: u64,
5439) -> Result<(), SuperviseError> {
5440 let (enabled, schedule) = {
5441 let mut state = lock_snapshot(snapshot)?;
5442 let enabled = state.enabled;
5443 let schedule = if enabled {
5444 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5445 } else {
5446 None
5447 };
5448 (enabled, schedule)
5449 };
5450
5451 if !enabled {
5452 return Err(SuperviseError::Disabled {
5453 module_id: spec.module_id.clone(),
5454 });
5455 }
5456
5457 if schedule.is_none() {
5458 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5459 error!(
5460 module_id = %spec.module_id,
5461 status = ?status,
5462 detail,
5463 max_restarts = runtime.restart_policy.max_restarts,
5464 window_secs = runtime.restart_policy.window.as_secs(),
5465 reason = %runtime.restart_policy.budget_exhausted_detail(),
5466 "health restart budget exhausted; marking module failed"
5467 );
5468 let stop_notice = begin_forwarding_drain_if_configured(
5469 spec,
5470 runtime,
5471 registry,
5472 snapshot,
5473 Some(true),
5474 RouteCloseReason::Disable,
5475 )
5476 .await?;
5477 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5478 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5479 })?;
5480 let result = drain_optional_child(
5481 &spec.module_id,
5482 spec.protocol,
5483 stop_notice,
5484 registry,
5485 runtime.forwarding.as_deref(),
5486 snapshot,
5487 &runtime.terminal_ring,
5488 &runtime.spawn_events,
5489 child,
5490 runtime.drain_timeout,
5491 ModuleState::Failed,
5492 Some(true),
5493 &runtime.failure_context(true),
5494 )
5495 .await;
5496 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5497 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5498 state.failed_retry.schedule(&spec.module_id);
5499 }
5500 })?;
5501 result?;
5502 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5503 return Ok(());
5504 }
5505
5506 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5507 let mut restart_count = 0;
5508 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5509 restart_count = state.crash_restarts.len();
5510 state.state = ModuleState::Unresponsive;
5511 state.health.status = status;
5512 state.health.last_action = Some(HealthAction::Restart.to_string());
5513 state.health.last_action_ms = Some(now_ms);
5514 })?;
5515 warn!(
5516 module_id = %spec.module_id,
5517 status = ?status,
5518 detail,
5519 restart_count,
5520 restart_in_window = schedule.restart_in_window,
5521 delay_ms = schedule.delay.as_millis() as u64,
5522 "health-triggered module restart"
5523 );
5524
5525 let stop_notice = begin_forwarding_drain_if_configured(
5526 spec,
5527 runtime,
5528 registry,
5529 snapshot,
5530 Some(true),
5531 RouteCloseReason::Restart,
5532 )
5533 .await?;
5534 drain_optional_child(
5535 &spec.module_id,
5536 spec.protocol,
5537 stop_notice,
5538 registry,
5539 runtime.forwarding.as_deref(),
5540 snapshot,
5541 &runtime.terminal_ring,
5542 &runtime.spawn_events,
5543 child,
5544 runtime.drain_timeout,
5545 ModuleState::Restarting,
5546 Some(true),
5547 &runtime.failure_context(true),
5548 )
5549 .await?;
5550 schedule_respawn(
5551 runtime,
5552 snapshot,
5553 &spec.module_id,
5554 schedule.delay,
5555 RespawnKind::Spawn,
5556 true,
5557 )
5558}
5559
5560fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5561 if let Some(reply) = runtime
5562 .deferred_reload_reply
5563 .lock()
5564 .unwrap_or_else(|p| p.into_inner())
5565 .take()
5566 {
5567 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5568 module_id: module_id.to_string(),
5569 reason: reason.to_string(),
5570 }));
5571 }
5572}
5573
5574fn schedule_respawn(
5575 runtime: &SupervisorRuntimeConfig,
5576 snapshot: &SharedSnapshot,
5577 module_id: &str,
5578 delay: Duration,
5579 kind: RespawnKind,
5580 daemon_initiated: bool,
5581) -> Result<(), SuperviseError> {
5582 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5583 update_snapshot(snapshot, Some(module_id), |state| {
5584 state.respawn_pending = true
5585 })?;
5586 *runtime
5587 .scheduled_respawn
5588 .lock()
5589 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5590 deadline: Instant::now() + delay,
5591 kind,
5592 daemon_initiated,
5593 });
5594 Ok(())
5595}
5596
5597fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5598 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5599 state.health.last_action = Some(action);
5600 state.health.last_action_ms = Some(now_ms);
5601 });
5602}
5603
5604fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5605 match status {
5606 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5607 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5608 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5609 }
5610}
5611
5612fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5624 let metrics = metrics?;
5625 match serde_json::to_vec(&metrics) {
5626 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5627 "truncated": true,
5628 "original_bytes": encoded.len(),
5629 })),
5630 Ok(_) | Err(_) => Some(metrics),
5631 }
5632}
5633
5634fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5640 if cadence.is_zero() {
5641 return Duration::ZERO;
5642 }
5643 let cadence_ms = cadence.as_millis() as u64;
5644 if cadence_ms == 0 {
5660 return cadence;
5661 }
5662 let jitter_span = (cadence_ms / 10).max(1);
5677 let hash = module_id.as_bytes().iter().fold(
5678 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5679 |acc, byte| {
5680 acc.wrapping_mul(1099511628211)
5681 .wrapping_add(u64::from(*byte))
5682 },
5683 );
5684 cadence + Duration::from_millis(hash % jitter_span)
5685}
5686
5687#[cfg(test)]
5688mod tests {
5689 use super::*;
5690
5691 #[test]
5692 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5693 let handle = SupervisorHandle::new();
5694 let module_id = "readded-tombstone";
5695 handle.record_rescan_removal(module_id);
5696 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5697
5698 handle.apply_identity_configuration(&ModuleSpec {
5699 module_id: module_id.to_string(),
5700 program: PathBuf::from("/test/module"),
5701 args: Vec::new(),
5702 env: Vec::new(),
5703 reserved: false,
5704 reserved_prefixes: Vec::new(),
5705 protocol: ModuleProtocol::Subc,
5706 overlap: Default::default(),
5707 });
5708
5709 assert!(
5710 handle.removal_tombstone_age_ms(module_id).is_none(),
5711 "a re-added module must not retain a stale removal tombstone"
5712 );
5713 }
5714
5715 #[derive(Debug, PartialEq, Eq)]
5718 struct OwnerInSpawnWindow {
5719 module_id: String,
5720 configured: bool,
5721 on_roster: bool,
5722 admission_refusal: Option<&'static str>,
5723 }
5724
5725 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5736 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5737 use crate::scopes::ScopeTable;
5738 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5739
5740 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5741 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5742 module_id: module_id.to_string(),
5743 program,
5744 args: Vec::new(),
5745 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5746 .into_iter()
5747 .map(|key| (key.to_string(), dir.path().display().to_string()))
5748 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5749 .collect(),
5750 reserved: false,
5751 reserved_prefixes: Vec::new(),
5752 protocol: ModuleProtocol::Subc,
5753 overlap: Default::default(),
5754 };
5755 let live = super::terminal_history_tests::fake_aft_stub_path();
5756 let missing = dir.path().join("definitely-missing-module");
5757
5758 let handle = SupervisorHandle::new();
5759 let mut supervisor =
5760 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5761 .with_handle(handle.clone());
5762 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5763 let hook_handle = handle.clone();
5764 let hook_observed = Arc::clone(&observed);
5765 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5766 let configured = hook_handle.is_configured(module_id);
5769 let selector = ScopeSelector::new(
5770 Principal::Reserved {
5771 module_id: module_id.to_string(),
5772 },
5773 "s".to_string(),
5774 Some(1),
5775 );
5776 let carrier = Principal::Reserved {
5777 module_id: "carrier".to_string(),
5778 };
5779 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5780 .admit(&carrier, module_id, &selector, configured)
5781 {
5782 Ok(_) => None,
5783 Err(refusal) => Some(refusal.code),
5784 };
5785 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5786 module_id: module_id.to_string(),
5787 configured,
5788 on_roster: hook_handle.get(module_id).is_some(),
5789 admission_refusal,
5790 });
5791 })));
5792
5793 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5794 let configured = supervisor
5795 .supervise_configured(stub("configured", live.clone()), true)
5796 .unwrap();
5797 let with_health = supervisor
5798 .supervise_configured_with_health(
5799 stub("with-health", live.clone()),
5800 true,
5801 HealthConfig::default(),
5802 None,
5803 RestartPolicy::default(),
5804 )
5805 .unwrap();
5806 let failed = supervisor
5809 .supervise_configured_with_health(
5810 stub("failed-spawn", missing.clone()),
5811 true,
5812 HealthConfig::default(),
5813 None,
5814 RestartPolicy::default(),
5815 )
5816 .unwrap();
5817 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5820
5821 let expected = [
5822 "plain",
5823 "configured",
5824 "with-health",
5825 "failed-spawn",
5826 "spawn-error",
5827 ]
5828 .into_iter()
5829 .map(|module_id| OwnerInSpawnWindow {
5830 module_id: module_id.to_string(),
5831 configured: true,
5832 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5833 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5834 })
5835 .collect::<Vec<_>>();
5836 assert_eq!(*observed.lock().unwrap(), expected);
5837
5838 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5839 assert!(
5840 handle.get(module_id).is_some(),
5841 "{module_id} is on the roster"
5842 );
5843 assert!(
5844 handle.is_configured(module_id),
5845 "{module_id} stays configured"
5846 );
5847 }
5848 assert!(handle.get("spawn-error").is_none());
5849 assert!(
5850 !handle.is_configured("spawn-error"),
5851 "a plain spawn that failed must not leave its module marked configured"
5852 );
5853
5854 handle.retire("failed-spawn");
5856 assert!(!handle.is_configured("failed-spawn"));
5857
5858 for module in [plain, configured, with_health] {
5859 module.stop().await.unwrap();
5860 }
5861 drop(failed);
5862 }
5863
5864 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5871 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5872 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5873 let spec = ModuleSpec {
5874 module_id: "spawned-not-admitted".to_string(),
5875 program: super::terminal_history_tests::fake_aft_stub_path(),
5876 args: Vec::new(),
5877 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5878 .into_iter()
5879 .map(|key| (key.to_string(), dir.path().display().to_string()))
5880 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5881 .collect(),
5882 reserved: false,
5883 reserved_prefixes: Vec::new(),
5884 protocol: ModuleProtocol::Subc,
5885 overlap: Default::default(),
5886 };
5887 let handle = SupervisorHandle::new();
5888 let mut supervisor =
5889 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5890 .with_handle(handle.clone());
5891 let observed = Arc::new(Mutex::new(None));
5892 let hook_handle = handle.clone();
5893 let hook_observed = Arc::clone(&observed);
5894 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5895 let state = hook_handle
5896 .get(module_id)
5897 .expect("a configured module is on the roster before it is spawned")
5898 .state()
5899 .unwrap();
5900 *hook_observed.lock().unwrap() = Some(state);
5901 })));
5902
5903 let module = supervisor.supervise_configured(spec, true).unwrap();
5904 assert_eq!(
5905 *observed.lock().unwrap(),
5906 Some(ModuleState::Starting),
5907 "between the spawn and its admission the supervisor still reports starting"
5908 );
5909 assert_eq!(module.state().unwrap(), ModuleState::Running);
5910 module.stop().await.unwrap();
5911 }
5912
5913 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5914 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5915 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5916 snapshot.process_alive = true;
5917 snapshot.pid = Some(41);
5918 snapshot.spawned_at_ms = Some(42);
5919 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5920 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5921 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5922 device: 43,
5923 inode: 44,
5924 #[cfg(windows)]
5925 windows_image: None,
5926 });
5927 })
5928 .unwrap();
5929 snapshot
5930 }
5931
5932 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5933 let snapshot = lock_snapshot(snapshot).unwrap();
5934 assert!(!snapshot.process_alive);
5935 assert_eq!(snapshot.pid, None);
5936 assert_eq!(snapshot.spawned_at_ms, None);
5937 assert_eq!(snapshot.spawned_from, None);
5938 assert_eq!(snapshot.launch_spec_digest, None);
5939 assert_eq!(snapshot.spawned_file_identity, None);
5940 }
5941
5942 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5943 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5944 let supervisor =
5945 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5946 let mut runtime = supervisor.runtime_config();
5947 runtime.test_seed_stale_facts_before_enable_spawn = true;
5948 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5949 let mut child = None;
5950 let spec = ModuleSpec {
5951 module_id: "failed-enable-clears-facts".to_string(),
5952 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5953 args: Vec::new(),
5954 env: Vec::new(),
5955 reserved: false,
5956 reserved_prefixes: Vec::new(),
5957 protocol: ModuleProtocol::Subc,
5958 overlap: Default::default(),
5959 };
5960
5961 let result = set_child_enabled(
5962 &spec,
5963 &runtime,
5964 &supervisor.registry,
5965 &supervisor.process_liveness,
5966 &snapshot,
5967 &mut child,
5968 true,
5969 &runtime.failure_context(false),
5970 )
5971 .await;
5972
5973 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5974 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5975 assert_snapshot_process_facts_cleared(&snapshot);
5976 }
5977
5978 #[tokio::test]
5979 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5980 let supervisor =
5981 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5982 let runtime = supervisor.runtime_config();
5983 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5984 ModuleState::Restarting,
5985 true,
5986 )));
5987 let spec = ModuleSpec {
5988 module_id: "start-stranded-restarting".to_string(),
5989 program: super::terminal_history_tests::fake_aft_stub_path(),
5990 args: Vec::new(),
5991 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5992 reserved: false,
5993 reserved_prefixes: Vec::new(),
5994 protocol: ModuleProtocol::None,
5995 overlap: Default::default(),
5996 };
5997 let mut child = None;
5998 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5999 assert!(!super::set_child_enabled(
6000 &spec,
6001 &runtime,
6002 &Registry::default(),
6003 &supervisor.process_liveness,
6004 &snapshot,
6005 &mut child,
6006 true,
6007 &runtime.failure_context(false),
6008 )
6009 .await
6010 .unwrap());
6011 assert!(child.is_none());
6012 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6013 assert!(super::set_child_enabled(
6014 &spec,
6015 &runtime,
6016 &Registry::default(),
6017 &supervisor.process_liveness,
6018 &snapshot,
6019 &mut child,
6020 true,
6021 &runtime.failure_context(false),
6022 )
6023 .await
6024 .unwrap());
6025 assert_eq!(
6026 lock_snapshot(&snapshot).unwrap().state,
6027 ModuleState::Running
6028 );
6029 let mut child = child.unwrap();
6030 child.start_kill().unwrap();
6031 child.wait().await.unwrap();
6032 }
6033
6034 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6035 async fn failed_reload_spawn_clears_current_process_facts() {
6036 let supervisor =
6037 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6038 let mut runtime = supervisor.runtime_config();
6039 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6040 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6041 let mut child = None;
6042 let spec = ModuleSpec {
6043 module_id: "failed-reload-clears-facts".to_string(),
6044 program: PathBuf::from("/unused/failed-reload-module"),
6045 args: Vec::new(),
6046 env: Vec::new(),
6047 reserved: false,
6048 reserved_prefixes: Vec::new(),
6049 protocol: ModuleProtocol::Subc,
6050 overlap: Default::default(),
6051 };
6052
6053 let result = handle_reload_spawn_failure(
6054 &spec,
6055 &runtime,
6056 &supervisor.process_liveness,
6057 &snapshot,
6058 &mut child,
6059 "forced reload spawn failure".to_string(),
6060 )
6061 .await;
6062
6063 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6064 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6065 assert_snapshot_process_facts_cleared(&snapshot);
6066 }
6067
6068 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6069 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6070 let supervisor =
6071 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6072 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6073 let module = supervisor.supervised_module(
6074 ModuleSpec {
6075 module_id: "drop-clears-facts".to_string(),
6076 program: PathBuf::from("/unused/drop-module"),
6077 args: Vec::new(),
6078 env: Vec::new(),
6079 reserved: false,
6080 reserved_prefixes: Vec::new(),
6081 protocol: ModuleProtocol::Subc,
6082 overlap: Default::default(),
6083 },
6084 supervisor.runtime_config(),
6085 Arc::clone(&snapshot),
6086 None,
6087 );
6088 assert!(!module
6089 .inner
6090 .monitor
6091 .lock()
6092 .unwrap()
6093 .as_ref()
6094 .unwrap()
6095 .is_finished());
6096
6097 drop(module);
6098
6099 assert_eq!(
6100 lock_snapshot(&snapshot).unwrap().state,
6101 ModuleState::Stopped
6102 );
6103 assert_snapshot_process_facts_cleared(&snapshot);
6104 }
6105
6106 #[cfg(unix)]
6107 #[tokio::test]
6108 async fn rescan_preserves_running_protocol_until_respawn() {
6109 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6110 let initial = ModuleSpec {
6111 module_id: "rescan-protocol".into(),
6112 program: PathBuf::from("/bin/sleep"),
6113 args: vec!["60".into()],
6114 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6115 .into_iter()
6116 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6117 .collect(),
6118 reserved: false,
6119 reserved_prefixes: vec![],
6120 protocol: ModuleProtocol::None,
6121 overlap: Default::default(),
6122 };
6123 let supervisor =
6124 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6125 let module = supervisor.spawn(initial.clone()).unwrap();
6126 assert!(module.status().unwrap().live);
6127 let mut next = initial;
6128 next.protocol = ModuleProtocol::Subc;
6129 module
6130 .update_configuration(next.clone(), HealthConfig::default(), None)
6131 .await
6132 .unwrap();
6133 assert!(
6134 module.status().unwrap().live,
6135 "rescan must not require HELLO from the old non-wire process"
6136 );
6137 let runtime = supervisor.runtime_config();
6138 let action = on_child_exit(
6139 &next,
6140 RestartPolicy::default(),
6141 &supervisor.registry,
6142 &module.inner.snapshot,
6143 &runtime.terminal_ring,
6144 &runtime.spawn_events,
6145 &runtime.child_roster,
6146 ExitReport {
6147 kind: ExitKind::Clean,
6148 code: Some(0),
6149 signal: None,
6150 at_ms: unix_ms_now(),
6151 },
6152 true,
6153 )
6154 .await;
6155 assert!(
6156 matches!(action, NextAction::Restart { .. }),
6157 "the old non-wire process's clean exit must restart"
6158 );
6159 module.drain().await.unwrap();
6160 }
6161
6162 #[cfg(unix)]
6163 #[tokio::test]
6164 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6165 use std::os::unix::fs::PermissionsExt;
6166 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6167 let script = dir.join("module.sh");
6168 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6169 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6170 let record_path = dir.join("live-children.json");
6171 let supervisor =
6172 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6173 .with_live_children_record(&record_path);
6174 for (program, args) in [
6175 (PathBuf::from("sleep"), vec!["60".into()]),
6176 (script, vec![]),
6177 ] {
6178 let spec = ModuleSpec {
6179 module_id: "image-identity".into(),
6180 program: program.clone(),
6181 args,
6182 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6183 .into_iter()
6184 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6185 .collect(),
6186 reserved: false,
6187 reserved_prefixes: vec![],
6188 protocol: ModuleProtocol::None,
6189 overlap: Default::default(),
6190 };
6191 let module = supervisor.spawn(spec).unwrap();
6192 #[cfg(target_os = "macos")]
6193 {
6194 let deadline = Instant::now() + Duration::from_secs(5);
6197 while crate::live_children::read_record(&record_path)
6198 .unwrap()
6199 .iter()
6200 .all(|entry| entry.executable.is_none())
6201 {
6202 assert!(Instant::now() < deadline, "module image was not confirmed");
6203 tokio::time::sleep(Duration::from_millis(5)).await;
6204 }
6205 }
6206 let entry = crate::live_children::read_record(&record_path)
6207 .unwrap()
6208 .pop()
6209 .unwrap();
6210 let observed = subc_os::Process::open(entry.pid)
6211 .unwrap()
6212 .unwrap()
6213 .observe()
6214 .unwrap();
6215 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6216 module.drain().await.unwrap();
6217 assert_eq!(
6218 verdict,
6219 crate::live_children::IdentityVerdict::Matches,
6220 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6221 );
6222 }
6223 }
6224
6225 #[cfg(unix)]
6226 fn http_fixture(
6227 dir: &std::path::Path,
6228 url: &str,
6229 threshold: u32,
6230 ) -> crate::daemon_config::ConfiguredModule {
6231 let path = dir.join("subc.jsonc");
6232 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6233 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6234 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6235 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6236 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6237 }}}).to_string()).unwrap();
6238 crate::daemon_config::load(&path)
6239 .unwrap()
6240 .unwrap()
6241 .modules
6242 .pop()
6243 .unwrap()
6244 }
6245
6246 #[cfg(unix)]
6247 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6248 timeout(Duration::from_secs(5), async {
6249 loop {
6250 if module.status().unwrap().health.status == status {
6251 break;
6252 }
6253 sleep(Duration::from_millis(5)).await;
6254 }
6255 })
6256 .await
6257 .unwrap_or_else(|_| {
6258 panic!(
6259 "expected {status:?}, got {:?}",
6260 module.status().unwrap().health
6261 )
6262 });
6263 }
6264
6265 #[cfg(unix)]
6266 #[tokio::test]
6267 async fn http_health_status_flips_ok_failing_ok() {
6268 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6269 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6270 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6271 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6272 let serving_status = status.clone();
6273 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6274 let server = tokio::spawn(async move {
6275 loop {
6276 let (mut stream, _) = listener.accept().await.unwrap();
6277 let mut request = [0u8; 2048];
6278 let count = stream.read(&mut request).await.unwrap();
6279 assert!(count > 0, "a probe must send an HTTP request");
6280 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6281 let body = if code == 200 {
6282 "ready"
6283 } else {
6284 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6285 };
6286 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6287 let _ = stream.write_all(response.as_bytes()).await;
6288 }
6289 });
6290 let configured = http_fixture(&dir, &url, 1000);
6291 let module =
6292 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6293 .supervise_configured_with_health(
6294 configured.module_spec(),
6295 true,
6296 configured.health,
6297 configured.drain_timeout_ms,
6298 configured.restart,
6299 )
6300 .unwrap();
6301 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6302 status.store(503, std::sync::atomic::Ordering::SeqCst);
6303 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6304 assert!(module
6305 .status()
6306 .unwrap()
6307 .health
6308 .detail
6309 .unwrap()
6310 .contains("scratch failure"));
6311 status.store(200, std::sync::atomic::Ordering::SeqCst);
6312 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6313 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6314 let before = module.status().unwrap();
6315 let (spec, mut health) = module.configuration().unwrap();
6316 health.http = None;
6317 module
6318 .update_configuration(spec.clone(), health.clone(), Some(10))
6319 .await
6320 .unwrap();
6321 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6322 health.http = Some(url);
6323 module
6324 .update_configuration(spec, health, Some(10))
6325 .await
6326 .unwrap();
6327 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6328 assert_eq!(
6329 module.status().unwrap().pid,
6330 before.pid,
6331 "changing a probe must apply live, not restart its process"
6332 );
6333 let (spec, mut health) = module.configuration().unwrap();
6334 health.failure_threshold = 2;
6335 module
6336 .update_configuration(spec, health, Some(10))
6337 .await
6338 .unwrap();
6339 status.store(503, std::sync::atomic::Ordering::SeqCst);
6340 timeout(Duration::from_secs(5), async {
6341 while module.status().unwrap().spawn_generation == before.spawn_generation {
6342 sleep(Duration::from_millis(5)).await;
6343 }
6344 })
6345 .await
6346 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6347 module.drain().await.unwrap();
6348 server.abort();
6349 }
6350
6351 #[cfg(unix)]
6352 #[tokio::test]
6353 async fn http_health_replacement_recovers_after_one_restart() {
6354 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6355 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6356 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6357 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6358 let mut configured = http_fixture(&dir, &url, 2);
6359 configured.restart.max_restarts = 2;
6360 let module =
6361 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6362 .supervise_configured_with_health(
6363 configured.module_spec(),
6364 true,
6365 configured.health,
6366 configured.drain_timeout_ms,
6367 configured.restart,
6368 )
6369 .unwrap();
6370 let first_generation = module.status().unwrap().spawn_generation;
6371 let observed = module.clone();
6372 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6373 let server = tokio::spawn(async move {
6374 let mut generation = first_generation;
6375 let mut probes = 0;
6376 loop {
6377 let (mut stream, _) = listener.accept().await.unwrap();
6378 let mut request = [0; 2048];
6379 assert!(stream.read(&mut request).await.unwrap() > 0);
6380 let current = observed.status().unwrap().spawn_generation;
6381 if current != generation {
6382 generation = current;
6383 probes = 0;
6384 }
6385 probes += 1;
6386 let healthy = generation != first_generation && probes > 1;
6389 let code = if healthy { 200 } else { 503 };
6390 let response = format!(
6391 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6392 );
6393 let _ = stream.write_all(response.as_bytes()).await;
6394 if healthy {
6395 healthy_tx.send(()).await.unwrap();
6396 }
6397 }
6398 });
6399 timeout(Duration::from_secs(5), async {
6400 let mut healthy_probes = 0;
6401 loop {
6402 if module.state().unwrap() == ModuleState::Failed {
6403 break;
6404 }
6405 while healthy_rx.try_recv().is_ok() {
6406 healthy_probes += 1;
6407 }
6408 if healthy_probes >= 3
6409 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6410 {
6411 break;
6412 }
6413 sleep(Duration::from_millis(5)).await;
6414 }
6415 })
6416 .await
6417 .expect("the replacement must settle healthy without exhausting its restart budget");
6418 let status = module.status().unwrap();
6419 module.drain().await.unwrap();
6420 server.abort();
6421 assert_eq!(status.state, ModuleState::Running);
6422 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6423 assert_eq!(status.health.consecutive_failures, 0);
6424 assert_eq!(status.lifetime_restarts, 1);
6425 assert_eq!(status.restart_count, 1);
6426 }
6427
6428 #[cfg(unix)]
6429 #[tokio::test]
6430 async fn http_health_sustained_failure_exhausts_restart_budget() {
6431 use std::collections::BTreeMap;
6432 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6433 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6434 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6435 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6436 let mut configured = http_fixture(&dir, &url, 2);
6437 configured.restart.max_restarts = 2;
6438 let module =
6439 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6440 .supervise_configured_with_health(
6441 configured.module_spec(),
6442 true,
6443 configured.health,
6444 configured.drain_timeout_ms,
6445 configured.restart,
6446 )
6447 .unwrap();
6448 let observed = module.clone();
6449 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6450 let served = probes.clone();
6451 let server = tokio::spawn(async move {
6452 loop {
6453 let (mut stream, _) = listener.accept().await.unwrap();
6454 let mut request = [0; 2048];
6455 assert!(stream.read(&mut request).await.unwrap() > 0);
6456 *served
6457 .lock()
6458 .unwrap()
6459 .entry(observed.status().unwrap().spawn_generation)
6460 .or_default() += 1;
6461 let _ = stream
6462 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6463 .await;
6464 }
6465 });
6466 timeout(Duration::from_secs(5), async {
6467 while module.state().unwrap() != ModuleState::Failed {
6468 sleep(Duration::from_millis(5)).await;
6469 }
6470 })
6471 .await
6472 .expect("sustained failures must still exhaust the health restart budget");
6473 let status = module.status().unwrap();
6474 server.abort();
6475 assert_eq!(status.restart_count, 2);
6476 assert_eq!(status.lifetime_restarts, 2);
6477 assert!(!status.process_alive);
6478 assert_eq!(
6479 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6480 vec![2, 2, 2]
6481 );
6482 }
6483
6484 #[cfg(unix)]
6485 #[tokio::test]
6486 async fn spawn_clears_the_previous_process_health() {
6487 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6488 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6489 configured.health.cadence = Duration::from_secs(3600);
6490 let module =
6491 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6492 .supervise_configured_with_health(
6493 configured.module_spec(),
6494 false,
6495 configured.health,
6496 configured.drain_timeout_ms,
6497 configured.restart,
6498 )
6499 .unwrap();
6500 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6501 state.health.status = SupervisorHealthStatus::Unresponsive;
6502 state.health.consecutive_failures = 3;
6503 state.health.detail = Some("old process refused connections".into());
6504 state.health.metrics = Some(serde_json::json!({"old": true}));
6505 state.health.last_probe_ms = Some(123);
6506 state.health.last_action = Some("restart".into());
6507 state.health.last_action_ms = Some(124);
6508 })
6509 .unwrap();
6510 module.set_enabled(true).await.unwrap();
6511 let health = module.status().unwrap().health;
6512 module.drain().await.unwrap();
6513 assert_eq!(health.consecutive_failures, 0);
6514 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6515 assert_eq!(health.detail, None);
6516 assert_eq!(health.metrics, None);
6517 assert_eq!(health.last_probe_ms, None);
6518 assert_eq!(health.last_action.as_deref(), Some("restart"));
6519 assert_eq!(health.last_action_ms, Some(124));
6520 }
6521
6522 #[cfg(unix)]
6523 #[tokio::test]
6524 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6525 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6526 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6527 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6528 drop(unused);
6529 let configured = http_fixture(&dir, &url, 2);
6530 let module =
6531 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6532 .supervise_configured_with_health(
6533 configured.module_spec(),
6534 true,
6535 configured.health,
6536 configured.drain_timeout_ms,
6537 configured.restart,
6538 )
6539 .unwrap();
6540 let before = module.status().unwrap().spawn_generation;
6541 timeout(Duration::from_secs(5), async {
6542 loop {
6543 let status = module.status().unwrap();
6544 if status.spawn_generation > before {
6545 assert!(status.lifetime_restarts > 0);
6546 break;
6547 }
6548 sleep(Duration::from_millis(5)).await;
6549 }
6550 })
6551 .await
6552 .expect("sustained HTTP refusal must trigger the health restart policy");
6553 module.drain().await.unwrap();
6554 }
6555
6556 #[cfg(unix)]
6557 #[tokio::test]
6558 async fn http_health_timeout_honours_deadline() {
6559 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6560 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6561 let server = tokio::spawn(async move {
6562 let _held = listener.accept().await.unwrap();
6563 std::future::pending::<()>().await;
6564 });
6565 let error = timeout(
6566 Duration::from_secs(1),
6567 probe_http_health(&url, Duration::from_millis(10)),
6568 )
6569 .await
6570 .expect("the probe must enforce its own deadline")
6571 .unwrap_err();
6572 server.abort();
6573 assert!(error.to_string().contains("timed out"));
6574 }
6575
6576 #[cfg(unix)]
6577 #[tokio::test]
6578 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6579 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6580 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6581 let url = format!(
6582 "http://localhost:{}/healthz",
6583 listener.local_addr().unwrap().port()
6584 );
6585 let server = tokio::spawn(async move {
6586 let (mut stream, _) = listener.accept().await.unwrap();
6587 let mut request = [0u8; 2048];
6588 assert!(stream.read(&mut request).await.unwrap() > 0);
6589 stream
6590 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6591 .await
6592 .unwrap();
6593 });
6594 assert_eq!(
6599 probe_http_health(&url, Duration::from_secs(10))
6600 .await
6601 .unwrap()
6602 .status,
6603 HealthStatus::Ok
6604 );
6605 server.await.unwrap();
6606 }
6607
6608 #[cfg(unix)]
6609 #[tokio::test]
6610 async fn http_health_timeout_keeps_partial_status_and_body() {
6611 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6612 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6613 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6614 let server = tokio::spawn(async move {
6615 let (mut stream, _) = listener.accept().await.unwrap();
6616 let mut request = [0u8; 2048];
6617 assert!(stream.read(&mut request).await.unwrap() > 0);
6618 stream
6619 .write_all(
6620 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6621 )
6622 .await
6623 .unwrap();
6624 std::future::pending::<()>().await;
6625 });
6626 let error = probe_http_health(&url, Duration::from_secs(1))
6627 .await
6628 .unwrap_err()
6629 .to_string();
6630 server.abort();
6631 assert!(
6632 error.contains("timed out")
6633 && error.contains("503 Unavailable")
6634 && error.contains("partial diagnostic"),
6635 "{error}"
6636 );
6637 }
6638
6639 #[cfg(unix)]
6640 #[tokio::test]
6641 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6642 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6643 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6644 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6645 let server = tokio::spawn(async move {
6646 let (mut stream, _) = listener.accept().await.unwrap();
6647 let mut request = [0u8; 2048];
6648 assert!(stream.read(&mut request).await.unwrap() > 0);
6649 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6650 let response = format!(
6651 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6652 body.len()
6653 );
6654 stream.write_all(response.as_bytes()).await.unwrap();
6655 });
6656 let error = probe_http_health(&url, Duration::from_secs(1))
6657 .await
6658 .unwrap_err()
6659 .to_string();
6660 server.await.unwrap();
6661 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6662 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6663 assert!(!error.contains("not-in-diagnostic"));
6664 }
6665
6666 #[cfg(unix)]
6667 #[tokio::test]
6668 async fn http_health_real_nats_server_monitoring() {
6669 if std::process::Command::new("nats-server")
6670 .arg("--version")
6671 .env("XDG_DATA_HOME", std::env::temp_dir())
6672 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6673 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6674 .output()
6675 .is_err()
6676 {
6677 eprintln!(
6678 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6679 );
6680 return;
6681 }
6682 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6683 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6684 let port = monitor.local_addr().unwrap().port();
6685 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6686 let client_port = client.local_addr().unwrap().port();
6687 let config = dir.join("server.conf");
6688 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6689 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6690 configured.program = PathBuf::from("nats-server");
6691 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6692 drop(monitor);
6693 drop(client);
6694 let module =
6695 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6696 .supervise_configured_with_health(
6697 configured.module_spec(),
6698 true,
6699 configured.health,
6700 configured.drain_timeout_ms,
6701 configured.restart,
6702 )
6703 .unwrap();
6704 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6705 module.drain().await.unwrap();
6706 }
6707
6708 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6709 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6710 let supervisor =
6711 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6712 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6713 let initial = ModuleSpec {
6714 module_id: "rescan-preserves-spawn-facts".to_string(),
6715 program: PathBuf::from("/spawned/module"),
6716 args: Vec::new(),
6717 env: Vec::new(),
6718 reserved: false,
6719 reserved_prefixes: Vec::new(),
6720 protocol: ModuleProtocol::Subc,
6721 overlap: Default::default(),
6722 };
6723 let module = supervisor.supervised_module(
6724 initial.clone(),
6725 supervisor.runtime_config(),
6726 snapshot,
6727 None,
6728 );
6729 let before = module.status().unwrap();
6730 let mut replacement = initial;
6731 replacement.program = PathBuf::from("/rescanned/replacement-module");
6732
6733 module
6734 .update_configuration(replacement, HealthConfig::default(), None)
6735 .await
6736 .unwrap();
6737
6738 let after = module.status().unwrap();
6739 assert_eq!(after.pid, before.pid);
6740 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6741 assert_eq!(after.spawned_from, before.spawned_from);
6742 drop(module);
6743 }
6744}
6745
6746fn unix_ms_now() -> u64 {
6747 SystemTime::now()
6748 .duration_since(UNIX_EPOCH)
6749 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6750 .unwrap_or(0)
6751}
6752
6753async fn supervise_loop(
6754 mut spec: ModuleSpec,
6755 mut runtime: SupervisorRuntimeConfig,
6756 registry: Arc<Registry>,
6757 process_liveness: Arc<SupervisorProcessLiveness>,
6758 snapshot: SharedSnapshot,
6759 mut child: Option<SupervisedChild>,
6760 mut commands: mpsc::Receiver<SupervisorCommand>,
6761) {
6762 let mut health_probe = HealthProbeRuntime::default();
6763 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6767 Ok(changes) => Some(changes),
6768 Err(err) => {
6769 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6772 None
6773 }
6774 };
6775 let mut pending_respawn: Option<PendingRespawn> = None;
6779 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6782 let mut previous_instance_recheck = tokio::time::interval_at(
6783 Instant::now() + Duration::from_secs(5),
6784 Duration::from_secs(5),
6785 );
6786 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6787 loop {
6788 #[cfg(test)]
6789 {
6790 lock_snapshot(&snapshot).unwrap().actor_select = None;
6791 }
6792 #[cfg(target_os = "macos")]
6793 if let Some(active) = child.as_mut() {
6794 active.confirm_privacy_exec().await;
6795 }
6796 if let Some(scheduled) = runtime
6797 .scheduled_respawn
6798 .lock()
6799 .unwrap_or_else(|p| p.into_inner())
6800 .take()
6801 {
6802 runtime.recovery_is_daemon = scheduled.daemon_initiated;
6803 pending_respawn = Some(scheduled);
6804 }
6805 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6806 pending_respawn = None;
6807 cancel_deferred_reload(
6808 &runtime,
6809 &spec.module_id,
6810 "respawn cancelled by a supervisor command",
6811 );
6812 }
6813 if child.is_none()
6814 && pending_respawn.is_none()
6815 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6816 {
6817 cancel_deferred_reload(
6818 &runtime,
6819 &spec.module_id,
6820 "respawn cancelled before a replacement was spawned",
6821 );
6822 let mut stranded = false;
6823 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6824 state.respawn_pending = false;
6825 state.coalesced_restart_pending = false;
6826 if matches!(
6827 state.state,
6828 ModuleState::Restarting
6829 | ModuleState::Starting
6830 | ModuleState::Draining
6831 | ModuleState::Unresponsive
6832 ) {
6833 error!(module_id = %spec.module_id, state = ?state.state,
6834 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6835 stranded = true;
6836 }
6837 });
6838 if stranded {
6839 fail_snapshot(
6840 &snapshot,
6841 Some(&spec.module_id),
6842 None,
6843 &runtime.failure_context(runtime.recovery_is_daemon),
6844 );
6845 }
6846 }
6847 if let Some(command) = requeued.pop_front() {
6848 if !handle_supervisor_command(
6849 command,
6850 &mut spec,
6851 &mut runtime,
6852 ®istry,
6853 &process_liveness,
6854 &snapshot,
6855 &mut child,
6856 &mut commands,
6857 &mut requeued,
6858 )
6859 .await
6860 {
6861 return;
6862 }
6863 if child.is_some() || !respawn_still_pending(&snapshot) {
6864 pending_respawn = None;
6865 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6866 state.respawn_pending = false
6867 });
6868 }
6869 continue;
6870 }
6871 if child.is_some() {
6872 runtime.recovery_is_daemon = true;
6873 #[cfg(test)]
6874 {
6875 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6876 }
6877 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6878 let active = child.as_mut().expect("child checked above");
6879 if runtime.child_roster.is_closed() || active.registered_since_spawn() {
6880 active.registration_deadline = None;
6881 }
6882 let registration_deadline = active.registration_deadline;
6883 let no_checks = runtime.health.http.is_none()
6884 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
6885 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
6886 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
6887 state
6888 .failed_retry
6889 .uptime_reset_at(no_checks, runtime.restart_policy.window)
6890 });
6891 let wake_at = match (health_probe.wake_at(), uptime_reset) {
6892 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
6893 (probe, None) => probe,
6894 };
6895 let wake_at = match (wake_at, registration_deadline) {
6896 (Some(wake), Some(register)) => Some(wake.min(register)),
6897 (wake, register) => wake.or(register),
6898 };
6899 #[cfg(test)]
6900 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
6901 let probe_sleep = supervisor_sleep(wake_at);
6902 tokio::pin!(probe_sleep);
6903 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6908 #[cfg(test)]
6909 {
6910 let registration_pending = wire_child
6911 && registration_changes
6912 .as_ref()
6913 .is_some_and(|changes| changes.has_changed().unwrap());
6914 let mut state = lock_snapshot(&snapshot).unwrap();
6915 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6916 generation: state.spawn_generation,
6917 turn: state.actor_turns,
6918 registered_connection: health_probe.registered_connection,
6919 next_probe_at: health_probe.next_probe_at,
6920 wake_after,
6921 });
6922 }
6923 let active_child = child.as_mut().expect("child checked above");
6924 tokio::select! {
6925 wait_result = active_child.wait() => {
6926 if let Ok(mut state) = lock_snapshot(&snapshot) {
6927 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
6928 }
6929 #[cfg(test)]
6930 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6931 let exit_report = match wait_result {
6940 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6941 Err(err) => {
6942 active_child.drain_stderr(&spec.module_id).await;
6943 fail_after_wait_error(
6944 &spec.module_id,
6945 &snapshot,
6946 &runtime.terminal_ring,
6947 &runtime.spawn_events,
6948 format!("failed to wait for supervised module: {err}"),
6949 &runtime.failure_context(true),
6950 );
6951 untrack_if_registration_released(
6952 &process_liveness,
6953 ®istry,
6954 &spec.module_id,
6955 &snapshot,
6956 );
6957 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6958 child = None;
6959 continue;
6960 }
6961 };
6962 active_child.drain_stderr(&spec.module_id).await;
6963
6964 let next = on_child_exit(
6965 &spec,
6966 runtime.restart_policy,
6967 ®istry,
6968 &snapshot,
6969 &runtime.terminal_ring,
6970 &runtime.spawn_events,
6971 &runtime.child_roster,
6972 exit_report,
6973 true,
6974 ).await;
6975 runtime.start_requester = None;
6976 runtime.pending_activity_restart = false;
6977 active_child.release_roster();
6980 match next {
6981 NextAction::Stop { registration_released } => {
6982 if registration_released {
6983 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6984 }
6985 child = None;
6986 }
6987 NextAction::Restart { schedule } => {
6988 let delay = schedule.map_or(
6989 runtime.restart_policy.delay_for_restart(0),
6990 |schedule| schedule.delay,
6991 );
6992 if let Some(schedule) = schedule {
6993 log_crash_respawn(&spec.module_id, schedule);
6994 }
6995 child = None;
7003 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7004 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7005 }
7006 }
7007 }
7008 command = commands.recv() => {
7009 #[cfg(test)]
7010 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7011 let Some(command) = command else {
7012 return;
7013 };
7014 if !handle_supervisor_command(
7015 command,
7016 &mut spec,
7017 &mut runtime,
7018 ®istry,
7019 &process_liveness,
7020 &snapshot,
7021 &mut child,
7022 &mut commands,
7023 &mut requeued,
7024 ).await {
7025 return;
7026 }
7027 }
7028 _ = async {
7029 match registration_changes.as_mut() {
7030 Some(changes) => { let _ = changes.changed().await; }
7031 None => std::future::pending().await,
7032 }
7033 }, if wire_child => {
7034 #[cfg(test)]
7035 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7036 }
7037 _ = &mut probe_sleep => {
7038 #[cfg(test)]
7039 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7040 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7041 active_child.registration_deadline = None;
7044 if !runtime.child_roster.is_closed()
7045 && !active_child.registered_since_spawn()
7046 && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7047 {
7048 warn!(module_id = %spec.module_id, pid = active_child.pid,
7049 wait_secs = runtime.register_timeout.as_secs(),
7050 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7051 active_child.spawn_failure = Some("registration_timeout".into());
7052 if let Err(error) = stop_unregistered_child(active_child).await {
7053 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7054 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7055 &runtime.spawn_events, format!("registration_timeout: {error}"),
7056 &runtime.failure_context(true));
7057 child = None;
7058 }
7059 continue;
7062 }
7063 }
7064 if health_probe.due() {
7065 run_health_probe_cycle(
7066 &spec,
7067 &runtime,
7068 ®istry,
7069 &process_liveness,
7070 &snapshot,
7071 &mut child,
7072 ).await;
7073 if child.is_none() {
7074 runtime.start_requester = None;
7075 runtime.pending_activity_restart = false;
7076 }
7077 if child.is_some() {
7078 health_probe.schedule_next(&spec, runtime.health.cadence);
7079 }
7080 }
7081 }
7082 }
7083 } else if let Some(pending) = pending_respawn {
7084 tokio::select! {
7085 _ = sleep_until(pending.deadline) => {
7086 pending_respawn = None;
7087 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7088 if !respawn_still_pending(&snapshot) {
7092 continue;
7093 }
7094 if runtime.child_roster.is_closed() {
7099 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7100 state.state = ModuleState::Stopped;
7101 });
7102 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7103 continue;
7104 }
7105 if let Err(err) = release_dead_registration(
7106 ®istry,
7107 runtime.forwarding.as_deref(),
7108 &snapshot,
7109 &spec.module_id,
7110 &runtime.failure_context(runtime.recovery_is_daemon),
7111 ).await {
7112 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7113 continue;
7114 }
7115
7116 if matches!(pending.kind, RespawnKind::Reload) {
7117 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7118 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
7119 if result.is_ok() && runtime.pending_activity_restart {
7120 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7121 }
7122 runtime.pending_activity_restart = false;
7123 if let Some(reply) = reply { let _ = reply.send(result); }
7124 continue;
7125 }
7126 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7127 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7128 Ok(next_child) => {
7129 child = Some(next_child);
7130 if runtime.pending_activity_restart {
7131 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7132 runtime.pending_activity_restart = false;
7133 }
7134 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7135 }
7136 Err(err) => {
7137 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7138 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7139 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7140 }
7141 }
7142 }
7143 command = commands.recv() => {
7144 let Some(command) = command else {
7145 return;
7146 };
7147 if !handle_supervisor_command(
7148 command,
7149 &mut spec,
7150 &mut runtime,
7151 ®istry,
7152 &process_liveness,
7153 &snapshot,
7154 &mut child,
7155 &mut commands,
7156 &mut requeued,
7157 ).await {
7158 return;
7159 }
7160 if child.is_some() || !respawn_still_pending(&snapshot) {
7165 pending_respawn = None;
7166 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7167 }
7168 }
7169 }
7170 } else {
7171 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7172 (state.enabled && state.state == ModuleState::Failed)
7173 .then_some(state.failed_retry.pending)
7174 .flatten()
7175 });
7176 let command = if lock_snapshot(&snapshot)
7177 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7178 {
7179 tokio::select! {
7180 command = commands.recv() => command,
7181 _ = previous_instance_recheck.tick() => {
7182 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7183 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7184 if enabled && !runtime.child_roster.is_closed() {
7185 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7186 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7187 Ok(next_child) => child = Some(next_child),
7188 Err(error) => {
7189 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7190 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7191 }
7192 }
7193 } else {
7194 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7195 state.waiting_for_previous_pid = None;
7196 state.waiting_for_previous_detail = None;
7197 });
7198 }
7199 }
7200 continue;
7201 }
7202 }
7203 } else {
7204 tokio::select! {
7205 biased;
7206 command = commands.recv() => command,
7207 _ = async {
7208 match slow_retry {
7209 Some(retry) => sleep_until(retry.deadline).await,
7210 None => std::future::pending().await,
7211 }
7212 } => {
7213 let retry = slow_retry.expect("only a scheduled slow retry can wake");
7214 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7215 state.failed_retry.pending = None;
7216 });
7217 if runtime.child_roster.is_closed() {
7218 continue;
7219 }
7220 info!(module_id = %spec.module_id, attempt = retry.attempt,
7221 delay_secs = retry.delay.as_secs(), "starting failed module retry");
7222 if let Err(error) = set_child_enabled(
7223 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
7224 &runtime.failure_context(true),
7225 ).await {
7226 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7227 }
7228 continue;
7229 }
7230 }
7231 };
7232 let Some(command) = command else {
7233 return;
7234 };
7235 if !handle_supervisor_command(
7236 command,
7237 &mut spec,
7238 &mut runtime,
7239 ®istry,
7240 &process_liveness,
7241 &snapshot,
7242 &mut child,
7243 &mut commands,
7244 &mut requeued,
7245 )
7246 .await
7247 {
7248 return;
7249 }
7250 }
7251 }
7252}
7253
7254async fn supervisor_sleep(wake_at: Option<Instant>) {
7257 match wake_at {
7258 Some(deadline) => sleep_until(deadline).await,
7259 None => std::future::pending().await,
7260 }
7261}
7262
7263fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7264 info!(
7265 module_id,
7266 restart_in_window = schedule.restart_in_window,
7267 delay_ms = schedule.delay.as_millis() as u64,
7268 "respawning after crash"
7269 );
7270}
7271
7272fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7278 matches!(
7279 lock_snapshot(snapshot),
7280 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7281 )
7282}
7283
7284enum NextAction {
7285 Stop {
7286 registration_released: bool,
7287 },
7288 Restart {
7289 schedule: Option<CrashRestartSchedule>,
7290 },
7291}
7292
7293#[allow(clippy::too_many_arguments)]
7294async fn handle_supervisor_command(
7295 command: SupervisorCommand,
7296 spec: &mut ModuleSpec,
7297 runtime: &mut SupervisorRuntimeConfig,
7298 registry: &Arc<Registry>,
7299 process_liveness: &SupervisorProcessLiveness,
7300 snapshot: &SharedSnapshot,
7301 child: &mut Option<SupervisedChild>,
7302 commands: &mut mpsc::Receiver<SupervisorCommand>,
7303 requeued: &mut VecDeque<SupervisorCommand>,
7304) -> bool {
7305 if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7306 runtime.recovery_is_daemon = false;
7307 }
7308 if matches!(
7309 command,
7310 SupervisorCommand::Drain { .. }
7311 | SupervisorCommand::Retire { .. }
7312 | SupervisorCommand::SetEnabled { .. }
7313 | SupervisorCommand::Restart { .. }
7314 ) {
7315 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7316 state.failed_retry.cancel()
7317 });
7318 }
7319 let command =
7320 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7321 match command {
7322 SupervisorCommand::Restart { reply, .. }
7323 | SupervisorCommand::Reload { reply, .. }
7324 | SupervisorCommand::Swap { reply, .. } => {
7325 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7326 return true;
7327 }
7328 SupervisorCommand::SetEnabled {
7329 enabled: true,
7330 reply,
7331 ..
7332 } => {
7333 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7334 return true;
7335 }
7336 other => other,
7337 }
7338 } else {
7339 command
7340 };
7341 match command {
7342 SupervisorCommand::Drain { requester, reply } => {
7343 let result = drain_optional_child(
7346 &spec.module_id,
7347 spec.protocol,
7348 StopNotice::NotSent,
7349 registry,
7350 runtime.forwarding.as_deref(),
7351 snapshot,
7352 &runtime.terminal_ring,
7353 &runtime.spawn_events,
7354 child,
7355 runtime.drain_timeout,
7356 ModuleState::Stopped,
7357 None,
7358 &FailureContext::operator(),
7359 )
7360 .await;
7361 let registration_released = result.is_ok();
7362 if registration_released {
7363 runtime.record_activity_transition(
7364 "subc.module_stopped",
7365 &spec.module_id,
7366 requester,
7367 );
7368 }
7369 let _ = reply.send(result);
7370 if registration_released {
7371 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7372 }
7373 false
7374 }
7375 SupervisorCommand::Retire { requester, reply } => {
7376 let result = async {
7377 let stop_notice = begin_forwarding_drain_if_configured(
7378 spec,
7379 runtime,
7380 registry,
7381 snapshot,
7382 None,
7383 RouteCloseReason::Disable,
7384 )
7385 .await?;
7386 drain_optional_child(
7387 &spec.module_id,
7388 spec.protocol,
7389 stop_notice,
7390 registry,
7391 runtime.forwarding.as_deref(),
7392 snapshot,
7393 &runtime.terminal_ring,
7394 &runtime.spawn_events,
7395 child,
7396 runtime.drain_timeout,
7397 ModuleState::Stopped,
7398 None,
7399 &FailureContext::operator(),
7400 )
7401 .await
7402 }
7403 .await;
7404 let registration_released = result.is_ok();
7405 if registration_released {
7406 runtime.record_activity_transition(
7407 "subc.module_stopped",
7408 &spec.module_id,
7409 requester,
7410 );
7411 }
7412 let _ = reply.send(result);
7413 if registration_released {
7414 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7415 }
7416 false
7417 }
7418 SupervisorCommand::Restart {
7419 requester,
7420 drain_timeout_ms,
7421 received_at_generation,
7422 queued_at,
7423 reply,
7424 } => {
7425 info!(
7429 module_id = %spec.module_id,
7430 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7431 "restart command dequeued"
7432 );
7433 let validation = match lock_snapshot(snapshot) {
7445 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7446 module_id: spec.module_id.clone(),
7447 }),
7448 Ok(_) => Ok(()),
7449 Err(err) => Err(err),
7450 };
7451 let initiated = validation.is_ok();
7452 let _ = reply.send(validation);
7453 let satisfied_by_generation = if initiated
7467 && child
7468 .as_ref()
7469 .is_some_and(|child| child.spawned_at > queued_at)
7470 {
7471 lock_snapshot(snapshot).ok().and_then(|state| {
7472 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7473 })
7474 } else {
7475 None
7476 };
7477 let satisfied_by_pending = initiated
7478 && child.is_none()
7479 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7480 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7484 if pending {
7485 state.coalesced_restart_pending = true;
7486 }
7487 pending
7488 });
7489 if satisfied_by_pending {
7490 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7491 } else if let Some(generation) = satisfied_by_generation {
7492 info!(
7493 module_id = %spec.module_id,
7494 received_at_generation,
7495 "restart already satisfied by generation {generation}; not restarting again"
7496 );
7497 } else if initiated {
7498 let drain_timeout = drain_timeout_ms
7501 .map(Duration::from_millis)
7502 .unwrap_or(runtime.drain_timeout);
7503 if let Err(err) = restart_child(
7504 spec,
7505 runtime,
7506 registry,
7507 process_liveness,
7508 snapshot,
7509 child,
7510 drain_timeout,
7511 )
7512 .await
7513 {
7514 warn!(
7515 module_id = %spec.module_id,
7516 error = %err,
7517 "operator restart failed after initiation ack; module state carries the outcome"
7518 );
7519 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7520 state.state = ModuleState::Failed;
7521 clear_current_process_facts(state);
7522 });
7523 } else {
7524 runtime.start_requester = requester;
7525 runtime.pending_activity_restart = true;
7526 }
7527 }
7528 true
7529 }
7530 SupervisorCommand::Reload { requester, reply } => {
7531 let result =
7532 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7533 if result.is_ok() {
7534 runtime.start_requester = requester;
7535 runtime.pending_activity_restart = true;
7536 }
7537 if result.is_ok()
7538 && runtime
7539 .scheduled_respawn
7540 .lock()
7541 .unwrap_or_else(|p| p.into_inner())
7542 .as_ref()
7543 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7544 {
7545 *runtime
7546 .deferred_reload_reply
7547 .lock()
7548 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7549 } else {
7550 let _ = reply.send(result);
7551 }
7552 true
7553 }
7554 SupervisorCommand::SetEnabled {
7555 enabled,
7556 requester,
7557 reply,
7558 } => {
7559 let previous_requester = runtime.start_requester.clone();
7560 if enabled {
7561 runtime.start_requester = requester.clone();
7562 }
7563 let result = set_child_enabled(
7564 spec,
7565 runtime,
7566 registry,
7567 process_liveness,
7568 snapshot,
7569 child,
7570 enabled,
7571 &runtime.failure_context(false),
7572 )
7573 .await;
7574 if matches!(result, Ok(true)) {
7575 runtime.pending_activity_restart = false;
7576 if enabled {
7577 runtime.start_requester = requester;
7578 } else {
7579 runtime.start_requester = None;
7580 runtime.record_activity_transition(
7581 "subc.module_disabled",
7582 &spec.module_id,
7583 requester,
7584 );
7585 }
7586 } else {
7587 runtime.start_requester = previous_requester;
7588 }
7589 let _ = reply.send(result);
7590 true
7591 }
7592 SupervisorCommand::UpdateConfiguration {
7593 spec: next_spec,
7594 health,
7595 drain_timeout_ms,
7596 reply,
7597 } => {
7598 if let Some(handle) = &runtime.supervisor_handle {
7599 handle.apply_identity_configuration(&next_spec);
7600 }
7601 *spec = next_spec;
7602 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7603 state.configuration_updated_since_spawn = true;
7604 });
7605 let health_changed = runtime.health != health;
7606 runtime.health = health;
7607 if health_changed {
7610 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7611 state.health = ModuleHealthStatus::default();
7612 });
7613 }
7614 runtime.drain_timeout = drain_timeout_ms
7615 .map(Duration::from_millis)
7616 .unwrap_or(runtime.default_drain_timeout);
7617 *runtime
7618 .effective_drain_timeout
7619 .lock()
7620 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7621 let _ = reply.send(());
7622 true
7623 }
7624 SupervisorCommand::Swap {
7625 requester,
7626 ready_timeout,
7627 reply,
7628 } => {
7629 let mut swap_runtime = runtime.clone();
7630 swap_runtime.start_requester = requester;
7631 let before = lock_snapshot(snapshot)
7632 .ok()
7633 .map(|state| state.spawn_generation);
7634 let end = swap::run_swap(
7635 spec,
7636 &swap_runtime,
7637 registry,
7638 process_liveness,
7639 snapshot,
7640 child,
7641 commands,
7642 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7643 reply,
7644 )
7645 .await;
7646 if before
7647 != lock_snapshot(snapshot)
7648 .ok()
7649 .map(|state| state.spawn_generation)
7650 {
7651 runtime.start_requester = swap_runtime.start_requester;
7652 }
7653 requeued.extend(end.requeue);
7654 true
7655 }
7656 }
7657}
7658
7659async fn restart_child(
7660 spec: &ModuleSpec,
7661 runtime: &SupervisorRuntimeConfig,
7662 registry: &Registry,
7663 process_liveness: &SupervisorProcessLiveness,
7664 snapshot: &SharedSnapshot,
7665 child: &mut Option<SupervisedChild>,
7666 drain_timeout: Duration,
7667) -> Result<(), SuperviseError> {
7668 if !lock_snapshot(snapshot)?.enabled {
7670 return Err(SuperviseError::Disabled {
7671 module_id: spec.module_id.clone(),
7672 });
7673 }
7674 let stop_notice = begin_forwarding_drain_with_timeout(
7675 spec,
7676 runtime,
7677 registry,
7678 snapshot,
7679 None,
7680 RouteCloseReason::Restart,
7681 drain_timeout,
7682 )
7683 .await?;
7684
7685 if child.is_some() {
7686 drain_optional_child(
7687 &spec.module_id,
7688 spec.protocol,
7689 stop_notice,
7690 registry,
7691 runtime.forwarding.as_deref(),
7692 snapshot,
7693 &runtime.terminal_ring,
7694 &runtime.spawn_events,
7695 child,
7696 drain_timeout,
7697 ModuleState::Restarting,
7698 Some(true),
7699 &FailureContext::operator(),
7700 )
7701 .await?;
7702 } else {
7703 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7704 state.enabled = true;
7705 state.state = ModuleState::Restarting;
7706 clear_current_process_facts(state);
7707 })?;
7708 release_dead_registration(
7709 registry,
7710 runtime.forwarding.as_deref(),
7711 snapshot,
7712 &spec.module_id,
7713 &runtime.failure_context(false),
7714 )
7715 .await?;
7716 }
7717
7718 reset_restart_count(snapshot, &spec.module_id)?;
7719 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7720 schedule_respawn(
7721 runtime,
7722 snapshot,
7723 &spec.module_id,
7724 runtime.restart_policy.backoff,
7725 RespawnKind::Spawn,
7726 false,
7727 )
7728}
7729
7730async fn reload_child(
7731 spec: &ModuleSpec,
7732 runtime: &SupervisorRuntimeConfig,
7733 registry: &Registry,
7734 process_liveness: &SupervisorProcessLiveness,
7735 snapshot: &SharedSnapshot,
7736 child: &mut Option<SupervisedChild>,
7737) -> Result<(), SuperviseError> {
7738 if !lock_snapshot(snapshot)?.enabled {
7740 return Err(SuperviseError::Disabled {
7741 module_id: spec.module_id.clone(),
7742 });
7743 }
7744 let stop_notice = begin_forwarding_drain(
7745 spec,
7746 runtime,
7747 registry,
7748 snapshot,
7749 Some(true),
7750 RouteCloseReason::Reload,
7751 )
7752 .await?;
7753
7754 if child.is_some() {
7755 drain_optional_child(
7756 &spec.module_id,
7757 spec.protocol,
7758 stop_notice,
7759 registry,
7760 runtime.forwarding.as_deref(),
7761 snapshot,
7762 &runtime.terminal_ring,
7763 &runtime.spawn_events,
7764 child,
7765 runtime.drain_timeout,
7766 ModuleState::Restarting,
7767 Some(true),
7768 &FailureContext::operator(),
7769 )
7770 .await?;
7771 } else {
7772 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7773 state.enabled = true;
7774 state.state = ModuleState::Restarting;
7775 clear_current_process_facts(state);
7776 })?;
7777 release_dead_registration(
7778 registry,
7779 runtime.forwarding.as_deref(),
7780 snapshot,
7781 &spec.module_id,
7782 &runtime.failure_context(false),
7783 )
7784 .await?;
7785 }
7786
7787 reset_restart_count(snapshot, &spec.module_id)?;
7788 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7789 schedule_respawn(
7790 runtime,
7791 snapshot,
7792 &spec.module_id,
7793 runtime.restart_policy.backoff,
7794 RespawnKind::Reload,
7795 false,
7796 )
7797}
7798
7799async fn finish_reload_child(
7800 spec: &ModuleSpec,
7801 runtime: &SupervisorRuntimeConfig,
7802 registry: &Registry,
7803 process_liveness: &SupervisorProcessLiveness,
7804 snapshot: &SharedSnapshot,
7805 child: &mut Option<SupervisedChild>,
7806) -> Result<(), SuperviseError> {
7807 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7808 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7809 Ok(next_child) => next_child,
7810 Err(err) => {
7811 return handle_reload_spawn_failure(
7812 spec,
7813 runtime,
7814 process_liveness,
7815 snapshot,
7816 child,
7817 format!("new child failed to spawn: {err}"),
7818 )
7819 .await;
7820 }
7821 };
7822 *child = Some(next_child);
7823
7824 let wait_outcome = {
7825 let active_child = child.as_mut().expect("new reload child was just stored");
7826 wait_for_registration_after_reload(
7827 registry,
7828 &spec.module_id,
7829 snapshot,
7830 active_child,
7831 REGISTRY_RELEASE_TIMEOUT,
7832 )
7833 .await?
7834 };
7835
7836 match wait_outcome {
7837 RegistrationWaitOutcome::Registered => {
7838 child
7839 .as_mut()
7840 .expect("registered reload child")
7841 .registration_deadline = None;
7842 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7843 Ok(())
7844 }
7845 RegistrationWaitOutcome::Exited(exit_report) => {
7846 if let Some(active_child) = child.as_mut() {
7847 active_child.drain_stderr(&spec.module_id).await;
7848 }
7849 let mut exited_child = child.take().expect("exited reload child is still stored");
7852 #[cfg(test)]
7853 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7854 gate.reached.notify_one();
7855 gate.resume.notified().await;
7856 }
7857 let result = handle_reload_child_registration_failure(
7858 spec,
7859 runtime,
7860 registry,
7861 process_liveness,
7862 snapshot,
7863 child,
7864 ReloadRegistrationFailure {
7865 exit_report: registration_failure_exit_report(exit_report),
7866 reason: exited_child
7867 .spawn_failure
7868 .clone()
7869 .unwrap_or_else(|| "new child exited before registering".to_string()),
7870 },
7871 )
7872 .await;
7873 exited_child.release_roster();
7874 result
7875 }
7876 RegistrationWaitOutcome::TimedOut => {
7877 let mut timed_out_child = child
7878 .take()
7879 .expect("timed-out reload child is still running");
7880 timed_out_child
7881 .start_kill()
7882 .map_err(|source| SuperviseError::Kill {
7883 module_id: spec.module_id.clone(),
7884 source,
7885 })?;
7886 let status = timed_out_child
7887 .wait()
7888 .await
7889 .map_err(|source| SuperviseError::Wait {
7890 module_id: spec.module_id.clone(),
7891 source,
7892 })?;
7893 timed_out_child.drain_stderr(&spec.module_id).await;
7894 handle_reload_child_registration_failure(
7895 spec,
7896 runtime,
7897 registry,
7898 process_liveness,
7899 snapshot,
7900 child,
7901 ReloadRegistrationFailure {
7902 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7903 snapshot,
7904 &timed_out_child,
7905 &status,
7906 )),
7907 reason: format!(
7908 "new child did not register within {:?}",
7909 REGISTRY_RELEASE_TIMEOUT
7910 ),
7911 },
7912 )
7913 .await
7914 }
7915 }
7916}
7917
7918#[allow(clippy::too_many_arguments)]
7919async fn set_child_enabled(
7920 spec: &ModuleSpec,
7921 runtime: &SupervisorRuntimeConfig,
7922 registry: &Registry,
7923 process_liveness: &SupervisorProcessLiveness,
7924 snapshot: &SharedSnapshot,
7925 child: &mut Option<SupervisedChild>,
7926 enabled: bool,
7927 failure: &FailureContext,
7928) -> Result<bool, SuperviseError> {
7929 let (current_enabled, current_state, respawn_pending) = {
7930 let state = lock_snapshot(snapshot)?;
7931 (state.enabled, state.state, state.respawn_pending)
7932 };
7933 let revive_terminal = enabled
7941 && current_enabled
7942 && child.is_none()
7943 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7944 || (current_state == ModuleState::Restarting && !respawn_pending));
7945 if current_enabled == enabled && !revive_terminal {
7946 return Ok(false);
7947 }
7948
7949 if enabled {
7950 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7951 state.enabled = true;
7952 state.state = ModuleState::Starting;
7953 clear_current_process_facts(state);
7954 })?;
7955 #[cfg(test)]
7956 if runtime.test_seed_stale_facts_before_enable_spawn {
7957 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7958 state.process_alive = true;
7959 state.pid = Some(41);
7960 state.spawned_at_ms = Some(42);
7961 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7962 state.spawned_file_identity = Some(SpawnedFileIdentity {
7963 device: 43,
7964 inode: 44,
7965 #[cfg(windows)]
7966 windows_image: None,
7967 });
7968 })?;
7969 }
7970 release_dead_registration(
7971 registry,
7972 runtime.forwarding.as_deref(),
7973 snapshot,
7974 &spec.module_id,
7975 failure,
7976 )
7977 .await?;
7978 reset_restart_count(snapshot, &spec.module_id)?;
7979 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7980 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7981 Ok(next_child) => next_child,
7982 Err(err) => {
7983 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7984 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7985 return Err(err);
7986 }
7987 };
7988 *child = Some(next_child);
7989 debug!(module_id = %spec.module_id, "supervised module enabled");
7990 Ok(true)
7991 } else {
7992 let stop_notice = begin_forwarding_drain_if_configured(
7993 spec,
7994 runtime,
7995 registry,
7996 snapshot,
7997 Some(false),
7998 RouteCloseReason::Disable,
7999 )
8000 .await?;
8001 drain_optional_child(
8002 &spec.module_id,
8003 spec.protocol,
8004 stop_notice,
8005 registry,
8006 runtime.forwarding.as_deref(),
8007 snapshot,
8008 &runtime.terminal_ring,
8009 &runtime.spawn_events,
8010 child,
8011 runtime.drain_timeout,
8012 ModuleState::Disabled,
8013 Some(false),
8014 failure,
8015 )
8016 .await?;
8017 debug!(module_id = %spec.module_id, "supervised module disabled");
8018 Ok(true)
8019 }
8020}
8021
8022#[allow(clippy::too_many_arguments)]
8023async fn on_child_exit(
8024 spec: &ModuleSpec,
8025 policy: RestartPolicy,
8026 registry: &Registry,
8027 snapshot: &SharedSnapshot,
8028 terminal_ring: &Arc<Mutex<TerminalRing>>,
8029 spawn_events: &SpawnEventFeed,
8030 roster: &ChildRoster,
8031 exit_report: ExitReport,
8032 daemon_initiated: bool,
8033) -> NextAction {
8034 if roster.is_closed() {
8040 return on_child_exit_during_daemon_shutdown(
8041 spec,
8042 registry,
8043 snapshot,
8044 terminal_ring,
8045 spawn_events,
8046 exit_report,
8047 )
8048 .await;
8049 }
8050 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8066 && running_protocol(spec, snapshot) == ModuleProtocol::None;
8067 match exit_report.kind {
8068 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8069 info!(
8070 module_id = %spec.module_id,
8071 exit_code = ?exit_report.code,
8072 exit_signal = ?exit_report.signal,
8073 "supervised module exited cleanly"
8074 );
8075 if let Err(err) = publish_after_terminal_record(
8076 &spec.module_id,
8077 || {
8078 record_terminal(
8079 &spec.module_id,
8080 terminal_ring,
8081 spawn_events,
8082 &exit_report,
8083 TerminalDisposition::Stopped,
8084 )
8085 },
8086 || {
8087 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8088 state.state = ModuleState::Stopped;
8089 clear_current_process_facts(state);
8090 state.last_exit = Some(exit_report.clone());
8091 })
8092 },
8093 ) {
8094 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8095 }
8096 let registration_released = match wait_for_registration_release(
8097 registry,
8098 &spec.module_id,
8099 REGISTRY_RELEASE_TIMEOUT,
8100 )
8101 .await
8102 {
8103 Ok(()) => true,
8104 Err(err) => {
8105 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8106 false
8107 }
8108 };
8109 NextAction::Stop {
8110 registration_released,
8111 }
8112 }
8113 ExitKind::Clean | ExitKind::Crash => {
8114 if unrequested_clean_exit_of_protocol_none {
8115 warn!(
8116 module_id = %spec.module_id,
8117 exit_code = ?exit_report.code,
8118 exit_signal = ?exit_report.signal,
8119 "protocol-none module exited cleanly without a stop request; handling it as a crash"
8120 );
8121 } else {
8122 warn!(
8123 module_id = %spec.module_id,
8124 exit_code = ?exit_report.code,
8125 exit_signal = ?exit_report.signal,
8126 "supervised module exited abnormally (crash)"
8127 );
8128 }
8129 let mut restart_schedule = None;
8130 let mut disposition = TerminalDisposition::Disabled;
8131 let mut final_state = ModuleState::Disabled;
8132 let mut disposition_detail = lock_snapshot(snapshot)
8136 .ok()
8137 .and_then(|mut state| state.spawn_failure.take());
8138 let now = Instant::now();
8139 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8140 if state.enabled {
8141 if let Some(schedule) = state.next_crash_restart(&policy, now) {
8142 state.respawn_pending = true;
8148 final_state = ModuleState::Restarting;
8149 restart_schedule = Some(schedule);
8150 disposition = TerminalDisposition::Restarting;
8151 } else {
8152 final_state = ModuleState::Failed;
8153 disposition = TerminalDisposition::Failed;
8154 let budget = policy.budget_exhausted_detail();
8155 disposition_detail =
8156 Some(disposition_detail.take().map_or_else(
8157 || budget.clone(),
8158 |cause| format!("{cause}; {budget}"),
8159 ));
8160 }
8161 } else {
8162 disposition = TerminalDisposition::Disabled;
8163 }
8164 }) {
8165 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8166 return NextAction::Stop {
8167 registration_released: false,
8168 };
8169 }
8170 if disposition == TerminalDisposition::Failed {
8171 error!(
8176 module_id = %spec.module_id,
8177 max_restarts = policy.max_restarts,
8178 window_secs = policy.window.as_secs(),
8179 "module stopped: {}",
8180 policy.budget_exhausted_detail()
8181 );
8182 }
8183 if let Err(err) = publish_after_terminal_record(
8184 &spec.module_id,
8185 || {
8186 record_terminal_with_detail(
8187 &spec.module_id,
8188 terminal_ring,
8189 spawn_events,
8190 &exit_report,
8191 disposition,
8192 disposition_detail,
8193 )
8194 },
8195 || {
8196 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8197 state.state = final_state;
8198 clear_current_process_facts(state);
8199 if final_state == ModuleState::Failed
8200 && daemon_initiated
8201 && !roster.is_closed()
8202 {
8203 state.failed_retry.schedule(&spec.module_id);
8204 }
8205 state.last_exit = Some(exit_report.clone());
8206 })
8207 },
8208 ) {
8209 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8210 }
8211
8212 if let Some(schedule) = restart_schedule {
8213 NextAction::Restart {
8214 schedule: Some(schedule),
8215 }
8216 } else {
8217 let registration_released = match wait_for_registration_release(
8218 registry,
8219 &spec.module_id,
8220 REGISTRY_RELEASE_TIMEOUT,
8221 )
8222 .await
8223 {
8224 Ok(()) => true,
8225 Err(err) => {
8226 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8227 false
8228 }
8229 };
8230 NextAction::Stop {
8231 registration_released,
8232 }
8233 }
8234 }
8235 ExitKind::DeliberateSeverance => {
8236 warn!(
8237 module_id = %spec.module_id,
8238 exit_code = ?exit_report.code,
8239 exit_signal = ?exit_report.signal,
8240 "supervised module exited after deliberate connection severance"
8241 );
8242 let should_restart = match lock_snapshot(snapshot) {
8243 Ok(state) => state.enabled,
8244 Err(err) => {
8245 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8246 return NextAction::Stop {
8247 registration_released: false,
8248 };
8249 }
8250 };
8251 let final_state = if should_restart {
8252 ModuleState::Restarting
8253 } else {
8254 ModuleState::Disabled
8255 };
8256 if let Err(err) = publish_after_terminal_record(
8257 &spec.module_id,
8258 || {
8259 record_terminal(
8260 &spec.module_id,
8261 terminal_ring,
8262 spawn_events,
8263 &exit_report,
8264 terminal_disposition(final_state),
8265 )
8266 },
8267 || {
8268 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8269 state.state = final_state;
8270 clear_current_process_facts(state);
8271 state.last_exit = Some(exit_report.clone());
8272 state.lifetime_restarts += 1;
8273 })
8274 },
8275 ) {
8276 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8277 return NextAction::Stop {
8278 registration_released: false,
8279 };
8280 }
8281
8282 if should_restart {
8283 NextAction::Restart { schedule: None }
8284 } else {
8285 let registration_released = match wait_for_registration_release(
8286 registry,
8287 &spec.module_id,
8288 REGISTRY_RELEASE_TIMEOUT,
8289 )
8290 .await
8291 {
8292 Ok(()) => true,
8293 Err(err) => {
8294 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8295 false
8296 }
8297 };
8298 NextAction::Stop {
8299 registration_released,
8300 }
8301 }
8302 }
8303 }
8304}
8305
8306async fn on_child_exit_during_daemon_shutdown(
8307 spec: &ModuleSpec,
8308 registry: &Registry,
8309 snapshot: &SharedSnapshot,
8310 terminal_ring: &Arc<Mutex<TerminalRing>>,
8311 spawn_events: &SpawnEventFeed,
8312 exit_report: ExitReport,
8313) -> NextAction {
8314 info!(
8315 module_id = %spec.module_id,
8316 exit_code = ?exit_report.code,
8317 exit_signal = ?exit_report.signal,
8318 exit_kind = ?exit_report.kind,
8319 "supervised module exited during daemon shutdown; not restarting it"
8320 );
8321 if let Err(err) = publish_after_terminal_record(
8322 &spec.module_id,
8323 || {
8324 record_terminal(
8325 &spec.module_id,
8326 terminal_ring,
8327 spawn_events,
8328 &exit_report,
8329 TerminalDisposition::DaemonShutdown,
8330 )
8331 },
8332 || {
8333 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8334 state.state = ModuleState::Stopped;
8335 clear_current_process_facts(state);
8336 state.last_exit = Some(exit_report.clone());
8337 })
8338 },
8339 ) {
8340 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8341 }
8342 let registration_released =
8343 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8344 .await
8345 .is_ok();
8346 NextAction::Stop {
8347 registration_released,
8348 }
8349}
8350
8351fn publish_after_terminal_record<T>(
8356 module_id: &str,
8357 record: impl FnOnce(),
8358 publish: impl FnOnce() -> T,
8359) -> T {
8360 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8361 let published = publish();
8362 if let Err(panic) = recorded {
8363 error!(
8364 module_id,
8365 "terminal recording panicked; published final state before resuming unwind"
8366 );
8367 std::panic::resume_unwind(panic);
8368 }
8369 published
8370}
8371
8372fn fail_after_wait_error(
8373 module_id: &str,
8374 snapshot: &SharedSnapshot,
8375 terminal_ring: &Arc<Mutex<TerminalRing>>,
8376 spawn_events: &SpawnEventFeed,
8377 detail: String,
8378 failure: &FailureContext,
8379) {
8380 let report = wait_error_exit_report();
8381 publish_after_terminal_record(
8382 module_id,
8383 || {
8384 record_terminal_with_detail(
8385 module_id,
8386 terminal_ring,
8387 spawn_events,
8388 &report,
8389 TerminalDisposition::Failed,
8390 Some(detail),
8391 )
8392 },
8393 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8394 );
8395}
8396
8397#[cfg(test)]
8398fn record_wait_error_terminal(
8399 module_id: &str,
8400 terminal_ring: &Arc<Mutex<TerminalRing>>,
8401 spawn_events: &SpawnEventFeed,
8402) {
8403 record_terminal(
8404 module_id,
8405 terminal_ring,
8406 spawn_events,
8407 &wait_error_exit_report(),
8408 TerminalDisposition::Failed,
8409 );
8410}
8411
8412fn record_terminal(
8413 module_id: &str,
8414 terminal_ring: &Arc<Mutex<TerminalRing>>,
8415 spawn_events: &SpawnEventFeed,
8416 exit_report: &ExitReport,
8417 disposition: TerminalDisposition,
8418) {
8419 record_terminal_with_detail(
8420 module_id,
8421 terminal_ring,
8422 spawn_events,
8423 exit_report,
8424 disposition,
8425 None,
8426 );
8427}
8428
8429fn durable_terminal_history_of(
8433 terminal_ring: &Mutex<TerminalRing>,
8434 module_id: &str,
8435) -> subc_control::TerminalHistory {
8436 let read = terminal_ring
8437 .lock()
8438 .unwrap_or_else(|p| p.into_inner())
8439 .capture_durable_history();
8440 read.read(module_id)
8441}
8442
8443fn record_terminal_with_detail(
8444 module_id: &str,
8445 terminal_ring: &Arc<Mutex<TerminalRing>>,
8446 spawn_events: &SpawnEventFeed,
8447 exit_report: &ExitReport,
8448 disposition: TerminalDisposition,
8449 disposition_detail: Option<String>,
8450) {
8451 #[cfg(test)]
8452 terminal_history_tests::before_terminal_record(spawn_events);
8453 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8454 let record = TerminalRecord {
8455 exit_code: exit_report.code,
8456 exit_signal: exit_report.signal,
8457 at_ms: exit_report.at_ms,
8458 disposition,
8459 exit_kind: exit_report.kind.into(),
8460 disposition_detail,
8461 };
8462 terminal_ring
8463 .lock()
8464 .unwrap_or_else(|poisoned| poisoned.into_inner())
8465 .record_exit(module_id, record);
8466}
8467
8468fn untrack_if_registration_released(
8469 process_liveness: &SupervisorProcessLiveness,
8470 registry: &Registry,
8471 module_id: &str,
8472 snapshot: &SharedSnapshot,
8473) {
8474 match registry.get_module(module_id) {
8475 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8476 Ok(Some(_)) => {}
8477 Err(err) => {
8478 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8479 }
8480 }
8481}
8482
8483#[cfg(test)]
8497fn apply_wire_spawn_args(
8498 command: &mut Command,
8499 spec: &ModuleSpec,
8500 connection_file_path: Option<&std::path::Path>,
8501 handle: Option<&SupervisorHandle>,
8502) -> Result<Option<NonceHandoff>, SuperviseError> {
8503 apply_wire_spawn_args_for_role(
8504 command,
8505 spec,
8506 connection_file_path,
8507 handle,
8508 SpawnRole::Plain,
8509 )
8510}
8511
8512#[cfg(unix)]
8517type NonceHandoff = subc_os::LaunchNonceHandoff;
8518#[cfg(windows)]
8519type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8520#[cfg(not(any(unix, windows)))]
8521type NonceHandoff = std::convert::Infallible;
8522
8523fn apply_wire_spawn_args_for_role(
8544 command: &mut Command,
8545 spec: &ModuleSpec,
8546 connection_file_path: Option<&std::path::Path>,
8547 handle: Option<&SupervisorHandle>,
8548 role: SpawnRole,
8549) -> Result<Option<NonceHandoff>, SuperviseError> {
8550 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8551 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8556 #[cfg(windows)]
8557 {
8558 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8559 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8560 }
8561 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8563 if spec.protocol == ModuleProtocol::None {
8564 return Ok(None);
8565 }
8566 if let Some(connection_file_path) = connection_file_path {
8567 command.arg(SUBC_ARG).arg(connection_file_path);
8568 }
8569
8570 let nonce = generate_launch_nonce()?;
8574 if let Some(handle) = handle {
8575 match role {
8576 SpawnRole::Plain => {
8577 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8578 if spec.reserved {
8579 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8580 }
8581 }
8582 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8583 }
8584 }
8585 #[cfg(unix)]
8586 let handoff = {
8587 let handoff =
8588 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8589 program: spec.program.clone(),
8590 source,
8591 cgroup_path: None,
8592 })?;
8593 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8594 Some(handoff)
8595 };
8596 #[cfg(windows)]
8597 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8598 #[cfg(not(any(unix, windows)))]
8599 let handoff = None;
8600 #[cfg(not(unix))]
8604 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8605 Ok(handoff)
8606}
8607
8608fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8609 command.env_remove(CK_LOG_ENV);
8610 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8617 for (key, value) in &spec.env {
8618 if matches!(
8622 key.as_str(),
8623 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8624 ) || key == SUBC_SPAWN_ROLE_ENV
8625 {
8626 continue;
8627 }
8628 command.env(key, value);
8629 }
8630}
8631
8632#[cfg(windows)]
8633fn prepare_windows_nonce_pipe(
8634 command: &mut Command,
8635 spec: &ModuleSpec,
8636 nonce: &str,
8637) -> Option<NonceHandoff> {
8638 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8639 Ok(handoff) => {
8640 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8641 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8646 Some(handoff)
8647 }
8648 Err(error) => {
8649 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8650 None
8651 }
8652 }
8653}
8654
8655#[cfg(windows)]
8656fn start_windows_nonce_pipe(
8657 handoff: Option<NonceHandoff>,
8658 pid: u32,
8659 spec: &ModuleSpec,
8660) -> Option<subc_os::LaunchNoncePipeDelivery> {
8661 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8665 Ok(delivery) => Some(delivery),
8666 Err(error) => {
8667 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8668 None
8669 }
8670 })
8671}
8672
8673fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8674 #[cfg(windows)]
8675 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8676 return Some(
8677 if snapshot
8678 .pipe_consumed
8679 .as_ref()
8680 .is_some_and(|consumed| consumed.get().is_some())
8681 {
8682 "pipe"
8683 } else {
8684 "env"
8685 }
8686 .to_owned(),
8687 );
8688 }
8689 let _ = snapshot;
8690 None
8691}
8692
8693#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8696enum SpawnRole {
8697 Plain,
8698 SwapCandidate,
8699}
8700
8701fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8704 if role == SpawnRole::SwapCandidate {
8705 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8706 }
8707}
8708
8709fn previous_instance_error(
8710 spec: &ModuleSpec,
8711 pid: u32,
8712 reason: crate::live_children::RetainedReason,
8713) -> SuperviseError {
8714 SuperviseError::Spawn {
8715 program: spec.program.clone(),
8716 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8717 cgroup_path: None,
8718 }
8719}
8720
8721fn spawn_child(
8722 spec: &ModuleSpec,
8723 connection_file_path: Option<&std::path::Path>,
8724 handle: Option<&SupervisorHandle>,
8725 ring: &Arc<Mutex<StderrRing>>,
8726 capture_logs_dir: Option<&std::path::Path>,
8727 roster: &ChildRoster,
8728 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8729) -> Result<SupervisedChild, SuperviseError> {
8730 spawn_child_in_slot(
8731 spec,
8732 connection_file_path,
8733 handle,
8734 ring,
8735 capture_logs_dir,
8736 roster,
8737 #[cfg(target_os = "linux")]
8738 cgroup_placement,
8739 SpawnRole::Plain,
8740 false,
8741 )
8742}
8743
8744#[allow(clippy::too_many_arguments)]
8757fn spawn_child_in_slot(
8758 spec: &ModuleSpec,
8759 connection_file_path: Option<&std::path::Path>,
8760 handle: Option<&SupervisorHandle>,
8761 ring: &Arc<Mutex<StderrRing>>,
8762 capture_logs_dir: Option<&std::path::Path>,
8763 roster: &ChildRoster,
8764 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8765 role: SpawnRole,
8766 alternate_slot: bool,
8767) -> Result<SupervisedChild, SuperviseError> {
8768 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8769 return Err(previous_instance_error(spec, pid, reason));
8770 }
8771 if roster.is_closed() {
8772 return Err(SuperviseError::Spawn {
8773 program: spec.program.clone(),
8774 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8775 cgroup_path: None,
8776 });
8777 }
8778 #[cfg(target_os = "linux")]
8779 let cgroup_name = {
8780 if cgroup_placement.is_none() {
8785 swap::cgroup_name(&spec.module_id, alternate_slot)
8786 } else {
8787 let nonce = generate_launch_nonce()?;
8788 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8789 let mut end = spec.module_id.len().min(64);
8792 while !spec.module_id.is_char_boundary(end) {
8793 end -= 1;
8794 }
8795 format!(
8796 "{}_{suffix}",
8797 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8798 )
8799 }
8800 };
8801 #[cfg(not(target_os = "linux"))]
8802 let _ = alternate_slot;
8803 #[cfg(target_os = "macos")]
8804 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8805 #[cfg(windows)]
8806 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8807 program: spec.program.clone(),
8808 source,
8809 cgroup_path: None,
8810 })?;
8811 #[cfg(windows)]
8812 let mut command = Command::new(image_capture.path());
8813 #[cfg(not(any(target_os = "macos", windows)))]
8814 let mut command = Command::new(&spec.program);
8815 command.args(&spec.args);
8816 apply_child_env(&mut command, spec);
8846 apply_spawn_role(&mut command, role);
8847 let nonce_handoff =
8848 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8849
8850 #[cfg(target_os = "linux")]
8851 let cgroup_path = cgroup_placement
8852 .map(|placement| placement.module_path(&cgroup_name))
8853 .transpose()
8854 .map_err(|source| SuperviseError::Cgroup {
8855 module_id: spec.module_id.clone(),
8856 source,
8857 })?;
8858 #[cfg(not(target_os = "linux"))]
8859 let cgroup_path: Option<PathBuf> = None;
8860 #[cfg(target_os = "linux")]
8861 if let Some(path) = &cgroup_path {
8862 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8863 if let Some(placement) = cgroup_placement {
8864 remove_module_cgroup(placement, &cgroup_name);
8865 }
8866 return Err(error);
8867 }
8868 }
8869
8870 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8871 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8872 match ChildOutputSink::open(&path, capture_retention(spec)) {
8873 Ok(sink) => sink,
8874 Err(error) => {
8875 warn!(
8876 module_id = %spec.module_id,
8877 path = %path.display(),
8878 error = %error,
8879 "could not open child output capture file; forwarding to stderr"
8880 );
8881 ChildOutputSink::Stderr
8882 }
8883 }
8884 } else {
8885 ChildOutputSink::Stderr
8886 };
8887
8888 command.stdout(Stdio::piped());
8889 command.stderr(Stdio::piped());
8890 command.kill_on_drop(true);
8891 #[cfg(unix)]
8908 command.process_group(0);
8909 command.stdin(Stdio::null());
8910 #[cfg(unix)]
8914 if let Some(handoff) = nonce_handoff {
8915 handoff.install_last(command.as_std_mut());
8916 }
8917 #[cfg(not(any(unix, windows)))]
8918 let _ = nonce_handoff;
8919
8920 #[cfg(windows)]
8925 {
8926 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8927 program: spec.program.clone(),
8928 source,
8929 cgroup_path: None,
8930 })?;
8931 subc_jobobject::suspend_on_create_async(&mut command);
8932 }
8933 let spawned_at = Instant::now();
8937 let mut child = match command.spawn() {
8938 Ok(child) => child,
8939 Err(source) => {
8940 #[cfg(target_os = "linux")]
8941 if let Some(placement) = cgroup_placement {
8942 remove_module_cgroup(placement, &cgroup_name);
8943 }
8944 return Err(SuperviseError::Spawn {
8945 program: spec.program.clone(),
8946 source,
8947 cgroup_path,
8948 });
8949 }
8950 };
8951 #[cfg(test)]
8952 spawn_record_tests::after_create(&child, spec);
8953 #[cfg(target_os = "macos")]
8958 drop(exec_ack);
8959
8960 #[cfg(windows)]
8965 let windows_image = match image_capture.bind_tokio(&child) {
8966 Ok(image) => Arc::new(image),
8967 Err(source) => {
8968 kill_and_reap_rejected_child(&mut child);
8969 return Err(SuperviseError::Spawn {
8970 program: spec.program.clone(),
8971 source,
8972 cgroup_path: None,
8973 });
8974 }
8975 };
8976 #[cfg(windows)]
8979 let job = contain_spawned_child(&mut child, spec)?;
8980 let spawned_at_ms = unix_ms_now();
8981 #[cfg(not(windows))]
8982 let spawned_from = spec.program.clone();
8983 #[cfg(windows)]
8984 let spawned_from = windows_image.path().to_path_buf();
8985 #[cfg(not(windows))]
8986 let spawned_file_identity = spawned_file_identity(&spawned_from);
8987 #[cfg(windows)]
8988 let spawned_file_identity = Some(SpawnedFileIdentity {
8989 device: 0,
8990 inode: 0,
8991 windows_image: Some(Arc::clone(&windows_image)),
8992 });
8993 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8994 program: spec.program.clone(),
8995 source: io::Error::other("spawned child exposed no live pid"),
8996 cgroup_path: cgroup_path.clone(),
8997 })?;
8998 #[cfg(windows)]
8999 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9000 #[cfg(not(windows))]
9001 let process_start_time = crate::provenance::process_start_time(pid);
9002 #[cfg(windows)]
9003 let process_start_time = Some(windows_image.start_time());
9004 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9005 #[cfg(all(test, target_os = "macos"))]
9006 privacy_exec_boundary_tests::before_image_sample(spec, pid);
9007 #[cfg(not(windows))]
9012 let recorded_image = observe_spawned_image(pid);
9013 #[cfg(target_os = "macos")]
9017 let recorded_image = if privacy_exec.is_some() {
9018 None
9019 } else {
9020 recorded_image
9021 };
9022 #[cfg(target_os = "linux")]
9023 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9024 #[cfg(not(target_os = "linux"))]
9025 let recorded_cgroup_name = None;
9026 let roster_guard = roster.admit(
9027 spec.module_id.clone(),
9028 pid,
9029 spec.protocol,
9030 process_start_time,
9031 crate::child_roster::RecordedIdentity {
9032 #[cfg(not(windows))]
9033 start_time: recorded_image.map(|image| image.start_time),
9034 #[cfg(windows)]
9035 start_time: process_start_time,
9036 #[cfg(not(windows))]
9037 executable: recorded_image
9038 .and_then(|image| image.executable)
9039 .map(crate::live_children::ExecutableIdentity::from),
9040 #[cfg(windows)]
9041 executable: Some(crate::live_children::ExecutableIdentity::from(
9042 windows_image.identity(),
9043 )),
9044 cgroup_name: recorded_cgroup_name,
9045 #[cfg(target_os = "linux")]
9046 cgroup_placement: cgroup_placement.cloned(),
9047 },
9048 );
9049 let roster_guard = match roster_guard {
9050 Ok(guard) => guard,
9051 Err(failure) => {
9052 return failure.with_cleanup(|source| {
9053 #[cfg(windows)]
9054 if let Some(job) = &job {
9055 let _ = job.terminate();
9056 }
9057 #[cfg(target_os = "linux")]
9058 kill_module_cgroup(cgroup_placement, &cgroup_name);
9059 kill_and_reap_rejected_child(&mut child);
9060 #[cfg(target_os = "linux")]
9061 if let Some(placement) = cgroup_placement {
9062 remove_module_cgroup(placement, &cgroup_name);
9063 }
9064 Err(SuperviseError::Spawn {
9065 program: spec.program.clone(),
9066 source: io::Error::other(format!(
9067 "could not durably record module child: {source}"
9068 )),
9069 cgroup_path,
9070 })
9071 });
9072 }
9073 };
9074 if roster.is_closed() {
9079 #[cfg(target_os = "linux")]
9081 kill_module_cgroup(cgroup_placement, &cgroup_name);
9082 if let Err(error) = child.start_kill() {
9083 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9084 }
9085 #[cfg(target_os = "linux")]
9086 if let Some(placement) = cgroup_placement {
9087 while matches!(child.try_wait(), Ok(None)) {
9091 std::thread::yield_now();
9092 }
9093 if matches!(
9094 subc_cgroup::kill_module(Some(placement), &cgroup_name),
9095 subc_cgroup::KillOutcome::Killed
9096 ) {
9097 if let Ok(path) = placement.module_path(&cgroup_name) {
9098 while std::fs::read_to_string(path.join("cgroup.events"))
9099 .ok()
9100 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9101 {
9102 std::thread::yield_now();
9103 }
9104 }
9105 }
9106 remove_module_cgroup(placement, &cgroup_name);
9107 }
9108 drop(roster_guard);
9109 return Err(SuperviseError::Spawn {
9110 program: spec.program.clone(),
9111 source: io::Error::other(
9112 "the daemon began shutting down while this process was starting; ended it",
9113 ),
9114 cgroup_path,
9115 });
9116 }
9117
9118 #[cfg(windows)]
9119 if let Err(error) = resume_suspended_child(pid, spec) {
9120 kill_and_reap_rejected_child(&mut child);
9121 return Err(error);
9122 }
9123
9124 let stdout_pump = match child.stdout.take() {
9125 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9126 None => {
9127 warn!(
9128 module_id = %spec.module_id,
9129 "spawned child exposed no stdout pipe; file capture will be incomplete"
9130 );
9131 None
9132 }
9133 };
9134 let stderr_pump = match child.stderr.take() {
9135 Some(stderr) => {
9136 let generation = ring
9137 .lock()
9138 .unwrap_or_else(|poisoned| poisoned.into_inner())
9139 .begin_process();
9140 Some(StderrPump {
9141 task: tokio::spawn(pump_stderr_to(
9142 stderr,
9143 Arc::clone(ring),
9144 generation,
9145 output_sink,
9146 )),
9147 generation,
9148 })
9149 }
9150 None => {
9151 ring.lock()
9155 .unwrap_or_else(|poisoned| poisoned.into_inner())
9156 .mark_not_captured("stderr pipe was not available on spawn");
9157 warn!(
9158 module_id = %spec.module_id,
9159 "spawned child exposed no stderr pipe; tail will be unavailable"
9160 );
9161 None
9162 }
9163 };
9164
9165 let exec = Arc::new(crate::activity::ExecOutcome::new());
9166 #[cfg(target_os = "macos")]
9167 let pending_exec = privacy_exec.is_some();
9168 #[cfg(not(target_os = "macos"))]
9169 let pending_exec = false;
9170 if !pending_exec {
9171 exec.finish(crate::activity::ExecState::CommittedExec);
9172 }
9173 Ok(SupervisedChild {
9174 child,
9175 exec: crate::activity::ExecOwner(exec),
9176 #[cfg(windows)]
9177 nonce_delivery,
9178 protocol: spec.protocol,
9179 #[cfg(target_os = "linux")]
9180 module_id: cgroup_name,
9181 #[cfg(target_os = "linux")]
9182 cgroup_placement: cgroup_placement.cloned(),
9183 #[cfg(windows)]
9184 job,
9185 stdout_pump,
9186 stderr_pump,
9187 stderr_ring: Arc::clone(ring),
9188 spawned_at,
9189 registration_deadline: None,
9190 registration_watch: None,
9191 spawned_at_ms,
9192 spawned_from,
9193 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9194 spawned_file_identity,
9195 process_start_time,
9196 process_identity,
9197 pid,
9198 roster_guard: Some(roster_guard),
9199 #[cfg(target_os = "macos")]
9200 privacy_exec,
9201 #[cfg(target_os = "macos")]
9202 report_ready: Arc::new(OnceLock::new()),
9203 spawn_failure: None,
9204 })
9205}
9206
9207#[cfg(test)]
9208mod spawn_record_tests {
9209 use super::*;
9210
9211 thread_local! {
9212 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9213 }
9214
9215 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9216 LAST_PID.set(child.id());
9217 #[cfg(windows)]
9218 if let Some((_, address)) = _spec
9219 .env
9220 .iter()
9221 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9222 {
9223 use std::io::{Read, Write};
9224 let mut stream = std::net::TcpStream::connect(address).unwrap();
9225 stream
9226 .write_all(&child.id().unwrap().to_ne_bytes())
9227 .unwrap();
9228 let mut release = [0];
9229 stream.read_exact(&mut release).unwrap();
9230 }
9231 }
9232
9233 #[cfg(windows)]
9234 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9235 if spec
9236 .env
9237 .iter()
9238 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9239 {
9240 Err(io::Error::other("injected module job assignment failure"))
9241 } else {
9242 Ok(())
9243 }
9244 }
9245
9246 fn spec() -> ModuleSpec {
9247 ModuleSpec {
9248 module_id: "spawn-record-worker".into(),
9249 #[cfg(unix)]
9250 program: "/bin/sleep".into(),
9251 #[cfg(windows)]
9252 program: std::env::current_exe().unwrap(),
9253 #[cfg(unix)]
9254 args: vec!["60".into()],
9255 #[cfg(windows)]
9256 args: vec![
9257 "--exact".into(),
9258 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9259 "--nocapture".into(),
9260 ],
9261 env: vec![],
9262 reserved: false,
9263 reserved_prefixes: vec![],
9264 protocol: ModuleProtocol::None,
9265 overlap: Default::default(),
9266 }
9267 }
9268
9269 #[tokio::test]
9270 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9271 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9272 let supervisor =
9273 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9274 .with_live_children_record(home.join("live-children.json"));
9275 crate::live_children::fail_next_file_sync();
9276 let result = supervisor.spawn(spec());
9277 let failed = result.is_err();
9278 if let Ok(module) = result {
9279 module.stop().await.unwrap();
9280 }
9281 assert!(failed, "record failure must fail spawn");
9282 let pid = LAST_PID.get().expect("the real OS child was created");
9283 assert!(
9284 subc_os::Process::open(pid).unwrap().is_none(),
9285 "child {pid} survived rejected admission or was not reaped"
9286 );
9287 }
9288
9289 #[cfg(target_os = "macos")]
9290 #[test]
9291 #[ignore = "manual on-device latency measurement"]
9292 fn durable_record_write_latency_50() {
9293 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9294 let path = home.join("live-children.json");
9295 let children = vec![crate::live_children::LiveChild {
9296 module_id: "latency-worker".into(),
9297 pid: std::process::id(),
9298 protocol: ModuleProtocol::None,
9299 start_time: Some(1),
9300 executable: None,
9301 cgroup_name: None,
9302 }];
9303 let mut samples = Vec::new();
9304 for _ in 0..50 {
9305 let start = std::time::Instant::now();
9306 crate::live_children::write_record(&path, &children).unwrap();
9307 samples.push(start.elapsed());
9308 }
9309 samples.sort();
9310 eprintln!(
9311 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9312 (samples[24] + samples[25]) / 2,
9313 samples[47]
9314 );
9315 }
9316
9317 #[cfg(windows)]
9318 #[test]
9319 fn windows_store_touch_fixture() {
9320 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9321 std::fs::write(path, b"module ran").unwrap();
9322 }
9323 }
9324
9325 #[cfg(windows)]
9326 #[tokio::test]
9327 async fn windows_daemon_fixture() {
9328 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9329 return;
9330 };
9331 let mut spec = spec();
9332 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9333 let supervisor =
9334 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9335 let _module = supervisor.spawn(spec).unwrap();
9336 }
9337
9338 #[cfg(windows)]
9339 #[test]
9340 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9341 use std::io::Read;
9342 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9343 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9346 command
9347 .args([
9348 "--exact",
9349 "supervise::spawn_record_tests::windows_daemon_fixture",
9350 "--nocapture",
9351 ])
9352 .env(
9353 "SUBC_TEST_DAEMON_BARRIER",
9354 listener.local_addr().unwrap().to_string(),
9355 );
9356 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9357 listener.set_nonblocking(true).unwrap();
9358 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9359 let mut stream = loop {
9360 if let Ok((stream, _)) = listener.accept() {
9361 break stream;
9362 }
9363 assert!(
9364 daemon.child.try_wait().unwrap().is_none(),
9365 "daemon failed before rendezvous"
9366 );
9367 assert!(
9368 std::time::Instant::now() < deadline,
9369 "daemon did not reach CreateProcess failpoint"
9370 );
9371 std::thread::yield_now();
9372 };
9373 stream
9374 .set_read_timeout(Some(Duration::from_secs(30)))
9375 .unwrap();
9376 let mut pid = [0; 4];
9377 stream.read_exact(&mut pid).unwrap();
9378 let pid = u32::from_ne_bytes(pid);
9379 assert!(subc_jobobject::process_exists(pid));
9380 daemon.child.kill().unwrap();
9383 daemon.child.wait().unwrap();
9384 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9385 if !exited {
9386 daemon.job.terminate().unwrap();
9387 }
9388 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9389 }
9390
9391 #[cfg(windows)]
9392 #[tokio::test]
9393 async fn windows_containment_failure_never_resumes() {
9394 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9395 let sentinel = home.join("store-touched");
9396 let mut spec = spec();
9397 spec.env
9398 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9399 spec.env.push((
9400 "SUBC_TEST_STORE_TOUCH".into(),
9401 sentinel.display().to_string(),
9402 ));
9403 let supervisor =
9404 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9405 let result = supervisor.spawn(spec);
9406 if let Ok(module) = result {
9409 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9410 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9411 tokio::task::yield_now().await;
9412 }
9413 })
9414 .await;
9415 let _ = module.stop().await;
9416 panic!(
9417 "assignment failure resumed a module; store touched={}",
9418 sentinel.exists()
9419 );
9420 }
9421 assert!(
9422 !sentinel.exists(),
9423 "refused module executed store-touch sentinel"
9424 );
9425 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9426 }
9427
9428 #[cfg(windows)]
9429 #[tokio::test]
9430 async fn windows_nested_daemon_fixture() {
9431 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9432 return;
9433 };
9434 let mut spec = spec();
9435 spec.env.push((
9436 "SUBC_TEST_STORE_TOUCH".into(),
9437 sentinel.to_str().unwrap().into(),
9438 ));
9439 let supervisor =
9440 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9441 let module = supervisor
9442 .spawn(spec)
9443 .expect("per-module job must nest below daemon and host jobs");
9444 tokio::time::timeout(Duration::from_secs(10), async {
9445 while !PathBuf::from(&sentinel).exists() {
9446 tokio::task::yield_now().await;
9447 }
9448 })
9449 .await
9450 .unwrap();
9451 module.stop().await.unwrap();
9452 }
9453
9454 #[cfg(windows)]
9455 #[test]
9456 fn windows_module_job_nests_under_host_job() {
9457 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9458 let sentinel = home.join("nested-store-touched");
9459 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9460 command
9461 .args([
9462 "--exact",
9463 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9464 "--nocapture",
9465 ])
9466 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9467 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9468 assert!(daemon.child.wait().unwrap().success());
9469 assert!(sentinel.exists(), "nested protocol:none module never ran");
9470 }
9471}
9472
9473#[cfg(target_os = "linux")]
9474pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9475 use subc_cgroup::KillOutcome;
9476 match subc_cgroup::kill_module(placement, module_id) {
9477 KillOutcome::Killed => {}
9478 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9479 debug!(
9480 module_id,
9481 "cgroup tree kill unavailable; using direct-child kill"
9482 );
9483 }
9484 KillOutcome::IoError { path, error } => {
9485 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9486 }
9487 }
9488}
9489
9490#[cfg(windows)]
9498fn contain_spawned_child(
9499 child: &mut Child,
9500 spec: &ModuleSpec,
9501) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9502 let assignment = || {
9503 let job = subc_jobobject::JobObject::new()?;
9504 #[cfg(test)]
9505 spawn_record_tests::assignment_failpoint(spec)?;
9506 job.assign(child)?;
9507 Ok::<_, io::Error>(job)
9508 };
9509 match assignment() {
9510 Ok(job) => Ok(Some(job)),
9511 Err(source) => {
9512 kill_and_reap_rejected_child(child);
9513 Err(SuperviseError::Spawn {
9514 program: spec.program.clone(),
9515 source: io::Error::other(format!(
9516 "Windows module job containment failed: {source}"
9517 )),
9518 cgroup_path: None,
9519 })
9520 }
9521 }
9522}
9523
9524fn kill_and_reap_rejected_child(child: &mut Child) {
9527 #[cfg(unix)]
9528 if let Some(pid) = child
9529 .id()
9530 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9531 {
9532 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9533 }
9534 let _ = child.start_kill();
9535 while matches!(child.try_wait(), Ok(None)) {
9536 std::thread::yield_now();
9537 }
9538}
9539
9540#[cfg(windows)]
9545fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9546 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9547 let _ = std::process::Command::new("taskkill.exe")
9551 .args(["/PID", &pid.to_string(), "/T", "/F"])
9552 .stdin(Stdio::null())
9553 .stdout(Stdio::null())
9554 .stderr(Stdio::null())
9555 .status();
9556 return Err(SuperviseError::Spawn {
9557 program: spec.program.clone(),
9558 source,
9559 cgroup_path: None,
9560 });
9561 }
9562 Ok(())
9563}
9564
9565#[cfg(target_os = "linux")]
9566fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9567 match placement.remove_module(module_id) {
9568 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9569 Err(error) => warn!(
9570 module_id,
9571 error = %error,
9572 "could not remove module cgroup after process exit; continuing teardown"
9573 ),
9574 }
9575}
9576
9577#[cfg(target_os = "linux")]
9578async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9579 if matches!(
9583 subc_cgroup::kill_module(Some(placement), module_id),
9584 subc_cgroup::KillOutcome::Killed
9585 ) {
9586 if let Ok(path) = placement.module_path(module_id) {
9587 while std::fs::read_to_string(path.join("cgroup.events"))
9588 .ok()
9589 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9590 {
9591 sleep(Duration::from_millis(1)).await;
9592 }
9593 }
9594 }
9595 remove_module_cgroup(placement, module_id);
9596}
9597
9598#[cfg(target_os = "linux")]
9599fn apply_cgroup_placement(
9600 command: &mut Command,
9601 spec: &ModuleSpec,
9602 path: &std::path::Path,
9603) -> Result<(), SuperviseError> {
9604 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9605 module_id: spec.module_id.clone(),
9606 source,
9607 })
9608}
9609
9610fn capture_retention(spec: &ModuleSpec) -> Retention {
9611 let defaults = Retention::default();
9612 let value = |name: &str| {
9613 spec.env
9614 .iter()
9615 .rev()
9616 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9617 };
9618 Retention {
9619 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9620 .and_then(|value| value.parse().ok())
9621 .unwrap_or(defaults.max_file_mb),
9622 keep: value(CAPTURE_KEEP_ENV)
9623 .and_then(|value| value.parse().ok())
9624 .unwrap_or(defaults.keep),
9625 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9626 .and_then(|value| value.parse().ok())
9627 .unwrap_or(defaults.max_age_days),
9628 }
9629}
9630
9631fn generate_launch_nonce() -> Result<String, SuperviseError> {
9634 let mut bytes = [0u8; 32];
9635 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9636 reason: source.to_string(),
9637 })?;
9638 let mut hex = String::with_capacity(64);
9639 for b in bytes {
9640 use std::fmt::Write;
9641 let _ = write!(hex, "{b:02x}");
9642 }
9643 Ok(hex)
9644}
9645
9646fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9649 if a.len() != b.len() {
9650 return false;
9651 }
9652 let mut diff = 0u8;
9653 for (x, y) in a.iter().zip(b.iter()) {
9654 diff |= x ^ y;
9655 }
9656 diff == 0
9657}
9658
9659#[cfg(not(windows))]
9663fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9664 let process = subc_os::Process::open(pid).ok().flatten()?;
9665 #[cfg(target_os = "linux")]
9666 return sample_after_linux_exec(
9667 || {
9668 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9669 let committed = linux_exec_committed(&stat);
9670 #[cfg(test)]
9671 spawn_image_tests::report_exec_state(committed);
9672 committed
9673 },
9674 || process.observe(),
9675 );
9676 #[cfg(not(target_os = "linux"))]
9677 process.observe()
9678}
9679
9680#[cfg(target_os = "linux")]
9684fn linux_exec_committed(stat: &str) -> Option<bool> {
9685 const PF_FORKNOEXEC: u64 = 0x40;
9686 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9687 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9688}
9689
9690#[cfg(target_os = "linux")]
9691fn sample_after_linux_exec(
9692 mut exec_committed: impl FnMut() -> Option<bool>,
9693 observe: impl FnOnce() -> Option<subc_os::Observation>,
9694) -> Option<subc_os::Observation> {
9695 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9696 while !exec_committed()? {
9697 if std::time::Instant::now() >= deadline {
9701 return None;
9702 }
9703 std::thread::sleep(Duration::from_micros(50));
9704 }
9705 observe()
9706}
9707
9708#[cfg(all(test, target_os = "linux"))]
9709mod spawn_image_tests {
9710 use super::*;
9711 use std::cell::{Cell, RefCell};
9712
9713 enum ExecEvent {
9714 Uncommitted,
9715 Image(Option<subc_os::Observation>),
9716 }
9717
9718 thread_local! {
9719 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9720 }
9721
9722 pub(super) fn report_exec_state(committed: Option<bool>) {
9723 if committed == Some(false) {
9724 EXEC_EVENTS.with(|events| {
9725 if let Some(events) = events.borrow_mut().take() {
9726 let _ = events.send(ExecEvent::Uncommitted);
9727 }
9728 });
9729 }
9730 }
9731
9732 #[test]
9733 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9734 use std::io::{Read, Write};
9735 use std::os::fd::AsRawFd;
9736
9737 let descriptor = std::fs::File::open("/dev/null").unwrap();
9738 let mut command = std::process::Command::new("/bin/sleep");
9739 command.arg("30");
9740 let (mut ready, mut release) =
9741 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9742 .unwrap();
9743 let spawn = std::thread::spawn(move || command.spawn());
9744 let mut message = [0u8; 8];
9745 ready.read_exact(&mut message).unwrap();
9746 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9747 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9748 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9749 let (events, received) = std::sync::mpsc::channel();
9750 let observer = std::thread::spawn(move || {
9751 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9752 let image = observe_spawned_image(pid);
9753 let _ = events.send(ExecEvent::Image(image));
9754 });
9755 let first = received.recv_timeout(Duration::from_secs(5));
9759 release.write_all(&[1]).unwrap();
9760 let mut child = spawn.join().unwrap().unwrap();
9761 let after = match first {
9762 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9763 Ok(ExecEvent::Image(image)) => image,
9764 _ => None,
9765 },
9766 Ok(ExecEvent::Image(image)) => image,
9767 Err(_) => None,
9768 };
9769 observer.join().unwrap();
9770 child.kill().unwrap();
9771 child.wait().unwrap();
9772
9773 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9774 assert_eq!(
9775 inherited.unwrap().executable,
9776 subc_os::file_identity(&std::env::current_exe().unwrap())
9777 );
9778 assert_eq!(
9779 after.unwrap().executable,
9780 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9781 );
9782 }
9783
9784 #[test]
9785 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9786 let committed = Cell::new(false);
9787 let inherited = subc_os::Observation {
9788 start_time: 23,
9789 executable: Some(subc_os::FileIdentity {
9790 device: 29,
9791 inode: 31,
9792 }),
9793 };
9794 let module = subc_os::Observation {
9795 start_time: 23,
9796 executable: Some(subc_os::FileIdentity {
9797 device: 29,
9798 inode: 37,
9799 }),
9800 };
9801 let polls = Cell::new(0);
9802 let image = sample_after_linux_exec(
9803 || {
9804 polls.set(polls.get() + 1);
9805 committed.set(polls.get() == 2);
9808 Some(committed.get())
9809 },
9810 || Some(if committed.get() { module } else { inherited }),
9811 );
9812 assert_eq!(image, Some(module));
9813 assert_eq!(polls.get(), 2);
9814 }
9815
9816 #[test]
9817 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9818 assert_eq!(
9819 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9820 Some(false)
9821 );
9822 assert_eq!(
9823 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9824 Some(true)
9825 );
9826 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9827 assert_eq!(linux_exec_committed("17 (a) R"), None);
9828 }
9829
9830 #[test]
9831 fn unreadable_exec_state_never_samples_an_inherited_image() {
9832 assert_eq!(
9833 sample_after_linux_exec(
9834 || None,
9835 || panic!("must not sample before exec confirmation")
9836 ),
9837 None
9838 );
9839 }
9840
9841 #[test]
9842 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9843 assert_eq!(
9844 sample_after_linux_exec(
9845 || Some(false),
9846 || panic!("must not sample after an exec timeout")
9847 ),
9848 None
9849 );
9850 }
9851}
9852
9853#[cfg(windows)]
9856fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9857 let mut program = spec.program.clone();
9858 if program.is_absolute() || program.components().count() > 1 {
9859 match subc_os::ExecutableCapture::open(&program) {
9862 Ok(capture) => return Ok(capture),
9863 Err(error)
9864 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9865 Err(error) => return Err(error),
9866 }
9867 program.set_extension("exe");
9868 return subc_os::ExecutableCapture::open(&program);
9869 }
9870 if program.extension().is_none() {
9871 program.set_extension("exe");
9872 }
9873 let custom_path = spec
9874 .env
9875 .iter()
9876 .rev()
9877 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9878 let mut directories = Vec::new();
9879 if let Some(parent) = std::env::current_exe()?.parent() {
9882 directories.push(parent.to_path_buf());
9883 }
9884 directories.push(std::env::current_dir()?);
9885 if let Some(windows) = std::env::var_os("SystemRoot") {
9886 let windows = PathBuf::from(windows);
9887 directories.push(windows.join("System32"));
9888 directories.push(windows.clone());
9889 }
9890 let path = custom_path
9891 .map(|(_, value)| std::ffi::OsString::from(value))
9892 .or_else(|| std::env::var_os("PATH"));
9893 if let Some(path) = path {
9894 directories.extend(std::env::split_paths(&path));
9895 }
9896 for directory in directories {
9897 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9898 Ok(capture) => return Ok(capture),
9899 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9900 Err(error) => return Err(error),
9901 }
9902 }
9903 Err(io::Error::new(
9904 io::ErrorKind::NotFound,
9905 "resolved executable not found",
9906 ))
9907}
9908
9909fn spawn_and_mark_running(
9910 spec: &ModuleSpec,
9911 runtime: &SupervisorRuntimeConfig,
9912 snapshot: &SharedSnapshot,
9913) -> Result<SupervisedChild, SuperviseError> {
9914 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9915 let registration_watch = watch_registration(&runtime.registry, spec);
9916 let child = spawn_child(
9917 spec,
9918 runtime.connection_file_path.as_deref(),
9919 runtime.supervisor_handle.as_ref(),
9920 &runtime.stderr_ring,
9921 runtime.capture_logs_dir.as_deref(),
9922 &runtime.child_roster,
9923 #[cfg(target_os = "linux")]
9924 runtime.cgroup_placement.as_ref(),
9925 )?;
9926 let mut child = child;
9927 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
9928 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9929 if let Some(spawn) = activity_spawn {
9930 spawn.complete(snapshot);
9931 }
9932 Ok(child)
9933}
9934
9935struct RegistrationWatch {
9936 changes: watch::Receiver<u64>,
9937 baseline: u64,
9938}
9939
9940fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
9944 if spec.protocol != ModuleProtocol::Subc {
9945 return None;
9946 }
9947 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
9948 let baseline = *changes.borrow();
9949 Some(RegistrationWatch { changes, baseline })
9950}
9951
9952fn arm_registration_deadline(
9953 child: &mut SupervisedChild,
9954 budget: Duration,
9955 registration_watch: Option<RegistrationWatch>,
9956) {
9957 child.registration_deadline =
9958 (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
9959 child.registration_watch = registration_watch;
9960}
9961
9962async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
9963 #[cfg(unix)]
9964 {
9965 let group = unreaped_registration_group(child)?;
9968 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
9969 sleep(crate::child_roster::TERM_TO_KILL).await;
9974 let group = unreaped_registration_group(child)?;
9975 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
9976 }
9977 child.start_kill()?;
9980 child.wait().await?;
9981 Ok(())
9982}
9983
9984#[cfg(unix)]
9985fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
9986 let group = i32::try_from(child.pid)
9989 .ok()
9990 .filter(|pid| *pid > 1)
9991 .and_then(rustix::process::Pid::from_raw)
9992 .ok_or_else(|| {
9993 io::Error::new(
9994 io::ErrorKind::InvalidInput,
9995 "invalid supervised process group",
9996 )
9997 })?;
9998 if child.child.id() != Some(child.pid) {
9999 return Err(io::Error::new(
10000 io::ErrorKind::InvalidInput,
10001 "refusing process-group signal after supervised child was reaped",
10002 ));
10003 }
10004 Ok(group)
10005}
10006
10007enum RegistrationWaitOutcome {
10008 Registered,
10009 Exited(ExitReport),
10010 TimedOut,
10011}
10012
10013struct ReloadRegistrationFailure {
10014 exit_report: ExitReport,
10015 reason: String,
10016}
10017
10018#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10019enum BusyGaugeObservation {
10020 Quiescent,
10021 Busy,
10022 Omitted,
10023}
10024
10025#[derive(Debug, Clone, PartialEq, Eq)]
10027enum GaugeVerdict {
10028 Zero,
10030 NonZero(u64),
10032 NotACount(String),
10036 Missing,
10038 ProbeFailed(&'static str),
10041}
10042
10043impl GaugeVerdict {
10044 fn reads_quiet(&self) -> bool {
10045 matches!(self, Self::Zero)
10046 }
10047}
10048
10049impl fmt::Display for GaugeVerdict {
10050 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10051 match self {
10052 Self::Zero => f.write_str("zero"),
10053 Self::NonZero(value) => write!(f, "non_zero({value})"),
10054 Self::NotACount(value) => write!(f, "not_a_count({value})"),
10055 Self::Missing => f.write_str("missing_from_health_body"),
10056 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10057 }
10058 }
10059}
10060
10061const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10064
10065fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10067 let metrics = metrics.and_then(Value::as_object);
10068 gauges
10069 .iter()
10070 .map(
10071 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10072 None => GaugeVerdict::Missing,
10073 Some(value) => match value.as_u64() {
10074 Some(0) => GaugeVerdict::Zero,
10075 Some(count) => GaugeVerdict::NonZero(count),
10076 None => {
10077 let mut text = value.to_string();
10078 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10079 let mut end = GAUGE_VALUE_TEXT_LIMIT;
10080 while !text.is_char_boundary(end) {
10081 end -= 1;
10082 }
10083 text.truncate(end);
10084 text.push_str("...");
10085 }
10086 GaugeVerdict::NotACount(text)
10087 }
10088 },
10089 },
10090 )
10091 .collect()
10092}
10093
10094fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10097 if verdicts
10098 .iter()
10099 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10100 {
10101 BusyGaugeObservation::Omitted
10102 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10103 BusyGaugeObservation::Quiescent
10104 } else {
10105 BusyGaugeObservation::Busy
10106 }
10107}
10108
10109#[derive(Debug, Clone, Default, PartialEq, Eq)]
10114struct GaugeTrace {
10115 latest: Option<(GaugeVerdict, Duration)>,
10116 last_value: Option<(GaugeVerdict, Duration)>,
10117}
10118
10119#[derive(Debug, Clone, Default, PartialEq, Eq)]
10125struct DrainGaugeTrace {
10126 probes: u32,
10128 gauges: Vec<(String, GaugeTrace)>,
10130}
10131
10132impl DrainGaugeTrace {
10133 fn new(gauges: &[String]) -> Self {
10134 Self {
10135 probes: 0,
10136 gauges: gauges
10137 .iter()
10138 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10139 .collect(),
10140 }
10141 }
10142
10143 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10145 self.probes = self.probes.saturating_add(1);
10146 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10147 trace.latest = Some((verdict.clone(), at));
10148 if matches!(
10149 verdict,
10150 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10151 ) {
10152 trace.last_value = Some((verdict.clone(), at));
10153 }
10154 }
10155 }
10156}
10157
10158impl fmt::Display for DrainGaugeTrace {
10163 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10164 if self.gauges.is_empty() {
10165 return f.write_str("none declared");
10166 }
10167 for (index, (name, trace)) in self.gauges.iter().enumerate() {
10168 if index > 0 {
10169 f.write_str(" | ")?;
10170 }
10171 write!(f, "{name}: ")?;
10172 match &trace.latest {
10173 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10174 None => f.write_str("never probed")?,
10175 }
10176 match &trace.last_value {
10177 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10178 None => f.write_str("; no value ever read")?,
10179 }
10180 }
10181 Ok(())
10182 }
10183}
10184
10185pub(crate) fn declared_busy_gauges(
10186 registry: &Registry,
10187 module_id: &str,
10188) -> Result<Vec<String>, SuperviseError> {
10189 busy_gauges_of(
10190 registry
10191 .get_module(module_id)
10192 .map_err(SuperviseError::Registry)?,
10193 )
10194}
10195
10196fn declared_busy_gauges_for_connection(
10200 registry: &Registry,
10201 connection_id: ConnectionId,
10202) -> Result<Vec<String>, SuperviseError> {
10203 busy_gauges_of(
10204 registry
10205 .get_module_by_connection(connection_id)
10206 .map_err(SuperviseError::Registry)?,
10207 )
10208}
10209
10210fn busy_gauges_of(
10211 registration: Option<crate::registry::ModuleRegistration>,
10212) -> Result<Vec<String>, SuperviseError> {
10213 let Some(registration) = registration else {
10214 return Ok(Vec::new());
10215 };
10216 let Some(self_signals) = registration.manifest.self_signals else {
10217 return Ok(Vec::new());
10218 };
10219
10220 let mut gauges = Vec::new();
10221 for declaration in self_signals {
10222 if declaration.kind != SelfSignalKind::Busy {
10223 continue;
10224 }
10225 match declaration.anchored_to {
10226 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10227 gauges.extend(declared)
10228 }
10229 _ => {
10230 gauges.push(String::new());
10233 }
10234 }
10235 }
10236 Ok(gauges)
10237}
10238
10239#[allow(clippy::too_many_arguments)]
10248async fn wait_for_forwarding_quiescence(
10249 forwarding: &ForwardingTable,
10250 module_id: &str,
10251 runtime: &SupervisorRuntimeConfig,
10252 endpoint: crate::ModuleEndpointId,
10253 deadline: Instant,
10254 busy_gauges: &[String],
10255 scope: DrainScope,
10256 trace: &mut DrainGaugeTrace,
10257) -> Result<bool, SuperviseError> {
10258 let wait_started = Instant::now();
10259 let mut gauges_quiescent = busy_gauges.is_empty();
10260 let mut next_probe_at = wait_started;
10261 let mut omission_reported = false;
10262
10263 loop {
10264 let now = Instant::now();
10265 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10266 let report = match scope {
10267 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10268 DrainScope::Endpoint(endpoint) => {
10269 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10270 }
10271 };
10272 let verdicts = match &report {
10273 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10274 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10275 };
10276 trace.record(&verdicts, wait_started.elapsed());
10277 gauges_quiescent = match report {
10278 Ok(_) => match busy_gauge_observation(&verdicts) {
10279 BusyGaugeObservation::Quiescent => true,
10280 BusyGaugeObservation::Busy => false,
10281 BusyGaugeObservation::Omitted => {
10282 if !omission_reported {
10285 forwarding
10286 .counters()
10287 .increment_drains_with_undeclared_gauge();
10288 let missing = busy_gauges
10289 .iter()
10290 .zip(&verdicts)
10291 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10292 .map(|(gauge, _)| gauge.as_str())
10293 .collect::<Vec<_>>()
10294 .join(",");
10295 warn!(
10296 module_id,
10297 missing_gauges = %missing,
10298 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10299 );
10300 omission_reported = true;
10301 }
10302 false
10303 }
10304 },
10305 Err(err) => {
10306 warn!(
10307 module_id,
10308 error = %err,
10309 "drain health.check did not produce declared busy gauges; treating module as busy"
10310 );
10311 false
10312 }
10313 };
10314 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10317 }
10318
10319 let in_flight = forwarding
10320 .endpoint_in_flight_count(endpoint)
10321 .map_err(SuperviseError::Forwarding)?;
10322 if in_flight == 0 && gauges_quiescent {
10323 return Ok(true);
10324 }
10325
10326 let now = Instant::now();
10327 if now >= deadline {
10328 return Ok(false);
10329 }
10330 let mut wait = deadline
10331 .saturating_duration_since(now)
10332 .min(REGISTRY_RELEASE_POLL);
10333 if !busy_gauges.is_empty() {
10334 wait = wait.min(next_probe_at.saturating_duration_since(now));
10335 }
10336 sleep(wait).await;
10337 }
10338}
10339
10340fn drain_busy_reprobe_interval() -> Duration {
10344 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10345}
10346
10347fn warn_drain_timed_out(
10362 module_id: &str,
10363 drain_timeout: Duration,
10364 reason: RouteCloseReason,
10365 holdouts: &crate::forwarding::DrainHoldouts,
10366 gauges: &DrainGaugeTrace,
10367) {
10368 warn!(
10369 module_id = %module_id,
10370 waited = ?drain_timeout,
10371 ?reason,
10372 held_requests = holdouts.requests,
10373 held_routes = holdouts.routes,
10374 total_routes = holdouts.total_routes,
10375 top_connections = ?holdouts.top_connections,
10376 held = %holdouts
10379 .held
10380 .iter()
10381 .map(|(channel, corr)| format!("{channel}:{corr}"))
10382 .collect::<Vec<_>>()
10383 .join(","),
10384 gauge_probes = gauges.probes,
10385 busy_gauges = %gauges,
10386 "route drain timed out before request quiescence; forcing teardown"
10387 );
10388}
10389
10390fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10398 match wait_result {
10399 Ok(drained) => *drained,
10400 Err(_) => false,
10401 }
10402}
10403
10404fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10405 for released in released_routes {
10406 let frame = match Frame::build_with_version(
10407 released.negotiated_ver,
10408 FrameType::Goodbye,
10409 control_flags(),
10410 released.channel,
10411 released.epoch,
10412 0,
10413 Vec::new(),
10414 ) {
10415 Ok(frame) => frame,
10416 Err(err) => {
10417 warn!(
10418 route_channel = released.channel,
10419 error = %err,
10420 "failed to build supervisor drain route GOODBYE frame"
10421 );
10422 continue;
10423 }
10424 };
10425 if !released.close_on_delivery_failure() {
10426 crate::forwarding::send_module_route_goodbye(
10427 &forwarding.counters(),
10428 &released.sink,
10429 frame,
10430 released.module_id.as_deref(),
10431 "supervisor drain",
10432 );
10433 continue;
10434 }
10435 if let Err(err) = released.sink.try_send(frame) {
10436 warn!(
10437 target_connection_id = released.connection_id.get(),
10438 route_channel = released.channel,
10439 error = %err,
10440 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10441 );
10442 let _ = forwarding.escalate_client_delivery_failure(
10443 released.connection_id,
10444 released.channel,
10445 released.epoch,
10446 CloseReason::new(
10447 "route_goodbye_delivery_failed",
10448 format!(
10449 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10450 released.channel
10451 ),
10452 ),
10453 crate::forwarding::UndeliveredFrame {
10454 module_id: released.module_id.as_deref(),
10455 sink: &released.sink,
10456 },
10457 );
10458 }
10459 }
10460}
10461
10462fn send_module_draining(
10463 module_id: &str,
10464 reason: RouteCloseReason,
10465 deadline_ms: u64,
10466 target: &ModuleDrainTarget,
10467) {
10468 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10469 reason,
10470 deadline_ms,
10471 }) {
10472 Ok(body) => body,
10473 Err(err) => {
10474 warn!(
10475 module_id,
10476 error = %err,
10477 "failed to encode module draining command"
10478 );
10479 return;
10480 }
10481 };
10482 let frame = match Frame::build_with_version(
10483 target.negotiated_ver,
10484 FrameType::Push,
10485 control_flags(),
10486 0,
10487 0,
10488 0,
10489 body,
10490 ) {
10491 Ok(frame) => frame,
10492 Err(err) => {
10493 warn!(
10494 module_id,
10495 error = %err,
10496 "failed to build module draining command frame"
10497 );
10498 return;
10499 }
10500 };
10501 if let Err(err) = target.sink.try_send(frame) {
10502 warn!(
10503 module_id,
10504 target_connection_id = target.endpoint.connection_id.get(),
10505 error = %err,
10506 "module draining command was not delivered to peer"
10507 );
10508 }
10509}
10510
10511fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10513 match Frame::build_with_version(
10514 negotiated_ver,
10515 FrameType::Goodbye,
10516 control_flags(),
10517 0,
10518 0,
10519 0,
10520 Vec::new(),
10521 ) {
10522 Ok(frame) => Some(frame),
10523 Err(err) => {
10524 warn!(
10525 module_id,
10526 error = %err,
10527 "failed to build module GOODBYE frame"
10528 );
10529 None
10530 }
10531 }
10532}
10533
10534#[cfg(unix)]
10548async fn send_module_goodbyes_for_daemon_shutdown(
10549 forwarding: &Arc<ForwardingTable>,
10550 reason: &CloseReason,
10551 wait_for_flush: bool,
10552) {
10553 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10554 let targets = match forwarding.module_connections() {
10555 Ok(targets) => targets,
10556 Err(err) => {
10557 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10558 return;
10559 }
10560 };
10561 let deadline = Instant::now() + GOODBYE_BUDGET;
10562 let mut sends = tokio::task::JoinSet::new();
10563 for target in targets {
10564 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10565 continue;
10566 };
10567 if !wait_for_flush {
10568 if let Err(err) = target.sink.try_send(frame) {
10569 debug!(
10570 module_id = %target.module_id,
10571 error = %err,
10572 "shutdown module GOODBYE was not queued"
10573 );
10574 }
10575 continue;
10576 }
10577 let forwarding = Arc::clone(forwarding);
10578 let reason = reason.clone();
10579 sends.spawn(async move {
10580 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10581 Ok(Ok(())) => {}
10582 Ok(Err(err)) => debug!(
10583 module_id = %target.module_id,
10584 error = %err,
10585 "module connection closed before its shutdown GOODBYE was written"
10586 ),
10587 Err(_) => warn!(
10588 module_id = %target.module_id,
10589 budget = ?GOODBYE_BUDGET,
10590 "shutdown module GOODBYE was not written within its budget; closing anyway"
10591 ),
10592 }
10593 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10594 });
10595 }
10596 while sends.join_next().await.is_some() {}
10598}
10599
10600fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10601 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10602 return;
10603 };
10604 if let Err(err) = target.sink.try_send(frame) {
10605 warn!(
10606 module_id,
10607 target_connection_id = target.endpoint.connection_id.get(),
10608 error = %err,
10609 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10610 );
10611 forwarding.request_connection_close(
10612 target.endpoint.connection_id,
10613 CloseReason::new(
10614 "module_goodbye_delivery_failed",
10615 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10616 ),
10617 );
10618 }
10619}
10620
10621#[derive(Clone, Copy)]
10622struct ForwardingDrainContext<'a> {
10623 spec: &'a ModuleSpec,
10624 runtime: &'a SupervisorRuntimeConfig,
10625 registry: &'a Registry,
10626 scope: DrainScope,
10627}
10628
10629#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10631enum DrainScope {
10632 Active,
10635 Endpoint(crate::ModuleEndpointId),
10640}
10641
10642#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10650enum StopNotice {
10651 SentOverConnection,
10654 NoConnection,
10658 NotSent,
10662}
10663
10664async fn begin_forwarding_drain(
10665 spec: &ModuleSpec,
10666 runtime: &SupervisorRuntimeConfig,
10667 registry: &Registry,
10668 snapshot: &SharedSnapshot,
10669 enabled: Option<bool>,
10670 reason: RouteCloseReason,
10671) -> Result<StopNotice, SuperviseError> {
10672 let Some(forwarding) = runtime.forwarding.as_ref() else {
10673 return Err(SuperviseError::ReloadUnavailable {
10674 module_id: spec.module_id.clone(),
10675 reason: "supervisor was not configured with a forwarding table".to_string(),
10676 });
10677 };
10678
10679 begin_forwarding_drain_with(
10680 forwarding,
10681 ForwardingDrainContext {
10682 spec,
10683 runtime,
10684 registry,
10685 scope: DrainScope::Active,
10686 },
10687 snapshot,
10688 enabled,
10689 reason,
10690 runtime.drain_timeout,
10691 )
10692 .await
10693}
10694
10695async fn begin_forwarding_drain_if_configured(
10696 spec: &ModuleSpec,
10697 runtime: &SupervisorRuntimeConfig,
10698 registry: &Registry,
10699 snapshot: &SharedSnapshot,
10700 enabled: Option<bool>,
10701 reason: RouteCloseReason,
10702) -> Result<StopNotice, SuperviseError> {
10703 begin_forwarding_drain_with_timeout(
10704 spec,
10705 runtime,
10706 registry,
10707 snapshot,
10708 enabled,
10709 reason,
10710 runtime.drain_timeout,
10711 )
10712 .await
10713}
10714
10715async fn begin_forwarding_drain_with_timeout(
10719 spec: &ModuleSpec,
10720 runtime: &SupervisorRuntimeConfig,
10721 registry: &Registry,
10722 snapshot: &SharedSnapshot,
10723 enabled: Option<bool>,
10724 reason: RouteCloseReason,
10725 drain_timeout: Duration,
10726) -> Result<StopNotice, SuperviseError> {
10727 let Some(forwarding) = runtime.forwarding.as_ref() else {
10728 return Ok(StopNotice::NotSent);
10729 };
10730
10731 begin_forwarding_drain_with(
10732 forwarding,
10733 ForwardingDrainContext {
10734 spec,
10735 runtime,
10736 registry,
10737 scope: DrainScope::Active,
10738 },
10739 snapshot,
10740 enabled,
10741 reason,
10742 drain_timeout,
10743 )
10744 .await
10745}
10746
10747async fn begin_forwarding_drain_with(
10748 forwarding: &ForwardingTable,
10749 context: ForwardingDrainContext<'_>,
10750 snapshot: &SharedSnapshot,
10751 enabled: Option<bool>,
10752 reason: RouteCloseReason,
10753 drain_timeout: Duration,
10754) -> Result<StopNotice, SuperviseError> {
10755 let ForwardingDrainContext {
10756 spec,
10757 runtime,
10758 registry,
10759 scope,
10760 } = context;
10761 debug_assert_ne!(reason, RouteCloseReason::Crash);
10762 let terminal = matches!(reason, RouteCloseReason::Disable);
10763 let drain_started_at = Instant::now();
10764 let drain_deadline = drain_started_at + drain_timeout;
10765 let deadline_ms =
10766 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10767 let busy_gauges = match scope {
10768 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10769 DrainScope::Endpoint(endpoint) => {
10770 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10771 }
10772 };
10773
10774 let gate_started = Instant::now();
10777 let drain_target = match scope {
10778 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10779 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10780 }
10781 .map_err(SuperviseError::Forwarding)?;
10782 info!(
10787 module_id = %spec.module_id,
10788 ?reason,
10789 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10790 connected = drain_target.is_some(),
10791 "module drain began; route admission closed"
10792 );
10793 if scope == DrainScope::Active {
10794 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10795 state.state = ModuleState::Draining;
10796 state.draining_to_replace =
10797 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10798 if let Some(enabled) = enabled {
10799 state.enabled = enabled;
10800 }
10801 })?;
10802 }
10803
10804 let Some(target) = drain_target.as_ref() else {
10805 return Ok(StopNotice::NoConnection);
10809 };
10810 {
10811 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10812 #[cfg(feature = "test-support")]
10813 forwarding.wait_before_route_closing_for_test().await;
10814 let routes = forwarding
10815 .endpoint_routes(target.endpoint)
10816 .map_err(SuperviseError::Forwarding)?;
10817 let routes_notified = routes.len();
10818 crate::control::send_route_control_pushes(
10819 forwarding,
10820 routes.clone(),
10821 ClientControlPush::RouteClosing {
10822 module_id: spec.module_id.clone(),
10823 channels: Vec::new(),
10824 reason,
10825 },
10826 );
10827 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10828
10829 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10835 let wait_result = wait_for_forwarding_quiescence(
10836 forwarding,
10837 &spec.module_id,
10838 runtime,
10839 target.endpoint,
10840 drain_deadline,
10841 &busy_gauges,
10842 scope,
10843 &mut gauge_trace,
10844 )
10845 .await;
10846 let drained = drained_after_quiescence_wait(&wait_result);
10847 if let Err(err) = &wait_result {
10848 error!(
10849 module_id = %spec.module_id,
10850 ?reason,
10851 error = %err,
10852 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10853 );
10854 } else if !drained {
10855 let holdouts = forwarding
10857 .endpoint_drain_holdouts(target.endpoint)
10858 .unwrap_or_default();
10859 warn_drain_timed_out(
10860 &spec.module_id,
10861 drain_timeout,
10862 reason,
10863 &holdouts,
10864 &gauge_trace,
10865 );
10866 }
10867 crate::control::send_route_control_pushes(
10868 forwarding,
10869 routes,
10870 ClientControlPush::RouteClosed {
10871 module_id: spec.module_id.clone(),
10872 channels: Vec::new(),
10873 reason,
10874 drained,
10875 abandoned: target.abandoned_bindings.len() as u32,
10876 excluded_subscriptions: target.excluded_subscriptions,
10877 terminal: Some(terminal),
10878 },
10879 );
10880 wait_result?;
10881
10882 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10888 Ok(routes) => routes,
10889 Err(err) => {
10890 warn!(
10891 module_id = %spec.module_id,
10892 ?reason,
10893 error = %err,
10894 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10895 );
10896 send_module_goodbye(&spec.module_id, forwarding, target);
10897 return Err(SuperviseError::Forwarding(err));
10898 }
10899 };
10900 let route_goodbye_count = released_routes.len();
10901 send_route_goodbyes(forwarding, released_routes);
10902 send_module_goodbye(&spec.module_id, forwarding, target);
10903
10904 info!(
10910 module_id = %spec.module_id,
10911 ?reason,
10912 routes_notified,
10913 route_goodbyes = route_goodbye_count,
10914 abandoned_reservations = target.abandoned_bindings.len(),
10915 excluded_subscriptions = target.excluded_subscriptions,
10916 drained,
10917 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10918 );
10919 }
10920
10921 Ok(StopNotice::SentOverConnection)
10922}
10923
10924async fn wait_for_registration_after_reload(
10927 registry: &Registry,
10928 module_id: &str,
10929 snapshot: &SharedSnapshot,
10930 child: &mut SupervisedChild,
10931 wait: Duration,
10932) -> Result<RegistrationWaitOutcome, SuperviseError> {
10933 wait_for_slot_registration(
10934 registry,
10935 crate::registry::RegistrationSlot::Active(module_id),
10936 module_id,
10937 snapshot,
10938 child,
10939 wait,
10940 )
10941 .await
10942}
10943
10944async fn wait_for_slot_registration(
10952 registry: &Registry,
10953 slot: crate::registry::RegistrationSlot<'_>,
10954 module_id: &str,
10955 snapshot: &SharedSnapshot,
10956 child: &mut SupervisedChild,
10957 wait: Duration,
10958) -> Result<RegistrationWaitOutcome, SuperviseError> {
10959 let deadline = Instant::now() + wait;
10960 loop {
10961 if registry
10962 .registration(slot)
10963 .map_err(SuperviseError::Registry)?
10964 .is_some()
10965 {
10966 return Ok(RegistrationWaitOutcome::Registered);
10967 }
10968
10969 let now = Instant::now();
10970 if now >= deadline {
10971 return Ok(RegistrationWaitOutcome::TimedOut);
10972 }
10973 let remaining = deadline.saturating_duration_since(now);
10974 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10975
10976 tokio::select! {
10977 wait_result = child.wait() => {
10978 let status = wait_result.map_err(|source| SuperviseError::Wait {
10979 module_id: module_id.to_string(),
10980 source,
10981 })?;
10982 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10983 snapshot,
10984 child,
10985 &status,
10986 )));
10987 }
10988 _ = sleep(poll) => {}
10989 }
10990 }
10991}
10992
10993fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10994 if exit_report.kind != ExitKind::DeliberateSeverance {
10998 exit_report.kind = ExitKind::Crash;
10999 }
11000 exit_report
11001}
11002
11003async fn handle_reload_child_registration_failure(
11004 spec: &ModuleSpec,
11005 runtime: &SupervisorRuntimeConfig,
11006 registry: &Registry,
11007 process_liveness: &SupervisorProcessLiveness,
11008 snapshot: &SharedSnapshot,
11009 _child: &mut Option<SupervisedChild>,
11010 failure: ReloadRegistrationFailure,
11011) -> Result<(), SuperviseError> {
11012 let ReloadRegistrationFailure {
11013 exit_report,
11014 reason,
11015 } = failure;
11016 match on_child_exit(
11017 spec,
11018 runtime.restart_policy,
11019 registry,
11020 snapshot,
11021 &runtime.terminal_ring,
11022 &runtime.spawn_events,
11023 &runtime.child_roster,
11024 exit_report,
11025 runtime.recovery_is_daemon,
11026 )
11027 .await
11028 {
11029 NextAction::Stop {
11030 registration_released,
11031 } => {
11032 if registration_released {
11033 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11034 }
11035 }
11036 NextAction::Restart { schedule } => {
11037 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11038 schedule.delay
11039 });
11040 if let Some(schedule) = schedule {
11041 log_crash_respawn(&spec.module_id, schedule);
11042 }
11043 schedule_respawn(
11044 runtime,
11045 snapshot,
11046 &spec.module_id,
11047 delay,
11048 RespawnKind::Spawn,
11049 true,
11050 )?;
11051 }
11052 }
11053 Err(SuperviseError::ReloadFailed {
11054 module_id: spec.module_id.clone(),
11055 reason,
11056 })
11057}
11058
11059async fn handle_reload_spawn_failure(
11060 spec: &ModuleSpec,
11061 runtime: &SupervisorRuntimeConfig,
11062 process_liveness: &SupervisorProcessLiveness,
11063 snapshot: &SharedSnapshot,
11064 _child: &mut Option<SupervisedChild>,
11065 reason: String,
11066) -> Result<(), SuperviseError> {
11067 let now = Instant::now();
11068 let mut schedule = None;
11069 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11070 clear_current_process_facts(state);
11071 if state.enabled {
11072 schedule = state.next_crash_restart(&runtime.restart_policy, now);
11073 state.state = if schedule.is_some() {
11074 ModuleState::Restarting
11075 } else {
11076 if runtime
11077 .failure_context(runtime.recovery_is_daemon)
11078 .may_retry()
11079 {
11080 state.failed_retry.schedule(&spec.module_id);
11081 }
11082 ModuleState::Failed
11083 };
11084 } else {
11085 state.state = ModuleState::Disabled;
11086 }
11087 })?;
11088 if let Some(schedule) = schedule {
11089 schedule_respawn(
11090 runtime,
11091 snapshot,
11092 &spec.module_id,
11093 schedule.delay,
11094 RespawnKind::Spawn,
11095 true,
11096 )?;
11097 } else {
11098 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11099 }
11100 Err(SuperviseError::ReloadFailed {
11101 module_id: spec.module_id.clone(),
11102 reason,
11103 })
11104}
11105
11106fn control_flags() -> Flags {
11107 Flags::new(false, Priority::Passive, false)
11108}
11109
11110#[allow(clippy::too_many_arguments)]
11111async fn drain_optional_child(
11112 module_id: &str,
11113 protocol: ModuleProtocol,
11114 stop_notice: StopNotice,
11115 registry: &Registry,
11116 forwarding: Option<&ForwardingTable>,
11117 snapshot: &SharedSnapshot,
11118 terminal_ring: &Arc<Mutex<TerminalRing>>,
11119 spawn_events: &SpawnEventFeed,
11120 child: &mut Option<SupervisedChild>,
11121 drain_timeout: Duration,
11122 final_state: ModuleState,
11123 enabled: Option<bool>,
11124 failure: &FailureContext,
11125) -> Result<(), SuperviseError> {
11126 if let Some(child) = child.take() {
11127 drain_child_to_state(
11128 module_id,
11129 protocol,
11130 stop_notice,
11131 registry,
11132 forwarding,
11133 snapshot,
11134 terminal_ring,
11135 spawn_events,
11136 child,
11137 drain_timeout,
11138 final_state,
11139 enabled,
11140 failure,
11141 )
11142 .await
11143 } else {
11144 update_snapshot(snapshot, Some(module_id), |state| {
11145 state.state = final_state;
11146 if final_state == ModuleState::Failed {
11147 state.failed_retry.failed_at = Some(Instant::now());
11148 }
11149 if let Some(enabled) = enabled {
11150 state.enabled = enabled;
11151 }
11152 clear_current_process_facts(state);
11153 })?;
11154 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11155 }
11156}
11157
11158#[allow(clippy::too_many_arguments)]
11159async fn drain_child_to_state(
11160 module_id: &str,
11161 _protocol: ModuleProtocol,
11162 stop_notice: StopNotice,
11163 registry: &Registry,
11164 forwarding: Option<&ForwardingTable>,
11165 snapshot: &SharedSnapshot,
11166 terminal_ring: &Arc<Mutex<TerminalRing>>,
11167 spawn_events: &SpawnEventFeed,
11168 mut child: SupervisedChild,
11169 drain_timeout: Duration,
11170 final_state: ModuleState,
11171 enabled: Option<bool>,
11172 failure: &FailureContext,
11173) -> Result<(), SuperviseError> {
11174 let protocol = child.protocol;
11175 update_snapshot(snapshot, Some(module_id), |state| {
11176 state.state = ModuleState::Draining;
11177 state.draining_to_replace = final_state == ModuleState::Restarting;
11178 if let Some(enabled) = enabled {
11179 state.enabled = enabled;
11180 }
11181 })?;
11182
11183 if stop_notice != StopNotice::SentOverConnection {
11194 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11195 info!(
11196 module_id,
11197 pid = child.pid,
11198 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11199 "module has no connection yet; requesting stop by signal"
11200 );
11201 }
11202 request_graceful_stop(module_id, &child);
11203 }
11204
11205 let exit_report = match timeout(drain_timeout, child.wait()).await {
11206 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11207 Ok(Err(source)) => {
11208 fail_after_wait_error(
11209 module_id,
11210 snapshot,
11211 terminal_ring,
11212 spawn_events,
11213 format!("failed to wait during drain: {source}"),
11214 failure,
11215 );
11216 return Err(SuperviseError::Wait {
11217 module_id: module_id.to_string(),
11218 source,
11219 });
11220 }
11221 Err(_) => {
11222 warn!(
11235 module_id,
11236 pid = child.pid,
11237 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11238 reason = ?final_state,
11239 ?stop_notice,
11240 "drain budget expired before the module exited; killing it"
11241 );
11242 child.start_kill().map_err(|source| {
11243 fail_after_wait_error(
11244 module_id,
11245 snapshot,
11246 terminal_ring,
11247 spawn_events,
11248 format!("failed to kill during drain: {source}"),
11249 failure,
11250 );
11251 SuperviseError::Kill {
11252 module_id: module_id.to_string(),
11253 source,
11254 }
11255 })?;
11256 let status = child.wait().await.map_err(|source| {
11257 fail_after_wait_error(
11258 module_id,
11259 snapshot,
11260 terminal_ring,
11261 spawn_events,
11262 format!("failed to wait after drain kill: {source}"),
11263 failure,
11264 );
11265 SuperviseError::Wait {
11266 module_id: module_id.to_string(),
11267 source,
11268 }
11269 })?;
11270 classify_reaped_child_exit(snapshot, &child, &status)
11271 }
11272 };
11273
11274 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11275 publish_after_terminal_record(
11276 module_id,
11277 || {
11278 record_terminal_with_detail(
11279 module_id,
11280 terminal_ring,
11281 spawn_events,
11282 &exit_report,
11283 terminal_disposition(final_state),
11284 detail,
11285 )
11286 },
11287 || {
11288 update_snapshot(snapshot, Some(module_id), |state| {
11289 state.state = final_state;
11290 if final_state == ModuleState::Failed {
11291 state.failed_retry.failed_at = Some(Instant::now());
11292 }
11293 if let Some(enabled) = enabled {
11294 state.enabled = enabled;
11295 }
11296 clear_current_process_facts(state);
11297 state.last_exit = Some(exit_report.clone());
11298 if exit_report.kind == ExitKind::DeliberateSeverance {
11299 state.lifetime_restarts += 1;
11300 }
11301 })
11302 },
11303 )?;
11304 child.drain_stderr(module_id).await;
11305
11306 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11307}
11308
11309#[cfg(unix)]
11329fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
11330 let Some(pid) = child
11331 .id()
11332 .and_then(|pid| i32::try_from(pid).ok())
11333 .and_then(rustix::process::Pid::from_raw)
11334 else {
11335 debug!(
11336 module_id,
11337 "no pid to signal for teardown; falling through to the drain wait"
11338 );
11339 return;
11340 };
11341 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
11342 Ok(()) => debug!(
11343 module_id,
11344 "sent SIGTERM to a module nothing else asked to stop"
11345 ),
11346 Err(err) => debug!(
11347 module_id,
11348 error = %err,
11349 "SIGTERM to module failed; the drain wait and kill still apply"
11350 ),
11351 }
11352}
11353
11354#[cfg(not(unix))]
11362fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11363 debug!(
11364 module_id,
11365 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11366 );
11367}
11368
11369fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11370 match final_state {
11371 ModuleState::Stopped => TerminalDisposition::Stopped,
11372 ModuleState::Disabled => TerminalDisposition::Disabled,
11373 ModuleState::Restarting => TerminalDisposition::Restarting,
11374 ModuleState::Failed => TerminalDisposition::Failed,
11375 ModuleState::Starting
11376 | ModuleState::Running
11377 | ModuleState::Unresponsive
11378 | ModuleState::Draining => {
11379 unreachable!("terminal exits only finish in terminal or restarting states")
11380 }
11381 }
11382}
11383
11384async fn release_dead_registration(
11392 registry: &Registry,
11393 forwarding: Option<&ForwardingTable>,
11394 snapshot: &SharedSnapshot,
11395 module_id: &str,
11396 failure: &FailureContext,
11397) -> Result<(), SuperviseError> {
11398 let result = async {
11399 let registration = registry
11400 .get_module(module_id)
11401 .map_err(SuperviseError::Registry)?;
11402 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11403 Ok(()) => return Ok(()),
11404 Err(SuperviseError::RegistrationStillActive { .. }) => {}
11405 Err(err) => return Err(err),
11406 }
11407 let pid = lock_snapshot(snapshot)?.reaped_pid;
11408 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11409 warn!(
11410 module_id,
11411 pid,
11412 connection_id = registration.connection_id.get(),
11413 "reaped module registration outlived release grace; closing dead connection"
11414 );
11415 forwarding.request_connection_close(
11416 registration.connection_id,
11417 CloseReason::new(
11418 "supervised_process_reaped",
11419 format!("module '{module_id}' pid {pid} exited"),
11420 ),
11421 );
11422 wait_for_slot_registration_release(
11423 registry,
11424 crate::registry::RegistrationSlot::Connection(registration.connection_id),
11425 REGISTRY_RELEASE_TIMEOUT,
11426 )
11427 .await?;
11428 }
11429 wait_for_registration_release(registry, module_id, Duration::ZERO).await
11430 }
11431 .await;
11432 if let Err(err) = &result {
11433 fail_snapshot(snapshot, Some(module_id), None, failure);
11434 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11435 }
11436 result
11437}
11438
11439async fn wait_for_registration_release(
11442 registry: &Registry,
11443 module_id: &str,
11444 wait: Duration,
11445) -> Result<(), SuperviseError> {
11446 wait_for_slot_registration_release(
11447 registry,
11448 crate::registry::RegistrationSlot::Active(module_id),
11449 wait,
11450 )
11451 .await
11452}
11453
11454async fn wait_for_slot_registration_release(
11462 registry: &Registry,
11463 slot: crate::registry::RegistrationSlot<'_>,
11464 wait: Duration,
11465) -> Result<(), SuperviseError> {
11466 let deadline = Instant::now() + wait;
11467 let mut release_events = registration_release_events().subscribe();
11468 let still_active = |registration: &crate::registry::ModuleRegistration| {
11469 SuperviseError::RegistrationStillActive {
11470 module_id: registration.manifest.module_id.clone(),
11471 waited: wait,
11472 }
11473 };
11474 loop {
11475 let _observed_generation = *release_events.borrow_and_update();
11476 let Some(registration) = registry
11477 .registration(slot)
11478 .map_err(SuperviseError::Registry)?
11479 else {
11480 return Ok(());
11481 };
11482
11483 let now = Instant::now();
11484 if now >= deadline {
11485 return Err(still_active(®istration));
11486 }
11487
11488 let remaining = deadline.saturating_duration_since(now);
11489 match timeout(remaining, release_events.changed()).await {
11490 Ok(Ok(())) | Ok(Err(_)) => {}
11491 Err(_) => return Err(still_active(®istration)),
11492 }
11493 }
11494}
11495
11496#[cfg(test)]
11497mod slot_registration_wait_tests {
11498 use super::*;
11499 use crate::registry::{ConnectionId, RegistrationSlot};
11500 use subc_protocol::manifest::ModuleManifest;
11501
11502 #[tokio::test]
11503 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11504 let registry = Arc::new(Registry::default());
11505 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
11506 let runtime = supervisor.runtime_config();
11507 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11508 let spec = ModuleSpec {
11509 module_id: "enable-stale-registration".to_string(),
11510 program: PathBuf::from("/missing/enable-retry-test"),
11511 args: Vec::new(),
11512 env: Vec::new(),
11513 reserved: false,
11514 reserved_prefixes: Vec::new(),
11515 protocol: ModuleProtocol::Subc,
11516 overlap: Default::default(),
11517 };
11518 let connection = ConnectionId::new(90);
11519 registry
11520 .register_with_control_ops(
11521 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11522 1,
11523 connection,
11524 Vec::new(),
11525 )
11526 .unwrap();
11527 let mut child = None;
11528 let err = set_child_enabled(
11529 &spec,
11530 &runtime,
11531 ®istry,
11532 &supervisor.process_liveness,
11533 &snapshot,
11534 &mut child,
11535 true,
11536 &runtime.failure_context(false),
11537 )
11538 .await
11539 .unwrap_err();
11540 assert!(matches!(
11541 err,
11542 SuperviseError::RegistrationStillActive { .. }
11543 ));
11544 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11545 assert!(child.is_none());
11546 registry.deregister_connection(connection).unwrap();
11547 let err = set_child_enabled(
11548 &spec,
11549 &runtime,
11550 ®istry,
11551 &supervisor.process_liveness,
11552 &snapshot,
11553 &mut child,
11554 true,
11555 &runtime.failure_context(false),
11556 )
11557 .await
11558 .unwrap_err();
11559 assert!(
11560 matches!(err, SuperviseError::Spawn { .. }),
11561 "second enable must attempt a spawn: {err}"
11562 );
11563 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11564 }
11565
11566 const INCUMBENT: u64 = 1;
11567 const CANDIDATE: u64 = 2;
11568
11569 fn swapped_registry() -> Arc<Registry> {
11570 let registry = Arc::new(Registry::default());
11571 let manifest = ModuleManifest::builder("m", "0.1.0").build();
11572 registry
11573 .register_with_control_ops(
11574 manifest.clone(),
11575 1,
11576 ConnectionId::new(INCUMBENT),
11577 Vec::new(),
11578 )
11579 .unwrap();
11580 registry
11581 .register_candidate_with_control_ops(
11582 manifest,
11583 1,
11584 ConnectionId::new(CANDIDATE),
11585 Vec::new(),
11586 )
11587 .unwrap();
11588 registry
11589 }
11590
11591 #[tokio::test]
11595 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11596 let registry = swapped_registry();
11597 registry.promote_candidate("m").unwrap().unwrap();
11598
11599 assert!(matches!(
11600 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
11601 Err(SuperviseError::RegistrationStillActive { .. })
11602 ));
11603
11604 assert!(matches!(
11606 wait_for_slot_registration_release(
11607 ®istry,
11608 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11609 Duration::from_millis(50),
11610 )
11611 .await,
11612 Err(SuperviseError::RegistrationStillActive { .. })
11613 ));
11614
11615 let releaser = Arc::clone(®istry);
11616 let release = tokio::spawn(async move {
11617 sleep(Duration::from_millis(20)).await;
11618 releaser
11619 .deregister_connection(ConnectionId::new(INCUMBENT))
11620 .unwrap();
11621 notify_registration_release();
11622 });
11623 wait_for_slot_registration_release(
11624 ®istry,
11625 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11626 Duration::from_secs(5),
11627 )
11628 .await
11629 .expect("the incumbent's own registration is released");
11630 release.await.unwrap();
11631 assert!(registry.get_module("m").unwrap().is_some());
11632 }
11633
11634 #[tokio::test]
11637 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11638 let registry = swapped_registry();
11639 assert!(matches!(
11640 wait_for_slot_registration_release(
11641 ®istry,
11642 RegistrationSlot::Candidate("m"),
11643 Duration::from_millis(50),
11644 )
11645 .await,
11646 Err(SuperviseError::RegistrationStillActive { .. })
11647 ));
11648 registry
11649 .deregister_connection(ConnectionId::new(CANDIDATE))
11650 .unwrap();
11651 wait_for_slot_registration_release(
11652 ®istry,
11653 RegistrationSlot::Candidate("m"),
11654 Duration::from_millis(50),
11655 )
11656 .await
11657 .expect("a candidate slot with no candidate is released");
11658 assert!(registry
11659 .registration(RegistrationSlot::Active("m"))
11660 .unwrap()
11661 .is_some());
11662 }
11663}
11664
11665fn classify_exit(status: &ExitStatus) -> ExitReport {
11666 ExitReport {
11667 kind: if status.success() {
11668 ExitKind::Clean
11669 } else {
11670 ExitKind::Crash
11671 },
11672 code: status.code(),
11673 signal: exit_signal(status),
11674 at_ms: unix_ms_now(),
11675 }
11676}
11677
11678fn wait_error_exit_report() -> ExitReport {
11684 ExitReport {
11685 kind: ExitKind::Crash,
11686 code: None,
11687 signal: None,
11688 at_ms: unix_ms_now(),
11689 }
11690}
11691
11692#[cfg(unix)]
11693fn exit_signal(status: &ExitStatus) -> Option<i32> {
11694 use std::os::unix::process::ExitStatusExt;
11695
11696 status.signal()
11697}
11698
11699#[cfg(not(unix))]
11700fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11701 None
11702}
11703
11704fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11710 update_snapshot(snapshot, Some(module_id), |state| {
11711 state.clear_crash_restarts();
11712 })
11713}
11714
11715fn set_running(
11716 snapshot: &SharedSnapshot,
11717 child: &SupervisedChild,
11718 module_id: &str,
11719 spawn_events: &SpawnEventFeed,
11720) -> Result<(), SuperviseError> {
11721 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11722 module_id: Some(module_id.to_string()),
11723 })?;
11724 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11725 if std::mem::take(&mut state.coalesced_restart_pending) {
11726 let generation = state.spawn_generation;
11727 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11728 }
11729 state.drain_disposition_detail = None;
11730 state.spawn_failure = None;
11731 state.waiting_for_previous_pid = None;
11732 state.waiting_for_previous_detail = None;
11733 state.in_alternate_slot = false;
11736 state.configuration_updated_since_spawn = false;
11737 state.spawned_protocol = Some(child.protocol);
11738 state.state = ModuleState::Running;
11739 state.enabled = true;
11740 state.process_alive = true;
11741 state.pid = child.id();
11742 state.health.status = SupervisorHealthStatus::Unknown;
11745 state.health.consecutive_failures = 0;
11746 state.health.detail = None;
11747 state.health.metrics = None;
11748 state.health.last_probe_ms = None;
11749 state.failed_retry.healthy_since = None;
11750 state.failed_retry.failed_at = None;
11751 state.failed_retry.running_since = Some(child.spawned_at);
11752 #[cfg(windows)]
11753 {
11754 state.pipe_consumed = child
11755 .nonce_delivery
11756 .as_ref()
11757 .map(|delivery| delivery.consumed());
11758 }
11759 #[cfg(target_os = "macos")]
11760 {
11761 state.report_ready = Some(Arc::clone(&child.report_ready));
11762 }
11763 state.spawned_at_ms = Some(child.spawned_at_ms);
11764 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11765 state.spawned_from = Some(child.spawned_from.clone());
11766 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11767 state.spawned_file_identity = child.spawned_file_identity.clone();
11768 state.process_start_time = child.process_start_time;
11769 Ok(())
11770}
11771
11772fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11773 state.failed_retry.healthy_since = None;
11774 state.failed_retry.running_since = None;
11775 state.waiting_for_previous_pid = None;
11776 state.waiting_for_previous_detail = None;
11777 state.process_alive = false;
11778 state.spawned_protocol = None;
11779 state.pid = None;
11780 #[cfg(windows)]
11781 {
11782 state.pipe_consumed = None;
11783 }
11784 #[cfg(target_os = "macos")]
11785 {
11786 state.report_ready = None;
11787 }
11788 state.spawned_at_ms = None;
11789 state.exec_outcome = None;
11790 state.spawned_from = None;
11791 state.launch_spec_digest = None;
11792 state.spawned_file_identity = None;
11793 state.process_start_time = None;
11794 state.deliberate_severance = None;
11795}
11796
11797#[cfg(test)]
11798fn record_deliberate_severance(
11799 snapshot: &SharedSnapshot,
11800 identity: ProcessIdentity,
11801) -> Result<(), SuperviseError> {
11802 update_snapshot(snapshot, None, |state| {
11803 state.deliberate_severance = Some(identity);
11804 })
11805}
11806
11807fn apply_deliberate_severance_marker(
11808 snapshot: &SharedSnapshot,
11809 exited_identity: Option<ProcessIdentity>,
11810 mut exit_report: ExitReport,
11811) -> ExitReport {
11812 let marker = lock_snapshot(snapshot)
11813 .ok()
11814 .and_then(|mut state| state.deliberate_severance.take());
11815 if marker.is_some() && marker == exited_identity {
11816 exit_report.kind = ExitKind::DeliberateSeverance;
11817 }
11818 exit_report
11819}
11820
11821fn classify_reaped_child_exit(
11822 snapshot: &SharedSnapshot,
11823 child: &SupervisedChild,
11824 status: &ExitStatus,
11825) -> ExitReport {
11826 let _ = update_snapshot(snapshot, None, |state| {
11827 state.reaped_pid = Some(child.pid);
11828 state.spawn_failure = child.spawn_failure.clone();
11829 });
11830 if child.spawn_failure.as_deref() == Some("registration_timeout") {
11831 registration_failure_exit_report(classify_exit(status))
11832 } else {
11833 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11834 }
11835}
11836
11837fn fail_snapshot(
11838 snapshot: &SharedSnapshot,
11839 module_id: Option<&str>,
11840 last_exit: Option<ExitReport>,
11841 failure: &FailureContext,
11842) {
11843 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11844 state.state = ModuleState::Failed;
11845 clear_current_process_facts(state);
11846 if failure.may_retry() && state.enabled {
11847 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
11848 } else if failure.shutdown.is_set() {
11849 state.failed_retry.cancel();
11850 }
11851 if let Some(last_exit) = last_exit {
11852 state.last_exit = Some(last_exit);
11853 }
11854 }) {
11855 error!(error = %err, "failed to mark supervisor state failed");
11856 }
11857}
11858
11859fn update_snapshot(
11860 snapshot: &SharedSnapshot,
11861 module_id: Option<&str>,
11862 update: impl FnOnce(&mut SupervisorSnapshot),
11863) -> Result<(), SuperviseError> {
11864 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11865 module_id: module_id.map(ToOwned::to_owned),
11866 })?;
11867 update(&mut state);
11868 Ok(())
11869}
11870
11871const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11872
11873fn lock_snapshot_for_control<'a>(
11874 snapshot: &'a SharedSnapshot,
11875 module_id: &str,
11876 caller: &'static str,
11877) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11878 let started_at = Instant::now();
11879 let guard = lock_snapshot(snapshot)?;
11880 let waited = started_at.elapsed();
11881 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11882 warn!(
11883 module_id = %module_id,
11884 waited_ms = waited.as_millis() as u64,
11885 caller = %caller,
11886 "slow snapshot lock"
11887 );
11888 }
11889 Ok(guard)
11890}
11891
11892fn lock_snapshot(
11893 snapshot: &SharedSnapshot,
11894) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11895 snapshot
11896 .lock()
11897 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11898}
11899
11900#[cfg(test)]
11901mod terminal_history_tests {
11902 use std::{
11903 path::PathBuf,
11904 sync::Arc,
11905 time::{Duration, Instant},
11906 };
11907
11908 use tokio::{sync::mpsc, time::sleep};
11909
11910 use super::{
11911 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11912 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11913 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11914 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11915 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11916 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11917 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11918 SupervisorHealthStatus, SupervisorSnapshot,
11919 };
11920 use super::Instant as ClockInstant;
11925 use crate::{
11926 registry::Registry,
11927 terminal_ring::{TerminalRing, TerminalRingConfig},
11928 };
11929 use std::sync::Mutex;
11930 use subc_control::TerminalDisposition;
11931
11932 pub(super) fn fake_aft_stub_path() -> PathBuf {
11937 let mut path = std::env::current_exe().expect("current_exe available in tests");
11938 path.pop();
11939 path.pop();
11940 path.push(if cfg!(windows) {
11941 "fake-aft-stub.exe"
11942 } else {
11943 "fake-aft-stub"
11944 });
11945 assert!(
11946 path.exists(),
11947 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11948 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11949 path.display()
11950 );
11951 path
11952 }
11953
11954 #[test]
11955 fn reserved_never_spawned_refuses_every_hello() {
11956 let supervisor = SupervisorHandle::default();
11961 supervisor.apply_identity_configuration(&ModuleSpec {
11962 module_id: "never-spawned".to_string(),
11963 program: PathBuf::from("/usr/bin/false"),
11964 args: Vec::new(),
11965 env: Vec::new(),
11966 reserved: true,
11967 reserved_prefixes: Vec::new(),
11968 protocol: ModuleProtocol::Subc,
11969 overlap: Default::default(),
11970 });
11971 assert!(
11972 supervisor
11973 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11974 .is_err(),
11975 "forged nonce must refuse on a reserved never-spawned id"
11976 );
11977 assert!(
11978 supervisor
11979 .reserved_hello_verification("never-spawned", None)
11980 .is_err(),
11981 "absent nonce must refuse on a reserved never-spawned id"
11982 );
11983 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11985 supervisor.apply_identity_configuration(&ModuleSpec {
11986 module_id: "never-spawned".to_string(),
11987 program: PathBuf::from("/usr/bin/false"),
11988 args: Vec::new(),
11989 env: Vec::new(),
11990 reserved: true,
11991 reserved_prefixes: Vec::new(),
11992 protocol: ModuleProtocol::Subc,
11993 overlap: Default::default(),
11994 });
11995 assert!(supervisor
11996 .reserved_hello_verification("never-spawned", Some("minted"))
11997 .is_ok());
11998 assert!(supervisor
11999 .reserved_hello_verification("never-spawned", Some("forged"))
12000 .is_err());
12001 }
12002
12003 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12006 let now = ClockInstant::now();
12007 for _ in 0..count {
12008 state.crash_restarts.push_back(now);
12009 }
12010 }
12011
12012 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12016 let aged = state
12017 .crash_restarts
12018 .front()
12019 .expect("a crash restart must be recorded before it can be aged")
12020 .checked_sub(window + Duration::from_secs(1))
12021 .expect("the test clock is far enough from its origin to age an instant");
12022 state.crash_restarts[0] = aged;
12023 }
12024
12025 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12026 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12027 seed_crash_restarts(&mut state, count);
12028 state
12029 }
12030
12031 #[test]
12032 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12033 let policy = RestartPolicy::new(3, Duration::ZERO);
12034 let now = ClockInstant::now();
12035 assert!(daemon_will_restart(
12036 &mut snapshot_with_restarts(true, 2),
12037 &policy,
12038 now
12039 ));
12040 assert!(!daemon_will_restart(
12041 &mut snapshot_with_restarts(true, 3),
12042 &policy,
12043 now
12044 ));
12045 assert!(!daemon_will_restart(
12046 &mut snapshot_with_restarts(false, 0),
12047 &policy,
12048 now
12049 ));
12050 }
12051
12052 #[test]
12053 fn crash_restart_backoff_escalates_with_in_window_count() {
12054 let policy = RestartPolicy::new(4, Duration::from_millis(100))
12055 .with_max_backoff(Duration::from_secs(30));
12056 let now = ClockInstant::now();
12057 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12058 let schedules = (0..4)
12059 .map(|_| {
12060 state
12061 .next_crash_restart(&policy, now)
12062 .expect("the test policy allows four crash restarts")
12063 })
12064 .collect::<Vec<_>>();
12065
12066 assert_eq!(
12067 schedules
12068 .iter()
12069 .map(|schedule| schedule.restart_in_window)
12070 .collect::<Vec<_>>(),
12071 vec![0, 1, 2, 3]
12072 );
12073 assert_eq!(
12074 schedules
12075 .iter()
12076 .map(|schedule| schedule.delay)
12077 .collect::<Vec<_>>(),
12078 vec![
12079 Duration::from_millis(100),
12080 Duration::from_secs(1),
12081 Duration::from_secs(10),
12082 Duration::from_secs(30),
12083 ]
12084 );
12085 }
12086
12087 #[test]
12088 fn crash_restart_backoff_resets_after_ring_clear() {
12089 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12090 let now = ClockInstant::now();
12091 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12092 assert_eq!(
12093 state.next_crash_restart(&policy, now).unwrap().delay,
12094 Duration::from_millis(100)
12095 );
12096 assert_eq!(
12097 state.next_crash_restart(&policy, now).unwrap().delay,
12098 Duration::from_secs(1)
12099 );
12100
12101 state.clear_crash_restarts();
12102 let schedule = state
12103 .next_crash_restart(&policy, now)
12104 .expect("a cleared ring must allow another restart");
12105 assert_eq!(schedule.restart_in_window, 0);
12106 assert_eq!(schedule.delay, Duration::from_millis(100));
12107 }
12108
12109 #[test]
12110 fn crash_restart_backoff_ignores_aged_restarts() {
12111 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12112 let now = ClockInstant::now();
12113 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12114 state
12115 .next_crash_restart(&policy, now)
12116 .expect("the first restart is allowed");
12117 state
12118 .next_crash_restart(&policy, now)
12119 .expect("the second restart is allowed");
12120 state.crash_restarts[0] = now
12121 .checked_sub(policy.window + Duration::from_secs(1))
12122 .expect("the fake clock can age a restart past the window");
12123
12124 let schedule = state
12125 .next_crash_restart(&policy, now)
12126 .expect("an aged restart must release its slot");
12127 assert_eq!(schedule.restart_in_window, 1);
12128 assert_eq!(schedule.delay, Duration::from_secs(1));
12129 assert_eq!(state.crash_restarts.len(), 2);
12130 }
12131
12132 #[test]
12136 fn a_budget_spent_before_the_window_no_longer_refuses() {
12137 let policy = RestartPolicy::new(3, Duration::ZERO);
12138 let mut state = snapshot_with_restarts(true, 3);
12139 let now = ClockInstant::now();
12140 assert!(!daemon_will_restart(&mut state, &policy, now));
12141
12142 assert!(daemon_will_restart(
12143 &mut state,
12144 &policy,
12145 now + policy.window + Duration::from_secs(1)
12146 ));
12147 assert!(
12148 state.crash_restarts.is_empty(),
12149 "reading the budget must drop the instants that left the window"
12150 );
12151 }
12152
12153 fn module_with_recovery_snapshot(
12154 state: ModuleState,
12155 enabled: bool,
12156 restart_count: u32,
12157 ) -> SupervisedModule {
12158 let registry = Arc::new(Registry::default());
12159 let supervisor =
12160 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
12161 let runtime = supervisor.runtime_config();
12162 let spec = ModuleSpec {
12163 module_id: "recovery-snapshot".to_string(),
12164 program: fake_aft_stub_path(),
12165 args: Vec::new(),
12166 env: Vec::new(),
12167 reserved: false,
12168 reserved_prefixes: Vec::new(),
12169 protocol: ModuleProtocol::Subc,
12170 overlap: Default::default(),
12171 };
12172 let mut snapshot = SupervisorSnapshot::new(state, enabled);
12173 seed_crash_restarts(&mut snapshot, restart_count);
12174 let (commands, _rx) = mpsc::channel(4);
12177 SupervisedModule {
12178 inner: Arc::new(SupervisedModuleInner {
12179 module_id: spec.module_id.clone(),
12180 registry,
12181 snapshot: Arc::new(Mutex::new(snapshot)),
12182 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12183 spec,
12184 health: runtime.health,
12185 })),
12186 stderr_ring: runtime.stderr_ring,
12187 terminal_ring: runtime.terminal_ring,
12188 commands,
12189 monitor: Mutex::new(None),
12190 restart_policy: runtime.restart_policy,
12191 effective_drain_timeout: runtime.effective_drain_timeout,
12192 provenance_probe: supervisor.provenance_probe.clone(),
12193 }),
12194 }
12195 }
12196
12197 #[cfg(target_os = "linux")]
12198 #[tokio::test]
12199 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12200 let supervisor =
12201 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12202 .with_cgroup_placement(None);
12203 let result = supervisor.spawn(ModuleSpec {
12204 module_id: "no-cgroup-placement".to_string(),
12205 program: fake_aft_stub_path(),
12206 args: Vec::new(),
12207 env: Vec::new(),
12208 reserved: false,
12209 reserved_prefixes: Vec::new(),
12210 protocol: ModuleProtocol::Subc,
12211 overlap: Default::default(),
12212 });
12213
12214 assert!(
12215 result.is_ok(),
12216 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12217 );
12218 }
12219
12220 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12221 async fn undecided_snapshot_uses_shared_restart_predicate() {
12222 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12223 .will_recover_after_connection_loss()
12224 .unwrap());
12225 assert!(
12226 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12227 .will_recover_after_connection_loss()
12228 .unwrap()
12229 );
12230 }
12231
12232 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12233 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12234 assert!(
12235 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12236 .will_recover_after_connection_loss()
12237 .unwrap()
12238 );
12239 }
12240
12241 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12242 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12243 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12244 .will_recover_after_connection_loss()
12245 .unwrap());
12246 assert!(
12247 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12248 .will_recover_after_connection_loss()
12249 .unwrap()
12250 );
12251 }
12252
12253 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12254 async fn warming_snapshot_is_limited_to_startup_phases() {
12255 for state in [
12256 ModuleState::Starting,
12257 ModuleState::Running,
12258 ModuleState::Restarting,
12259 ] {
12260 assert!(
12261 module_with_recovery_snapshot(state, true, 0)
12262 .is_warming()
12263 .unwrap(),
12264 "{state:?} should be warming"
12265 );
12266 }
12267 for state in [
12268 ModuleState::Unresponsive,
12269 ModuleState::Draining,
12270 ModuleState::Stopped,
12271 ModuleState::Failed,
12272 ModuleState::Disabled,
12273 ] {
12274 assert!(
12275 !module_with_recovery_snapshot(state, true, 0)
12276 .is_warming()
12277 .unwrap(),
12278 "{state:?} should not be warming"
12279 );
12280 }
12281 }
12282
12283 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12284 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12285 let registry = Arc::new(Registry::default());
12286 let supervisor =
12287 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
12288 let module = supervisor
12289 .spawn(ModuleSpec {
12290 module_id: "terminal-history".to_string(),
12291 program: fake_aft_stub_path(),
12292 args: Vec::new(),
12293 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12294 reserved: false,
12295 reserved_prefixes: Vec::new(),
12296 protocol: ModuleProtocol::Subc,
12297 overlap: Default::default(),
12298 })
12299 .unwrap();
12300
12301 let deadline = Instant::now() + Duration::from_secs(5);
12302 loop {
12303 let history = module.terminal_history();
12304 if history.entries.len() == 2 {
12305 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12306 assert_eq!(history.dropped, 0);
12307 assert_eq!(
12308 history
12309 .entries
12310 .iter()
12311 .map(|entry| entry.exit_code)
12312 .collect::<Vec<_>>(),
12313 vec![Some(23), Some(23)]
12314 );
12315 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
12316 return;
12317 }
12318 assert!(
12319 Instant::now() < deadline,
12320 "module did not retain two terminal exits: {history:?}"
12321 );
12322 sleep(Duration::from_millis(10)).await;
12323 }
12324 }
12325
12326 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12330 async fn disable_during_crash_backoff_cancels_pending_respawn() {
12331 let backoff = Duration::from_secs(2);
12332 let supervisor = Supervisor::new_for_test(
12333 Arc::new(Registry::default()),
12334 RestartPolicy::new(10, backoff),
12335 );
12336 let module = supervisor
12337 .spawn(ModuleSpec {
12338 module_id: "disable-during-backoff".to_string(),
12339 program: fake_aft_stub_path(),
12340 args: Vec::new(),
12341 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12342 reserved: false,
12343 reserved_prefixes: Vec::new(),
12344 protocol: ModuleProtocol::Subc,
12345 overlap: Default::default(),
12346 })
12347 .unwrap();
12348
12349 let deadline = Instant::now() + Duration::from_secs(5);
12351 loop {
12352 if module.status().unwrap().state == ModuleState::Restarting {
12353 break;
12354 }
12355 assert!(
12356 Instant::now() < deadline,
12357 "module never entered the crash backoff"
12358 );
12359 sleep(Duration::from_millis(10)).await;
12360 }
12361
12362 let started = Instant::now();
12363 module.set_enabled(false).await.unwrap();
12364 let waited = started.elapsed();
12365
12366 assert!(
12367 waited < backoff / 2,
12368 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12369 );
12370 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12371
12372 sleep(backoff + Duration::from_millis(500)).await;
12374 let status = module.status().unwrap();
12375 assert_eq!(status.state, ModuleState::Disabled);
12376 assert_eq!(
12377 status.spawn_generation, 1,
12378 "module respawned after the operator disabled it"
12379 );
12380 }
12381
12382 #[cfg(unix)]
12385 fn protocol_none_sigterm_exits_clean_spec(
12386 module_id: &str,
12387 dir: &std::path::Path,
12388 ) -> (ModuleSpec, PathBuf, PathBuf) {
12389 let ready = dir.join("ready");
12390 let marker = dir.join("sigterm");
12391 let spec = ModuleSpec {
12392 module_id: module_id.to_string(),
12393 program: fake_aft_stub_path(),
12394 args: Vec::new(),
12395 env: vec![
12396 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12397 (
12398 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12399 marker.display().to_string(),
12400 ),
12401 (
12402 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12403 ready.display().to_string(),
12404 ),
12405 ],
12406 reserved: false,
12407 reserved_prefixes: Vec::new(),
12408 protocol: ModuleProtocol::None,
12409 overlap: Default::default(),
12410 };
12411 (spec, ready, marker)
12412 }
12413
12414 #[cfg(unix)]
12418 async fn wait_for_file(path: &std::path::Path) {
12419 let deadline = Instant::now() + Duration::from_secs(10);
12420 while !path.exists() {
12421 assert!(
12422 Instant::now() < deadline,
12423 "{} never appeared",
12424 path.display()
12425 );
12426 sleep(Duration::from_millis(10)).await;
12427 }
12428 }
12429
12430 #[cfg(unix)]
12434 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12435 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12436 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12437 let (spec, ready, marker) =
12438 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12439 let supervisor = Supervisor::new_for_test(
12440 Arc::new(Registry::default()),
12441 RestartPolicy::new(3, Duration::ZERO),
12442 );
12443 let module = supervisor.spawn(spec).unwrap();
12444 wait_for_file(&ready).await;
12445 let deadline = Instant::now() + Duration::from_secs(10);
12449 let first_pid = loop {
12450 let status = module.status().unwrap();
12451 if status.state == ModuleState::Running {
12452 if let Some(pid) = status.pid {
12453 break pid;
12454 }
12455 }
12456 assert!(
12457 Instant::now() < deadline,
12458 "a running module must report its pid after exec confirmation: {status:?}"
12459 );
12460 sleep(Duration::from_millis(10)).await;
12461 };
12462
12463 rustix::process::kill_process(
12464 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12465 rustix::process::Signal::TERM,
12466 )
12467 .unwrap();
12468
12469 let deadline = Instant::now() + Duration::from_secs(10);
12470 let respawned = loop {
12471 let status = module.status().unwrap();
12472 if status.state == ModuleState::Running
12473 && status.pid.is_some_and(|pid| pid != first_pid)
12474 {
12475 break status;
12476 }
12477 assert!(
12478 Instant::now() < deadline,
12479 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12480 );
12481 sleep(Duration::from_millis(10)).await;
12482 };
12483 assert_eq!(respawned.spawn_generation, 2);
12484 assert!(
12485 marker.exists(),
12486 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12487 );
12488
12489 let history = module.terminal_history();
12490 assert_eq!(history.entries.len(), 1, "{history:?}");
12491 let entry = &history.entries[0];
12492 assert_eq!(entry.exit_code, Some(0));
12493 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12494 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12495
12496 module.stop().await.unwrap();
12497 }
12498
12499 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12503 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12504 let supervisor = Supervisor::new_for_test(
12505 Arc::new(Registry::default()),
12506 RestartPolicy::new(1, Duration::ZERO),
12507 );
12508 let module = supervisor
12509 .spawn(ModuleSpec {
12510 module_id: "none-clean-exit-budget".to_string(),
12511 program: fake_aft_stub_path(),
12512 args: Vec::new(),
12513 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12514 reserved: false,
12515 reserved_prefixes: Vec::new(),
12516 protocol: ModuleProtocol::None,
12517 overlap: Default::default(),
12518 })
12519 .unwrap();
12520
12521 let deadline = Instant::now() + Duration::from_secs(10);
12524 loop {
12525 let status = module.status().unwrap();
12526 if status.state == ModuleState::Failed {
12527 break;
12528 }
12529 assert!(
12530 Instant::now() < deadline,
12531 "module never exhausted its budget: {status:?} {:?}",
12532 module.terminal_history()
12533 );
12534 sleep(Duration::from_millis(10)).await;
12535 }
12536 let history = module.terminal_history();
12537 assert_eq!(
12538 history
12539 .entries
12540 .iter()
12541 .map(|entry| (entry.exit_code, entry.disposition.clone()))
12542 .collect::<Vec<_>>(),
12543 vec![
12544 (Some(0), TerminalDisposition::Restarting),
12545 (Some(0), TerminalDisposition::Failed),
12546 ]
12547 );
12548 let detail = history.entries[1]
12549 .disposition_detail
12550 .as_deref()
12551 .expect("a budget failure names the budget");
12552 assert!(detail.contains("max_restarts=1"), "{detail}");
12553 assert_eq!(module.status().unwrap().spawn_generation, 2);
12554 }
12555
12556 #[test]
12557 fn restart_budget_failure_is_published_after_its_terminal_record() {
12558 use std::io::{Read, Write};
12559 let supervisor = Supervisor::new_for_test(
12560 Arc::new(Registry::default()),
12561 RestartPolicy::new(0, Duration::ZERO),
12562 );
12563 let runtime = supervisor.runtime_config();
12564 let spec = ModuleSpec {
12565 protocol: ModuleProtocol::None,
12566 ..windowed_crash_spec("budget-publication-order")
12567 };
12568 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12569 ModuleState::Running,
12570 true,
12571 )));
12572 lock_snapshot(&snapshot).unwrap().process_alive = true;
12573 let reaped_snapshot = snapshot.clone();
12574 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12575 let history = runtime.terminal_ring.clone();
12576 let reap = std::thread::spawn(move || {
12577 tokio::runtime::Builder::new_current_thread()
12578 .enable_all()
12579 .build()
12580 .unwrap()
12581 .block_on(on_child_exit(
12582 &spec,
12583 runtime.restart_policy,
12584 &supervisor.registry,
12585 &reaped_snapshot,
12586 &runtime.terminal_ring,
12587 &runtime.spawn_events,
12588 &runtime.child_roster,
12589 ExitReport {
12590 kind: ExitKind::Clean,
12591 code: Some(0),
12592 signal: None,
12593 at_ms: 1,
12594 },
12595 true,
12596 ))
12597 });
12598 let mut reached = [0];
12599 barrier.read_exact(&mut reached).unwrap();
12600 assert_eq!(&reached, b"R");
12601 let (before_state, before_alive) = {
12602 let state = lock_snapshot(&snapshot).unwrap();
12603 (state.state, state.process_alive)
12604 };
12605 let before_history = history.lock().unwrap().snapshot();
12606 barrier.write_all(b"X").unwrap();
12607 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12608 assert!(before_history.entries.is_empty());
12609 assert_ne!(
12610 before_state,
12611 ModuleState::Failed,
12612 "Failed was visible before its terminal record could be written"
12613 );
12614 assert!(before_alive);
12615 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12616 let history = history.lock().unwrap().snapshot();
12617 assert_eq!(history.entries.len(), 1);
12618 assert_eq!(history.entries[0].exit_code, Some(0));
12619 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12620 }
12621
12622 #[test]
12623 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12624 use std::io::{Read, Write};
12625
12626 let supervisor = Supervisor::new_for_test(
12627 Arc::new(Registry::default()),
12628 RestartPolicy::new(3, Duration::ZERO),
12629 );
12630 let mut runtime = supervisor.runtime_config();
12631 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12632 let snapshot = module.inner.snapshot.clone();
12633 lock_snapshot(&snapshot).unwrap().process_alive = true;
12634 runtime.terminal_ring = module.inner.terminal_ring.clone();
12635 let spec = windowed_crash_spec(module.module_id());
12636 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12637 let reaped_snapshot = snapshot.clone();
12638 let reap = std::thread::spawn(move || {
12639 tokio::runtime::Builder::new_current_thread()
12640 .enable_all()
12641 .build()
12642 .unwrap()
12643 .block_on(on_child_exit(
12644 &spec,
12645 runtime.restart_policy,
12646 &supervisor.registry,
12647 &reaped_snapshot,
12648 &runtime.terminal_ring,
12649 &runtime.spawn_events,
12650 &runtime.child_roster,
12651 ExitReport {
12652 kind: ExitKind::Crash,
12653 code: Some(2),
12654 signal: None,
12655 at_ms: 1,
12656 },
12657 true,
12658 ))
12659 });
12660
12661 let mut reached = [0];
12665 barrier.read_exact(&mut reached).unwrap();
12666 assert_eq!(&reached, b"R");
12667 let before_status = module.status().unwrap();
12668 let before_history = module.terminal_history();
12669 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12670 barrier.write_all(b"X").unwrap();
12671 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12672
12673 assert_eq!(before_status.state, ModuleState::Running);
12674 assert!(before_status.process_alive);
12675 assert_eq!(before_status.restart_count, 3);
12676 assert!(before_history.entries.is_empty());
12677 assert!(
12678 before_recovers,
12679 "connection cleanup would report a terminal close for a scheduled crash restart"
12680 );
12681 let restarting = module.status().unwrap();
12682 assert_eq!(restarting.state, ModuleState::Restarting);
12683 assert_eq!(restarting.restart_count, 3);
12684 assert!(module.will_recover_after_connection_loss().unwrap());
12685 let history = module.terminal_history();
12686 assert_eq!(history.entries.len(), 1);
12687 assert_eq!(
12688 history.entries[0].disposition,
12689 TerminalDisposition::Restarting
12690 );
12691
12692 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12695 state.state = ModuleState::Running;
12696 state.respawn_pending = false;
12697 })
12698 .unwrap();
12699 assert!(!module.will_recover_after_connection_loss().unwrap());
12700 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12701 state.enabled = false;
12702 state.respawn_pending = true;
12703 })
12704 .unwrap();
12705 assert!(!module.will_recover_after_connection_loss().unwrap());
12706 }
12707
12708 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12709 use std::net::{TcpListener, TcpStream};
12710 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12711 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12712 let (barrier, _) = listener.accept().unwrap();
12713 for stream in [&writer, &barrier] {
12714 stream
12715 .set_read_timeout(Some(Duration::from_secs(10)))
12716 .unwrap();
12717 stream
12718 .set_write_timeout(Some(Duration::from_secs(10)))
12719 .unwrap();
12720 }
12721 *events.1.lock().unwrap() = Some(writer);
12722 barrier
12723 }
12724
12725 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12726 use std::io::{Read, Write};
12727 let barrier = events.1.lock().unwrap().take();
12728 if let Some(mut barrier) = barrier {
12729 barrier.write_all(b"R").unwrap();
12730 let mut release = [0];
12731 barrier.read_exact(&mut release).unwrap();
12732 assert_eq!(&release, b"X");
12733 }
12734 }
12735
12736 #[test]
12737 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12738 use std::io::{Read, Write};
12739
12740 let supervisor = Supervisor::new_for_test(
12741 Arc::new(Registry::default()),
12742 RestartPolicy::new(0, Duration::ZERO),
12743 );
12744 let mut runtime = supervisor.runtime_config();
12745 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12746 let snapshot = module.inner.snapshot.clone();
12747 runtime.terminal_ring = module.inner.terminal_ring.clone();
12748 let spec = ModuleSpec {
12749 module_id: module.module_id().into(),
12750 program: fake_aft_stub_path(),
12751 protocol: ModuleProtocol::None,
12752 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12753 ..windowed_crash_spec(module.module_id())
12754 };
12755 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12756 let reap = std::thread::spawn(move || {
12757 tokio::runtime::Builder::new_current_thread()
12758 .enable_all()
12759 .build()
12760 .unwrap()
12761 .block_on(async {
12762 let mut child =
12763 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12764 health_restart_child(
12765 &spec,
12766 &runtime,
12767 &supervisor.registry,
12768 &supervisor.process_liveness,
12769 &snapshot,
12770 &mut child,
12771 SupervisorHealthStatus::Failing,
12772 Some("health probe did not answer"),
12773 1,
12774 )
12775 .await
12776 .unwrap();
12777 assert!(child.is_none());
12778 });
12779 });
12780 let mut reached = [0];
12785 barrier.read_exact(&mut reached).unwrap();
12786 assert_eq!(&reached, b"R");
12787 let before_status = module.status().unwrap();
12788 let before_history = module.terminal_history();
12789 barrier.write_all(b"X").unwrap();
12790 reap.join().unwrap();
12791 assert!(before_history.entries.is_empty());
12792 assert_ne!(
12793 before_status.state,
12794 ModuleState::Failed,
12795 "Failed was visible before its terminal record could be written"
12796 );
12797 assert!(
12798 before_status.process_alive,
12799 "process death was visible before its terminal record could be written"
12800 );
12801 let failed = module.status().unwrap();
12802 assert_eq!(failed.state, ModuleState::Failed);
12803 assert!(!failed.process_alive);
12804 let history = module.terminal_history();
12805 assert_eq!(history.entries.len(), 1);
12806 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12807 assert_eq!(
12808 history.entries[0].disposition_detail.as_deref(),
12809 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12810 );
12811 }
12812
12813 #[test]
12814 fn observed_exits_publish_process_death_after_terminal_history() {
12815 use std::io::{Read, Write};
12816 for (kind, enabled, shutdown, wait_error, expected_state) in [
12817 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12818 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12819 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12820 (
12821 ExitKind::DeliberateSeverance,
12822 true,
12823 false,
12824 false,
12825 ModuleState::Restarting,
12826 ),
12827 #[cfg(unix)]
12828 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12829 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12830 ] {
12831 let supervisor = Supervisor::new_for_test(
12832 Arc::new(Registry::default()),
12833 RestartPolicy::new(1, Duration::ZERO),
12834 );
12835 let runtime = supervisor.runtime_config();
12836 #[cfg(unix)]
12837 if shutdown {
12838 runtime.child_roster.close();
12839 }
12840 #[cfg(not(unix))]
12841 let _ = shutdown;
12842 let spec = windowed_crash_spec("exit-publication-order");
12843 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12844 ModuleState::Running,
12845 enabled,
12846 )));
12847 lock_snapshot(&snapshot).unwrap().process_alive = true;
12848 let reaped_snapshot = snapshot.clone();
12849 let history = runtime.terminal_ring.clone();
12850 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12851 let reap = std::thread::spawn(move || {
12852 if wait_error {
12853 super::fail_after_wait_error(
12854 &spec.module_id,
12855 &reaped_snapshot,
12856 &runtime.terminal_ring,
12857 &runtime.spawn_events,
12858 "wait failed".into(),
12859 &runtime.failure_context(true),
12860 );
12861 } else {
12862 tokio::runtime::Builder::new_current_thread()
12863 .enable_all()
12864 .build()
12865 .unwrap()
12866 .block_on(on_child_exit(
12867 &spec,
12868 runtime.restart_policy,
12869 &supervisor.registry,
12870 &reaped_snapshot,
12871 &runtime.terminal_ring,
12872 &runtime.spawn_events,
12873 &runtime.child_roster,
12874 ExitReport {
12875 kind,
12876 code: Some(0),
12877 signal: None,
12878 at_ms: 1,
12879 },
12880 true,
12881 ));
12882 }
12883 });
12884 let mut reached = [0];
12885 barrier.read_exact(&mut reached).unwrap();
12886 assert_eq!(&reached, b"R");
12887 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12888 let before_history = history.lock().unwrap().snapshot();
12889 barrier.write_all(b"X").unwrap();
12890 reap.join().unwrap();
12891 assert!(
12892 before_alive,
12893 "{kind:?}: death was published before recording"
12894 );
12895 assert!(before_history.entries.is_empty());
12896 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12897 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12898 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12899 }
12900 }
12901
12902 #[tokio::test]
12903 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12904 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12905 let path = dir.join("terminals.jsonl");
12906 std::fs::create_dir(&path).unwrap();
12907 let supervisor = Supervisor::new_for_test(
12908 Arc::new(Registry::default()),
12909 RestartPolicy::new(0, Duration::ZERO),
12910 )
12911 .with_terminal_journal(path, "health-budget-journal-failure".into());
12912 let runtime = supervisor.runtime_config();
12913 let spec = ModuleSpec {
12914 program: fake_aft_stub_path(),
12915 protocol: ModuleProtocol::None,
12916 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12917 ..windowed_crash_spec("health-budget-journal-failure")
12918 };
12919 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12920 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12921 health_restart_child(
12922 &spec,
12923 &runtime,
12924 &supervisor.registry,
12925 &supervisor.process_liveness,
12926 &snapshot,
12927 &mut child,
12928 SupervisorHealthStatus::Failing,
12929 None,
12930 1,
12931 )
12932 .await
12933 .unwrap();
12934 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12935 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12936 let history = runtime
12937 .terminal_ring
12938 .lock()
12939 .unwrap()
12940 .durable_history(&spec.module_id);
12941 assert!(history.journal_write_failures > 0);
12942 assert_eq!(history.entries.len(), 1);
12943 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12944 }
12945
12946 #[test]
12947 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12948 let supervisor = Supervisor::new_for_test(
12949 Arc::new(Registry::default()),
12950 RestartPolicy::new(0, Duration::ZERO),
12951 );
12952 let runtime = supervisor.runtime_config();
12953 let spec = ModuleSpec {
12954 program: fake_aft_stub_path(),
12955 protocol: ModuleProtocol::None,
12956 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12957 ..windowed_crash_spec("health-budget-recording-panic")
12958 };
12959 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12960 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12961 tokio::runtime::Builder::new_current_thread()
12962 .enable_all()
12963 .build()
12964 .unwrap()
12965 .block_on(async {
12966 let mut child =
12967 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12968 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12969 health_restart_child(
12970 &spec,
12971 &runtime,
12972 &supervisor.registry,
12973 &supervisor.process_liveness,
12974 &snapshot,
12975 &mut child,
12976 SupervisorHealthStatus::Failing,
12977 None,
12978 1,
12979 )
12980 .await
12981 .unwrap();
12982 });
12983 }));
12984 let panic = result.expect_err("recording must still unwind");
12985 assert_eq!(
12986 panic.downcast_ref::<String>().map(String::as_str),
12987 Some("spawn event sequence exhausted")
12988 );
12989 let state = lock_snapshot(&snapshot).unwrap();
12990 assert_eq!(state.state, ModuleState::Failed);
12991 assert!(!state.process_alive);
12992 assert!(state.last_exit.is_some());
12993 }
12994
12995 #[tokio::test]
12996 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12997 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12998 let path = dir.join("terminals.jsonl");
12999 std::fs::create_dir(&path).unwrap();
13000 let supervisor = Supervisor::new_for_test(
13001 Arc::new(Registry::default()),
13002 RestartPolicy::new(0, Duration::ZERO),
13003 )
13004 .with_terminal_journal(path, "budget-journal-failure".into());
13005 let runtime = supervisor.runtime_config();
13006 let spec = windowed_crash_spec("budget-journal-failure");
13007 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13008 assert!(matches!(
13009 on_child_exit(
13010 &spec,
13011 runtime.restart_policy,
13012 &supervisor.registry,
13013 &snapshot,
13014 &runtime.terminal_ring,
13015 &runtime.spawn_events,
13016 &runtime.child_roster,
13017 crash_exit_report(1),
13018 true,
13019 )
13020 .await,
13021 NextAction::Stop { .. }
13022 ));
13023 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13024 let history = runtime
13025 .terminal_ring
13026 .lock()
13027 .unwrap()
13028 .durable_history(&spec.module_id);
13029 assert!(history.journal_write_failures > 0);
13030 assert_eq!(history.entries.len(), 1);
13031 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13032 }
13033
13034 #[test]
13035 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13036 let supervisor = Supervisor::new_for_test(
13037 Arc::new(Registry::default()),
13038 RestartPolicy::new(0, Duration::ZERO),
13039 );
13040 let runtime = supervisor.runtime_config();
13041 let spec = windowed_crash_spec("budget-recording-panic");
13042 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13043 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13044 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13047 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13048 tokio::runtime::Builder::new_current_thread()
13049 .enable_all()
13050 .build()
13051 .unwrap()
13052 .block_on(on_child_exit(
13053 &spec,
13054 runtime.restart_policy,
13055 &supervisor.registry,
13056 &snapshot,
13057 &runtime.terminal_ring,
13058 &runtime.spawn_events,
13059 &runtime.child_roster,
13060 crash_exit_report(1),
13061 true,
13062 ))
13063 }));
13064 let panic = result.err().expect("recording must still unwind");
13065 assert_eq!(
13066 panic.downcast_ref::<String>().map(String::as_str),
13067 Some("spawn event sequence exhausted")
13068 );
13069 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13070 }
13071
13072 #[cfg(unix)]
13075 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13076 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13077 for disable in [false, true] {
13078 let label = if disable {
13079 "none-requested-disable"
13080 } else {
13081 "none-requested-stop"
13082 };
13083 let dir = cortexkit_test_support::ScratchDir::new(label);
13084 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13085 let supervisor = Supervisor::new_for_test(
13086 Arc::new(Registry::default()),
13087 RestartPolicy::new(3, Duration::ZERO),
13088 );
13089 let module = supervisor.spawn(spec).unwrap();
13090 wait_for_file(&ready).await;
13091
13092 if disable {
13093 module.set_enabled(false).await.unwrap();
13094 } else {
13095 module.stop().await.unwrap();
13096 }
13097 assert!(
13098 marker.exists(),
13099 "{label}: the child must have left through its SIGTERM handler with exit 0"
13100 );
13101
13102 sleep(Duration::from_millis(500)).await;
13105 let status = module.status().unwrap();
13106 let expected = if disable {
13107 ModuleState::Disabled
13108 } else {
13109 ModuleState::Stopped
13110 };
13111 assert_eq!(status.state, expected, "{label}");
13112 assert_eq!(
13113 status.spawn_generation, 1,
13114 "{label}: respawned after a requested stop"
13115 );
13116 let history = module.terminal_history();
13117 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13118 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13119 assert_ne!(
13120 history.entries[0].disposition,
13121 TerminalDisposition::Restarting,
13122 "{label}"
13123 );
13124 }
13125 }
13126
13127 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13130 async fn subc_wire_clean_exit_is_still_a_stop() {
13131 let supervisor = Supervisor::new_for_test(
13132 Arc::new(Registry::default()),
13133 RestartPolicy::new(3, Duration::ZERO),
13134 );
13135 let module = supervisor
13136 .spawn(ModuleSpec {
13137 module_id: "wire-clean-exit".to_string(),
13138 program: fake_aft_stub_path(),
13139 args: Vec::new(),
13140 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13141 reserved: false,
13142 reserved_prefixes: Vec::new(),
13143 protocol: ModuleProtocol::Subc,
13144 overlap: Default::default(),
13145 })
13146 .unwrap();
13147
13148 let deadline = Instant::now() + Duration::from_secs(10);
13149 while module.terminal_history().entries.is_empty() {
13150 assert!(Instant::now() < deadline, "module never exited");
13151 sleep(Duration::from_millis(10)).await;
13152 }
13153 sleep(Duration::from_millis(500)).await;
13155 let status = module.status().unwrap();
13156 assert_eq!(status.state, ModuleState::Stopped);
13157 assert_eq!(status.spawn_generation, 1);
13158 let history = module.terminal_history();
13159 assert_eq!(history.entries.len(), 1, "{history:?}");
13160 assert_eq!(history.entries[0].exit_code, Some(0));
13161 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13162 }
13163
13164 #[cfg(unix)]
13165 #[tokio::test]
13166 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13167 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13168 let record = dir.join("live-children.json");
13169 let supervisor = Supervisor::new_for_test(
13170 Arc::new(Registry::default()),
13171 RestartPolicy::new(0, Duration::ZERO),
13172 );
13173 let mut runtime = supervisor.runtime_config();
13174 runtime.child_roster.record_to(record.clone());
13175 let gate = Arc::new(super::ReloadExitRecordGate::default());
13176 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13177 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13178 let spec = ModuleSpec {
13179 module_id: "reload-exit-roster".into(),
13180 program: fake_aft_stub_path(),
13181 args: Vec::new(),
13182 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13183 reserved: false,
13184 reserved_prefixes: Vec::new(),
13185 protocol: ModuleProtocol::Subc,
13186 overlap: Default::default(),
13187 };
13188 let mut child = None;
13189 let reload = super::finish_reload_child(
13190 &spec,
13191 &runtime,
13192 &supervisor.registry,
13193 &supervisor.process_liveness,
13194 &snapshot,
13195 &mut child,
13196 );
13197 tokio::pin!(reload);
13198 tokio::select! {
13199 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13200 _ = gate.reached.notified() => {}
13201 }
13202 assert!(runtime
13203 .terminal_ring
13204 .lock()
13205 .unwrap()
13206 .snapshot()
13207 .entries
13208 .is_empty());
13209 assert_eq!(
13210 crate::live_children::read_record(&record).unwrap().len(),
13211 1,
13212 "shutdown must still wait for the reaped child until its terminal record exists"
13213 );
13214 runtime.child_roster.close();
13215 gate.resume.notify_one();
13216 assert!(reload.await.is_err());
13217 assert!(crate::live_children::read_record(&record)
13218 .unwrap()
13219 .is_empty());
13220 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13221 assert_eq!(history.entries.len(), 1);
13222 assert_eq!(
13223 history.entries[0].disposition,
13224 TerminalDisposition::DaemonShutdown
13225 );
13226 }
13227
13228 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13232 async fn every_restart_increment_path_advances_lifetime_count() {
13233 let supervisor = Supervisor::new_for_test(
13234 Arc::new(Registry::default()),
13235 RestartPolicy::new(1, Duration::ZERO),
13236 );
13237 let runtime = supervisor.runtime_config();
13238 let spec = ModuleSpec {
13239 module_id: "lifetime-increment-path".to_string(),
13240 program: PathBuf::from("/unused/lifetime-increment-path"),
13241 args: Vec::new(),
13242 env: Vec::new(),
13243 reserved: false,
13244 reserved_prefixes: Vec::new(),
13245 protocol: ModuleProtocol::Subc,
13246 overlap: Default::default(),
13247 };
13248
13249 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13250 assert!(matches!(
13251 on_child_exit(
13252 &spec,
13253 runtime.restart_policy,
13254 &supervisor.registry,
13255 &crash_snapshot,
13256 &runtime.terminal_ring,
13257 &runtime.spawn_events,
13258 &runtime.child_roster,
13259 ExitReport {
13260 kind: ExitKind::Crash,
13261 code: Some(1),
13262 signal: None,
13263 at_ms: 1,
13264 },
13265 true,
13266 )
13267 .await,
13268 NextAction::Restart { schedule: _ }
13269 ));
13270 let (crash_restarts, crash_lifetime) = {
13271 let state = lock_snapshot(&crash_snapshot).unwrap();
13272 (state.crash_restarts.len(), state.lifetime_restarts)
13273 };
13274 assert_eq!(crash_restarts, 1);
13275 assert_eq!(crash_lifetime, 1);
13276
13277 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13278 let mut health_child = None;
13279 assert!(matches!(
13280 health_restart_child(
13281 &spec,
13282 &runtime,
13283 &supervisor.registry,
13284 &supervisor.process_liveness,
13285 &health_snapshot,
13286 &mut health_child,
13287 SupervisorHealthStatus::Failing,
13288 None,
13289 2,
13290 )
13291 .await,
13292 Ok(())
13293 ));
13294 assert!(health_child.is_none());
13295 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13296 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13297 let (health_restarts, health_lifetime) = {
13298 let state = lock_snapshot(&health_snapshot).unwrap();
13299 (state.crash_restarts.len(), state.lifetime_restarts)
13300 };
13301 assert_eq!(health_restarts, 1);
13302 assert_eq!(health_lifetime, 1);
13303
13304 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13305 let mut reload_child = None;
13306 assert!(matches!(
13307 handle_reload_spawn_failure(
13308 &spec,
13309 &runtime,
13310 &supervisor.process_liveness,
13311 &reload_snapshot,
13312 &mut reload_child,
13313 "forced reload spawn failure".to_string(),
13314 )
13315 .await,
13316 Err(SuperviseError::ReloadFailed { .. })
13317 ));
13318 let (reload_restarts, reload_lifetime) = {
13319 let state = lock_snapshot(&reload_snapshot).unwrap();
13320 (state.crash_restarts.len(), state.lifetime_restarts)
13321 };
13322 assert_eq!(reload_restarts, 1);
13323 assert_eq!(reload_lifetime, 1);
13324 }
13325
13326 #[tokio::test]
13327 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
13328 let supervisor = Supervisor::new_for_test(
13329 Arc::new(Registry::default()),
13330 RestartPolicy::new(3, Duration::ZERO),
13331 );
13332 let runtime = supervisor.runtime_config();
13333 let spec = ModuleSpec {
13334 module_id: "deliberately-severed".to_string(),
13335 program: PathBuf::from("/unused/deliberately-severed"),
13336 args: Vec::new(),
13337 env: Vec::new(),
13338 reserved: false,
13339 reserved_prefixes: Vec::new(),
13340 protocol: ModuleProtocol::Subc,
13341 overlap: Default::default(),
13342 };
13343 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13344 let process = ProcessIdentity {
13345 pid: 41,
13346 start_time: 101,
13347 };
13348 record_deliberate_severance(&snapshot, process).unwrap();
13349 let exit_report = apply_deliberate_severance_marker(
13350 &snapshot,
13351 Some(process),
13352 ExitReport {
13353 kind: ExitKind::Crash,
13354 code: Some(1),
13355 signal: None,
13356 at_ms: 1,
13357 },
13358 );
13359 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
13360
13361 assert!(matches!(
13362 on_child_exit(
13363 &spec,
13364 runtime.restart_policy,
13365 &supervisor.registry,
13366 &snapshot,
13367 &runtime.terminal_ring,
13368 &runtime.spawn_events,
13369 &runtime.child_roster,
13370 exit_report,
13371 true,
13372 )
13373 .await,
13374 NextAction::Restart { schedule: _ }
13375 ));
13376 let state = lock_snapshot(&snapshot).unwrap();
13377 assert_eq!(state.lifetime_restarts, 1);
13378 assert_eq!(state.crash_restarts.len(), 0);
13379 }
13380
13381 #[tokio::test]
13382 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13383 let supervisor = Supervisor::new_for_test(
13384 Arc::new(Registry::default()),
13385 RestartPolicy::new(3, Duration::ZERO),
13386 );
13387 let runtime = supervisor.runtime_config();
13388 let spec = ModuleSpec {
13389 module_id: "genuine-crash".to_string(),
13390 program: PathBuf::from("/unused/genuine-crash"),
13391 args: Vec::new(),
13392 env: Vec::new(),
13393 reserved: false,
13394 reserved_prefixes: Vec::new(),
13395 protocol: ModuleProtocol::Subc,
13396 overlap: Default::default(),
13397 };
13398 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13399
13400 assert!(matches!(
13401 on_child_exit(
13402 &spec,
13403 runtime.restart_policy,
13404 &supervisor.registry,
13405 &snapshot,
13406 &runtime.terminal_ring,
13407 &runtime.spawn_events,
13408 &runtime.child_roster,
13409 ExitReport {
13410 kind: ExitKind::Crash,
13411 code: Some(1),
13412 signal: None,
13413 at_ms: 1,
13414 },
13415 true,
13416 )
13417 .await,
13418 NextAction::Restart { schedule: _ }
13419 ));
13420 let state = lock_snapshot(&snapshot).unwrap();
13421 assert_eq!(state.lifetime_restarts, 1);
13422 assert_eq!(state.crash_restarts.len(), 1);
13423 }
13424
13425 fn crash_exit_report(at_ms: u64) -> ExitReport {
13426 ExitReport {
13427 kind: ExitKind::Crash,
13428 code: Some(1),
13429 signal: None,
13430 at_ms,
13431 }
13432 }
13433
13434 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13435 ModuleSpec {
13436 module_id: module_id.to_string(),
13437 program: PathBuf::from("/unused").join(module_id),
13438 args: Vec::new(),
13439 env: Vec::new(),
13440 reserved: false,
13441 reserved_prefixes: Vec::new(),
13442 protocol: ModuleProtocol::Subc,
13443 overlap: Default::default(),
13444 }
13445 }
13446
13447 #[tokio::test]
13453 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13454 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13455 let supervisor = Supervisor::new_for_test(
13456 Arc::new(Registry::default()),
13457 RestartPolicy::new(2, Duration::ZERO),
13458 );
13459 let runtime = supervisor.runtime_config();
13460 let spec = windowed_crash_spec("crash-loop-in-window");
13461 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13462
13463 for attempt in 1..=2 {
13464 assert!(
13465 matches!(
13466 on_child_exit(
13467 &spec,
13468 runtime.restart_policy,
13469 &supervisor.registry,
13470 &snapshot,
13471 &runtime.terminal_ring,
13472 &runtime.spawn_events,
13473 &runtime.child_roster,
13474 crash_exit_report(attempt),
13475 true,
13476 )
13477 .await,
13478 NextAction::Restart { schedule: _ }
13479 ),
13480 "crash {attempt} is inside the budget and must respawn"
13481 );
13482 }
13483
13484 assert!(matches!(
13485 on_child_exit(
13486 &spec,
13487 runtime.restart_policy,
13488 &supervisor.registry,
13489 &snapshot,
13490 &runtime.terminal_ring,
13491 &runtime.spawn_events,
13492 &runtime.child_roster,
13493 crash_exit_report(3),
13494 true,
13495 )
13496 .await,
13497 NextAction::Stop { .. }
13498 ));
13499
13500 {
13501 let state = lock_snapshot(&snapshot).unwrap();
13502 assert_eq!(state.state, ModuleState::Failed);
13503 assert_eq!(state.crash_restarts.len(), 2);
13504 assert_eq!(state.lifetime_restarts, 2);
13505 }
13506
13507 let history = runtime
13508 .terminal_ring
13509 .lock()
13510 .expect("terminal ring is not poisoned")
13511 .snapshot();
13512 let last = history
13513 .entries
13514 .last()
13515 .expect("the refused crash is retained");
13516 assert_eq!(last.disposition, TerminalDisposition::Failed);
13517 assert_eq!(
13518 last.disposition_detail.as_deref(),
13519 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13520 );
13521
13522 let captured = crate::router::test_log::captured_logs(&logs);
13523 assert!(
13524 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13525 "the stop must be logged with its window: {captured}"
13526 );
13527 }
13528
13529 #[tokio::test]
13537 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13538 let supervisor = Supervisor::new_for_test(
13539 Arc::new(Registry::default()),
13540 RestartPolicy::new(2, Duration::ZERO),
13541 );
13542 let runtime = supervisor.runtime_config();
13543 let spec = windowed_crash_spec("crash-across-windows");
13544 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13545
13546 for attempt in 1..=2 {
13547 assert!(matches!(
13548 on_child_exit(
13549 &spec,
13550 runtime.restart_policy,
13551 &supervisor.registry,
13552 &snapshot,
13553 &runtime.terminal_ring,
13554 &runtime.spawn_events,
13555 &runtime.child_roster,
13556 crash_exit_report(attempt),
13557 true,
13558 )
13559 .await,
13560 NextAction::Restart { schedule: _ }
13561 ));
13562 }
13563
13564 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13567 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13568 })
13569 .unwrap();
13570
13571 assert!(
13572 matches!(
13573 on_child_exit(
13574 &spec,
13575 runtime.restart_policy,
13576 &supervisor.registry,
13577 &snapshot,
13578 &runtime.terminal_ring,
13579 &runtime.spawn_events,
13580 &runtime.child_roster,
13581 crash_exit_report(3),
13582 true,
13583 )
13584 .await,
13585 NextAction::Restart { schedule: _ }
13586 ),
13587 "a crash older than the window must not hold a budget slot"
13588 );
13589
13590 let state = lock_snapshot(&snapshot).unwrap();
13591 assert_eq!(state.state, ModuleState::Restarting);
13592 assert_eq!(
13593 state.crash_restarts.len(),
13594 2,
13595 "the aged instant is dropped and the new one takes its place"
13596 );
13597 assert_eq!(
13598 state.lifetime_restarts, 3,
13599 "the ledger counts every restart, including the ones the window forgot"
13600 );
13601 }
13602
13603 #[tokio::test]
13608 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13609 let supervisor = Supervisor::new_for_test(
13610 Arc::new(Registry::default()),
13611 RestartPolicy::new(2, Duration::ZERO),
13612 );
13613 let runtime = supervisor.runtime_config();
13614 let spec = windowed_crash_spec("operator-cleared-budget");
13615 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13616
13617 for attempt in 1..=2 {
13618 assert!(matches!(
13619 on_child_exit(
13620 &spec,
13621 runtime.restart_policy,
13622 &supervisor.registry,
13623 &snapshot,
13624 &runtime.terminal_ring,
13625 &runtime.spawn_events,
13626 &runtime.child_roster,
13627 crash_exit_report(attempt),
13628 true,
13629 )
13630 .await,
13631 NextAction::Restart { schedule: _ }
13632 ));
13633 }
13634
13635 reset_restart_count(&snapshot, &spec.module_id).unwrap();
13636 {
13637 let state = lock_snapshot(&snapshot).unwrap();
13638 assert!(
13639 state.crash_restarts.is_empty(),
13640 "an operator restart returns the full budget"
13641 );
13642 assert_eq!(
13643 state.lifetime_restarts, 2,
13644 "clearing the budget must not unmake the crashes"
13645 );
13646 }
13647
13648 assert!(
13649 matches!(
13650 on_child_exit(
13651 &spec,
13652 runtime.restart_policy,
13653 &supervisor.registry,
13654 &snapshot,
13655 &runtime.terminal_ring,
13656 &runtime.spawn_events,
13657 &runtime.child_roster,
13658 crash_exit_report(3),
13659 true,
13660 )
13661 .await,
13662 NextAction::Restart { schedule: _ }
13663 ),
13664 "the cleared budget must be spendable again"
13665 );
13666 let state = lock_snapshot(&snapshot).unwrap();
13667 assert_eq!(state.crash_restarts.len(), 1);
13668 assert_eq!(state.lifetime_restarts, 3);
13669 }
13670
13671 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13672 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13673 let severed = ProcessIdentity {
13674 pid: 41,
13675 start_time: 101,
13676 };
13677 let successor = ProcessIdentity {
13678 pid: 41,
13679 start_time: 202,
13680 };
13681 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13682 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13683 state.pid = Some(successor.pid);
13684 state.process_start_time = Some(successor.start_time);
13685 })
13686 .unwrap();
13687 assert!(!module.record_deliberate_severance(severed).unwrap());
13688
13689 let exit_report = apply_deliberate_severance_marker(
13690 &module.inner.snapshot,
13691 Some(successor),
13692 ExitReport {
13693 kind: ExitKind::Crash,
13694 code: Some(1),
13695 signal: None,
13696 at_ms: 1,
13697 },
13698 );
13699
13700 assert_eq!(exit_report.kind, ExitKind::Crash);
13701 }
13702
13703 #[tokio::test]
13704 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13705 let registry = Registry::default();
13706 let supervisor = Supervisor::new_for_test(
13707 Arc::new(Registry::default()),
13708 RestartPolicy::new(3, Duration::ZERO),
13709 );
13710 let runtime = supervisor.runtime_config();
13711 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13712 let spec = ModuleSpec {
13713 module_id: "drain-deliberate-severance".to_string(),
13714 program: fake_aft_stub_path(),
13715 args: Vec::new(),
13716 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13717 reserved: false,
13718 reserved_prefixes: Vec::new(),
13719 protocol: ModuleProtocol::Subc,
13720 overlap: Default::default(),
13721 };
13722 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13723 let process = ProcessIdentity {
13724 pid: 41,
13725 start_time: 101,
13726 };
13727 child.process_identity = Some(process);
13728 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13729 state.pid = Some(process.pid);
13730 state.process_start_time = Some(process.start_time);
13731 })
13732 .unwrap();
13733 record_deliberate_severance(&snapshot, process).unwrap();
13734
13735 drain_child_to_state(
13736 &spec.module_id,
13737 spec.protocol,
13738 StopNotice::SentOverConnection,
13741 ®istry,
13742 None,
13743 &snapshot,
13744 &runtime.terminal_ring,
13745 &runtime.spawn_events,
13746 child,
13747 Duration::from_secs(1),
13748 ModuleState::Stopped,
13749 Some(false),
13750 &super::FailureContext::operator(),
13751 )
13752 .await
13753 .unwrap();
13754
13755 let state = lock_snapshot(&snapshot).unwrap();
13756 assert_eq!(
13757 state.last_exit.as_ref().map(|exit| exit.kind),
13758 Some(ExitKind::DeliberateSeverance)
13759 );
13760 assert_eq!(state.lifetime_restarts, 1);
13761 assert_eq!(state.crash_restarts.len(), 0);
13762 drop(state);
13763 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13764 assert_eq!(
13765 history.entries[0].exit_kind,
13766 subc_control::TerminalExitKind::DeliberateSeverance
13767 );
13768 }
13769
13770 #[tokio::test]
13771 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13772 let registry = Registry::default();
13773 let supervisor = Supervisor::new_for_test(
13774 Arc::new(Registry::default()),
13775 RestartPolicy::new(3, Duration::ZERO),
13776 );
13777 let runtime = supervisor.runtime_config();
13778 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13779 let spec = ModuleSpec {
13780 module_id: "ordinary-drain".to_string(),
13781 program: fake_aft_stub_path(),
13782 args: Vec::new(),
13783 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13784 reserved: false,
13785 reserved_prefixes: Vec::new(),
13786 protocol: ModuleProtocol::Subc,
13787 overlap: Default::default(),
13788 };
13789 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13790
13791 drain_child_to_state(
13792 &spec.module_id,
13793 spec.protocol,
13794 StopNotice::SentOverConnection,
13797 ®istry,
13798 None,
13799 &snapshot,
13800 &runtime.terminal_ring,
13801 &runtime.spawn_events,
13802 child,
13803 Duration::from_secs(1),
13804 ModuleState::Stopped,
13805 Some(false),
13806 &super::FailureContext::operator(),
13807 )
13808 .await
13809 .unwrap();
13810
13811 let state = lock_snapshot(&snapshot).unwrap();
13812 assert_eq!(
13813 state.last_exit.as_ref().map(|exit| exit.kind),
13814 Some(ExitKind::Crash)
13815 );
13816 assert_eq!(state.lifetime_restarts, 0);
13817 assert_eq!(state.crash_restarts.len(), 0);
13818 }
13819
13820 #[test]
13821 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13822 assert!(!include_str!("server.rs")
13828 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13829 }
13830
13831 #[test]
13838 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13839 assert!(drained_after_quiescence_wait(&Ok(true)));
13840 assert!(!drained_after_quiescence_wait(&Ok(false)));
13841 assert!(!drained_after_quiescence_wait(&Err(
13842 SuperviseError::StatePoisoned { module_id: None }
13843 )));
13844 }
13845
13846 #[test]
13855 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13856 let ring = Arc::new(Mutex::new(TerminalRing::new(
13857 TerminalRingConfig::default(),
13858 0,
13859 )));
13860 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13861
13862 let snapshot = ring.lock().unwrap().snapshot();
13863 assert_eq!(snapshot.entries.len(), 1);
13864 let entry = &snapshot.entries[0];
13865 assert_eq!(entry.exit_code, None);
13866 assert_eq!(entry.exit_signal, None);
13867 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13868 }
13869
13870 #[test]
13871 fn wait_error_exit_path_preserves_spawn_event_density() {
13872 let feed = super::SpawnEventFeed::default();
13873 feed.configure_incarnation("wait-error-density".to_string());
13874 feed.emit_spawned("wait-error", 41, 1);
13875 let ring = Arc::new(Mutex::new(TerminalRing::new(
13876 TerminalRingConfig::default(),
13877 0,
13878 )));
13879
13880 record_wait_error_terminal("wait-error", &ring, &feed);
13881 feed.emit_spawned("after-wait-error", 42, 2);
13882
13883 let state = feed.0.lock().unwrap();
13884 let sequences = state
13885 .events
13886 .iter()
13887 .map(|event| event.cursor.seq)
13888 .collect::<Vec<_>>();
13889 assert_eq!(sequences, vec![1, 2, 3]);
13890 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13891 assert_eq!(state.events[1].exit_code, None);
13892 assert_eq!(state.events[1].exit_signal, None);
13893 }
13894
13895 #[test]
13899 fn wait_error_exit_report_is_classified_as_a_crash() {
13900 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13901 }
13902}
13903
13904#[cfg(all(test, unix))]
13905mod restart_ordering_tests {
13906 use super::*;
13907 use std::io::Write;
13908 use tokio::io::AsyncReadExt;
13909
13910 #[test]
13911 #[ignore = "child process fixture"]
13912 fn serving_child() {
13913 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13914 return;
13915 };
13916 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13917 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13918 for stream in listener.incoming() {
13919 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13920 }
13921 }
13922
13923 async fn restart_before_publication(requests: usize) {
13924 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13925 let address_file = home.path().join("address");
13926 let spec = ModuleSpec {
13927 module_id: "restart-serving-child".into(),
13928 program: std::env::current_exe().unwrap(),
13929 args: vec![
13930 "--exact".into(),
13931 "supervise::restart_ordering_tests::serving_child".into(),
13932 "--ignored".into(),
13933 "--nocapture".into(),
13934 ],
13935 env: vec![(
13936 "RESTART_TEST_ADDRESS_FILE".into(),
13937 address_file.display().to_string(),
13938 )],
13939 reserved: false,
13940 reserved_prefixes: vec![],
13941 protocol: ModuleProtocol::None,
13942 overlap: Default::default(),
13943 };
13944 let mut supervisor =
13945 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13946 let (spawned_tx, spawned_rx) = oneshot::channel();
13947 let spawned_tx = Mutex::new(Some(spawned_tx));
13948 let (release_tx, release_rx) = std::sync::mpsc::channel();
13949 let release_rx = Mutex::new(release_rx);
13950 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13953 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13954 release_rx
13955 .lock()
13956 .unwrap()
13957 .recv_timeout(Duration::from_secs(20))
13958 .unwrap();
13959 })));
13960 let registered = supervisor
13961 .register_configured_with_health(
13962 spec.clone(),
13963 true,
13964 HealthConfig::default(),
13965 None,
13966 RestartPolicy::new(3, Duration::ZERO),
13967 DEFAULT_REGISTER_TIMEOUT,
13968 )
13969 .unwrap();
13970 let module = registered.module.clone();
13971 let startup = tokio::task::spawn_blocking(move || {
13972 supervisor.start_registered_configured(registered, None, true)
13973 });
13974 spawned_rx.await.unwrap();
13975 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13976 let address = loop {
13977 if let Ok(address) = std::fs::read_to_string(&address_file) {
13978 if !address.is_empty() {
13979 break address;
13980 }
13981 }
13982 tokio::time::sleep(Duration::from_millis(1)).await;
13983 };
13984 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13985 let mut reply = String::new();
13986 stream.read_to_string(&mut reply).await.unwrap();
13987 reply.trim().parse::<u32>().unwrap()
13988 })
13989 .await
13990 .expect("child must serve before generation publication");
13991 assert_eq!(
13992 lock_snapshot(&module.inner.snapshot)
13993 .unwrap()
13994 .spawn_generation,
13995 0
13996 );
13997 let mut replies = Vec::new();
13998 for _ in 0..requests {
13999 let mut restart = Box::pin(module.restart(Some(0)));
14000 std::future::poll_fn(|cx| {
14003 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14004 std::task::Poll::Ready(())
14005 })
14006 .await;
14007 replies.push(restart);
14008 }
14009 release_tx.send(()).unwrap();
14010 startup.await.unwrap().unwrap();
14011 for reply in replies {
14012 reply.await.unwrap();
14013 }
14014 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14017 loop {
14018 if lock_snapshot(&module.inner.snapshot)
14019 .unwrap()
14020 .spawn_generation
14021 > 1
14022 {
14023 break;
14024 }
14025 tokio::time::sleep(Duration::from_millis(1)).await;
14026 }
14027 })
14028 .await;
14029 module.update_spec_for_test(spec).await.unwrap();
14033 let (generation, pid) = {
14034 let state = lock_snapshot(&module.inner.snapshot).unwrap();
14035 (state.spawn_generation, state.pid)
14036 };
14037 module.stop().await.unwrap();
14038 replacement.expect("restart must publish a replacement for the already-serving child");
14039 assert_ne!(
14040 pid,
14041 Some(old_pid),
14042 "a serving child predating the restart must be replaced"
14043 );
14044 assert_eq!(generation, 2, "queued restarts must share one replacement");
14045 }
14046
14047 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14048 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14049 restart_before_publication(1).await;
14050 }
14051
14052 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14053 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14054 restart_before_publication(2).await;
14055 }
14056
14057 #[tokio::test]
14058 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14059 let supervisor =
14060 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14061 let registered = supervisor
14062 .register_configured_with_health(
14063 ModuleSpec {
14064 module_id: "queued-before-replacement".into(),
14065 program: "/bin/sleep".into(),
14070 args: vec!["600".into()],
14071 env: vec![],
14072 reserved: false,
14073 reserved_prefixes: vec![],
14074 protocol: ModuleProtocol::None,
14075 overlap: Default::default(),
14076 },
14077 true,
14078 HealthConfig::default(),
14079 None,
14080 RestartPolicy::default(),
14081 DEFAULT_REGISTER_TIMEOUT,
14082 )
14083 .unwrap();
14084 let RegisteredConfiguredModule {
14085 mut spec,
14086 mut runtime,
14087 module,
14088 mut commands,
14089 } = registered;
14090 let snapshot = &module.inner.snapshot;
14091 let mut restart = Box::pin(module.restart(Some(0)));
14092 std::future::poll_fn(|cx| {
14093 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14094 std::task::Poll::Ready(())
14095 })
14096 .await;
14097 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14100 let replacement_pid = child.as_ref().unwrap().id();
14101 let command = commands.recv().await.unwrap();
14102 assert!(
14103 handle_supervisor_command(
14104 command,
14105 &mut spec,
14106 &mut runtime,
14107 &supervisor.registry,
14108 &supervisor.process_liveness,
14109 snapshot,
14110 &mut child,
14111 &mut commands,
14112 &mut VecDeque::new(),
14113 )
14114 .await
14115 );
14116 restart.await.unwrap();
14117 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14118 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14119 if let Some(mut child) = child {
14120 child.start_kill().unwrap();
14121 child.wait().await.unwrap();
14122 }
14123 assert_eq!(
14124 kept_pid, replacement_pid,
14125 "a newer replacement must survive"
14126 );
14127 assert!(
14128 !pending,
14129 "a satisfied restart must not schedule another spawn"
14130 );
14131 }
14132}
14133
14134#[cfg(all(test, unix))]
14135mod health_event_tests {
14136 use super::*;
14137 use cortexkit_test_support::ScratchDir;
14138 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14139
14140 struct Actor {
14141 module: SupervisedModule,
14142 registry: Arc<Registry>,
14143 forwarding: Arc<ForwardingTable>,
14144 spec: ModuleSpec,
14145 health: HealthConfig,
14146 rx: mpsc::Receiver<crate::router::OutboundFrame>,
14147 roster: ChildRoster,
14148 _home: ScratchDir,
14149 }
14150
14151 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14152 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14156 let tick = Instant::now();
14157 loop {
14158 if let Some(value) = observe() {
14159 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14160 return value;
14161 }
14162 assert!(
14163 std::time::Instant::now() < deadline,
14164 "timed out waiting for {reason}"
14165 );
14166 tokio::task::yield_now().await;
14167 }
14168 }
14169
14170 impl Actor {
14171 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14172 Self::start_with_policy(
14173 protocol,
14174 cadence,
14175 DEFAULT_REGISTER_TIMEOUT,
14176 RestartPolicy::new(3, Duration::ZERO),
14177 )
14178 .await
14179 }
14180
14181 async fn start_with_policy(
14182 protocol: ModuleProtocol,
14183 cadence: Duration,
14184 register_timeout: Duration,
14185 policy: RestartPolicy,
14186 ) -> Self {
14187 let home = ScratchDir::new("health-events");
14188 let registry = Arc::new(Registry::default());
14189 let forwarding = Arc::new(ForwardingTable::default());
14190 let health = HealthConfig {
14191 cadence,
14192 deadline: Duration::from_secs(3600),
14193 ..HealthConfig::default()
14194 };
14195 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14196 .with_forwarding(forwarding.clone())
14197 .with_health_config(health.clone());
14198 let spec = ModuleSpec {
14199 module_id: "event-health-child".into(),
14200 program: PathBuf::from("/bin/sleep"),
14208 args: vec!["600".into()],
14211 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14212 .into_iter()
14213 .map(|key| (key.into(), home.path().display().to_string()))
14214 .collect(),
14215 reserved: false,
14216 reserved_prefixes: vec![],
14217 protocol,
14218 overlap: Default::default(),
14219 };
14220 let module = supervisor
14221 .supervise_configured_requested(
14222 spec.clone(),
14223 true,
14224 health.clone(),
14225 Some(0),
14226 policy,
14227 register_timeout,
14228 None,
14229 )
14230 .unwrap();
14231 let (_, rx) = mpsc::channel(8);
14232 let actor = Self {
14233 module,
14234 registry,
14235 forwarding,
14236 spec,
14237 health,
14238 rx,
14239 roster: supervisor.child_roster.clone(),
14240 _home: home,
14241 };
14242 actor
14243 .wait_for_select(|checkpoint| checkpoint.generation > 0)
14244 .await;
14245 actor
14246 }
14247
14248 async fn expire_registration_wait(&self, remaining: Duration) {
14249 tokio::time::advance(remaining).await;
14250 wait_for("registration timeout TERM grace", || {
14251 lock_snapshot(&self.module.inner.snapshot)
14252 .unwrap()
14253 .actor_select
14254 .is_none()
14255 .then_some(())
14256 })
14257 .await;
14258 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14259 }
14260
14261 fn turns(&self) -> u64 {
14262 lock_snapshot(&self.module.inner.snapshot)
14263 .unwrap()
14264 .actor_turns
14265 }
14266
14267 async fn wait_for_select(
14268 &self,
14269 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14270 ) -> ActorSelectCheckpoint {
14271 wait_for("actor select checkpoint", || {
14272 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14273 state.actor_select.filter(|checkpoint| {
14274 checkpoint.generation == state.spawn_generation
14275 && state.state == ModuleState::Running
14276 && state.process_alive
14277 && state.reported_pid().is_some()
14278 && expected(checkpoint)
14279 })
14280 })
14281 .await
14282 }
14283
14284 fn hello(&mut self, connection: u64, health: bool) {
14285 let manifest =
14286 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14287 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14288 .build();
14289 let connection = ConnectionId::new(connection);
14290 let (tx, rx) = mpsc::channel(8);
14291 self.rx = rx;
14292 self.forwarding
14293 .register_module_connection(
14294 connection,
14295 self.spec.module_id.clone(),
14296 subc_protocol::PROTOCOL_VERSION,
14297 Concurrency::ModuleManaged,
14298 FrameSink::new(tx),
14299 )
14300 .unwrap();
14301 self.registry
14302 .register_with_control_ops(
14303 manifest,
14304 subc_protocol::PROTOCOL_VERSION,
14305 connection,
14306 if health {
14307 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
14308 } else {
14309 vec![]
14310 },
14311 )
14312 .unwrap();
14313 }
14314
14315 async fn probe(&mut self) -> crate::router::OutboundFrame {
14316 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
14317 Ok(frame) => Some(frame),
14318 Err(mpsc::error::TryRecvError::Empty) => None,
14319 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
14320 })
14321 .await;
14322 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14323 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14324 frame
14325 }
14326
14327 fn no_probe(&mut self) {
14328 assert!(matches!(
14329 self.rx.try_recv(),
14330 Err(mpsc::error::TryRecvError::Empty)
14331 ));
14332 }
14333
14334 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
14335 self.forwarding
14336 .complete_module_control_rpc(
14337 ConnectionId::new(connection),
14338 frame.header.corr,
14339 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14340 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14341 status: HealthStatus::Ok,
14342 detail: None,
14343 metrics: None,
14344 }),
14345 )
14346 .unwrap();
14347 }
14348 }
14349
14350 #[tokio::test(start_paused = true)]
14351 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
14352 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14353 let initial = actor.module.status().unwrap();
14354 tokio::time::advance(Duration::from_secs(119)).await;
14355 tokio::task::yield_now().await;
14356 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
14357 assert!(actor.module.terminal_history().entries.is_empty());
14358 actor.expire_registration_wait(Duration::from_secs(1)).await;
14359 actor
14360 .wait_for_select(|at| at.generation > initial.spawn_generation)
14361 .await;
14362 let status = actor.module.status().unwrap();
14363 assert_ne!(status.pid, initial.pid);
14364 assert!(!cortexkit_test_support::process_alive(
14365 initial.pid.unwrap() as i32
14366 ));
14367 assert_eq!(status.restart_count, 1);
14368 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
14369 let entries = actor.module.terminal_history().entries;
14370 assert_eq!(entries.len(), 1);
14371 assert_eq!(
14372 entries[0].disposition_detail.as_deref(),
14373 Some("registration_timeout")
14374 );
14375 actor.module.stop().await.unwrap();
14376 }
14377
14378 #[tokio::test(start_paused = true)]
14379 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
14380 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14381 let pid = actor.module.status().unwrap().pid;
14382 tokio::time::advance(Duration::from_secs(119)).await;
14383 actor.hello(1, false);
14384 let registered = actor
14385 .wait_for_select(|at| at.registered_connection.is_some())
14386 .await;
14387 assert_eq!(
14388 registered.wake_after, None,
14389 "HELLO must cancel the deadline"
14390 );
14391 tokio::time::advance(Duration::from_secs(300)).await;
14392 tokio::task::yield_now().await;
14393 assert_eq!(actor.module.status().unwrap().pid, pid);
14394 assert!(actor.module.terminal_history().entries.is_empty());
14395 actor
14397 .registry
14398 .deregister_connection(ConnectionId::new(1))
14399 .unwrap();
14400 actor
14401 .wait_for_select(|at| at.registered_connection.is_none())
14402 .await;
14403 tokio::time::advance(Duration::from_secs(300)).await;
14404 tokio::task::yield_now().await;
14405 assert_eq!(actor.module.status().unwrap().pid, pid);
14406 assert!(actor.module.terminal_history().entries.is_empty());
14407 actor.module.stop().await.unwrap();
14408
14409 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14410 let pid = transient.module.status().unwrap().pid;
14411 tokio::time::advance(Duration::from_secs(119)).await;
14412 let turns = transient.turns();
14413 transient.hello(1, false);
14416 transient
14417 .registry
14418 .deregister_connection(ConnectionId::new(1))
14419 .unwrap();
14420 let parked = transient.wait_for_select(|at| at.turn > turns).await;
14421 assert_eq!(
14422 parked.wake_after, None,
14423 "transient HELLO must cancel the deadline"
14424 );
14425 tokio::time::advance(Duration::from_secs(300)).await;
14426 tokio::task::yield_now().await;
14427 assert_eq!(transient.module.status().unwrap().pid, pid);
14428 assert!(transient.module.terminal_history().entries.is_empty());
14429 transient.module.stop().await.unwrap();
14430 }
14431
14432 #[tokio::test(start_paused = true)]
14433 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
14434 let actor = Actor::start_with_policy(
14435 ModuleProtocol::Subc,
14436 Duration::from_secs(30),
14437 Duration::from_secs(10),
14438 RestartPolicy::new(1, Duration::ZERO),
14439 )
14440 .await;
14441 let generation = actor.module.status().unwrap().spawn_generation;
14442 actor
14443 .expire_registration_wait(Duration::from_secs(10))
14444 .await;
14445 actor.wait_for_select(|at| at.generation > generation).await;
14446 actor
14447 .expire_registration_wait(Duration::from_secs(10))
14448 .await;
14449 let retry = wait_for("failed retry schedule", || {
14450 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
14451 (state.state == ModuleState::Failed)
14452 .then_some(state.failed_retry.pending)
14453 .flatten()
14454 })
14455 .await;
14456 assert_eq!(retry.delay, Duration::from_secs(600));
14457 assert_eq!(retry.attempt, 1);
14458 let entries = actor.module.terminal_history().entries;
14459 assert_eq!(entries.len(), 2);
14460 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
14461 assert!(entries[1]
14462 .disposition_detail
14463 .as_deref()
14464 .unwrap()
14465 .contains("registration_timeout"));
14466 tokio::time::advance(Duration::from_secs(599)).await;
14467 tokio::task::yield_now().await;
14468 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
14469 tokio::time::advance(Duration::from_secs(1)).await;
14470 actor
14471 .wait_for_select(|at| at.generation > generation + 1)
14472 .await;
14473 actor
14475 .expire_registration_wait(Duration::from_secs(10))
14476 .await;
14477 wait_for("slow retry registration timeout", || {
14478 (actor.module.terminal_history().entries.len() == 3).then_some(())
14479 })
14480 .await;
14481 actor.module.stop().await.unwrap();
14482 }
14483
14484 #[tokio::test(start_paused = true)]
14485 async fn registration_timeout_honours_ten_second_override() {
14486 let actor = Actor::start_with_policy(
14487 ModuleProtocol::Subc,
14488 Duration::from_secs(30),
14489 Duration::from_secs(10),
14490 RestartPolicy::new(3, Duration::ZERO),
14491 )
14492 .await;
14493 let generation = actor.module.status().unwrap().spawn_generation;
14494 tokio::time::advance(Duration::from_secs(9)).await;
14495 tokio::task::yield_now().await;
14496 assert!(actor.module.terminal_history().entries.is_empty());
14497 actor.expire_registration_wait(Duration::from_secs(1)).await;
14498 actor.wait_for_select(|at| at.generation > generation).await;
14499 assert_eq!(actor.module.terminal_history().entries.len(), 1);
14500 actor.module.stop().await.unwrap();
14501 }
14502
14503 #[tokio::test]
14504 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
14505 use std::os::unix::process::{CommandExt, ExitStatusExt};
14506
14507 struct Helper(std::process::Child);
14508 impl Drop for Helper {
14509 fn drop(&mut self) {
14510 if matches!(self.0.try_wait(), Ok(None)) {
14513 let _ = self.0.kill();
14514 let _ = self.0.wait();
14515 }
14516 }
14517 }
14518
14519 let home = ScratchDir::new("registration-timeout-helper");
14520 let supervisor = Supervisor::new_for_test(
14521 Arc::new(Registry::default()),
14522 RestartPolicy::new(0, Duration::ZERO),
14523 );
14524 let spec = ModuleSpec {
14525 module_id: "registration-timeout-group".into(),
14526 program: "/bin/sleep".into(),
14527 args: vec!["120".into()],
14528 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14529 .into_iter()
14530 .map(|key| (key.into(), home.path().display().to_string()))
14531 .collect(),
14532 reserved: false,
14533 reserved_prefixes: vec![],
14534 protocol: ModuleProtocol::Subc,
14535 overlap: Default::default(),
14536 };
14537 let module = supervisor
14538 .supervise_configured_requested(
14539 spec,
14540 true,
14541 HealthConfig::default(),
14542 None,
14543 RestartPolicy::new(0, Duration::ZERO),
14544 Duration::from_secs(10),
14545 None,
14546 )
14547 .unwrap();
14548 let leader = timeout(Duration::from_secs(5), async {
14549 loop {
14550 if let Some(pid) = module.status().unwrap().pid {
14551 break pid;
14552 }
14553 sleep(Duration::from_millis(5)).await;
14554 }
14555 })
14556 .await
14557 .expect("supervised group leader must be reported before the deadline");
14558 assert!(leader > 1);
14559 let ready = home.join("helper.ready");
14560 let mut helper = Helper(
14563 std::process::Command::new("/bin/sh")
14564 .args([
14565 "-c",
14566 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
14567 "helper",
14568 ])
14569 .arg(&ready)
14570 .process_group(i32::try_from(leader).unwrap())
14571 .stdout(std::process::Stdio::null())
14572 .stderr(std::process::Stdio::null())
14573 .spawn()
14574 .unwrap(),
14575 );
14576 timeout(Duration::from_secs(5), async {
14577 while !ready.exists() {
14578 assert!(helper.0.try_wait().unwrap().is_none());
14579 sleep(Duration::from_millis(5)).await;
14580 }
14581 })
14582 .await
14583 .expect("helper must install TERM ignore before the registration deadline");
14584 let helper_pid =
14585 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
14586 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
14587 sleep(Duration::from_millis(20)).await;
14588 assert!(
14589 helper.0.try_wait().unwrap().is_none(),
14590 "helper must ignore TERM"
14591 );
14592
14593 timeout(Duration::from_secs(20), async {
14594 while module.state().unwrap() != ModuleState::Failed {
14595 sleep(Duration::from_millis(5)).await;
14596 }
14597 })
14598 .await
14599 .expect("registration timeout must stop and reap the group leader");
14600 let helper_exit = timeout(Duration::from_secs(3), async {
14601 loop {
14602 if let Some(status) = helper.0.try_wait().unwrap() {
14603 break status;
14604 }
14605 sleep(Duration::from_millis(5)).await;
14606 }
14607 })
14608 .await
14609 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
14610 assert_eq!(helper_exit.signal(), Some(9));
14611 assert!(!cortexkit_test_support::process_alive(
14612 i32::try_from(leader).unwrap()
14613 ));
14614 assert_eq!(
14615 module.status().unwrap().last_exit.unwrap().kind,
14616 ExitKind::Crash
14617 );
14618 assert!(module.terminal_history().entries[0]
14619 .disposition_detail
14620 .as_deref()
14621 .unwrap()
14622 .contains("registration_timeout"));
14623 module.stop().await.unwrap();
14624 }
14625
14626 #[tokio::test(start_paused = true)]
14627 async fn registration_timeout_does_not_count_an_earlier_exit() {
14628 let actor = Actor::start_with_policy(
14629 ModuleProtocol::Subc,
14630 Duration::from_secs(30),
14631 DEFAULT_REGISTER_TIMEOUT,
14632 RestartPolicy::new(0, Duration::ZERO),
14633 )
14634 .await;
14635 let pid = actor.module.status().unwrap().pid.unwrap();
14636 rustix::process::kill_process(
14637 rustix::process::Pid::from_raw(pid as i32).unwrap(),
14638 rustix::process::Signal::TERM,
14639 )
14640 .unwrap();
14641 wait_for("early exit", || {
14642 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
14643 })
14644 .await;
14645 tokio::time::advance(Duration::from_secs(121)).await;
14646 tokio::task::yield_now().await;
14647 let entries = actor.module.terminal_history().entries;
14648 assert_eq!(entries.len(), 1);
14649 assert!(!entries[0]
14650 .disposition_detail
14651 .as_deref()
14652 .unwrap()
14653 .contains("registration_timeout"));
14654 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
14655 actor.module.stop().await.unwrap();
14656 }
14657
14658 #[tokio::test(start_paused = true)]
14659 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
14660 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14661 let pid = actor.module.status().unwrap().pid;
14662 tokio::time::advance(Duration::from_secs(119)).await;
14663 actor.roster.close();
14664 let turns = actor.turns();
14665 tokio::time::advance(Duration::from_secs(300)).await;
14666 let parked = actor.wait_for_select(|at| at.turn > turns).await;
14667 assert_eq!(parked.wake_after, None);
14668 assert_eq!(actor.module.status().unwrap().pid, pid);
14669 assert!(actor.module.terminal_history().entries.is_empty());
14670 actor.module.stop().await.unwrap();
14671 let status = actor.module.status().unwrap();
14672 assert_eq!(status.restart_count, 0);
14673 assert_eq!(
14674 actor.module.terminal_history().entries[0].disposition,
14675 TerminalDisposition::DaemonShutdown
14676 );
14677 }
14678
14679 #[tokio::test(start_paused = true)]
14680 async fn no_health_children_park_for_sixty_seconds() {
14681 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14682 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14683 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14684 unadvertised.hello(1, false);
14685 for _ in 0..6000 {
14690 tokio::time::advance(Duration::from_millis(10)).await;
14691 tokio::task::yield_now().await;
14692 }
14693 for actor in [&none, &unregistered, &unadvertised] {
14694 assert!(
14695 actor.turns() <= 5,
14696 "no-health actor woke {} times",
14697 actor.turns()
14698 );
14699 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
14700 }
14701 }
14702
14703 #[tokio::test(start_paused = true)]
14704 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
14705 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
14709 let tick = Instant::now();
14710 actor.hello(1, true);
14711 actor.probe().await;
14712 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
14713 }
14714
14715 #[tokio::test(start_paused = true)]
14716 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
14717 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14718 let hello_tick = Instant::now();
14719 actor.hello(1, true);
14720 let armed = actor
14721 .wait_for_select(|checkpoint| {
14722 checkpoint.registered_connection == Some(ConnectionId::new(1))
14723 && checkpoint.next_probe_at.is_some()
14724 })
14725 .await;
14726 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14727 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
14728 let first_deadline = hello_tick + delay;
14729 assert_eq!(armed.next_probe_at, Some(first_deadline));
14730 tokio::time::advance(Duration::from_secs(29)).await;
14731 let turns = actor.turns();
14732 actor
14733 .registry
14734 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
14735 .unwrap();
14736 let updated = actor
14737 .wait_for_select(|checkpoint| checkpoint.turn > turns)
14738 .await;
14739 assert_eq!(
14740 updated.next_probe_at,
14741 Some(first_deadline),
14742 "catalog update must not reset cadence"
14743 );
14744 actor.no_probe();
14745 tokio::time::advance(first_deadline - Instant::now()).await;
14746 let frame = actor.probe().await;
14747 actor.answer(1, frame);
14748 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
14749 let next_deadline = Instant::now() + next;
14750 let rearmed = actor
14751 .wait_for_select(|checkpoint| {
14752 checkpoint
14753 .next_probe_at
14754 .is_some_and(|deadline| deadline > first_deadline)
14755 })
14756 .await;
14757 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14758 tokio::time::advance(next - Duration::from_nanos(1)).await;
14759 actor.no_probe();
14760 tokio::time::advance(Duration::from_nanos(1)).await;
14761 actor.probe().await;
14762 }
14763
14764 #[tokio::test(start_paused = true)]
14765 async fn disconnect_disarms_and_reconnect_rearms_health() {
14766 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14767 actor.hello(1, true);
14768 actor
14769 .wait_for_select(|checkpoint| {
14770 checkpoint.registered_connection == Some(ConnectionId::new(1))
14771 && checkpoint.next_probe_at.is_some()
14772 })
14773 .await;
14774 let turns = actor.turns();
14775 let tick = Instant::now();
14776 actor
14777 .registry
14778 .deregister_connection(ConnectionId::new(1))
14779 .unwrap();
14780 let disarmed = actor
14781 .wait_for_select(|checkpoint| {
14782 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14783 })
14784 .await;
14785 assert_eq!(disarmed.next_probe_at, None);
14786 assert_eq!(disarmed.wake_after, None);
14787 assert_eq!(
14788 Instant::now(),
14789 tick,
14790 "disconnect must disarm in the notification tick"
14791 );
14792 tokio::time::advance(Duration::from_secs(60)).await;
14793 actor.no_probe();
14794 let reconnect_tick = Instant::now();
14795 actor.hello(2, true);
14796 let rearmed = actor
14797 .wait_for_select(|checkpoint| {
14798 checkpoint.registered_connection == Some(ConnectionId::new(2))
14799 && checkpoint.next_probe_at.is_some()
14800 })
14801 .await;
14802 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14803 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14804 tokio::time::advance(delay).await;
14805 actor.probe().await;
14806 }
14807
14808 #[tokio::test(start_paused = true)]
14809 async fn reregistration_without_health_disarms_the_old_schedule() {
14810 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14811 actor.hello(1, true);
14812 actor
14813 .wait_for_select(|checkpoint| {
14814 checkpoint.registered_connection == Some(ConnectionId::new(1))
14815 && checkpoint.next_probe_at.is_some()
14816 })
14817 .await;
14818 actor
14819 .registry
14820 .deregister_connection(ConnectionId::new(1))
14821 .unwrap();
14822 actor.hello(2, false);
14823 let disarmed = actor
14824 .wait_for_select(|checkpoint| {
14825 checkpoint.registered_connection == Some(ConnectionId::new(2))
14826 })
14827 .await;
14828 assert_eq!(disarmed.next_probe_at, None);
14829 assert_eq!(disarmed.wake_after, None);
14830 tokio::time::advance(Duration::from_secs(60)).await;
14831 actor.no_probe();
14832 }
14833
14834 #[tokio::test(start_paused = true)]
14835 async fn restart_waits_for_its_own_hello_before_rearming_health() {
14836 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14837 actor.hello(1, true);
14838 let armed = actor
14839 .wait_for_select(|checkpoint| {
14840 checkpoint.registered_connection == Some(ConnectionId::new(1))
14841 && checkpoint.next_probe_at.is_some()
14842 })
14843 .await;
14844 actor
14848 .registry
14849 .deregister_connection(ConnectionId::new(1))
14850 .unwrap();
14851 actor.module.restart(Some(0)).await.unwrap();
14852 let parked = actor
14853 .wait_for_select(|checkpoint| {
14854 checkpoint.generation > armed.generation
14855 && checkpoint.registered_connection.is_none()
14856 && checkpoint.next_probe_at.is_none()
14857 })
14858 .await;
14859 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
14860 actor.rx.close();
14861 while actor.rx.try_recv().is_ok() {}
14864 let turns = actor.turns();
14865 tokio::time::advance(Duration::from_secs(60)).await;
14866 let after = actor.turns();
14867 eprintln!(
14868 "restart park turns: before={turns} after={after} delta={}",
14869 after - turns
14870 );
14871 assert_eq!(after, turns, "replacement must park before HELLO");
14872 let hello_tick = Instant::now();
14873 actor.hello(2, true);
14874 let rearmed = actor
14875 .wait_for_select(|checkpoint| {
14876 checkpoint.registered_connection == Some(ConnectionId::new(2))
14877 && checkpoint.next_probe_at.is_some()
14878 })
14879 .await;
14880 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14881 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14882 tokio::time::advance(delay).await;
14883 actor.probe().await;
14884 }
14885
14886 #[tokio::test(start_paused = true)]
14887 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14888 use tokio::{
14889 io::{AsyncReadExt, AsyncWriteExt},
14890 net::TcpListener,
14891 };
14892 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14893 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14894 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14895 let (started_tx, mut started_rx) = mpsc::channel(8);
14896 let peer = tokio::spawn(async move {
14897 loop {
14898 let (mut socket, _) = listener.accept().await.unwrap();
14899 let mut request = [0; 1024];
14900 assert!(socket.read(&mut request).await.unwrap() > 0);
14901 started_tx.send(Instant::now()).await.unwrap();
14902 socket
14903 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14904 .await
14905 .unwrap();
14906 }
14907 });
14908 actor
14909 .module
14910 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14911 .await
14912 .unwrap();
14913 let first_tick = Instant::now();
14914 let first = actor
14915 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
14916 .await;
14917 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
14918 assert!(started_rx.try_recv().is_err());
14919
14920 tokio::time::advance(
14923 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
14924 )
14925 .await;
14926 actor.module.restart(Some(0)).await.unwrap();
14927 let spawn_tick = Instant::now();
14928 let replacement = actor
14929 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
14930 .await;
14931 let deadline = replacement.next_probe_at.unwrap();
14932 assert!(
14933 deadline >= spawn_tick + actor.health.cadence,
14934 "a replacement must not inherit its predecessor's probe deadline"
14935 );
14936 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
14937 assert!(
14938 started_rx.try_recv().is_err(),
14939 "no HTTP probe during startup grace"
14940 );
14941 tokio::time::advance(deadline - Instant::now()).await;
14942 let landed = wait_for("first replacement HTTP probe", || {
14943 started_rx.try_recv().ok()
14944 })
14945 .await;
14946 assert!(landed >= spawn_tick + actor.health.cadence);
14947 wait_for("healthy replacement", || {
14948 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14949 .then_some(())
14950 })
14951 .await;
14952 peer.abort();
14953 }
14954
14955 #[tokio::test(start_paused = true)]
14956 async fn rescan_adds_and_removes_http_health_without_polling() {
14957 use tokio::{
14958 io::{AsyncReadExt, AsyncWriteExt},
14959 net::TcpListener,
14960 };
14961 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14962 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14963 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
14964 actor.health.cadence = Duration::from_millis(1);
14966 let (started_tx, mut started_rx) = mpsc::channel(8);
14967 let peer = tokio::spawn(async move {
14968 loop {
14969 let (mut socket, _) = listener.accept().await.unwrap();
14970 let mut request = [0; 1024];
14971 assert!(socket.read(&mut request).await.unwrap() > 0);
14972 started_tx.send(()).await.unwrap();
14973 socket
14974 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14975 .await
14976 .unwrap();
14977 }
14978 });
14979 let tick = Instant::now();
14980 let turns = actor.turns();
14981 actor
14982 .module
14983 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14984 .await
14985 .unwrap();
14986 let armed = actor
14987 .wait_for_select(|checkpoint| {
14988 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
14989 })
14990 .await;
14991 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
14992 assert_eq!(Instant::now(), tick);
14993 tokio::time::advance(Duration::from_millis(1)).await;
14994 let mut started = false;
14995 wait_for("successful HTTP probe", || {
14996 started |= started_rx.try_recv().is_ok();
14997 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14998 .then_some(())
14999 })
15000 .await;
15001 assert!(started, "added HTTP check must start at its first deadline");
15002 assert_eq!(
15003 actor.module.status().unwrap().health.status,
15004 SupervisorHealthStatus::Ok
15005 );
15006 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15007 actor.health.http = None;
15008 let turns = actor.turns();
15009 actor
15010 .module
15011 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15012 .await
15013 .unwrap();
15014 let parked = actor
15015 .wait_for_select(|checkpoint| {
15016 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15017 })
15018 .await;
15019 assert_eq!(parked.wake_after, None);
15020 let turns = actor.turns();
15021 tokio::time::advance(Duration::from_secs(60)).await;
15022 assert_eq!(
15023 actor.turns(),
15024 turns,
15025 "removed HTTP check must park the actor"
15026 );
15027 assert!(
15028 started_rx.try_recv().is_err(),
15029 "removed HTTP check must stay disarmed"
15030 );
15031 assert_eq!(
15032 actor.module.status().unwrap().health,
15033 ModuleHealthStatus::default()
15034 );
15035 peer.abort();
15036 }
15037}
15038
15039#[cfg(test)]
15040mod drain_busy_reprobe_tests {
15041 use super::*;
15045 use std::collections::BTreeMap;
15046 use std::sync::Mutex as StdMutex;
15047 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15048
15049 const MODULE_ID: &str = "drain-reprobe-module";
15050 const CONNECTION: u64 = 7;
15051 const GAUGE: &str = "runs_in_flight";
15052 const CADENCE: Duration = Duration::from_secs(30);
15055 const CEILING: Duration = Duration::from_secs(30);
15056
15057 #[derive(Clone, Copy)]
15059 enum Answer {
15060 BusyUntil { until: Instant, latency: Duration },
15063 OmitsGauge,
15065 Silent,
15067 }
15068
15069 struct Drain {
15070 forwarding: Arc<ForwardingTable>,
15071 runtime: SupervisorRuntimeConfig,
15072 endpoint: crate::ModuleEndpointId,
15073 probes: Arc<StdMutex<Vec<Instant>>>,
15075 responder: tokio::task::JoinHandle<()>,
15076 }
15077
15078 impl Drain {
15079 fn start(answer: Answer) -> Self {
15080 let registry = Arc::new(Registry::default());
15081 let forwarding = Arc::new(ForwardingTable::default());
15082 let health = HealthConfig {
15083 cadence: CADENCE,
15084 ..HealthConfig::default()
15085 };
15086 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15087 .with_forwarding(forwarding.clone())
15088 .with_health_config(health)
15089 .runtime_config();
15090 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15091 let endpoint = forwarding
15092 .register_module_connection(
15093 ConnectionId::new(CONNECTION),
15094 MODULE_ID.to_string(),
15095 subc_protocol::PROTOCOL_VERSION,
15096 Concurrency::ModuleManaged,
15097 FrameSink::new(tx),
15098 )
15099 .unwrap();
15100 let probes = Arc::new(StdMutex::new(Vec::new()));
15101 let responder = tokio::spawn({
15102 let forwarding = forwarding.clone();
15103 let probes = probes.clone();
15104 async move {
15105 while let Some(frame) = rx.recv().await {
15106 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15107 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15108 probes.lock().unwrap().push(Instant::now());
15109 let metrics = match answer {
15110 Answer::BusyUntil { until, latency } => {
15111 sleep(latency).await;
15112 let busy = u64::from(Instant::now() < until);
15113 serde_json::json!({ GAUGE: busy })
15114 }
15115 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15116 Answer::Silent => continue,
15117 };
15118 let _ = forwarding.complete_module_control_rpc(
15121 ConnectionId::new(CONNECTION),
15122 frame.header.corr,
15123 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15124 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15125 status: HealthStatus::Ok,
15126 detail: None,
15127 metrics: Some(metrics),
15128 }),
15129 );
15130 }
15131 }
15132 });
15133 Self {
15134 forwarding,
15135 runtime,
15136 endpoint,
15137 probes,
15138 responder,
15139 }
15140 }
15141
15142 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15145 let (drained, elapsed, _) = self.run_traced(ceiling).await;
15146 (drained, elapsed)
15147 }
15148
15149 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15151 let started = Instant::now();
15152 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15153 let drained = wait_for_forwarding_quiescence(
15154 &self.forwarding,
15155 MODULE_ID,
15156 &self.runtime,
15157 self.endpoint,
15158 started + ceiling,
15159 &[GAUGE.to_string()],
15160 DrainScope::Active,
15161 &mut trace,
15162 )
15163 .await
15164 .unwrap();
15165 (drained, started.elapsed(), trace)
15166 }
15167
15168 fn probes(&self) -> Vec<Instant> {
15169 self.probes.lock().unwrap().clone()
15170 }
15171 }
15172
15173 impl Drop for Drain {
15174 fn drop(&mut self) {
15175 self.responder.abort();
15176 }
15177 }
15178
15179 #[tokio::test(start_paused = true)]
15180 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15181 let drain = Drain::start(Answer::BusyUntil {
15182 until: Instant::now() + Duration::from_secs(1),
15183 latency: Duration::ZERO,
15184 });
15185 let (drained, elapsed) = drain.run(CEILING).await;
15186 assert!(
15187 drained,
15188 "gauges that read zero during the drain must let it finish drained, \
15189 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15190 );
15191 assert!(
15192 elapsed < Duration::from_secs(3),
15193 "drain took {elapsed:?} after the gauges went quiet at 1s"
15194 );
15195 let probes = drain.probes();
15196 assert!(
15197 probes.len() >= 2,
15198 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15199 );
15200 }
15201
15202 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15207 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15208 let drain = Drain::start(Answer::BusyUntil {
15209 until: Instant::now() + Duration::from_millis(600),
15210 latency: Duration::ZERO,
15211 });
15212 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15213 assert!(drained, "the gauge fell at 600ms: {trace}");
15214 assert!(
15215 elapsed < Duration::from_secs(3),
15216 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15217 );
15218 let probes = drain.probes();
15219 assert!(
15220 probes.len() >= 2 && trace.probes as usize == probes.len(),
15221 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15222 );
15223 }
15224
15225 #[tokio::test(start_paused = true)]
15226 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15227 let started = Instant::now();
15228 let drain = Drain::start(Answer::BusyUntil {
15229 until: started + Duration::from_secs(3600),
15230 latency: Duration::ZERO,
15231 });
15232 let (drained, elapsed) = drain.run(CEILING).await;
15233 assert!(!drained, "a module busy throughout must not drain");
15234 assert!(
15235 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15236 "a busy module is held until the ceiling and no longer: {elapsed:?}"
15237 );
15238 let probes = drain.probes();
15241 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15242 assert!(
15243 probes.len() as u128 <= max_probes,
15244 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15245 probes.len()
15246 );
15247 for pair in probes.windows(2) {
15248 assert!(
15249 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15250 "consecutive probes {:?} apart",
15251 pair[1] - pair[0]
15252 );
15253 }
15254 }
15255
15256 #[tokio::test(start_paused = true)]
15257 async fn a_slow_probe_is_never_overlapped_by_the_next() {
15258 let latency = Duration::from_secs(1);
15261 let drain = Drain::start(Answer::BusyUntil {
15262 until: Instant::now() + Duration::from_secs(3600),
15263 latency,
15264 });
15265 let (drained, _) = drain.run(Duration::from_secs(5)).await;
15266 assert!(!drained);
15267 let probes = drain.probes();
15268 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15269 for pair in probes.windows(2) {
15270 assert!(
15271 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15272 "a probe started {:?} after the previous one, before it could finish",
15273 pair[1] - pair[0]
15274 );
15275 }
15276 }
15277
15278 #[tokio::test(start_paused = true)]
15279 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15280 let drain = Drain::start(Answer::Silent);
15281 let ceiling = Duration::from_secs(2);
15282 let (drained, elapsed) = drain.run(ceiling).await;
15283 assert!(!drained, "an unanswered probe counts as busy");
15284 assert!(
15285 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15286 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15287 );
15288 assert_eq!(
15289 drain.probes().len(),
15290 1,
15291 "the unanswered probe holds the only slot until the ceiling"
15292 );
15293 }
15294
15295 #[derive(Clone, Default)]
15297 struct LogCapture {
15298 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
15299 }
15300
15301 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
15302 fn on_event(
15303 &self,
15304 event: &tracing::Event<'_>,
15305 _context: tracing_subscriber::layer::Context<'_, S>,
15306 ) {
15307 struct Fields(BTreeMap<String, String>);
15308 impl tracing::field::Visit for Fields {
15309 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
15310 self.0
15311 .insert(field.name().to_string(), format!("{value:?}"));
15312 }
15313 }
15314 let mut fields = Fields(BTreeMap::new());
15315 event.record(&mut fields);
15316 self.events.lock().unwrap().push(fields.0);
15317 }
15318 }
15319
15320 impl LogCapture {
15321 fn install(&self) -> tracing::subscriber::DefaultGuard {
15322 use tracing_subscriber::layer::SubscriberExt;
15323 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
15324 }
15325
15326 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
15327 self.events
15328 .lock()
15329 .unwrap()
15330 .iter()
15331 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
15332 .cloned()
15333 .collect()
15334 }
15335 }
15336
15337 #[tokio::test(start_paused = true)]
15342 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
15343 let capture = LogCapture::default();
15344 let _guard = capture.install();
15345 let drain = Drain::start(Answer::OmitsGauge);
15346 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
15347 assert!(!drained, "a gauge never reported cannot read quiet");
15348 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
15349
15350 let missing = capture.with_message("lacks a declared busy gauge");
15351 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
15352 assert_eq!(
15353 missing[0].get("missing_gauges").map(String::as_str),
15354 Some(GAUGE)
15355 );
15356
15357 warn_drain_timed_out(
15358 MODULE_ID,
15359 Duration::from_secs(2),
15360 RouteCloseReason::Restart,
15361 &Default::default(),
15362 &trace,
15363 );
15364 let timeouts = capture.with_message("route drain timed out");
15365 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
15366 let gauges = &timeouts[0]["busy_gauges"];
15367 assert!(
15368 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
15369 "the timeout line must name the gauge and why it read busy: {gauges}"
15370 );
15371 assert!(
15372 gauges.ends_with("; no value ever read"),
15373 "no value was ever read for the gauge: {gauges}"
15374 );
15375 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
15376 }
15377
15378 #[tokio::test(start_paused = true)]
15382 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
15383 let busy = Drain::start(Answer::BusyUntil {
15384 until: Instant::now() + Duration::from_secs(3600),
15385 latency: Duration::ZERO,
15386 });
15387 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
15388 let rendered = trace.to_string();
15389 assert!(
15390 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
15391 && rendered.contains("; last value non_zero(1) @"),
15392 "{rendered}"
15393 );
15394
15395 let silent = Drain::start(Answer::Silent);
15396 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
15397 assert_eq!(
15400 trace.to_string(),
15401 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
15402 );
15403 assert_eq!(trace.probes, 1);
15404 }
15405
15406 #[test]
15408 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
15409 let gauges = [GAUGE.to_string()];
15410 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
15411 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
15412 assert_eq!(
15413 busy_gauge_observation(&verdicts),
15414 BusyGaugeObservation::Busy
15415 );
15416 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
15417 assert_eq!(
15418 busy_gauge_observation(&zero),
15419 BusyGaugeObservation::Quiescent
15420 );
15421 let none = gauge_verdicts(None, &gauges);
15422 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
15423 }
15424}
15425
15426#[cfg(test)]
15427mod health_evidence_tests {
15428 use super::{HealthProbeError, HealthProbeEvidence};
15429 use std::collections::HashSet;
15430
15431 #[test]
15439 fn only_a_dead_lane_is_proof_of_death() {
15440 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
15441 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
15445 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
15446 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
15447 }
15448
15449 #[test]
15455 fn every_evidence_class_has_a_distinct_label() {
15456 let labels = [
15457 HealthProbeError::lane_dead("").label(),
15458 HealthProbeError::no_answer("").label(),
15459 HealthProbeError::bad_answer("").label(),
15460 HealthProbeError::misconfigured("").label(),
15461 ];
15462 let unique: HashSet<_> = labels.iter().collect();
15463 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
15464 }
15465
15466 #[test]
15472 fn classification_preserves_the_original_message() {
15473 let err = HealthProbeError::no_answer("module did not answer within 5s");
15474 assert_eq!(err.to_string(), "module did not answer within 5s");
15475 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15476 }
15477}
15478
15479#[cfg(test)]
15480mod health_tombstone_tests {
15481 use std::{path::PathBuf, sync::Arc, time::Duration};
15482
15483 use subc_protocol::{
15484 manifest::Concurrency,
15485 session::{HealthStatus, ModuleControlResponse},
15486 };
15487 use tokio::sync::mpsc;
15488
15489 use super::{
15490 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
15491 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
15492 };
15493 use crate::{
15494 control::ControlHandler,
15495 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
15496 registry::{ConnectionId, Registry},
15497 router::FrameSink,
15498 };
15499
15500 struct ProbeHarness {
15501 spec: ModuleSpec,
15502 runtime: SupervisorRuntimeConfig,
15503 forwarding: Arc<ForwardingTable>,
15504 module_connection: ConnectionId,
15505 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15506 handler: ControlHandler,
15507 module: super::SupervisedModule,
15508 }
15509
15510 fn probe_harness() -> ProbeHarness {
15511 let registry = Arc::new(Registry::default());
15512 let forwarding = Arc::new(ForwardingTable::default());
15513 let supervisor_handle = super::SupervisorHandle::new();
15514 let health = HealthConfig {
15515 http: None,
15516 cadence: Duration::from_secs(30),
15517 deadline: Duration::from_secs(5),
15518 failure_threshold: 3,
15519 on_degraded: HealthAction::Report,
15520 on_failing: HealthAction::Report,
15521 critical: false,
15522 };
15523 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15524 .with_forwarding(Arc::clone(&forwarding))
15525 .with_handle(supervisor_handle.clone())
15526 .with_health_config(health);
15527 let spec = ModuleSpec {
15528 module_id: "late-health-module".to_string(),
15529 program: PathBuf::from("disabled-module"),
15530 args: Vec::new(),
15531 env: Vec::new(),
15532 reserved: false,
15533 reserved_prefixes: Vec::new(),
15534 protocol: ModuleProtocol::Subc,
15535 overlap: Default::default(),
15536 };
15537 let module = supervisor
15538 .supervise_configured(spec.clone(), false)
15539 .unwrap();
15540 let runtime = supervisor.runtime_config();
15541 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
15542 .with_supervisor(supervisor_handle);
15543 let module_connection = ConnectionId::new(700);
15544 let (module_tx, module_rx) = mpsc::channel(8);
15545 forwarding
15546 .register_module_connection(
15547 module_connection,
15548 spec.module_id.clone(),
15549 subc_protocol::PROTOCOL_VERSION,
15550 Concurrency::ModuleManaged,
15551 FrameSink::new(module_tx),
15552 )
15553 .unwrap();
15554
15555 ProbeHarness {
15556 spec,
15557 runtime,
15558 forwarding,
15559 module_connection,
15560 module_rx,
15561 handler,
15562 module,
15563 }
15564 }
15565
15566 async fn finish_after(
15567 harness: &mut ProbeHarness,
15568 stall: Duration,
15569 ) -> ModuleControlRpcCompletion {
15570 assert!(stall > harness.runtime.health.deadline);
15571 let deadline = harness.runtime.health.deadline;
15572 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15573 let answer = async {
15574 let frame = harness.module_rx.recv().await.expect("health.check frame");
15575 tokio::time::advance(deadline).await;
15576 tokio::task::yield_now().await;
15577 tokio::time::advance(stall - deadline).await;
15578 harness
15579 .forwarding
15580 .complete_module_control_rpc(
15581 harness.module_connection,
15582 frame.header.corr,
15583 Some("health.check"),
15584 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15585 status: HealthStatus::Ok,
15586 detail: None,
15587 metrics: None,
15588 }),
15589 )
15590 .unwrap()
15591 };
15592 let (probe_result, completion) = tokio::join!(probe, answer);
15593 let err = probe_result.expect_err("probe must miss its deadline");
15594 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15595 completion
15596 }
15597
15598 async fn time_out_without_answer(harness: &mut ProbeHarness) {
15599 let deadline = harness.runtime.health.deadline;
15600 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15601 let exhaust_deadline = async {
15602 let _frame = harness.module_rx.recv().await.expect("health.check frame");
15603 tokio::time::advance(deadline).await;
15604 tokio::task::yield_now().await;
15605 };
15606 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
15607 let err = probe_result.expect_err("probe must miss its deadline");
15608 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15609 }
15610
15611 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
15612 let registry = Arc::clone(&harness.module.inner.registry);
15613 let snapshot = Arc::clone(&harness.module.inner.snapshot);
15614 let process_liveness = super::SupervisorProcessLiveness::default();
15615 let mut child = None;
15616 let cycle = super::run_health_probe_cycle(
15617 &harness.spec,
15618 &harness.runtime,
15619 ®istry,
15620 &process_liveness,
15621 &snapshot,
15622 &mut child,
15623 );
15624 let peer = async {
15625 let frame = harness.module_rx.recv().await.expect("health.check frame");
15626 if answer {
15627 harness
15628 .forwarding
15629 .complete_module_control_rpc(
15630 harness.module_connection,
15631 frame.header.corr,
15632 Some("health.check"),
15633 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15634 status: HealthStatus::Ok,
15635 detail: None,
15636 metrics: Some(serde_json::json!({"ready": true})),
15637 }),
15638 )
15639 .unwrap();
15640 } else {
15641 tokio::time::advance(harness.runtime.health.deadline).await;
15642 tokio::task::yield_now().await;
15643 }
15644 };
15645 tokio::join!(cycle, peer);
15646 }
15647
15648 #[tokio::test(start_paused = true)]
15649 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
15650 let mut harness = probe_harness();
15651 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
15655 monitor.abort();
15656 let _ = monitor.await;
15657 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
15658 state.enabled = true;
15659 state.state = super::ModuleState::Running;
15660 state.process_alive = true;
15661 })
15662 .unwrap();
15663
15664 run_probe_cycle(&mut harness, true).await;
15665 assert_eq!(
15666 harness.module.status().unwrap().health.status,
15667 super::SupervisorHealthStatus::Ok
15668 );
15669
15670 for failures in 1..harness.runtime.health.failure_threshold {
15671 run_probe_cycle(&mut harness, false).await;
15672 let status = harness.module.status().unwrap();
15673 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15674 assert_eq!(status.health.consecutive_failures, failures);
15675 assert!(status.health.last_probe_ms.is_some());
15676 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
15677 assert!(status.health.metrics.is_none());
15678 assert_eq!(status.state, super::ModuleState::Running);
15679 assert!(status.process_alive);
15680 assert_eq!(status.restart_count, 0);
15681 assert_eq!(status.lifetime_restarts, 0);
15682 assert!(status.health.last_action.is_none());
15683 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15684 }
15685
15686 run_probe_cycle(&mut harness, true).await;
15687 let recovered = harness.module.status().unwrap();
15688 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
15689 assert_eq!(recovered.health.consecutive_failures, 0);
15690 assert!(recovered.health.detail.is_none());
15691 assert_eq!(
15692 recovered.health.metrics,
15693 Some(serde_json::json!({"ready": true}))
15694 );
15695 assert_eq!(recovered.lifetime_restarts, 0);
15696
15697 for failures in 1..=harness.runtime.health.failure_threshold {
15698 run_probe_cycle(&mut harness, false).await;
15699 let status = harness.module.status().unwrap();
15700 assert_eq!(status.health.consecutive_failures, failures);
15701 if failures < harness.runtime.health.failure_threshold {
15702 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15703 assert_eq!(status.state, super::ModuleState::Running);
15704 assert_eq!(status.lifetime_restarts, 0);
15705 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15706 } else {
15707 assert_eq!(
15708 status.health.status,
15709 super::SupervisorHealthStatus::Unresponsive
15710 );
15711 assert_eq!(status.state, super::ModuleState::Restarting);
15712 assert_eq!(status.restart_count, 1);
15713 assert_eq!(status.lifetime_restarts, 1);
15714 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
15715 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
15716 }
15717 }
15718 }
15719
15720 #[tokio::test(start_paused = true)]
15721 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
15722 let mut harness = probe_harness();
15723
15724 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
15725 let first_latency = match &first {
15726 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15727 other => panic!("late answer was not retained: {other:?}"),
15728 };
15729 assert!(harness.handler.observe_module_control_completion(first));
15730
15731 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
15732 let second_latency = match &second {
15733 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15734 other => panic!("late answer was not retained: {other:?}"),
15735 };
15736 assert!(harness.handler.observe_module_control_completion(second));
15737
15738 assert_eq!(first_latency, Duration::from_secs(8));
15739 assert_eq!(
15740 second_latency - first_latency,
15741 Duration::from_secs(3),
15742 "latency must grow linearly with the additional stall"
15743 );
15744 let health = harness.module.status().unwrap().health;
15745 assert_eq!(health.late_answer_count, 2);
15746 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
15747 }
15748
15749 #[tokio::test(start_paused = true)]
15757 async fn late_answer_clears_the_consecutive_failure_streak() {
15758 let mut harness = probe_harness();
15759
15760 time_out_without_answer(&mut harness).await;
15762 harness
15763 .module
15764 .record_health_probe_failure_for_test("[no-answer] test miss")
15765 .unwrap();
15766 assert_eq!(
15767 harness.module.status().unwrap().health.consecutive_failures,
15768 1,
15769 "precondition: the miss must be on the streak before the late answer"
15770 );
15771
15772 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15774 assert!(matches!(
15775 late,
15776 ModuleControlRpcCompletion::LateHealthAnswer { .. }
15777 ));
15778 assert!(harness.handler.observe_module_control_completion(late));
15779
15780 let health = harness.module.status().unwrap().health;
15781 assert_eq!(
15782 health.consecutive_failures, 0,
15783 "a late answer is an answer: the streak must reset"
15784 );
15785 assert_eq!(health.late_answer_count, 1);
15786 }
15787
15788 #[tokio::test(start_paused = true)]
15789 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15790 let mut harness = probe_harness();
15791
15792 for _ in 0..20 {
15793 time_out_without_answer(&mut harness).await;
15794 assert_eq!(
15795 harness.forwarding.health_probe_tombstone_count().unwrap(),
15796 1
15797 );
15798 }
15799 }
15800}
15801
15802#[cfg(test)]
15803mod child_env_tests {
15804 #[cfg(windows)]
15805 use super::{
15806 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15807 };
15808 #[cfg(windows)]
15809 use std::sync::{Arc, OnceLock};
15810 #[cfg(windows)]
15811 #[test]
15812 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15813 let spec = spec(Vec::new());
15814 let mut command = Command::new("unused");
15815 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15816 .unwrap()
15817 .unwrap();
15818 let name = command
15819 .as_std()
15820 .get_envs()
15821 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15822 .and_then(|(_, value)| value)
15823 .unwrap();
15824 assert_eq!(name, OsStr::new(handoff.name()));
15825 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15826 assert!(command.as_std().get_envs().any(|(key, value)| key
15827 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15828 && value == Some(OsStr::new("env"))));
15829 assert!(command
15830 .as_std()
15831 .get_envs()
15832 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15833 }
15834
15835 #[cfg(windows)]
15836 #[test]
15837 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15838 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15839 snapshot.process_alive = true;
15840 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15841 assert_eq!(
15842 observed_launch_nonce_source(&snapshot).as_deref(),
15843 Some("env")
15844 );
15845 let consumed = Arc::new(OnceLock::new());
15846 snapshot.pipe_consumed = Some(consumed.clone());
15847 assert_eq!(
15848 observed_launch_nonce_source(&snapshot).as_deref(),
15849 Some("env")
15850 );
15851 consumed.set(()).unwrap();
15852 assert_eq!(
15853 observed_launch_nonce_source(&snapshot).as_deref(),
15854 Some("pipe")
15855 );
15856 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15857 assert_eq!(
15858 observed_launch_nonce_source(&snapshot).as_deref(),
15859 Some("env")
15860 );
15861 clear_current_process_facts(&mut snapshot);
15862 assert_eq!(observed_launch_nonce_source(&snapshot), None);
15863 }
15864 use super::{
15865 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15866 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15867 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15868 };
15869 use std::{ffi::OsStr, path::PathBuf};
15870 use tokio::process::Command;
15871
15872 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15873 ModuleSpec {
15874 module_id: "env-plan".to_string(),
15875 program: PathBuf::from("/nonexistent"),
15876 args: Vec::new(),
15877 env,
15878 reserved: false,
15879 reserved_prefixes: Vec::new(),
15880 protocol: ModuleProtocol::Subc,
15881 overlap: Default::default(),
15882 }
15883 }
15884
15885 #[test]
15886 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15887 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15888 let digest = LaunchSpecDigest::from_spec(&base);
15889 let mut reordered = base.clone();
15890 reordered.env.reverse();
15891 reordered.module_id = "different-id".into();
15892 reordered.reserved = true;
15893 reordered.overlap = Default::default();
15894 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15895 let mut delimiter_collision = base.clone();
15896 delimiter_collision.env[0] = ("A=B".into(), "C".into());
15897 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15898 let mut changed = base.clone();
15899 changed.protocol = ModuleProtocol::None;
15900 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15901 changed = base.clone();
15902 changed.args.push("--flag".into());
15903 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15904 changed = base.clone();
15905 changed.program = PathBuf::from("/other");
15906 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15907 changed = base.clone();
15908 changed.env[1].1 = "other".into();
15909 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15910 assert!(!format!("{digest:?}").contains("B=C"));
15911 assert!(!format!("{digest:?}").contains("[u8"));
15912 }
15913
15914 #[cfg(unix)]
15915 #[test]
15916 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
15917 use std::os::unix::ffi::OsStringExt;
15918 let mut left = spec(Vec::new());
15919 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
15920 let mut right = left.clone();
15921 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
15922 assert_eq!(
15923 left.program.to_string_lossy(),
15924 right.program.to_string_lossy()
15925 );
15926 assert_ne!(
15927 LaunchSpecDigest::from_spec(&left),
15928 LaunchSpecDigest::from_spec(&right)
15929 );
15930 }
15931
15932 #[test]
15946 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
15947 let mut command = Command::new("/nonexistent");
15948 apply_child_env(&mut command, &spec(Vec::new()));
15949 let removed = command
15950 .as_std()
15951 .get_envs()
15952 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
15953 assert!(
15954 removed,
15955 "ambient CK_LOG must be explicitly removed for an unconfigured module"
15956 );
15957
15958 let mut configured = Command::new("/nonexistent");
15959 apply_child_env(
15960 &mut configured,
15961 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
15962 );
15963 let effective = configured
15964 .as_std()
15965 .get_envs()
15966 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
15967 .last()
15968 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15969 assert_eq!(
15970 effective,
15971 Some(Some("debug".to_string())),
15972 "a module's configured CK_LOG must survive the ambient removal"
15973 );
15974 }
15975
15976 #[test]
15985 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
15986 let connection_file = std::path::Path::new("/run/subc-connection.json");
15987 let handle = SupervisorHandle::new();
15988
15989 let mut none_spec = spec(Vec::new());
15990 none_spec.protocol = ModuleProtocol::None;
15991 let mut none = Command::new("/nonexistent");
15992 let none_handoff =
15993 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
15994 .expect("protocol-none spawn args apply");
15995 assert!(
15996 none_handoff.is_none(),
15997 "protocol:none spawn must not receive a nonce descriptor"
15998 );
15999 #[cfg(windows)]
16000 assert!(!none.as_std().get_envs().any(|(key, value)| key
16001 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16002 && value.is_some()));
16003 assert!(
16004 !none.as_std().get_envs().any(|(key, value)| key
16005 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16006 && value.is_some()),
16007 "protocol:none spawn must not name a nonce descriptor"
16008 );
16009 let none_args: Vec<String> = none
16010 .as_std()
16011 .get_args()
16012 .map(|a| a.to_string_lossy().into_owned())
16013 .collect();
16014 assert!(
16015 !none_args.iter().any(|a| a == SUBC_ARG),
16016 "protocol:none argv must not carry --subc; got {none_args:?}"
16017 );
16018 let none_has_nonce = none
16019 .as_std()
16020 .get_envs()
16021 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16022 assert!(
16023 !none_has_nonce,
16024 "protocol:none spawn must not receive a launch nonce"
16025 );
16026 let none_has_module_id = none
16027 .as_std()
16028 .get_envs()
16029 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16030 assert!(
16031 none_has_module_id,
16032 "SUBC_MODULE_ID is inert and stays on every path"
16033 );
16034 assert!(
16035 handle.spawn_nonce(&none_spec.module_id).is_none(),
16036 "no nonce record for a process that will never present one"
16037 );
16038
16039 let wire_spec = spec(Vec::new());
16041 let mut wire = Command::new("/nonexistent");
16042 let wire_handoff =
16043 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16044 .expect("subc-wire spawn args apply");
16045 let wire_fd_env = wire
16046 .as_std()
16047 .get_envs()
16048 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16049 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16050 #[cfg(unix)]
16051 assert_eq!(
16052 wire_fd_env,
16053 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16054 "a subc-wire spawn names the pipe it will receive at descriptor 3"
16055 );
16056 #[cfg(windows)]
16057 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16058 #[cfg(not(any(unix, windows)))]
16059 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16060 let wire_args: Vec<String> = wire
16061 .as_std()
16062 .get_args()
16063 .map(|a| a.to_string_lossy().into_owned())
16064 .collect();
16065 assert_eq!(
16066 wire_args,
16067 vec![
16068 SUBC_ARG.to_string(),
16069 connection_file.to_string_lossy().into_owned()
16070 ],
16071 "a subc-wire spawn still carries --subc <path>"
16072 );
16073 assert_eq!(
16074 wire.as_std()
16075 .get_envs()
16076 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16077 !cfg!(unix),
16078 "only Windows supplies the environment nonce"
16079 );
16080 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16081 }
16082
16083 #[test]
16093 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16094 let role = |command: &Command| {
16095 command
16096 .as_std()
16097 .get_envs()
16098 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16099 .last()
16100 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16101 };
16102 let forged = spec(vec![(
16103 SUBC_SPAWN_ROLE_ENV.to_string(),
16104 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16105 )]);
16106
16107 let mut plain = Command::new("/nonexistent");
16108 apply_child_env(&mut plain, &forged);
16109 apply_spawn_role(&mut plain, SpawnRole::Plain);
16110 assert_eq!(
16111 role(&plain),
16112 Some(None),
16113 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16114 );
16115
16116 let mut candidate = Command::new("/nonexistent");
16117 apply_child_env(&mut candidate, &spec(Vec::new()));
16118 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16119 assert_eq!(
16120 role(&candidate),
16121 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16122 );
16123 }
16124
16125 #[test]
16131 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16132 let mut command = Command::new("/nonexistent");
16133 apply_child_env(
16134 &mut command,
16135 &spec(vec![
16136 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16137 ("KEPT".to_string(), "yes".to_string()),
16138 ]),
16139 );
16140 let keys: Vec<String> = command
16141 .as_std()
16142 .get_envs()
16143 .filter(|(_, value)| value.is_some())
16144 .map(|(key, _)| key.to_string_lossy().into_owned())
16145 .collect();
16146 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16147 assert!(
16148 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16149 "daemon-private capture key leaked to the child: {keys:?}"
16150 );
16151 }
16152}
16153
16154#[cfg(test)]
16155mod jitter_tests {
16156 use super::jittered_health_delay;
16157 use std::{collections::HashSet, time::Duration};
16158
16159 const FLEET: [&str; 14] = [
16168 "aft",
16169 "alfonso-core",
16170 "magic-context",
16171 "broca",
16172 "thalamus",
16173 "quota",
16174 "engram",
16175 "plexus",
16176 "cerebellum",
16177 "astrocyte",
16178 "synapse",
16179 "subc-mcp",
16180 "cortexkit-credentials",
16181 "subc-federation",
16182 ];
16183
16184 #[test]
16192 fn probe_delays_disperse_across_the_fleet() {
16193 let cadence = Duration::from_secs(30);
16194 let delays: HashSet<Duration> = FLEET
16195 .iter()
16196 .map(|id| jittered_health_delay(id, 0, cadence))
16197 .collect();
16198 assert_eq!(
16199 delays.len(),
16200 FLEET.len(),
16201 "every supervised module must land on its own probe offset"
16202 );
16203 }
16204
16205 #[test]
16211 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16212 let cadence = Duration::from_secs(30);
16213 let span = cadence / 10;
16214 for id in FLEET {
16215 for probe_index in 0..8 {
16216 let delay = jittered_health_delay(id, probe_index, cadence);
16217 assert!(
16218 delay >= cadence,
16219 "{id}#{probe_index}: jitter must not shorten the cadence"
16220 );
16221 assert!(
16222 delay < cadence + span,
16223 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16224 );
16225 }
16226 }
16227 }
16228
16229 #[test]
16235 fn a_module_offset_is_stable_across_restarts() {
16236 let cadence = Duration::from_secs(30);
16237 for id in FLEET {
16238 assert_eq!(
16239 jittered_health_delay(id, 0, cadence),
16240 jittered_health_delay(id, 0, cadence),
16241 "{id}: the same module and probe index must produce the same offset"
16242 );
16243 }
16244 }
16245
16246 #[test]
16248 fn zero_cadence_yields_zero_delay() {
16249 assert_eq!(
16250 jittered_health_delay("aft", 0, Duration::ZERO),
16251 Duration::ZERO
16252 );
16253 }
16254}
16255
16256#[cfg(all(test, target_os = "linux"))]
16257mod cgroup_placement_tests {
16258 use super::{
16259 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16260 SuperviseError, SupervisedChild,
16261 };
16262 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16263 use cortexkit_test_support::ScratchDir;
16264 use std::{
16265 fs, io,
16266 path::{Path, PathBuf},
16267 sync::{Arc, Mutex},
16268 };
16269 use tokio::process::Command;
16270
16271 #[tokio::test]
16272 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16273 use super::*;
16274 let dir = ScratchDir::new("unique-spawn-cgroups");
16275 let root = PathBuf::from(format!(
16276 "/sys/fs/cgroup/subc-unique-test-{}-{}",
16277 std::process::id(),
16278 unix_ms_now()
16279 ));
16280 if let Err(error) = fs::create_dir(&root) {
16281 assert!(
16282 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16283 "required cgroup test cannot execute: {error}"
16284 );
16285 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16286 return;
16287 }
16288 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16289 let group_count = || {
16290 fs::read_dir(root.join("subc-modules"))
16291 .unwrap()
16292 .map(|entry| entry.unwrap().file_type().unwrap())
16293 .filter(|kind| kind.is_dir())
16294 .count()
16295 };
16296 let supervisor =
16297 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16298 .with_cgroup_placement(Some(placement.clone()));
16299 let runtime = supervisor.runtime_config();
16300 let mut spec = ModuleSpec {
16301 module_id: "unique-spawn".into(),
16302 program: PathBuf::from("/bin/sleep"),
16303 args: vec!["60".into()],
16304 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16305 .into_iter()
16306 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
16307 .collect(),
16308 reserved: false,
16309 reserved_prefixes: vec![],
16310 protocol: ModuleProtocol::None,
16311 overlap: Default::default(),
16312 };
16313 let spawn = |spec: &ModuleSpec| {
16314 spawn_child(
16315 spec,
16316 None,
16317 None,
16318 &runtime.stderr_ring,
16319 None,
16320 &runtime.child_roster,
16321 Some(&placement),
16322 )
16323 .unwrap()
16324 };
16325 let mut live = spawn(&spec);
16326 for _ in 0..3 {
16327 let next = spawn(&spec);
16329 assert_ne!(live.module_id, next.module_id);
16330 live.start_kill().unwrap();
16331 live.wait().await.unwrap();
16332 live = next;
16333 assert!(
16334 live.child.try_wait().unwrap().is_none(),
16335 "retiring the old slot must not kill the replacement"
16336 );
16337 assert_eq!(
16338 group_count(),
16339 1,
16340 "only the live spawn's cgroup should remain"
16341 );
16342 }
16343 supervisor.begin_daemon_shutdown();
16344 let reap = tokio::spawn(async move {
16345 live.wait().await.unwrap();
16346 });
16347 supervisor
16348 .end_children_for_daemon_shutdown(false, std::future::pending())
16349 .await;
16350 reap.await.unwrap();
16351 assert_eq!(group_count(), 0);
16352 spec.program = PathBuf::from("/bin/true");
16354 spec.args.clear();
16355 let fresh_roster = ChildRoster::default();
16356 let mut short = spawn_child(
16357 &spec,
16358 None,
16359 None,
16360 &runtime.stderr_ring,
16361 None,
16362 &fresh_roster,
16363 Some(&placement),
16364 )
16365 .unwrap();
16366 short.wait().await.unwrap();
16367 assert_eq!(group_count(), 0);
16368 spec.module_id = "_".repeat(255);
16369 let mut long_id = spawn_child(
16370 &spec,
16371 None,
16372 None,
16373 &runtime.stderr_ring,
16374 None,
16375 &fresh_roster,
16376 Some(&placement),
16377 )
16378 .unwrap();
16379 long_id.wait().await.unwrap();
16380 assert_eq!(
16381 group_count(),
16382 0,
16383 "valid long module IDs must not exceed cgroup NAME_MAX"
16384 );
16385 fs::remove_dir(root.join("subc-modules")).unwrap();
16386 fs::remove_dir(root).unwrap();
16387 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
16388 }
16389
16390 #[test]
16391 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
16392 let path = Path::new("/definitely-missing-subc-cgroup");
16393 let mut command = Command::new("true");
16394 let error = apply_cgroup_placement(
16395 &mut command,
16396 &ModuleSpec {
16397 module_id: "broken-cgroup".to_string(),
16398 program: PathBuf::from("true"),
16399 args: Vec::new(),
16400 env: Vec::new(),
16401 reserved: false,
16402 reserved_prefixes: Vec::new(),
16403 protocol: ModuleProtocol::Subc,
16404 overlap: Default::default(),
16405 },
16406 path,
16407 )
16408 .expect_err("a parent cgroup open failure must reject the supervised spawn");
16409 let reason = error.to_string();
16410
16411 assert!(
16412 matches!(error, SuperviseError::Cgroup { .. }),
16413 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
16414 );
16415 assert!(
16416 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
16417 "parent cgroup open failure must name cgroup.procs: {reason}"
16418 );
16419 }
16420
16421 #[tokio::test]
16422 async fn reaping_a_child_removes_its_empty_module_cgroup() {
16423 let root = ScratchDir::new("supervisor-reap-cgroup");
16424 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16425 let placement = subc_cgroup::prepare_at(&root)
16426 .expect("prepare scratch cgroup root")
16427 .expect("scratch root has a cgroup.procs marker");
16428 let module_id = "reaped-module";
16429 let module = placement
16430 .module_path(module_id)
16431 .expect("create scratch module cgroup");
16432 let child = Command::new("true")
16433 .env("XDG_DATA_HOME", root.path())
16434 .env("XDG_RUNTIME_DIR", root.path())
16435 .env("XDG_CONFIG_HOME", root.path())
16436 .spawn()
16437 .expect("spawn short-lived child");
16438 let pid = child.id().expect("spawned child has pid");
16439 let mut child = SupervisedChild {
16440 child,
16441 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
16442 protocol: ModuleProtocol::Subc,
16443 module_id: module_id.to_string(),
16444 cgroup_placement: Some(placement),
16445 stdout_pump: None,
16446 stderr_pump: None,
16447 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
16448 spawned_at: tokio::time::Instant::now(),
16449 registration_deadline: None,
16450 registration_watch: None,
16451 spawned_at_ms: 0,
16452 spawned_from: PathBuf::from("true"),
16453 launch_spec_digest: LaunchSpecDigest([0; 32]),
16454 spawned_file_identity: None,
16455 process_start_time: None,
16456 process_identity: None,
16457 pid,
16458 roster_guard: None,
16459 #[cfg(target_os = "macos")]
16460 privacy_exec: None,
16461 spawn_failure: None,
16462 };
16463
16464 child.wait().await.expect("reap short-lived child");
16465
16466 assert!(
16467 !module.exists(),
16468 "reaping the supervised child must remove its empty cgroup"
16469 );
16470 }
16471
16472 #[test]
16473 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
16474 let root = ScratchDir::new("supervisor-non-empty-cgroup");
16475 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16476 let placement = subc_cgroup::prepare_at(&root)
16477 .expect("prepare scratch cgroup root")
16478 .expect("scratch root has a cgroup.procs marker");
16479 let module = placement
16480 .module_path("surviving-module")
16481 .expect("create scratch module cgroup");
16482 fs::write(module.join("surviving-process"), b"still present")
16483 .expect("make scratch cgroup non-empty");
16484 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
16485
16486 remove_module_cgroup(&placement, "surviving-module");
16487
16488 let logs = crate::router::test_log::captured_logs(&logs);
16489 assert!(
16490 module.exists(),
16491 "failed removal must leave the cgroup intact"
16492 );
16493 assert!(
16494 logs.contains("could not remove module cgroup after process exit; continuing teardown")
16495 && logs.contains("surviving-module"),
16496 "best-effort removal must report the failure without returning it: {logs}"
16497 );
16498 }
16499
16500 #[test]
16501 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
16502 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
16503 let reason = SuperviseError::Spawn {
16504 program: PathBuf::from("/bin/true"),
16505 source: io::Error::from_raw_os_error(13),
16506 cgroup_path: Some(cgroup_path.clone()),
16507 }
16508 .to_string();
16509
16510 assert!(
16511 reason.contains(&cgroup_path.display().to_string()),
16512 "a pre_exec spawn failure must name the cgroup path: {reason}"
16513 );
16514 }
16515}
16516
16517#[cfg(test)]
16518mod spawn_subscriber_lag_tests {
16519 use super::*;
16520
16521 #[tokio::test]
16526 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
16527 let feed = SpawnEventFeed::default();
16528 feed.configure_incarnation("lag-incarnation".to_string());
16529 let (tx, mut rx) = mpsc::channel(1);
16532 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
16533 .expect("subscribe");
16534 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
16535 for index in 0..emitted {
16536 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
16537 tokio::task::yield_now().await;
16540 }
16541 assert_eq!(
16542 feed.subscriber_count(),
16543 0,
16544 "the lagged subscriber must be removed"
16545 );
16546
16547 let mut data = Vec::new();
16548 let mut last = None;
16549 loop {
16550 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16551 .await
16552 .expect("the forwarder must finish once the subscriber is dropped");
16553 let Some(outbound) = next else { break };
16554 let frame = outbound.frame;
16555 if frame.header.ty == FrameType::StreamData {
16556 assert!(last.is_none(), "no data may follow the terminal frame");
16557 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
16558 data.push(event.cursor.seq);
16559 } else {
16560 assert!(last.is_none(), "exactly one terminal frame");
16561 last = Some(frame);
16562 }
16563 }
16564 assert!(!data.is_empty(), "queued frames drain before the terminal");
16565 for pair in data.windows(2) {
16566 assert_eq!(
16567 pair[1],
16568 pair[0] + 1,
16569 "queued frames arrive dense and in order"
16570 );
16571 }
16572 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
16573 assert_eq!(terminal.header.ty, FrameType::Error);
16574 assert_eq!(terminal.header.corr, 7);
16575 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
16576 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
16577 let detail = body.detail.expect("lagged error carries detail");
16578 assert_eq!(
16579 detail["first_undelivered_cursor"]["seq"],
16580 data.last().unwrap() + 1,
16581 "the named cursor is the first event the subscriber did not receive"
16582 );
16583 assert_eq!(
16584 detail["first_undelivered_cursor"]["daemon_incarnation"],
16585 "lag-incarnation"
16586 );
16587 }
16588}
16589
16590#[cfg(test)]
16591mod terminal_history_read_concurrency_tests {
16592 use super::*;
16593 use crate::terminal_journal::read_pause;
16594 use cortexkit_test_support::ScratchDir;
16595 use std::sync::mpsc as std_mpsc;
16596
16597 fn journaled_ring(
16598 journal: &Arc<crate::terminal_journal::TerminalJournal>,
16599 ) -> Arc<Mutex<TerminalRing>> {
16600 Arc::new(Mutex::new(
16601 TerminalRing::new(TerminalRingConfig::default(), 1)
16602 .with_journal(Some(Arc::clone(journal))),
16603 ))
16604 }
16605
16606 fn crash(at_ms: u64) -> ExitReport {
16607 ExitReport {
16608 kind: ExitKind::Crash,
16609 code: Some(1),
16610 signal: None,
16611 at_ms,
16612 }
16613 }
16614
16615 fn record_within(
16618 module_id: &'static str,
16619 ring: &Arc<Mutex<TerminalRing>>,
16620 at_ms: u64,
16621 bound: Duration,
16622 ) -> bool {
16623 let ring = Arc::clone(ring);
16624 let (done, done_rx) = std_mpsc::channel();
16625 std::thread::spawn(move || {
16626 record_terminal(
16627 module_id,
16628 &ring,
16629 &SpawnEventFeed::default(),
16630 &crash(at_ms),
16631 TerminalDisposition::Restarting,
16632 );
16633 let _ = done.send(());
16634 });
16635 done_rx.recv_timeout(bound).is_ok()
16636 }
16637
16638 #[test]
16643 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
16644 let dir = ScratchDir::new("terminal-history-concurrent-read");
16645 let path = dir.join("terminals.jsonl");
16646 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
16647 path.clone(),
16648 "daemon".into(),
16649 ));
16650 let reader_ring = journaled_ring(&journal);
16651 let other_ring = journaled_ring(&journal);
16652 assert!(record_within(
16653 "reader-module",
16654 &reader_ring,
16655 10,
16656 Duration::from_secs(5)
16657 ));
16658
16659 let (started, release) = read_pause::install(&path);
16660 let reading = {
16661 let ring = Arc::clone(&reader_ring);
16662 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
16663 };
16664 started
16665 .recv_timeout(Duration::from_secs(5))
16666 .expect("the history read reached its pause");
16667
16668 let bound = Duration::from_secs(1);
16669 assert!(
16670 record_within("other-module", &other_ring, 20, bound),
16671 "another module's exit waited on a history read (journal writer held)"
16672 );
16673 assert!(
16674 record_within("reader-module", &reader_ring, 30, bound),
16675 "the read module's own exit waited on its history read (ring held)"
16676 );
16677
16678 drop(release);
16679 let paused = reading.join().unwrap();
16680 assert_eq!(
16681 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16682 vec![10],
16683 "an exit recorded after the read began lands in neither half of it"
16684 );
16685 assert_eq!(paused.journal_skipped_lines, 0);
16686 assert_eq!(paused.journal_read_errors, 0);
16687
16688 let after = durable_terminal_history_of(&reader_ring, "reader-module");
16689 assert_eq!(
16690 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16691 vec![10, 30],
16692 "the next read merges ring and journal with no duplicate"
16693 );
16694 assert_eq!(after.journal_skipped_lines, 0);
16695 }
16696}
16697
16698#[cfg(test)]
16703mod stderr_settle_tests {
16704 use std::{
16705 future::Future,
16706 io,
16707 pin::Pin,
16708 sync::{Arc, Mutex},
16709 task::{Context, Poll},
16710 time::Duration,
16711 };
16712
16713 use tokio::{
16714 io::{AsyncRead, ReadBuf},
16715 sync::oneshot,
16716 time::Instant,
16717 };
16718
16719 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
16720 use crate::stderr_tail::{
16721 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
16722 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
16723 };
16724
16725 const BOUND: Duration = Duration::from_millis(250);
16726
16727 struct HeldReader {
16731 before: Option<Vec<u8>>,
16732 gate: Option<oneshot::Receiver<()>>,
16733 after: io::Cursor<Vec<u8>>,
16734 }
16735
16736 impl AsyncRead for HeldReader {
16737 fn poll_read(
16738 mut self: Pin<&mut Self>,
16739 cx: &mut Context<'_>,
16740 buf: &mut ReadBuf<'_>,
16741 ) -> Poll<io::Result<()>> {
16742 if let Some(bytes) = self.before.take() {
16743 buf.put_slice(&bytes);
16744 return Poll::Ready(Ok(()));
16745 }
16746 if let Some(gate) = self.gate.as_mut() {
16747 match Pin::new(gate).poll(cx) {
16748 Poll::Pending => return Poll::Pending,
16749 Poll::Ready(_) => self.gate = None,
16750 }
16751 }
16752 Pin::new(&mut self.after).poll_read(cx, buf)
16753 }
16754 }
16755
16756 struct DiscardSink;
16757
16758 impl OutputSink for DiscardSink {
16759 fn write_line(&mut self, _line: &[u8]) {}
16760 }
16761
16762 fn line(text: &str) -> TailEntry {
16763 TailEntry::Line {
16764 text: text.to_string(),
16765 truncated: false,
16766 at_ms: None,
16767 }
16768 }
16769
16770 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16771 ring.lock().unwrap()
16772 }
16773
16774 fn held_pump(
16778 ring: &Arc<Mutex<StderrRing>>,
16779 before: &str,
16780 after: &str,
16781 ) -> (StderrPump, oneshot::Sender<()>) {
16782 let generation = lock(ring).begin_process();
16783 let (release, gate) = oneshot::channel();
16784 let reader = HeldReader {
16785 before: Some(before.as_bytes().to_vec()),
16786 gate: Some(gate),
16787 after: io::Cursor::new(after.as_bytes().to_vec()),
16788 };
16789 let task = tokio::spawn(pump_stderr_to(
16790 reader,
16791 Arc::clone(ring),
16792 generation,
16793 DiscardSink,
16794 ));
16795 (StderrPump { task, generation }, release)
16796 }
16797
16798 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16799 for _ in 0..1000 {
16800 if done(&lock(ring)) {
16801 return;
16802 }
16803 tokio::time::sleep(Duration::from_millis(1)).await;
16804 }
16805 panic!(
16806 "ring never reached the expected state: {:?}",
16807 lock(ring).snapshot(None, None)
16808 );
16809 }
16810
16811 #[tokio::test(start_paused = true)]
16812 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16813 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16814 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16815
16816 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16817 let before_release = lock(&ring).snapshot(None, None);
16818 assert!(
16819 matches!(before_release.capture, CaptureState::Incomplete { .. }),
16820 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16821 );
16822
16823 let next = lock(&ring).begin_process();
16826 lock(&ring).push_line_from(next, "next process booting");
16827 release.send(()).unwrap();
16828 wait_until(&ring, |ring| {
16829 ring.snapshot(None, None).capture == CaptureState::Captured
16830 })
16831 .await;
16832
16833 assert_eq!(
16834 untimed(lock(&ring).snapshot(None, None).entries),
16835 vec![
16836 line("booting"),
16837 line("config error: missing storage"),
16838 TailEntry::ProcessStart,
16839 line("next process booting"),
16840 ],
16841 "the crash's last line must survive a slow reader and stay in the crashed process's section"
16842 );
16843 }
16844
16845 #[tokio::test(start_paused = true)]
16846 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16847 ) {
16848 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16849 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16852
16853 let started = Instant::now();
16854 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16855 assert_eq!(
16856 started.elapsed(),
16857 BOUND,
16858 "the restart must wait exactly the bound for a pipe that stays open, no longer"
16859 );
16860
16861 let next = lock(&ring).begin_process();
16862 lock(&ring).push_line_from(next, "next process booting");
16863 tokio::time::sleep(Duration::from_secs(60)).await;
16864
16865 let snapshot = lock(&ring).snapshot(None, None);
16866 match &snapshot.capture {
16867 CaptureState::Incomplete { reason } => assert!(
16868 reason.contains("had not reached EOF") && reason.contains("250ms"),
16869 "the reason must say what is missing and after how long: {reason}"
16870 ),
16871 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16872 }
16873 assert_eq!(
16874 untimed(snapshot.entries),
16875 vec![
16876 line("parent exiting"),
16877 TailEntry::ProcessStart,
16878 line("next process booting"),
16879 ]
16880 );
16881 }
16882
16883 #[tokio::test(start_paused = true)]
16884 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16885 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16886 let (pump, release) = held_pump(&ring, "one\n", "two\n");
16887 release.send(()).unwrap();
16888
16889 settle_stderr_pump("clean", &ring, pump, BOUND).await;
16890
16891 let snapshot = lock(&ring).snapshot(None, None);
16892 assert_eq!(snapshot.capture, CaptureState::Captured);
16893 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16894 }
16895
16896 #[tokio::test(start_paused = true)]
16897 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16898 const BURST: usize = 150;
16899 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16900 let path = capture.join("two-pipe.stderr.log");
16901 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16902 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16903 let generation = lock(&ring).begin_process();
16904 let stdout = (0..BURST)
16905 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16906 .collect::<String>();
16907 let stderr = (0..BURST)
16908 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16909 .collect::<String>();
16910 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
16915 let (release, gate) = oneshot::channel();
16916 let reader = HeldReader {
16917 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
16918 gate: Some(gate),
16919 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
16920 };
16921 let (finished, completion) = oneshot::channel();
16922 let stdout_sink = sink.clone();
16923 let pump = tokio::spawn(async move {
16924 pump_stdout_to(reader, stdout_sink).await;
16925 finished.send(()).unwrap();
16926 });
16927 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
16928
16929 let started = Instant::now();
16930 settle_stdout_pump("two-pipe", pump, BOUND).await;
16931 assert_eq!(
16932 started.elapsed(),
16933 BOUND,
16934 "a late reader must not hold up restart"
16935 );
16936 assert_eq!(
16937 std::fs::read_to_string(&path).unwrap().lines().count(),
16938 BURST + 1
16939 );
16940
16941 release
16942 .send(())
16943 .expect("stdout reader must survive the exit drain bound");
16944 tokio::time::timeout(Duration::from_secs(1), completion)
16945 .await
16946 .expect("released stdout reader must reach EOF")
16947 .expect("stdout capture must finish at EOF, not be cancelled");
16948 let contents = std::fs::read_to_string(&path).unwrap();
16949 let mut lines = contents
16950 .lines()
16951 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
16952 .collect::<Vec<_>>();
16953 let mut expected = (0..BURST)
16954 .flat_map(|i| {
16955 [
16956 format!("out-{i:04}-{}", "o".repeat(64)),
16957 format!("err-{i:04}-{}", "e".repeat(64)),
16958 ]
16959 })
16960 .collect::<Vec<_>>();
16961 lines.sort_unstable();
16962 expected.sort_unstable();
16963 assert_eq!(
16964 lines, expected,
16965 "both pipes must retain every complete line exactly once"
16966 );
16967 }
16968}
16969
16970#[cfg(all(test, windows))]
16984mod job_containment_tests {
16985 use super::*;
16986 use cortexkit_test_support::ScratchDir;
16987 use std::{
16988 path::{Path, PathBuf},
16989 sync::{Arc, Mutex},
16990 time::{Duration, Instant},
16991 };
16992
16993 fn stub_path() -> PathBuf {
16999 let mut path = std::env::current_exe().expect("current_exe available in tests");
17000 path.pop();
17001 path.pop();
17002 path.push("fake-aft-stub.exe");
17003 assert!(
17004 path.exists(),
17005 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17006 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17007 path.display()
17008 );
17009 path
17010 }
17011
17012 fn read_grandchild_pid(path: &Path) -> u32 {
17014 let deadline = Instant::now() + Duration::from_secs(10);
17015 loop {
17016 if let Ok(contents) = std::fs::read_to_string(path) {
17017 if let Ok(pid) = contents.trim().parse() {
17018 return pid;
17019 }
17020 }
17021 assert!(
17022 Instant::now() < deadline,
17023 "the stub never recorded a grandchild pid at {}",
17024 path.display()
17025 );
17026 std::thread::sleep(Duration::from_millis(10));
17027 }
17028 }
17029
17030 struct Fixture {
17033 _dir: ScratchDir,
17034 module_id: String,
17035 grandchild: u32,
17036 child: Option<SupervisedChild>,
17037 registry: Arc<Registry>,
17038 snapshot: Arc<Mutex<SupervisorSnapshot>>,
17039 terminal_ring: Arc<Mutex<TerminalRing>>,
17040 spawn_events: SpawnEventFeed,
17041 }
17042
17043 fn fixture(label: &str, module_id: &str) -> Fixture {
17044 let dir = ScratchDir::new(label);
17045 let pid_file = dir.join("grandchild.pid");
17046 let supervisor = Supervisor::new_for_test(
17047 Arc::new(Registry::default()),
17048 RestartPolicy::new(3, Duration::ZERO),
17049 );
17050 let runtime = supervisor.runtime_config();
17051 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17052 let spec = ModuleSpec {
17053 module_id: module_id.to_string(),
17054 program: stub_path(),
17055 args: Vec::new(),
17059 env: vec![
17060 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17061 (
17062 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17063 pid_file.display().to_string(),
17064 ),
17065 ],
17066 reserved: false,
17067 reserved_prefixes: Vec::new(),
17068 protocol: ModuleProtocol::Subc,
17069 overlap: Default::default(),
17070 };
17071 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17072 .expect("spawn the supervised fixture");
17073 let grandchild = read_grandchild_pid(&pid_file);
17074 Fixture {
17075 _dir: dir,
17076 module_id: module_id.to_string(),
17077 grandchild,
17078 child: Some(child),
17079 registry: Arc::new(Registry::default()),
17080 snapshot,
17081 terminal_ring: Arc::clone(&runtime.terminal_ring),
17082 spawn_events: SpawnEventFeed::default(),
17083 }
17084 }
17085
17086 impl Fixture {
17087 async fn drain(&mut self) {
17089 let child = self
17090 .child
17091 .take()
17092 .expect("the fixture child is still present");
17093 drain_child_to_state(
17094 &self.module_id,
17095 ModuleProtocol::Subc,
17096 StopNotice::NotSent,
17099 &self.registry,
17100 None,
17101 &self.snapshot,
17102 &self.terminal_ring,
17103 &self.spawn_events,
17104 child,
17105 Duration::from_millis(500),
17106 ModuleState::Stopped,
17107 Some(false),
17108 &FailureContext::operator(),
17109 )
17110 .await
17111 .expect("drain the supervised fixture");
17112 }
17113 }
17114
17115 #[tokio::test]
17121 async fn teardown_reaps_the_grandchild() {
17122 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17123 let grandchild = fixture.grandchild;
17124
17125 assert!(
17126 subc_jobobject::process_exists(grandchild),
17127 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17128 );
17129
17130 fixture.drain().await;
17131
17132 assert!(
17133 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17134 "grandchild {grandchild} outlived module teardown: the tree was not contained"
17135 );
17136 }
17137
17138 #[test]
17151 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17152 let dir = ScratchDir::new("teardown-uncontained");
17153 let pid_file = dir.join("grandchild.pid");
17154 let mut child = std::process::Command::new(stub_path())
17155 .env("FAKE_AFT_NEVER_CONNECT", "1")
17156 .env(
17157 "FAKE_AFT_GRANDCHILD_PID_FILE",
17158 pid_file.display().to_string(),
17159 )
17160 .stdin(std::process::Stdio::null())
17161 .stdout(std::process::Stdio::null())
17162 .stderr(std::process::Stdio::null())
17163 .spawn()
17164 .expect("spawn the uncontained fixture");
17165 let grandchild = read_grandchild_pid(&pid_file);
17166
17167 child.kill().expect("kill the direct child");
17169 let _ = child.wait();
17170
17171 assert!(
17172 subc_jobobject::process_exists(grandchild),
17173 "grandchild {grandchild} died with the direct child, so this control no longer \
17174 distinguishes contained from uncontained teardown and the regression test is \
17175 passing vacuously"
17176 );
17177
17178 kill_tree(grandchild);
17181 }
17182
17183 #[tokio::test]
17192 async fn dropping_containment_reaps_the_grandchild() {
17193 let mut fixture = fixture("drop-containment", "tree-drop");
17194 let grandchild = fixture.grandchild;
17195
17196 assert!(subc_jobobject::process_exists(grandchild));
17197
17198 fixture.child.as_mut().expect("child present").job = None;
17200
17201 assert!(
17202 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17203 "grandchild {grandchild} survived the containment handle closing, so a daemon \
17204 crash would leave the tree behind"
17205 );
17206 }
17207
17208 fn kill_tree(pid: u32) {
17210 let _ = std::process::Command::new("taskkill.exe")
17211 .args(["/PID", &pid.to_string(), "/T", "/F"])
17212 .stdin(std::process::Stdio::null())
17213 .stdout(std::process::Stdio::null())
17214 .stderr(std::process::Stdio::null())
17215 .status();
17216 assert!(
17217 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17218 "could not clean up grandchild {pid}"
17219 );
17220 }
17221}
17222
17223#[cfg(test)]
17224mod privacy_trampoline_configuration_tests {
17225 #[tokio::test]
17226 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17227 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17228 #[cfg(target_os = "macos")]
17229 {
17230 let supervisor = super::Supervisor::new(
17231 std::sync::Arc::new(crate::Registry::default()),
17232 super::RestartPolicy::default(),
17233 );
17234 let error = supervisor.spawn(spec()).unwrap_err();
17235 assert!(
17236 error
17237 .to_string()
17238 .contains("no privacy trampoline configured"),
17239 "{error}"
17240 );
17241 }
17242 }
17243
17244 #[tokio::test]
17245 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17246 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17247 #[cfg(target_os = "macos")]
17248 {
17249 let supervisor = super::Supervisor::new(
17250 std::sync::Arc::new(crate::Registry::default()),
17251 super::RestartPolicy::default(),
17252 )
17253 .with_privacy_trampoline(std::env::current_exe().unwrap());
17254 let error = supervisor.spawn(spec()).unwrap_err();
17255 assert!(
17256 error
17257 .to_string()
17258 .contains("binary does not implement the privacy trampoline protocol"),
17259 "{error}"
17260 );
17261 }
17262 }
17263
17264 #[cfg(target_os = "macos")]
17265 fn spec() -> super::ModuleSpec {
17266 super::ModuleSpec {
17267 module_id: "privacy-configuration".into(),
17268 program: "/bin/sleep".into(),
17269 args: vec!["30".into()],
17270 env: vec![],
17271 reserved: false,
17272 reserved_prefixes: vec![],
17273 protocol: subc_control::ModuleProtocol::None,
17274 overlap: super::ModuleOverlap::Exclusive,
17275 }
17276 }
17277}
17278
17279#[cfg(test)]
17280mod privacy_exec_boundary_tests {
17281 #[cfg(target_os = "macos")]
17282 use super::*;
17283 #[cfg(target_os = "macos")]
17284 use std::{
17285 io::{Read, Write},
17286 net::{TcpListener, TcpStream},
17287 };
17288
17289 #[cfg(target_os = "macos")]
17292 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
17293 if let Some((_, path)) = spec
17294 .env
17295 .iter()
17296 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
17297 {
17298 let mut barrier = TcpStream::connect(path).unwrap();
17299 barrier
17300 .set_read_timeout(Some(Duration::from_secs(30)))
17301 .unwrap();
17302 barrier.write_all(&pid.to_ne_bytes()).unwrap();
17303 let mut release = [0];
17304 barrier.read_exact(&mut release).unwrap();
17305 assert_eq!(&release, b"X");
17306 }
17307 }
17308
17309 #[cfg(target_os = "macos")]
17310 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
17311 ModuleSpec {
17312 module_id: "privacy-boundary".into(),
17313 program: program.into(),
17314 args: args.iter().map(|arg| (*arg).into()).collect(),
17315 env: vec![],
17316 reserved: false,
17317 reserved_prefixes: vec![],
17318 protocol: ModuleProtocol::None,
17319 overlap: ModuleOverlap::Exclusive,
17320 }
17321 }
17322
17323 #[cfg(target_os = "macos")]
17324 async fn accept(listener: TcpListener) -> TcpStream {
17325 let listener = tokio::net::TcpListener::from_std({
17327 listener.set_nonblocking(true).unwrap();
17328 listener
17329 })
17330 .unwrap();
17331 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
17332 .await
17333 .unwrap()
17334 .unwrap();
17335 let stream = stream.into_std().unwrap();
17336 stream.set_nonblocking(false).unwrap();
17337 stream
17338 .set_read_timeout(Some(Duration::from_secs(30)))
17339 .unwrap();
17340 stream
17341 }
17342
17343 #[tokio::test]
17344 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17345 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
17346 #[cfg(target_os = "macos")]
17347 {
17348 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
17349 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17352 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17353 let record = root.join("live-children.json");
17354 let supervisor =
17355 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17356 .with_live_children_record(&record);
17357 let runtime = supervisor.runtime_config();
17358 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17359 let mut spec = spec("/bin/sleep", &["30"]);
17360 spec.env = vec![
17361 (
17362 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
17363 exec_listener.local_addr().unwrap().to_string(),
17364 ),
17365 (
17366 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
17367 sample_listener.local_addr().unwrap().to_string(),
17368 ),
17369 ];
17370 let spawn = tokio::task::spawn_blocking(move || {
17371 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
17372 });
17373 let mut sample = accept(sample_listener).await;
17374 let mut pid = [0; 4];
17375 sample.read_exact(&mut pid).unwrap();
17376 let pid = u32::from_ne_bytes(pid);
17377 let mut exec = accept(exec_listener).await;
17378 let mut ready = [0];
17379 exec.read_exact(&mut ready).unwrap();
17380 assert_eq!(&ready, b"R");
17381 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
17384 assert_eq!(
17385 observe_spawned_image(pid).unwrap().executable,
17386 Some(trampoline)
17387 );
17388 sample.write_all(b"X").unwrap();
17389 let mut child = spawn.await.unwrap();
17390 let early = crate::live_children::read_record(&record).unwrap();
17391 assert_eq!(early.len(), 1);
17392 assert_eq!(early[0].pid, pid);
17393 assert_eq!(
17394 early[0].executable, None,
17395 "unconfirmed trampoline image entered the roster"
17396 );
17397 assert!(child.report_ready.get().is_none());
17398 child.privacy_exec.as_mut().unwrap().deadline =
17401 tokio::time::Instant::now() + Duration::from_secs(30);
17402 exec.write_all(b"X").unwrap();
17403 child.confirm_privacy_exec().await;
17404 assert_eq!(child.spawn_failure, None);
17405 assert!(child.report_ready.get().is_some());
17406 let confirmed = crate::live_children::read_record(&record).unwrap();
17407 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
17408 assert_ne!(module, trampoline);
17409 assert_eq!(confirmed[0].executable, Some(module.into()));
17410 child.start_kill().unwrap();
17411 child.wait().await.unwrap();
17412 child.release_roster();
17413 }
17414 }
17415
17416 #[tokio::test]
17417 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17418 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
17419 #[cfg(target_os = "macos")]
17420 {
17421 let registry = Arc::new(Registry::default());
17422 let policy = RestartPolicy::new(0, Duration::ZERO);
17423 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
17424 let runtime = supervisor.runtime_config();
17425 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17426 let spec = spec("/bin/sh", &["-c", "exit 121"]);
17427 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17431 let pid = child.pid;
17432 tokio::task::spawn_blocking(move || {
17433 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
17434 })
17435 .await
17436 .unwrap()
17437 .unwrap();
17438 child.privacy_exec.as_mut().unwrap().deadline =
17439 tokio::time::Instant::now() + Duration::from_secs(30);
17440 let status = child.wait().await.unwrap();
17441 assert_eq!(status.code(), Some(121));
17442 assert!(child.privacy_exec.is_none());
17443 assert!(
17444 child.report_ready.get().is_none(),
17445 "an exited module must not publish a live pid"
17446 );
17447 let report = classify_reaped_child_exit(&snapshot, &child, &status);
17448 on_child_exit(
17449 &spec,
17450 policy,
17451 ®istry,
17452 &snapshot,
17453 &runtime.terminal_ring,
17454 &runtime.spawn_events,
17455 &runtime.child_roster,
17456 report,
17457 true,
17458 )
17459 .await;
17460 let state = lock_snapshot(&snapshot).unwrap();
17461 assert_eq!(state.state, ModuleState::Failed);
17462 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
17463 assert_eq!(state.reported_pid(), None);
17464 drop(state);
17465 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17466 assert_eq!(history.entries.len(), 1);
17467 let terminal = &history.entries[0];
17468 assert_eq!(terminal.exit_code, Some(121));
17469 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
17470 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
17471 assert_eq!(
17472 terminal.disposition_detail.as_deref(),
17473 Some(policy.budget_exhausted_detail().as_str()),
17474 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
17475 );
17476 assert_eq!(child.spawn_failure, None);
17477 child.release_roster();
17478 }
17479 }
17480}
17481
17482#[cfg(all(test, unix))]
17486mod launch_nonce_descriptor_tests {
17487 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
17488 use crate::stderr_tail::{StderrRing, StderrTailConfig};
17489 use cortexkit_test_support::ScratchDir;
17490 use std::{
17491 path::PathBuf,
17492 sync::{Arc, Mutex},
17493 time::{Duration, Instant},
17494 };
17495
17496 async fn probe(role: super::SpawnRole) {
17497 let scratch = ScratchDir::new("launch-nonce-descriptor");
17498 let fd_copy = scratch.join("from-descriptor");
17499 let env_copy = scratch.join("environment");
17500 let script = format!(
17501 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
17502 fd = fd_copy.display(), env = env_copy.display(),
17503 );
17504 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
17505 let spec = ModuleSpec {
17506 module_id: "nonce-descriptor-probe".to_string(),
17507 program: PathBuf::from("/bin/sh"),
17508 args: vec!["-c".to_string(), script],
17509 env: vec![
17510 xdg("XDG_DATA_HOME"),
17511 xdg("XDG_RUNTIME_DIR"),
17512 xdg("XDG_CONFIG_HOME"),
17513 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
17514 ],
17515 reserved: true,
17516 reserved_prefixes: Vec::new(),
17517 protocol: ModuleProtocol::Subc,
17518 overlap: Default::default(),
17519 };
17520 let handle = SupervisorHandle::new();
17521 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17522 let roster = ChildRoster::default();
17523 #[cfg(target_os = "macos")]
17524 {
17525 let path = super::test_privacy_trampoline();
17526 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
17527 }
17528 let child = super::spawn_child_in_slot(
17529 &spec,
17530 None,
17531 Some(&handle),
17532 &ring,
17533 None,
17534 &roster,
17535 #[cfg(target_os = "linux")]
17536 None,
17537 role,
17538 matches!(role, super::SpawnRole::SwapCandidate),
17539 )
17540 .expect("spawn probe");
17541 let deadline = Instant::now() + Duration::from_secs(10);
17542 while !(fd_copy.exists() && env_copy.exists()) {
17543 assert!(Instant::now() < deadline, "probe never wrote its copies");
17544 tokio::time::sleep(Duration::from_millis(20)).await;
17545 }
17546 let nonce = std::fs::read_to_string(fd_copy).unwrap();
17547 assert!(!nonce.is_empty());
17548 let environment = std::fs::read_to_string(env_copy).unwrap();
17549 assert!(environment
17550 .lines()
17551 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
17552 let copy = environment
17553 .lines()
17554 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
17555 assert_eq!(
17556 copy, None,
17557 "Unix children must never receive the environment nonce"
17558 );
17559 if matches!(role, super::SpawnRole::Plain) {
17560 assert_eq!(
17561 handle.spawn_nonce(&spec.module_id).as_deref(),
17562 Some(nonce.as_str())
17563 );
17564 }
17565 drop(child);
17566 }
17567
17568 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17569 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
17570 probe(super::SpawnRole::Plain).await;
17571 }
17572
17573 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17574 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
17575 probe(super::SpawnRole::SwapCandidate).await;
17576 }
17577}
17578
17579#[cfg(all(test, target_os = "linux"))]
17580mod cgroup_containment_tests {
17581 use super::*;
17582 use cortexkit_test_support::ScratchDir;
17583
17584 fn running(pid: u32) -> bool {
17585 std::fs::read_to_string(format!("/proc/{pid}/stat"))
17587 .ok()
17588 .and_then(|stat| {
17589 stat.rsplit_once(") ")
17590 .map(|(_, rest)| rest.starts_with('Z'))
17591 })
17592 .is_some_and(|zombie| !zombie)
17593 }
17594
17595 #[tokio::test]
17596 async fn linux_teardown_reaps_the_grandchild() {
17597 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
17598 }
17599
17600 #[tokio::test]
17601 async fn linux_shutdown_straggler_reaps_the_grandchild() {
17602 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
17603 }
17604
17605 async fn teardown_tree(test_name: &str, shutdown: bool) {
17606 let dir = ScratchDir::new(test_name);
17607 let root = PathBuf::from(format!(
17608 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
17609 std::process::id(),
17610 unix_ms_now()
17611 ));
17612 if let Err(error) = std::fs::create_dir(&root) {
17613 assert!(
17614 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17615 "required cgroup test cannot execute: {error}"
17616 );
17617 eprintln!(
17618 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
17619 root.display()
17620 );
17621 return;
17622 }
17623 let placement = subc_cgroup::prepare_at(&root)
17624 .expect("prepare isolated kernel cgroup")
17625 .expect("isolated cgroup is delegated");
17626 let module_id = "tree-teardown";
17627 let module = placement
17628 .module_path(module_id)
17629 .expect("create isolated module cgroup");
17630 if !module.join("cgroup.kill").exists() {
17631 std::fs::remove_dir(&module).unwrap();
17632 std::fs::remove_dir(root.join("subc-modules")).unwrap();
17633 std::fs::remove_dir(&root).unwrap();
17634 assert!(
17635 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17636 "required cgroup.kill interface unavailable"
17637 );
17638 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
17639 return;
17640 }
17641 let supervisor = Supervisor::new_for_test(
17642 Arc::new(Registry::default()),
17643 RestartPolicy::new(3, Duration::ZERO),
17644 )
17645 .with_cgroup_placement(Some(placement));
17646 let mut runtime = supervisor.runtime_config();
17647 runtime.child_roster = runtime
17648 .child_roster
17649 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
17650 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17651 let pid_file = dir.join("grandchild.pid");
17652 let spec = ModuleSpec {
17653 module_id: module_id.to_string(),
17654 program: PathBuf::from("/bin/sh"),
17655 args: vec![
17656 "-c".into(),
17657 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
17658 "fixture".into(),
17659 pid_file.display().to_string(),
17660 ],
17661 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17662 .into_iter()
17663 .map(|key| (key.to_string(), dir.display().to_string()))
17664 .collect(),
17665 reserved: false,
17666 reserved_prefixes: Vec::new(),
17667 protocol: ModuleProtocol::None,
17668 overlap: Default::default(),
17669 };
17670 let child =
17671 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
17672 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17673 let grandchild: u32 = loop {
17674 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
17675 if let Ok(pid) = contents.trim().parse() {
17676 break pid;
17677 }
17678 }
17679 assert!(
17680 tokio::time::Instant::now() < deadline,
17681 "grandchild pid was not recorded"
17682 );
17683 tokio::time::sleep(Duration::from_millis(10)).await;
17684 };
17685 assert!(
17686 running(grandchild),
17687 "grandchild must be alive before teardown"
17688 );
17689 if shutdown {
17690 let mut child = child;
17691 crate::child_roster::end_children_for_daemon_shutdown(
17692 &runtime.child_roster,
17693 false,
17694 std::future::pending(),
17695 )
17696 .await;
17697 child.wait().await.expect("reap shutdown straggler");
17698 } else {
17699 drain_child_to_state(
17700 module_id,
17701 ModuleProtocol::None,
17702 StopNotice::NotSent,
17703 &Registry::default(),
17704 None,
17705 &snapshot,
17706 &runtime.terminal_ring,
17707 &SpawnEventFeed::default(),
17708 child,
17709 Duration::from_millis(100),
17710 ModuleState::Stopped,
17711 Some(false),
17712 &FailureContext::operator(),
17713 )
17714 .await
17715 .expect("real supervisor teardown");
17716 }
17717 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
17718 while running(grandchild) && tokio::time::Instant::now() < deadline {
17719 tokio::time::sleep(Duration::from_millis(10)).await;
17720 }
17721 let survived = running(grandchild);
17722 if survived {
17724 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
17725 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
17726 tokio::time::sleep(Duration::from_millis(100)).await;
17727 }
17728 if module.exists() {
17729 std::fs::remove_dir(&module).expect("remove empty module cgroup");
17730 }
17731 std::fs::remove_dir(root.join("subc-modules")).unwrap();
17732 std::fs::remove_dir(&root).unwrap();
17733 assert!(
17734 !survived,
17735 "grandchild {grandchild} outlived module teardown"
17736 );
17737 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
17738 }
17739}
17740
17741#[cfg(all(test, unix))]
17744mod record_write_stall_tests {
17745 use super::*;
17746 use std::sync::atomic::{AtomicBool, Ordering};
17747 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
17748
17749 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
17752 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
17756
17757 fn sleeping_spec() -> ModuleSpec {
17769 ModuleSpec {
17770 module_id: "record-write-stall".into(),
17771 program: "/bin/sleep".into(),
17772 args: vec!["600".into()],
17773 env: vec![],
17774 reserved: false,
17775 reserved_prefixes: vec![],
17776 protocol: ModuleProtocol::None,
17777 overlap: Default::default(),
17778 }
17779 }
17780
17781 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17782 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17783 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17784 let supervisor =
17785 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17786 .with_live_children_record(home.join("live-children.json"));
17787 let module = supervisor.spawn(sleeping_spec()).unwrap();
17788 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17793 loop {
17794 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17795 .unwrap()
17796 .reported_pid()
17797 {
17798 return pid;
17799 }
17800 tokio::time::sleep(Duration::from_millis(10)).await;
17801 }
17802 })
17803 .await
17804 .expect("the first process was admitted as the module");
17805
17806 let key = vec![0x42; 32];
17809 let daemon_id = [0x24; 16];
17810 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17811 let port = listener.local_addr().unwrap().port();
17812 let server = tokio::spawn(crate::server::serve_listener(
17813 listener,
17814 Arc::new(crate::router::Router::with_default_self_handler()),
17815 crate::server::ServerAuth::with_limits(
17816 key.clone(),
17817 daemon_id,
17818 "test",
17819 crate::server::DEFAULT_AUTH_DEADLINE,
17820 4,
17821 ),
17822 ));
17823
17824 let (held_tx, held_rx) = std::sync::mpsc::channel();
17827 let armed = AtomicBool::new(true);
17828 supervisor
17829 .child_roster
17830 .set_record_write_hook_for_test(move |live| {
17831 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17832 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17833 let _ = held_tx.send(());
17834 std::thread::sleep(HELD_RECORD_WRITE);
17835 }
17836 });
17837
17838 let module_process = std::thread::spawn(move || {
17841 held_rx
17842 .recv_timeout(Duration::from_secs(30))
17843 .expect("the restart reached the replacement's record write");
17844 let conn = ConnectionInfo {
17845 schema: SCHEMA_VERSION,
17846 wire_version: None,
17847 endpoints: vec![Endpoint {
17848 host: "127.0.0.1".to_owned(),
17849 port,
17850 }],
17851 key,
17852 daemon_id,
17853 pid: std::process::id(),
17854 daemon_ver: "test".to_owned(),
17855 };
17856 tokio::runtime::Builder::new_current_thread()
17857 .enable_all()
17858 .build()
17859 .unwrap()
17860 .block_on(async {
17861 let started = Instant::now();
17862 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17863 .await
17864 .unwrap();
17865 let result =
17866 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17867 (result, started.elapsed())
17868 })
17869 });
17870
17871 module.restart(Some(0)).await.unwrap();
17872 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17873 .await
17874 .unwrap();
17875 assert!(
17876 auth.is_ok(),
17877 "the daemon must answer a module's handshake while a restart waits on the \
17878 live-children record write; got {auth:?} after {elapsed:?}"
17879 );
17880
17881 tokio::time::timeout(Duration::from_secs(20), async {
17882 while lock_snapshot(&module.inner.snapshot)
17883 .unwrap()
17884 .reported_pid()
17885 .is_none_or(|pid| pid == old_pid)
17886 {
17887 tokio::time::sleep(Duration::from_millis(10)).await;
17888 }
17889 })
17890 .await
17891 .expect("the restart published its replacement");
17892 module.stop().await.unwrap();
17893 server.abort();
17894 }
17895}