1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62mod slow_retry;
63use slow_retry::{FailedRetry, FailureContext};
64
65#[cfg(all(test, unix))]
66mod slow_retry_tests;
67
68pub const SUBC_ARG: &str = "--subc";
74
75const DEFAULT_MAX_RESTARTS: u32 = 3;
76const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
77const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
78const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
82pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
93pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
94const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
95const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
96const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
108const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
131pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
133const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
134pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
139
140#[cfg(target_os = "macos")]
141struct PrivacyExec {
142 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
143 deadline: tokio::time::Instant,
144 expected: Option<subc_os::FileIdentity>,
145 trampoline: Option<subc_os::FileIdentity>,
146 script: bool,
147 module_id: String,
148 #[cfg(test)]
149 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
150}
151
152#[cfg(all(test, target_os = "macos"))]
153pub(crate) fn test_privacy_trampoline() -> PathBuf {
154 let path = std::env::current_exe()
157 .unwrap()
158 .parent()
159 .unwrap()
160 .parent()
161 .unwrap()
162 .join("privacy-trampoline-fixture");
163 assert!(
167 path.exists(),
168 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
169 --bins --features test-support` or `cargo test -p subc-daemon` first",
170 path.display()
171 );
172 path
173}
174
175#[cfg(target_os = "macos")]
176fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
177 use std::io::Read;
178 let mut probe = std::process::Command::new(path)
179 .args(["__disclaim-exec", "--probe"])
180 .stdin(Stdio::null())
181 .stdout(Stdio::piped())
182 .stderr(Stdio::piped())
183 .spawn()
184 .map_err(|error| {
185 format!(
186 "privacy trampoline probe failed for {}: {error}",
187 path.display()
188 )
189 })?;
190 let deadline = std::time::Instant::now() + Duration::from_secs(5);
191 let status = loop {
192 match probe.try_wait() {
193 Ok(Some(status)) => break status,
194 Ok(None) if std::time::Instant::now() < deadline => {
195 std::thread::sleep(Duration::from_millis(5))
196 }
197 result => {
198 let _ = probe.kill();
199 let _ = probe.wait();
200 return Err(format!(
201 "privacy trampoline probe failed or timed out for {}: {result:?}",
202 path.display()
203 ));
204 }
205 }
206 };
207 let mut answer = String::new();
208 if let Some(stdout) = probe.stdout.take() {
209 let _ = stdout.take(256).read_to_string(&mut answer);
210 }
211 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
212 return Ok(());
213 }
214 let mut diagnostic = String::new();
215 if let Some(stderr) = probe.stderr.take() {
216 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
217 }
218 let cause = diagnostic
219 .trim()
220 .strip_prefix("ck-subc: own privacy identity refused: ")
221 .unwrap_or("binary does not implement the privacy trampoline protocol");
222 Err(format!(
223 "{cause}: probe of {} exited {status}",
224 path.display()
225 ))
226}
227
228#[cfg(target_os = "macos")]
229fn privacy_command(
230 spec: &ModuleSpec,
231 roster: &ChildRoster,
232) -> Result<
233 (
234 Command,
235 Option<PrivacyExec>,
236 subc_os::privacy_identity::ExecAcknowledgement,
237 ),
238 SuperviseError,
239> {
240 let failure = |cause: String| {
241 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
242 SuperviseError::Spawn {
243 program: spec.program.clone(),
244 source: io::Error::other(cause),
245 cgroup_path: None,
246 }
247 };
248 let trampoline = roster.privacy_trampoline().map_err(failure)?;
249 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
254 let path = spec
255 .env
256 .iter()
257 .find(|(key, _)| key == "PATH")
258 .map(|(_, value)| std::ffi::OsString::from(value))
259 .or_else(|| std::env::var_os("PATH"))
260 .unwrap_or_else(|| "/usr/bin:/bin".into());
261 std::env::split_paths(&path)
262 .map(|dir| dir.join(&spec.program))
263 .find(|path| path.is_file())
264 .unwrap_or_else(|| spec.program.clone())
265 } else {
266 spec.program.clone()
267 };
268 let expected = subc_os::file_identity(&program);
269 let trampoline_image = subc_os::file_identity(&trampoline);
270 let script = {
271 use std::io::Read;
272 let mut prefix = [0u8; 2];
273 std::fs::File::open(&program)
274 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
275 };
276 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
277 return Err(failure(
278 "privacy identity module executable is missing or is the trampoline itself".to_string(),
279 ));
280 }
281 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
282 .map_err(|error| failure(error.to_string()))?;
283 let reader =
284 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
285 let mut command = Command::new(&trampoline);
286 command
287 .arg("__disclaim-exec")
288 .arg(ack.fd().to_string())
289 .arg(&program);
290 ack.install(command.as_std_mut());
291 Ok((
292 command,
293 Some(PrivacyExec {
294 reader,
295 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
296 expected,
297 trampoline: trampoline_image,
298 script,
299 module_id: spec.module_id.clone(),
300 #[cfg(test)]
301 confirmation_pause: spec
302 .env
303 .iter()
304 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
305 .map(|(_, address)| (address.clone(), None)),
306 }),
307 ack,
308 ))
309}
310
311struct SupervisedChild {
312 child: Child,
313 exec: crate::activity::ExecOwner,
314 #[cfg(windows)]
315 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
316 #[cfg(target_os = "macos")]
317 privacy_exec: Option<PrivacyExec>,
318 #[cfg(target_os = "macos")]
325 report_ready: Arc<OnceLock<()>>,
326 spawn_failure: Option<String>,
329 protocol: ModuleProtocol,
334 #[cfg(target_os = "linux")]
340 module_id: String,
341 #[cfg(target_os = "linux")]
342 cgroup_placement: Option<subc_cgroup::Placement>,
343 #[cfg(windows)]
365 job: Option<subc_jobobject::JobObject>,
366 stdout_pump: Option<JoinHandle<()>>,
367 stderr_pump: Option<StderrPump>,
368 stderr_ring: Arc<Mutex<StderrRing>>,
369 spawned_at: Instant,
372 registration_deadline: Option<Instant>,
375 registration_watch: Option<RegistrationWatch>,
376 spawned_at_ms: u64,
377 spawned_from: PathBuf,
378 launch_spec_digest: LaunchSpecDigest,
379 spawned_file_identity: Option<SpawnedFileIdentity>,
380 process_start_time: Option<u64>,
381 process_identity: Option<ProcessIdentity>,
382 pid: u32,
383 roster_guard: Option<crate::child_roster::RosterGuard>,
386}
387
388impl SupervisedChild {
389 fn registered_since_spawn(&self) -> bool {
390 self.registration_watch
391 .as_ref()
392 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
393 }
394
395 fn id(&self) -> Option<u32> {
396 Some(self.pid)
397 }
398
399 fn process_identity(&self) -> Option<ProcessIdentity> {
400 self.process_identity
401 }
402
403 async fn wait(&mut self) -> io::Result<ExitStatus> {
404 #[cfg(target_os = "macos")]
405 self.confirm_privacy_exec().await;
406 let result = self.child.wait().await;
414 #[cfg(target_os = "linux")]
415 if result.is_ok() {
416 if let Some(placement) = self.cgroup_placement.as_ref() {
417 cleanup_reaped_cgroup(placement, &self.module_id).await;
418 self.cgroup_placement = None;
421 }
422 }
423 result
424 }
425
426 #[cfg(target_os = "macos")]
427 async fn confirm_privacy_exec(&mut self) {
428 let Some(pending) = &mut self.privacy_exec else {
429 return;
430 };
431 let result = tokio::time::timeout_at(pending.deadline, async {
432 let mut record = Vec::new();
433 loop {
434 let mut ready = pending.reader.readable().await?;
435 let read = ready.try_io(|reader| {
436 use std::io::Read;
437 let mut reader = reader.get_ref();
438 let mut buffer = [0u8; 256];
439 reader.read(&mut buffer).map(|count| (count, buffer))
440 });
441 match read {
442 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
443 Ok(Ok((count, buffer))) => {
444 if record.len() + count > 1024 {
445 return Err(io::Error::other(
446 "privacy exec refusal record is too long",
447 ));
448 }
449 record.extend_from_slice(&buffer[..count]);
450 }
451 Ok(Err(error)) => return Err(error),
452 Err(_) => continue,
453 }
454 }
455 })
456 .await;
457 #[cfg(test)]
458 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
459 use tokio::io::{AsyncReadExt, AsyncWriteExt};
460 if stream.is_none() {
461 *stream = Some(
462 tokio::net::TcpStream::connect(address.as_str())
463 .await
464 .unwrap(),
465 );
466 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
467 stream
468 .as_mut()
469 .unwrap()
470 .write_all(&self.pid.to_ne_bytes())
471 .await
472 .unwrap();
473 }
474 let mut reply = [0];
475 stream
476 .as_mut()
477 .unwrap()
478 .read_exact(&mut reply)
479 .await
480 .unwrap();
481 pending.confirmation_pause = None;
482 if &reply == b"F" {
483 Ok(Err(io::Error::other("injected late exec refusal")))
484 } else {
485 assert_eq!(&reply, b"X");
486 result
487 }
488 } else {
489 result
490 };
491 let pending = self.privacy_exec.as_ref().expect("pending exec");
494 let cause = match result {
495 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
496 Ok(Err(error)) => Some(format!(
497 "privacy identity exec acknowledgement failed: {error}"
498 )),
499 Ok(Ok(record)) if !record.is_empty() => Some(
500 std::str::from_utf8(&record)
501 .ok()
502 .and_then(|record| {
503 record
504 .trim()
505 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
506 })
507 .filter(|cause| !cause.is_empty())
508 .unwrap_or("invalid privacy exec refusal record")
509 .to_string(),
510 ),
511 Ok(Ok(_)) => match self.child.try_wait() {
512 Ok(Some(_status)) => None,
516 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
517 Ok(None) => {
518 let image = observe_spawned_image(self.pid);
519 if let Some(image) = image.filter(|image| {
520 image.executable.is_some()
521 && image.executable != pending.trampoline
522 && (image.executable == pending.expected || pending.script)
523 }) {
524 if let Err(error) = self
525 .roster_guard
526 .as_ref()
527 .map(|guard| guard.confirm_executable(image))
528 .transpose()
529 {
530 Some(format!(
531 "could not durably record confirmed module image: {error}"
532 ))
533 } else {
534 let _ = self.report_ready.set(());
535 info!(module_id = %pending.module_id, pid = self.pid,
536 "module spawned with own privacy identity (responsibility disclaimed)");
537 None
538 }
539 } else if image.is_none()
540 || image.is_some_and(|image| image.executable.is_none())
541 {
542 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
549 Ok(Ok(_status)) => None,
550 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
551 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
552 }
553 } else {
554 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
555 }
556 }
557 },
558 };
559 let pending = self.privacy_exec.take().expect("pending exec");
560 self.exec.0.finish(if cause.is_some() {
563 crate::activity::ExecState::Refused
564 } else {
565 crate::activity::ExecState::CommittedExec
566 });
567 if let Some(cause) = cause {
568 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
569 self.spawn_failure = Some(cause);
570 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
573 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
574 }
575 let _ = self.child.start_kill();
576 }
577 }
578
579 fn release_roster(&mut self) {
583 self.roster_guard = None;
584 }
585
586 fn start_kill(&mut self) -> io::Result<()> {
599 #[cfg(windows)]
600 if let Some(job) = &self.job {
601 if let Err(error) = job.terminate() {
602 debug!(
603 error = %error,
604 "job termination failed; the direct-child kill still owns the outcome"
605 );
606 }
607 }
608 #[cfg(target_os = "linux")]
609 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
610 self.child.start_kill()
611 }
612
613 async fn drain_stderr(&mut self, module_id: &str) {
614 if let Some(pump) = self.stdout_pump.take() {
615 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
616 }
617
618 let Some(pump) = self.stderr_pump.take() else {
619 return;
620 };
621 settle_stderr_pump(
622 module_id,
623 &self.stderr_ring,
624 pump,
625 STDERR_PUMP_DRAIN_TIMEOUT,
626 )
627 .await;
628 }
629}
630
631async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
635 match timeout(bound, &mut pump).await {
636 Ok(Ok(())) => {}
637 Ok(Err(error)) => {
638 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
639 }
640 Err(_) => {
641 drop(pump);
642 warn!(
643 module_id,
644 waited = ?bound,
645 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
646 );
647 }
648 }
649}
650
651struct StderrPump {
654 task: JoinHandle<()>,
655 generation: u64,
656}
657
658async fn settle_stderr_pump(
664 module_id: &str,
665 ring: &Arc<Mutex<StderrRing>>,
666 pump: StderrPump,
667 bound: Duration,
668) {
669 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
670 let StderrPump {
671 mut task,
672 generation,
673 } = pump;
674 lock().retire_pump(generation);
675 match timeout(bound, &mut task).await {
676 Ok(Ok(())) => {}
677 Ok(Err(err)) => {
678 let mut ring = lock();
679 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
680 ring.finish_pump(generation);
681 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
682 }
683 Err(_) => {
684 drop(task);
686 lock().mark_pump_late(
687 generation,
688 format!(
689 "stderr of the exited process had not reached EOF {bound:?} after it was \
690 retired (a descendant may still hold the pipe open); lines it still \
691 writes are kept in that process's section"
692 ),
693 );
694 warn!(
695 module_id,
696 waited = ?bound,
697 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
698 );
699 }
700 }
701}
702
703fn registration_release_events() -> &'static watch::Sender<u64> {
704 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
705 EVENTS.get_or_init(|| {
706 let (sender, _receiver) = watch::channel(0);
707 sender
708 })
709}
710
711pub(crate) fn notify_registration_release() {
712 let events = registration_release_events();
713 let next_generation = (*events.borrow()).wrapping_add(1);
714 events.send_replace(next_generation);
715}
716
717#[derive(Debug, Clone, PartialEq, Eq)]
719pub struct ModuleSpec {
720 pub module_id: String,
721 pub program: PathBuf,
722 pub args: Vec<String>,
723 pub env: Vec<(String, String)>,
724 pub reserved: bool,
729 pub reserved_prefixes: Vec<String>,
734 pub protocol: ModuleProtocol,
753 pub overlap: ModuleOverlap,
758}
759
760#[derive(Clone, PartialEq, Eq)]
763pub(crate) struct LaunchSpecDigest([u8; 32]);
764
765impl fmt::Debug for LaunchSpecDigest {
766 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
767 f.write_str("LaunchSpecDigest([redacted])")
768 }
769}
770
771impl LaunchSpecDigest {
772 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
773 fn field(hash: &mut Sha256, bytes: &[u8]) {
774 hash.update((bytes.len() as u64).to_le_bytes());
775 hash.update(bytes);
776 }
777 let mut hash = Sha256::new();
778 if let Some(program) = spec.program.to_str() {
779 field(&mut hash, b"utf8-program");
780 field(&mut hash, program.as_bytes());
781 } else {
782 #[cfg(unix)]
785 {
786 use std::os::unix::ffi::OsStrExt;
787 field(&mut hash, b"unix-program");
788 field(&mut hash, spec.program.as_os_str().as_bytes());
789 }
790 #[cfg(windows)]
791 {
792 use std::os::windows::ffi::OsStrExt;
793 field(&mut hash, b"windows-program");
794 let bytes = spec
795 .program
796 .as_os_str()
797 .encode_wide()
798 .flat_map(u16::to_le_bytes)
799 .collect::<Vec<_>>();
800 field(&mut hash, &bytes);
801 }
802 #[cfg(not(any(unix, windows)))]
803 field(&mut hash, spec.program.to_string_lossy().as_bytes());
804 }
805 hash.update((spec.args.len() as u64).to_le_bytes());
806 for arg in &spec.args {
807 field(&mut hash, arg.as_bytes());
808 }
809 let mut env = spec.env.iter().collect::<Vec<_>>();
810 env.sort_unstable();
811 hash.update((env.len() as u64).to_le_bytes());
812 for (key, value) in env {
813 field(&mut hash, key.as_bytes());
814 field(&mut hash, value.as_bytes());
815 }
816 field(
817 &mut hash,
818 match spec.protocol {
819 ModuleProtocol::Subc => b"subc",
820 ModuleProtocol::None => b"none",
821 },
822 );
823 Self(hash.finalize().into())
824 }
825}
826
827#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
834pub enum ModuleOverlap {
835 #[default]
837 Exclusive,
838 Safe,
850}
851
852impl ModuleOverlap {
853 pub fn as_str(self) -> &'static str {
854 match self {
855 Self::Exclusive => "exclusive",
856 Self::Safe => "safe",
857 }
858 }
859}
860
861pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
871pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
873pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
878
879#[derive(Debug, Clone, Copy, PartialEq, Eq)]
897pub struct RestartPolicy {
898 pub max_restarts: u32,
899 pub backoff: Duration,
902 pub max_backoff: Duration,
904 pub window: Duration,
908}
909
910impl RestartPolicy {
911 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
915 Self {
916 max_restarts,
917 backoff,
918 max_backoff: DEFAULT_MAX_BACKOFF,
919 window: DEFAULT_RESTART_WINDOW,
920 }
921 }
922
923 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
924 self.max_backoff = max_backoff;
925 self
926 }
927
928 pub fn with_window(mut self, window: Duration) -> Self {
929 self.window = window;
930 self
931 }
932
933 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
938 if self.backoff.is_zero() || self.max_backoff.is_zero() {
939 return Duration::ZERO;
940 }
941
942 let mut delay = self.backoff;
943 for _ in 0..restart_in_window {
944 if delay >= self.max_backoff {
945 return self.max_backoff;
946 }
947 delay = delay
948 .checked_mul(10)
949 .unwrap_or(self.max_backoff)
950 .min(self.max_backoff);
951 }
952 delay.min(self.max_backoff)
953 }
954
955 fn budget_exhausted_detail(&self) -> String {
960 format!(
961 "crash budget exhausted: max_restarts={} within window_secs={}",
962 self.max_restarts,
963 self.window.as_secs()
964 )
965 }
966}
967
968impl Default for RestartPolicy {
969 fn default() -> Self {
970 Self {
971 max_restarts: DEFAULT_MAX_RESTARTS,
972 backoff: DEFAULT_BACKOFF,
973 max_backoff: DEFAULT_MAX_BACKOFF,
974 window: DEFAULT_RESTART_WINDOW,
975 }
976 }
977}
978
979#[derive(Debug, Clone, Copy, PartialEq, Eq)]
980struct CrashRestartSchedule {
981 restart_in_window: u32,
982 delay: Duration,
983}
984
985fn daemon_will_restart(
992 state: &mut SupervisorSnapshot,
993 policy: &RestartPolicy,
994 now: Instant,
995) -> bool {
996 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
997}
998
999const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1000const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1001const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1002const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1003
1004#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1005pub enum HealthAction {
1006 Report,
1007 Restart,
1008 Alert,
1009}
1010
1011impl fmt::Display for HealthAction {
1012 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1013 f.write_str(match self {
1014 Self::Report => "report",
1015 Self::Restart => "restart",
1016 Self::Alert => "alert",
1017 })
1018 }
1019}
1020
1021#[derive(Debug, Clone, PartialEq, Eq)]
1022pub struct HealthConfig {
1023 pub http: Option<String>,
1027 pub cadence: Duration,
1028 pub deadline: Duration,
1029 pub failure_threshold: u32,
1030 pub on_degraded: HealthAction,
1031 pub on_failing: HealthAction,
1032 pub critical: bool,
1033}
1034
1035impl Default for HealthConfig {
1036 fn default() -> Self {
1037 Self {
1038 http: None,
1039 cadence: DEFAULT_HEALTH_CADENCE,
1040 deadline: DEFAULT_HEALTH_DEADLINE,
1041 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1042 on_degraded: HealthAction::Report,
1043 on_failing: HealthAction::Report,
1044 critical: false,
1045 }
1046 }
1047}
1048
1049#[derive(Debug, Clone, PartialEq)]
1067pub struct ModuleHealthStatus {
1068 pub status: SupervisorHealthStatus,
1069 pub last_probe_ms: Option<u64>,
1070 pub detail: Option<String>,
1071 pub metrics: Option<Value>,
1072 pub consecutive_failures: u32,
1073 pub late_answer_count: u64,
1076 pub last_late_answer_latency_ms: Option<u64>,
1078 pub last_action: Option<String>,
1079 pub last_action_ms: Option<u64>,
1083}
1084
1085impl Default for ModuleHealthStatus {
1086 fn default() -> Self {
1087 Self {
1088 status: SupervisorHealthStatus::Unknown,
1089 last_probe_ms: None,
1090 detail: None,
1091 metrics: None,
1092 consecutive_failures: 0,
1093 late_answer_count: 0,
1094 last_late_answer_latency_ms: None,
1095 last_action: None,
1096 last_action_ms: None,
1097 }
1098 }
1099}
1100
1101#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1103pub enum ModuleState {
1104 Starting,
1105 Running,
1106 Unresponsive,
1107 Restarting,
1108 Draining,
1109 Stopped,
1110 Failed,
1111 Disabled,
1112}
1113
1114impl fmt::Display for ModuleState {
1115 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1116 f.write_str(match self {
1117 Self::Starting => "starting",
1118 Self::Running => "running",
1119 Self::Unresponsive => "unresponsive",
1120 Self::Restarting => "restarting",
1121 Self::Draining => "draining",
1122 Self::Stopped => "stopped",
1123 Self::Failed => "failed",
1124 Self::Disabled => "disabled",
1125 })
1126 }
1127}
1128
1129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1131pub enum ExitKind {
1132 Clean,
1133 Crash,
1134 DeliberateSeverance,
1135}
1136
1137impl From<ExitKind> for TerminalExitKind {
1138 fn from(kind: ExitKind) -> Self {
1139 match kind {
1140 ExitKind::Clean => Self::Clean,
1141 ExitKind::Crash => Self::Crash,
1142 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1143 }
1144 }
1145}
1146
1147#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1150pub(crate) struct ProcessIdentity {
1151 pub(crate) pid: u32,
1152 pub(crate) start_time: u64,
1153}
1154
1155#[derive(Debug, Clone, PartialEq, Eq)]
1157pub struct ExitReport {
1158 pub kind: ExitKind,
1159 pub code: Option<i32>,
1160 pub signal: Option<i32>,
1161 pub at_ms: u64,
1162}
1163
1164#[derive(Debug, Clone, PartialEq)]
1167pub struct ModuleStatus {
1168 pub module_id: String,
1169 pub waiting_for_previous_pid: Option<u32>,
1171 pub waiting_for_previous_detail: Option<String>,
1173 pub launch_nonce_source: Option<String>,
1178 pub state: ModuleState,
1179 pub enabled: bool,
1180 pub process_alive: bool,
1181 pub registration_active: bool,
1182 pub protocol: ModuleProtocol,
1187 pub live: bool,
1198 pub restart_count: u32,
1202 pub lifetime_restarts: u32,
1206 pub spawn_generation: u64,
1207 pub max_restarts: u32,
1212 pub restart_window: Duration,
1216 pub drain_timeout: Duration,
1220 pub restart_backoff: Duration,
1221 pub restart_max_backoff: Duration,
1222 pub pid: Option<u32>,
1227 pub spawned_at_ms: Option<u64>,
1228 pub spawned_from: Option<PathBuf>,
1229 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1230 pub process_start_time: Option<u64>,
1231 pub last_exit: Option<ExitReport>,
1232 pub health: ModuleHealthStatus,
1233}
1234
1235#[cfg(test)]
1236#[derive(Debug, Clone, Copy, PartialEq)]
1237struct ActorSelectCheckpoint {
1238 generation: u64,
1239 turn: u64,
1240 registered_connection: Option<ConnectionId>,
1241 next_probe_at: Option<Instant>,
1242 wake_after: Option<Duration>,
1243}
1244
1245#[derive(Debug, Clone, PartialEq)]
1246struct SupervisorSnapshot {
1247 failed_retry: FailedRetry,
1248 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1249 #[cfg(windows)]
1250 pipe_consumed: Option<Arc<OnceLock<()>>>,
1251 #[cfg(test)]
1253 actor_turns: u64,
1254 #[cfg(test)]
1257 actor_select: Option<ActorSelectCheckpoint>,
1258 state: ModuleState,
1259 enabled: bool,
1260 process_alive: bool,
1261 spawned_protocol: Option<ModuleProtocol>,
1262 spawn_failure: Option<String>,
1263 waiting_for_previous_pid: Option<u32>,
1264 waiting_for_previous_detail: Option<String>,
1265 crash_restarts: VecDeque<Instant>,
1271 lifetime_restarts: u32,
1272 spawn_generation: u64,
1281 pid: Option<u32>,
1282 #[cfg(target_os = "macos")]
1283 report_ready: Option<Arc<OnceLock<()>>>,
1284 reaped_pid: Option<u32>,
1286 respawn_pending: bool,
1288 coalesced_restart_pending: bool,
1290 spawned_at_ms: Option<u64>,
1291 spawned_from: Option<PathBuf>,
1292 launch_spec_digest: Option<LaunchSpecDigest>,
1293 spawned_file_identity: Option<SpawnedFileIdentity>,
1294 process_start_time: Option<u64>,
1295 deliberate_severance: Option<ProcessIdentity>,
1296 last_exit: Option<ExitReport>,
1297 drain_disposition_detail: Option<String>,
1299 health: ModuleHealthStatus,
1300 in_alternate_slot: bool,
1305 draining_to_replace: bool,
1312 configuration_updated_since_spawn: bool,
1318}
1319
1320impl SupervisorSnapshot {
1321 fn reported_pid(&self) -> Option<u32> {
1326 #[cfg(target_os = "macos")]
1327 if self
1328 .report_ready
1329 .as_ref()
1330 .is_some_and(|ready| ready.get().is_none())
1331 {
1332 return None;
1333 }
1334 self.pid
1335 }
1336
1337 fn starting() -> Self {
1338 Self::new(ModuleState::Starting, true)
1339 }
1340
1341 fn disabled() -> Self {
1342 Self::new(ModuleState::Disabled, false)
1343 }
1344
1345 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1349 while let Some(oldest) = self.crash_restarts.front() {
1350 if now.duration_since(*oldest) > window {
1351 self.crash_restarts.pop_front();
1352 } else {
1353 break;
1354 }
1355 }
1356 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1357 }
1358
1359 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1365 self.crash_restarts.push_back(now);
1366 while self.crash_restarts.len() > policy.max_restarts as usize {
1367 self.crash_restarts.pop_front();
1368 }
1369 self.lifetime_restarts += 1;
1370 }
1371
1372 fn next_crash_restart(
1376 &mut self,
1377 policy: &RestartPolicy,
1378 now: Instant,
1379 ) -> Option<CrashRestartSchedule> {
1380 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1381 if restart_in_window >= policy.max_restarts {
1382 return None;
1383 }
1384 self.record_crash_restart(policy, now);
1385 Some(CrashRestartSchedule {
1386 restart_in_window,
1387 delay: policy.delay_for_restart(restart_in_window),
1388 })
1389 }
1390
1391 fn clear_crash_restarts(&mut self) {
1396 self.crash_restarts.clear();
1397 }
1398
1399 fn new(state: ModuleState, enabled: bool) -> Self {
1400 Self {
1401 failed_retry: FailedRetry::default(),
1402 exec_outcome: None,
1403 #[cfg(test)]
1404 actor_turns: 0,
1405 #[cfg(test)]
1406 actor_select: None,
1407 state,
1408 enabled,
1409 process_alive: false,
1410 #[cfg(windows)]
1411 pipe_consumed: None,
1412 spawned_protocol: None,
1413 spawn_failure: None,
1414 waiting_for_previous_pid: None,
1415 waiting_for_previous_detail: None,
1416 crash_restarts: VecDeque::new(),
1417 lifetime_restarts: 0,
1418 spawn_generation: 0,
1419 pid: None,
1420 #[cfg(target_os = "macos")]
1421 report_ready: None,
1422 reaped_pid: None,
1423 respawn_pending: false,
1424 coalesced_restart_pending: false,
1425 spawned_at_ms: None,
1426 spawned_from: None,
1427 launch_spec_digest: None,
1428 spawned_file_identity: None,
1429 process_start_time: None,
1430 deliberate_severance: None,
1431 last_exit: None,
1432 drain_disposition_detail: None,
1433 health: ModuleHealthStatus::default(),
1434 in_alternate_slot: false,
1435 draining_to_replace: false,
1436 configuration_updated_since_spawn: false,
1437 }
1438 }
1439}
1440
1441type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1442
1443type SpawnSubscriberKey = (ConnectionId, u64);
1444
1445#[derive(Debug)]
1446struct SpawnSubscriber {
1447 version: u8,
1448 frames: mpsc::Sender<Frame>,
1449 lagged: Option<oneshot::Sender<SpawnCursor>>,
1453}
1454
1455#[derive(Debug)]
1456struct SpawnEventState {
1457 daemon_incarnation: String,
1458 seq: u64,
1459 capacity: usize,
1460 live: HashMap<String, LiveSpawn>,
1461 generations: HashMap<String, u64>,
1462 events: VecDeque<SpawnEvent>,
1463 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1464}
1465
1466impl Default for SpawnEventState {
1467 fn default() -> Self {
1468 Self {
1469 daemon_incarnation: "unconfigured".to_string(),
1470 seq: 0,
1471 capacity: SPAWN_EVENT_RING_CAPACITY,
1472 live: HashMap::new(),
1473 generations: HashMap::new(),
1474 events: VecDeque::new(),
1475 subscribers: HashMap::new(),
1476 }
1477 }
1478}
1479
1480#[derive(Debug, Clone, Default)]
1481struct SpawnEventFeed(
1482 Arc<Mutex<SpawnEventState>>,
1483 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1484);
1485
1486#[derive(Debug, Clone, PartialEq, Eq)]
1487pub(crate) enum SpawnSubscribeRefusal {
1488 ForeignIncarnation { current: String },
1489 TooOld { oldest: SpawnCursor },
1490 Frame(String),
1491}
1492
1493impl SpawnEventFeed {
1494 fn configure_incarnation(&self, daemon_incarnation: String) {
1495 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1496 state.daemon_incarnation = daemon_incarnation;
1497 state.seq = 0;
1498 state.live.clear();
1499 state.generations.clear();
1500 state.events.clear();
1501 state.subscribers.clear();
1502 }
1503
1504 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1505 SpawnCursor {
1506 daemon_incarnation: state.daemon_incarnation.clone(),
1507 seq: state.seq,
1508 }
1509 }
1510
1511 fn snapshot(&self) -> SpawnSnapshot {
1512 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1513 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1514 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1515 SpawnSnapshot {
1516 cursor: Self::cursor(&state),
1517 ring_bound: state.capacity as u64,
1518 live,
1519 }
1520 }
1521
1522 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1523 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1524 let generation = state
1525 .generations
1526 .get(module_id)
1527 .copied()
1528 .unwrap_or(0)
1529 .checked_add(1)
1530 .expect("spawn generation exhausted");
1531 state.generations.insert(module_id.to_string(), generation);
1532 let live = LiveSpawn {
1533 module_id: module_id.to_string(),
1534 spawn_generation: generation,
1535 pid,
1536 spawned_at_ms,
1537 };
1538 state.live.insert(module_id.to_string(), live);
1539 Self::emit_locked(
1540 &mut state,
1541 SpawnEventKind::Spawned,
1542 module_id.to_string(),
1543 generation,
1544 pid,
1545 None,
1546 None,
1547 );
1548 generation
1549 }
1550
1551 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1552 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1553 let Some(live) = state.live.remove(module_id) else {
1554 warn!(
1555 module_id,
1556 "terminal record had no live spawn event identity"
1557 );
1558 return;
1559 };
1560 Self::emit_locked(
1561 &mut state,
1562 SpawnEventKind::Exited,
1563 module_id.to_string(),
1564 live.spawn_generation,
1565 live.pid,
1566 exit_code,
1567 exit_signal,
1568 );
1569 }
1570
1571 fn emit_superseded_exited(
1578 &self,
1579 module_id: &str,
1580 spawn_generation: u64,
1581 pid: u32,
1582 exit_code: Option<i32>,
1583 exit_signal: Option<i32>,
1584 ) {
1585 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1586 if state
1587 .live
1588 .get(module_id)
1589 .is_some_and(|live| live.spawn_generation == spawn_generation)
1590 {
1591 state.live.remove(module_id);
1592 }
1593 Self::emit_locked(
1594 &mut state,
1595 SpawnEventKind::Exited,
1596 module_id.to_string(),
1597 spawn_generation,
1598 pid,
1599 exit_code,
1600 exit_signal,
1601 );
1602 }
1603
1604 #[allow(clippy::too_many_arguments)]
1605 fn emit_locked(
1606 state: &mut SpawnEventState,
1607 kind: SpawnEventKind,
1608 module_id: String,
1609 spawn_generation: u64,
1610 pid: u32,
1611 exit_code: Option<i32>,
1612 exit_signal: Option<i32>,
1613 ) {
1614 state.seq = state
1615 .seq
1616 .checked_add(1)
1617 .expect("spawn event sequence exhausted");
1618 let event = SpawnEvent {
1619 cursor: Self::cursor(state),
1620 kind,
1621 module_id,
1622 spawn_generation,
1623 pid,
1624 exit_code,
1625 exit_signal,
1626 };
1627 state.events.push_back(event.clone());
1628 while state.events.len() > state.capacity {
1629 state.events.pop_front();
1630 }
1631 let body = match serde_json::to_vec(&event) {
1632 Ok(body) => body,
1633 Err(error) => {
1634 error!(%error, "failed to serialize supervisor spawn event");
1635 return;
1636 }
1637 };
1638 state.subscribers.retain(|(connection_id, corr), subscriber| {
1639 let frame = Frame::build_with_version(
1640 subscriber.version,
1641 FrameType::StreamData,
1642 control_flags(),
1643 0,
1644 0,
1645 *corr,
1646 body.clone(),
1647 );
1648 match frame {
1649 Ok(frame) => {
1650 if subscriber.frames.try_send(frame).is_ok() {
1651 true
1652 } else {
1653 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1654 if let Some(lagged) = subscriber.lagged.take() {
1655 let _ = lagged.send(event.cursor.clone());
1656 }
1657 false
1658 }
1659 }
1660 Err(error) => {
1661 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1662 false
1663 }
1664 }
1665 });
1666 }
1667
1668 fn subscribe(
1669 &self,
1670 connection_id: ConnectionId,
1671 corr: u64,
1672 version: u8,
1673 since: Option<SpawnCursor>,
1674 sink: FrameSink,
1675 ) -> Result<(), SpawnSubscribeRefusal> {
1676 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1677 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1678 {
1679 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1680 let replay = if let Some(since) = since {
1681 if since.daemon_incarnation != state.daemon_incarnation {
1682 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1683 current: state.daemon_incarnation.clone(),
1684 });
1685 }
1686 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1687 if since.seq < oldest.seq.saturating_sub(1) {
1688 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1689 }
1690 }
1691 state
1692 .events
1693 .iter()
1694 .filter(|event| event.cursor.seq > since.seq)
1695 .cloned()
1696 .collect::<Vec<_>>()
1697 } else {
1698 Vec::new()
1699 };
1700 for event in replay {
1701 let body = serde_json::to_vec(&event)
1702 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1703 let frame = Frame::build_with_version(
1704 version,
1705 FrameType::StreamData,
1706 control_flags(),
1707 0,
1708 0,
1709 corr,
1710 body,
1711 )
1712 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1713 frames
1714 .try_send(frame)
1715 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1716 }
1717 state.subscribers.insert(
1718 (connection_id, corr),
1719 SpawnSubscriber {
1720 version,
1721 frames,
1722 lagged: Some(lagged),
1723 },
1724 );
1725 }
1726 tokio::spawn(async move {
1737 while let Some(frame) = receiver.recv().await {
1738 if sink.send(frame).await.is_err() {
1739 return;
1740 }
1741 }
1742 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1743 return;
1744 };
1745 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1746 Ok(frame) => {
1747 let _ = sink.send(frame).await;
1748 }
1749 Err(error) => {
1750 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1751 }
1752 }
1753 });
1754 Ok(())
1755 }
1756
1757 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1758 let Some(subscriber) = self
1759 .0
1760 .lock()
1761 .unwrap_or_else(|p| p.into_inner())
1762 .subscribers
1763 .remove(&(connection_id, corr))
1764 else {
1765 return false;
1766 };
1767 if let Ok(frame) = Frame::build_with_version(
1768 subscriber.version,
1769 FrameType::StreamEnd,
1770 control_flags(),
1771 0,
1772 0,
1773 corr,
1774 Vec::new(),
1775 ) {
1776 tokio::spawn(async move {
1777 let _ = subscriber.frames.send(frame).await;
1778 });
1779 }
1780 true
1781 }
1782
1783 fn remove_connection(&self, connection_id: ConnectionId) {
1784 self.0
1785 .lock()
1786 .unwrap_or_else(|p| p.into_inner())
1787 .subscribers
1788 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1789 }
1790
1791 #[cfg(any(test, feature = "test-support"))]
1792 fn set_capacity(&self, capacity: usize) {
1793 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1794 }
1795
1796 #[cfg(any(test, feature = "test-support"))]
1797 fn subscriber_count(&self) -> usize {
1798 self.0
1799 .lock()
1800 .unwrap_or_else(|p| p.into_inner())
1801 .subscribers
1802 .len()
1803 }
1804}
1805
1806fn spawn_subscriber_lagged_frame(
1809 version: u8,
1810 corr: u64,
1811 first_undelivered: SpawnCursor,
1812) -> Result<Frame, String> {
1813 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1815 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1816 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1817 .to_string(),
1818 detail: Some(serde_json::json!({
1819 "first_undelivered_cursor": first_undelivered
1820 })),
1821 })
1822 .map_err(|error| error.to_string())?;
1823 Frame::build_with_version(
1824 version,
1825 FrameType::Error,
1826 crate::router::daemon_error_flags(),
1827 0,
1828 0,
1829 corr,
1830 body,
1831 )
1832 .map_err(|error| error.to_string())
1833}
1834
1835pub trait ModuleProcessLiveness: Send + Sync {
1836 fn process_live(&self, module_id: &str) -> Option<bool>;
1837
1838 fn process_replacing(&self, _module_id: &str) -> bool {
1844 false
1845 }
1846}
1847
1848#[derive(Debug, Clone, Default)]
1850pub struct SupervisorProcessLiveness {
1851 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1852}
1853
1854impl SupervisorProcessLiveness {
1855 pub fn new() -> Self {
1856 Self::default()
1857 }
1858
1859 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1860 let mut snapshots = self
1861 .snapshots
1862 .lock()
1863 .unwrap_or_else(|poisoned| poisoned.into_inner());
1864 snapshots.insert(module_id, snapshot);
1865 }
1866
1867 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1868 let mut snapshots = self
1869 .snapshots
1870 .lock()
1871 .unwrap_or_else(|poisoned| poisoned.into_inner());
1872 let is_current = snapshots
1873 .get(module_id)
1874 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1875 .unwrap_or(false);
1876 if is_current {
1877 snapshots.remove(module_id);
1878 }
1879 }
1880}
1881
1882impl ModuleProcessLiveness for SupervisorProcessLiveness {
1883 fn process_live(&self, module_id: &str) -> Option<bool> {
1884 let snapshot = {
1885 let snapshots = self
1886 .snapshots
1887 .lock()
1888 .unwrap_or_else(|poisoned| poisoned.into_inner());
1889 snapshots.get(module_id).cloned()
1890 }?;
1891 let snapshot = snapshot
1892 .lock()
1893 .unwrap_or_else(|poisoned| poisoned.into_inner());
1894 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1895 }
1896
1897 fn process_replacing(&self, module_id: &str) -> bool {
1898 let Some(snapshot) = self
1899 .snapshots
1900 .lock()
1901 .unwrap_or_else(|poisoned| poisoned.into_inner())
1902 .get(module_id)
1903 .cloned()
1904 else {
1905 return false;
1906 };
1907 let snapshot = snapshot
1908 .lock()
1909 .unwrap_or_else(|poisoned| poisoned.into_inner());
1910 snapshot.enabled
1911 && match snapshot.state {
1912 ModuleState::Restarting => true,
1913 ModuleState::Draining => snapshot.draining_to_replace,
1914 ModuleState::Starting
1915 | ModuleState::Running
1916 | ModuleState::Unresponsive
1917 | ModuleState::Stopped
1918 | ModuleState::Failed
1919 | ModuleState::Disabled => false,
1920 }
1921 }
1922}
1923
1924#[cfg(test)]
1925#[derive(Debug, Default)]
1926struct ReloadExitRecordGate {
1927 reached: tokio::sync::Notify,
1928 resume: tokio::sync::Notify,
1929}
1930
1931#[derive(Debug, Clone, Copy)]
1932enum RespawnKind {
1933 Spawn,
1934 Reload,
1935}
1936
1937#[derive(Debug, Clone, Copy)]
1938struct PendingRespawn {
1939 deadline: Instant,
1940 kind: RespawnKind,
1941 daemon_initiated: bool,
1942}
1943
1944type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1945
1946#[derive(Debug, Clone)]
1947struct SupervisorRuntimeConfig {
1948 registry: Arc<Registry>,
1949 recovery_is_daemon: bool,
1951 start_requester: Option<Principal>,
1952 pending_activity_restart: bool,
1953 provenance_probe: ExecutableIdentityProbe,
1954 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1956 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1958 restart_policy: RestartPolicy,
1959 register_timeout: Duration,
1960 drain_timeout: Duration,
1963 effective_drain_timeout: Arc<Mutex<Duration>>,
1966 default_drain_timeout: Duration,
1969 health: HealthConfig,
1970 connection_file_path: Option<PathBuf>,
1971 capture_logs_dir: Option<PathBuf>,
1972 forwarding: Option<Arc<ForwardingTable>>,
1973 supervisor_handle: Option<SupervisorHandle>,
1976 stderr_ring: Arc<Mutex<StderrRing>>,
1983 terminal_ring: Arc<Mutex<TerminalRing>>,
1984 spawn_events: SpawnEventFeed,
1985 child_roster: ChildRoster,
1986 #[cfg(target_os = "linux")]
1987 cgroup_placement: Option<subc_cgroup::Placement>,
1988 #[cfg(test)]
1989 test_seed_stale_facts_before_enable_spawn: bool,
1990 #[cfg(test)]
1991 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1992}
1993
1994impl SupervisorRuntimeConfig {
1995 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
1996 FailureContext {
1997 daemon_initiated,
1998 shutdown: self.child_roster.shutdown_flag(),
1999 }
2000 }
2001}
2002
2003#[derive(Debug, Clone, PartialEq, Eq)]
2004struct SupervisedConfiguration {
2005 spec: ModuleSpec,
2006 health: HealthConfig,
2007}
2008
2009#[derive(Debug, Clone, Default)]
2015pub struct SupervisorHandle {
2016 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2017 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2018 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2019 configured_ids: Arc<Mutex<HashSet<String>>>,
2031 spawn_events: SpawnEventFeed,
2032 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2043 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2049 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2053 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2061 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2067 promotion_observer: PromotionObserverSlot,
2069 operation_lock: Arc<AsyncMutex<()>>,
2073}
2074
2075pub(crate) trait SwapPromotionObserver: Send + Sync {
2084 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2085}
2086
2087#[derive(Clone, Default)]
2091struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2092
2093impl fmt::Debug for PromotionObserverSlot {
2094 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2095 f.write_str("PromotionObserverSlot")
2096 }
2097}
2098
2099#[derive(Debug, Clone)]
2101struct OpenSwap {
2102 candidate_nonce: String,
2105 incumbent_nonce: Option<String>,
2110 candidate_admitted: bool,
2114}
2115
2116#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2119pub(crate) enum SwapHelloAdmission {
2120 NotSwapping,
2123 Candidate,
2125 Refused,
2128}
2129
2130#[derive(Debug, Clone, PartialEq, Eq)]
2131pub(crate) enum ReservedHelloRejection {
2132 Exact {
2133 module_id: String,
2134 },
2135 Prefix {
2136 prefix: String,
2137 owner_module_id: String,
2138 },
2139}
2140
2141impl SupervisorHandle {
2142 pub fn new() -> Self {
2143 Self::default()
2144 }
2145
2146 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2147 self.spawn_events.snapshot()
2148 }
2149
2150 pub(crate) fn subscribe_spawns(
2151 &self,
2152 connection_id: ConnectionId,
2153 corr: u64,
2154 version: u8,
2155 since: Option<SpawnCursor>,
2156 sink: FrameSink,
2157 ) -> Result<(), SpawnSubscribeRefusal> {
2158 self.spawn_events
2159 .subscribe(connection_id, corr, version, since, sink)
2160 }
2161
2162 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2163 self.spawn_events.cancel(connection_id, corr)
2164 }
2165
2166 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2167 self.spawn_events.remove_connection(connection_id);
2168 }
2169
2170 #[cfg(any(test, feature = "test-support"))]
2171 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2172 assert!(capacity > 0, "spawn event capacity must be non-zero");
2173 self.spawn_events.set_capacity(capacity);
2174 }
2175
2176 #[cfg(any(test, feature = "test-support"))]
2177 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2178 self.spawn_events.subscriber_count()
2179 }
2180
2181 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2184 self.spawn_nonces
2185 .lock()
2186 .unwrap_or_else(|poisoned| poisoned.into_inner())
2187 .insert(module_id.to_string(), nonce);
2188 }
2189
2190 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2193 self.reserved_nonces
2194 .lock()
2195 .unwrap_or_else(|poisoned| poisoned.into_inner())
2196 .insert(module_id.to_string(), Some(nonce));
2197 }
2198
2199 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2201 let mut owners = self
2202 .reserved_prefix_owners
2203 .lock()
2204 .unwrap_or_else(|poisoned| poisoned.into_inner());
2205 owners.retain(|_, owner| owner != owner_module_id);
2206 for prefix in prefixes {
2207 owners.insert(prefix.clone(), owner_module_id.to_string());
2208 }
2209 }
2210
2211 #[cfg(test)]
2213 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2214 self.spawn_nonces
2215 .lock()
2216 .unwrap_or_else(|poisoned| poisoned.into_inner())
2217 .get(module_id)
2218 .cloned()
2219 }
2220
2221 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2222 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2223 let spawn_nonce = self
2224 .spawn_nonces
2225 .lock()
2226 .unwrap_or_else(|poisoned| poisoned.into_inner())
2227 .get(&spec.module_id)
2228 .cloned();
2229 let mut reserved_nonces = self
2230 .reserved_nonces
2231 .lock()
2232 .unwrap_or_else(|poisoned| poisoned.into_inner());
2233 if spec.reserved {
2234 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2239 }
2240 drop(reserved_nonces);
2241 self.removal_tombstones
2245 .lock()
2246 .unwrap_or_else(|poisoned| poisoned.into_inner())
2247 .remove(&spec.module_id);
2248 }
2249
2250 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2255 self.reserved_hello_verification(module_id, presented)
2256 .is_ok()
2257 }
2258
2259 pub(crate) fn reserved_hello_verification(
2262 &self,
2263 module_id: &str,
2264 presented: Option<&str>,
2265 ) -> Result<bool, ReservedHelloRejection> {
2266 let nonces = self
2267 .reserved_nonces
2268 .lock()
2269 .unwrap_or_else(|poisoned| poisoned.into_inner());
2270 if let Some(expected) = nonces.get(module_id) {
2271 let authorized = match expected {
2275 Some(expected) => {
2276 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2277 }
2278 None => false,
2279 };
2280 if authorized {
2281 return Ok(true);
2282 }
2283 return Err(ReservedHelloRejection::Exact {
2284 module_id: module_id.to_string(),
2285 });
2286 }
2287 drop(nonces);
2288
2289 let matched_prefix = self
2290 .reserved_prefix_owners
2291 .lock()
2292 .unwrap_or_else(|poisoned| poisoned.into_inner())
2293 .iter()
2294 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2295 .max_by_key(|(prefix, _)| prefix.len())
2296 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2297 let Some((prefix, owner_module_id)) = matched_prefix else {
2298 return Ok(false);
2299 };
2300
2301 let authorized = presented.is_some_and(|presented| {
2302 self.spawn_nonces
2303 .lock()
2304 .unwrap_or_else(|poisoned| poisoned.into_inner())
2305 .get(&owner_module_id)
2306 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2307 || self.swap_nonce_matches(&owner_module_id, presented)
2310 });
2311 if authorized {
2312 Ok(true)
2313 } else {
2314 Err(ReservedHelloRejection::Prefix {
2315 prefix,
2316 owner_module_id,
2317 })
2318 }
2319 }
2320
2321 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2326 if presented.is_empty() {
2327 return false;
2328 }
2329 let nonces = self
2330 .spawn_nonces
2331 .lock()
2332 .unwrap_or_else(|poisoned| poisoned.into_inner());
2333 let current = nonces
2334 .get(module_id)
2335 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2336 drop(nonces);
2337 current || self.swap_nonce_matches(module_id, presented)
2342 }
2343
2344 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2346 let swaps = self
2347 .swaps
2348 .lock()
2349 .unwrap_or_else(|poisoned| poisoned.into_inner());
2350 swaps.get(module_id).is_some_and(|swap| {
2351 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2352 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2353 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2354 })
2355 })
2356 }
2357
2358 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2361 let incumbent_nonce = self
2362 .spawn_nonces
2363 .lock()
2364 .unwrap_or_else(|poisoned| poisoned.into_inner())
2365 .get(module_id)
2366 .cloned();
2367 self.swaps
2368 .lock()
2369 .unwrap_or_else(|poisoned| poisoned.into_inner())
2370 .insert(
2371 module_id.to_string(),
2372 OpenSwap {
2373 candidate_nonce,
2374 incumbent_nonce,
2375 candidate_admitted: false,
2376 },
2377 );
2378 }
2379
2380 pub(crate) fn close_swap(&self, module_id: &str) {
2383 self.swaps
2384 .lock()
2385 .unwrap_or_else(|poisoned| poisoned.into_inner())
2386 .remove(module_id);
2387 }
2388
2389 pub(crate) fn set_swap_promotion_observer(
2392 &self,
2393 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2394 ) {
2395 *self
2396 .promotion_observer
2397 .0
2398 .lock()
2399 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2400 }
2401
2402 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2405 let observer = self
2406 .promotion_observer
2407 .0
2408 .lock()
2409 .unwrap_or_else(|poisoned| poisoned.into_inner())
2410 .as_ref()
2411 .and_then(std::sync::Weak::upgrade);
2412 if let Some(observer) = observer {
2413 observer.swap_promoted(registration);
2414 }
2415 }
2416
2417 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2419 self.swaps
2420 .lock()
2421 .unwrap_or_else(|poisoned| poisoned.into_inner())
2422 .contains_key(module_id)
2423 }
2424
2425 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2430 let candidate_nonce = self
2431 .swaps
2432 .lock()
2433 .unwrap_or_else(|poisoned| poisoned.into_inner())
2434 .get(module_id)
2435 .map(|swap| swap.candidate_nonce.clone());
2436 let Some(nonce) = candidate_nonce else {
2437 return;
2438 };
2439 self.set_spawn_nonce(module_id, nonce.clone());
2440 if reserved {
2441 self.set_reserved_nonce(module_id, nonce);
2442 }
2443 }
2444
2445 pub(crate) fn swap_hello_admission(
2460 &self,
2461 module_id: &str,
2462 presented: Option<&str>,
2463 ) -> SwapHelloAdmission {
2464 let swaps = self
2465 .swaps
2466 .lock()
2467 .unwrap_or_else(|poisoned| poisoned.into_inner());
2468 let Some(swap) = swaps.get(module_id) else {
2469 return SwapHelloAdmission::NotSwapping;
2470 };
2471 let Some(presented) = presented else {
2472 return SwapHelloAdmission::Refused;
2473 };
2474 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2475 return if swap.candidate_admitted {
2476 SwapHelloAdmission::Refused
2477 } else {
2478 SwapHelloAdmission::Candidate
2479 };
2480 }
2481 if swap
2482 .incumbent_nonce
2483 .as_deref()
2484 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2485 {
2486 return SwapHelloAdmission::NotSwapping;
2487 }
2488 SwapHelloAdmission::Refused
2489 }
2490
2491 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2494 if let Some(swap) = self
2495 .swaps
2496 .lock()
2497 .unwrap_or_else(|poisoned| poisoned.into_inner())
2498 .get_mut(module_id)
2499 {
2500 swap.candidate_admitted = true;
2501 }
2502 }
2503
2504 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2506 self.spawn_nonces
2507 .lock()
2508 .unwrap_or_else(|poisoned| poisoned.into_inner())
2509 .get(module_id)
2510 .cloned()
2511 }
2512
2513 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2515 self.reserved_nonces
2516 .lock()
2517 .unwrap_or_else(|poisoned| poisoned.into_inner())
2518 .get(module_id)
2519 .cloned()
2520 .flatten()
2521 }
2522
2523 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2524 self.mark_configured(module.module_id());
2528 let mut modules = self
2529 .modules
2530 .lock()
2531 .unwrap_or_else(|poisoned| poisoned.into_inner());
2532 modules.insert(module.module_id().to_string(), module)
2533 }
2534
2535 fn mark_configured(&self, module_id: &str) {
2539 self.configured_ids
2540 .lock()
2541 .unwrap_or_else(|poisoned| poisoned.into_inner())
2542 .insert(module_id.to_string());
2543 }
2544
2545 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2549 let modules = self
2550 .modules
2551 .lock()
2552 .unwrap_or_else(|poisoned| poisoned.into_inner());
2553 if !modules.contains_key(module_id) {
2554 self.configured_ids
2555 .lock()
2556 .unwrap_or_else(|poisoned| poisoned.into_inner())
2557 .remove(module_id);
2558 }
2559 }
2560
2561 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2569 self.configured_ids
2570 .lock()
2571 .unwrap_or_else(|poisoned| poisoned.into_inner())
2572 .contains(module_id)
2573 }
2574
2575 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2576 let modules = self
2577 .modules
2578 .lock()
2579 .unwrap_or_else(|poisoned| poisoned.into_inner());
2580 modules.get(module_id).cloned()
2581 }
2582
2583 pub(crate) fn record_late_health_answer(
2584 &self,
2585 module_id: &str,
2586 latency_ms: u64,
2587 ) -> Result<bool, SuperviseError> {
2588 let Some(module) = self.get(module_id) else {
2589 return Ok(false);
2590 };
2591 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2592 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2593 state.health.last_late_answer_latency_ms = Some(latency_ms);
2594 state.health.consecutive_failures = 0;
2602 })?;
2603 Ok(true)
2604 }
2605
2606 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2612 let Some(module) = self.get(module_id) else {
2613 return Ok(false);
2614 };
2615 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2616 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2617 return Ok(false);
2618 };
2619 drop(snapshot);
2620 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2621 }
2622
2623 pub fn list(&self) -> Vec<SupervisedModule> {
2624 let modules = self
2625 .modules
2626 .lock()
2627 .unwrap_or_else(|poisoned| poisoned.into_inner());
2628 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2629 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2630 modules
2631 }
2632
2633 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2634 self.activity_images
2635 .lock()
2636 .unwrap_or_else(|p| p.into_inner())
2637 .remove(module_id);
2638 self.spawn_nonces
2639 .lock()
2640 .unwrap_or_else(|poisoned| poisoned.into_inner())
2641 .remove(module_id);
2642 self.close_swap(module_id);
2643 let mut reserved_nonces = self
2644 .reserved_nonces
2645 .lock()
2646 .unwrap_or_else(|poisoned| poisoned.into_inner());
2647 if reserved_nonces.contains_key(module_id) {
2648 reserved_nonces.insert(module_id.to_string(), None);
2651 }
2652 drop(reserved_nonces);
2653 self.reserved_prefix_owners
2654 .lock()
2655 .unwrap_or_else(|poisoned| poisoned.into_inner())
2656 .retain(|_, owner| owner != module_id);
2657 let removed = self
2658 .modules
2659 .lock()
2660 .unwrap_or_else(|poisoned| poisoned.into_inner())
2661 .remove(module_id);
2662 self.configured_ids
2663 .lock()
2664 .unwrap_or_else(|poisoned| poisoned.into_inner())
2665 .remove(module_id);
2666 removed
2667 }
2668
2669 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2672 self.removal_tombstones
2673 .lock()
2674 .unwrap_or_else(|poisoned| poisoned.into_inner())
2675 .insert(module_id.to_string(), unix_ms_now());
2676 }
2677
2678 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2680 self.removal_tombstones
2681 .lock()
2682 .unwrap_or_else(|poisoned| poisoned.into_inner())
2683 .get(module_id)
2684 .copied()
2685 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2686 }
2687
2688 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2693 if self.get(module_id).is_some() {
2694 return false;
2695 }
2696 let mut reserved_nonces = self
2697 .reserved_nonces
2698 .lock()
2699 .unwrap_or_else(|poisoned| poisoned.into_inner());
2700 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2701 return false;
2702 }
2703 reserved_nonces.remove(module_id);
2704 true
2705 }
2706
2707 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2708 Arc::clone(&self.operation_lock)
2709 }
2710}
2711
2712#[derive(Debug, Clone)]
2714pub struct Supervisor {
2715 registry: Arc<Registry>,
2716 restart_policy: RestartPolicy,
2717 drain_timeout: Duration,
2718 connection_file_path: Option<PathBuf>,
2719 capture_logs_dir: Option<PathBuf>,
2720 forwarding: Option<Arc<ForwardingTable>>,
2721 process_liveness: Arc<SupervisorProcessLiveness>,
2722 supervisor_handle: Option<SupervisorHandle>,
2723 health: HealthConfig,
2724 daemon_start_clock: crate::clock::StartClock,
2725 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2726 spawn_events: SpawnEventFeed,
2727 provenance_probe: ExecutableIdentityProbe,
2728 child_roster: ChildRoster,
2731 #[cfg(target_os = "linux")]
2732 cgroup_placement: Option<subc_cgroup::Placement>,
2733 #[cfg(test)]
2734 test_after_first_spawn: AfterFirstSpawnHook,
2735}
2736
2737pub(crate) struct RegisteredConfiguredModule {
2741 spec: ModuleSpec,
2742 runtime: SupervisorRuntimeConfig,
2743 module: SupervisedModule,
2744 commands: mpsc::Receiver<SupervisorCommand>,
2745}
2746
2747#[cfg(test)]
2753#[derive(Clone, Default)]
2754struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2755
2756#[cfg(test)]
2757type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2758
2759#[cfg(test)]
2760impl fmt::Debug for AfterFirstSpawnHook {
2761 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2762 f.write_str("AfterFirstSpawnHook")
2763 }
2764}
2765
2766#[cfg(test)]
2767impl AfterFirstSpawnHook {
2768 fn run(&self, module_id: &str) {
2769 if let Some(hook) = &self.0 {
2770 hook(module_id);
2771 }
2772 }
2773}
2774
2775impl Supervisor {
2776 #[cfg(test)]
2777 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2778 let supervisor = Self::new(registry, policy);
2779 #[cfg(target_os = "macos")]
2780 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2781 supervisor
2782 }
2783 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2788 let path = path.into();
2789 #[cfg(target_os = "macos")]
2790 {
2791 let result = probe_privacy_trampoline(&path).map(|()| path);
2792 if let Err(cause) = &result {
2793 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2794 }
2795 self.child_roster.set_privacy_trampoline(result);
2796 }
2797 #[cfg(not(target_os = "macos"))]
2798 let _ = path;
2799 self
2800 }
2801 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2804 self.child_roster.shutdown_flag()
2805 }
2806
2807 #[cfg(unix)]
2818 pub(crate) fn begin_daemon_shutdown(&self) {
2819 self.child_roster.close();
2820 if let Some(journal) = &self.terminal_journal {
2821 journal.stamp_shutdown();
2822 }
2823 }
2824
2825 #[cfg(unix)]
2829 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2830 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2831 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2832 let Some(forwarding) = &self.forwarding else {
2833 return Ok(());
2834 };
2835 let notice_started = Instant::now();
2836 let modules = forwarding
2837 .begin_daemon_drain()
2838 .map_err(SuperviseError::Forwarding)?;
2839 let deadline_ms =
2840 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2841 let mut notices = tokio::task::JoinSet::new();
2842 let mut drains = Vec::new();
2843 for (module_id, endpoint) in modules {
2844 let Some(target) = forwarding
2845 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2846 .map_err(SuperviseError::Forwarding)?
2847 else {
2848 continue;
2849 };
2850 let routes = forwarding
2851 .endpoint_routes(target.endpoint)
2852 .map_err(SuperviseError::Forwarding)?;
2853 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2859 reason: RouteCloseReason::Restart,
2860 deadline_ms,
2861 })
2862 .expect("module draining serializes");
2863 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2864 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2865 for route in routes {
2866 let client = route.goodbye_target;
2867 if let Some((_, channels)) = clients
2868 .iter_mut()
2869 .find(|(existing, _)| existing.connection_id == client.connection_id)
2870 {
2871 channels.push(client.channel);
2872 } else {
2873 let channel = client.channel;
2874 clients.push((client, vec![channel]));
2875 }
2876 }
2877 for (client, mut channels) in clients {
2878 channels.sort_unstable();
2879 channels.dedup();
2880 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2881 module_id: module_id.clone(),
2882 channels,
2883 reason: RouteCloseReason::Restart,
2884 })
2885 .expect("route closing serializes");
2886 recipients.push((client.sink, client.negotiated_ver, closing));
2887 }
2888 for (sink, version, body) in recipients {
2889 notices.spawn(async move {
2890 let frame = Frame::build_with_version(
2891 version,
2892 FrameType::Push,
2893 control_flags(),
2894 0,
2895 0,
2896 0,
2897 body,
2898 )
2899 .expect("bounded lifecycle notice frame builds");
2900 sink.send_flushed(frame).await
2901 });
2902 }
2903 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2904 drains.push((module_id, target.endpoint, gauges));
2905 }
2906 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2909 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2910 if !matches!(result, Ok(Ok(()))) {
2911 warn!(?result, "daemon shutdown notice delivery failed");
2912 }
2913 }
2914 notices.abort_all();
2915 let notice_elapsed = notice_started.elapsed();
2916 let drain_started = Instant::now();
2917 let deadline = drain_started + DRAIN_BUDGET;
2918 let mut waits = tokio::task::JoinSet::new();
2919 for (module_id, endpoint, gauges) in drains {
2920 let forwarding = Arc::clone(forwarding);
2921 let mut runtime = self.runtime_config();
2922 runtime.health.cadence = Duration::from_millis(100);
2923 waits.spawn(async move {
2924 let wait_result = wait_for_forwarding_quiescence(
2925 &forwarding,
2926 &module_id,
2927 &runtime,
2928 endpoint,
2929 deadline,
2930 &gauges,
2931 DrainScope::Endpoint(endpoint),
2932 &mut DrainGaugeTrace::new(&gauges),
2933 )
2934 .await;
2935 forwarding
2939 .record_daemon_drain_result(
2940 endpoint,
2941 drained_after_quiescence_wait(&wait_result),
2942 )
2943 .map_err(SuperviseError::Forwarding)?;
2944 wait_result
2945 });
2946 }
2947 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2948 if !matches!(result, Ok(Ok(true))) {
2949 warn!(?result, "daemon shutdown drain did not reach quiescence");
2950 }
2951 }
2952 info!(
2955 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
2956 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
2957 "daemon shutdown forwarding drain complete"
2958 );
2959 Ok(())
2960 }
2961
2962 #[cfg(unix)]
2974 pub(crate) async fn end_children_for_daemon_shutdown(
2975 &self,
2976 already_escalated: bool,
2977 escalate: impl std::future::Future<Output = ()>,
2978 ) {
2979 tokio::pin!(escalate);
2980 let mut escalated = already_escalated;
2981 if let Some(forwarding) = &self.forwarding {
2982 let reason = CloseReason::new(
2983 "daemon_shutdown",
2984 "the daemon is exiting after its shutdown notice and drain",
2985 );
2986 if escalated {
2987 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2990 } else {
2991 tokio::select! {
2992 biased;
2993 _ = escalate.as_mut() => {
2994 info!("second SIGTERM: abandoning module GOODBYE delivery");
2995 escalated = true;
2996 }
2997 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2998 }
2999 }
3000 let closed = forwarding.close_all_connections(&reason);
3001 debug!(closed, "closed established connections for daemon shutdown");
3002 }
3003 let escalated_here = escalated && !already_escalated;
3007 let remaining_escalate = async move {
3008 if escalated_here {
3009 std::future::pending::<()>().await;
3010 } else {
3011 escalate.await;
3012 }
3013 };
3014 crate::child_roster::end_children_for_daemon_shutdown(
3015 &self.child_roster,
3016 escalated,
3017 remaining_escalate,
3018 )
3019 .await;
3020 }
3021
3022 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3023 Self {
3024 registry,
3025 restart_policy,
3026 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3027 connection_file_path: None,
3028 capture_logs_dir: None,
3029 forwarding: None,
3030 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3031 supervisor_handle: None,
3032 health: HealthConfig::default(),
3033 daemon_start_clock: crate::clock::StartClock::capture(),
3034 terminal_journal: None,
3035 spawn_events: SpawnEventFeed::default(),
3036 provenance_probe: ExecutableIdentityProbe::default(),
3037 child_roster: ChildRoster::default(),
3038 #[cfg(target_os = "linux")]
3039 cgroup_placement: None,
3040 #[cfg(test)]
3041 test_after_first_spawn: AfterFirstSpawnHook::default(),
3042 }
3043 }
3044
3045 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3046 self.drain_timeout = drain_timeout;
3047 self
3048 }
3049
3050 pub(crate) fn default_drain_timeout(&self) -> Duration {
3052 self.drain_timeout
3053 }
3054
3055 pub fn with_process_liveness(
3056 mut self,
3057 process_liveness: Arc<SupervisorProcessLiveness>,
3058 ) -> Self {
3059 self.process_liveness = process_liveness;
3060 self
3061 }
3062
3063 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3064 self.connection_file_path = Some(connection_file_path.into());
3065 self
3066 }
3067
3068 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3070 self.capture_logs_dir = Some(logs_dir.into());
3071 self
3072 }
3073
3074 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3077 self.spawn_events.configure_incarnation(daemon_incarnation);
3081 self
3082 }
3083
3084 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3087 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3088 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3089 path,
3090 daemon_incarnation,
3091 )));
3092 this
3093 }
3094
3095 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3096 self.forwarding = Some(forwarding);
3097 self
3098 }
3099
3100 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3101 self.spawn_events = supervisor_handle.spawn_events.clone();
3102 self.supervisor_handle = Some(supervisor_handle);
3103 self
3104 }
3105
3106 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3107 self.health = health;
3108 self
3109 }
3110
3111 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3115 self.child_roster.record_to(path.into());
3116 self
3117 }
3118
3119 #[cfg(target_os = "linux")]
3120 pub fn with_cgroup_placement(
3121 mut self,
3122 cgroup_placement: Option<subc_cgroup::Placement>,
3123 ) -> Self {
3124 self.cgroup_placement = cgroup_placement;
3125 self
3126 }
3127
3128 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3134 validate_spec(&spec)?;
3135 self.establish_identity(&spec);
3136
3137 let runtime = self.runtime_config();
3138 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3139 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3140 let registration_watch = watch_registration(&self.registry, &spec);
3141 let spawned = spawn_child(
3142 &spec,
3143 runtime.connection_file_path.as_deref(),
3144 self.supervisor_handle.as_ref(),
3145 &runtime.stderr_ring,
3146 runtime.capture_logs_dir.as_deref(),
3147 &runtime.child_roster,
3148 #[cfg(target_os = "linux")]
3149 runtime.cgroup_placement.as_ref(),
3150 );
3151 #[cfg(test)]
3152 self.test_after_first_spawn.run(&spec.module_id);
3153 let mut child = match spawned {
3154 Ok(child) => child,
3155 Err(err) => {
3156 self.abandon_unrostered(&spec.module_id);
3159 return Err(err);
3160 }
3161 };
3162 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3163 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3164 if let Some(spawn) = activity_spawn {
3165 spawn.complete(&snapshot);
3166 }
3167
3168 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3169 }
3170
3171 fn establish_identity(&self, spec: &ModuleSpec) {
3184 if let Some(supervisor_handle) = &self.supervisor_handle {
3185 supervisor_handle.apply_identity_configuration(spec);
3186 supervisor_handle.mark_configured(&spec.module_id);
3187 }
3188 }
3189
3190 fn abandon_unrostered(&self, module_id: &str) {
3193 if let Some(supervisor_handle) = &self.supervisor_handle {
3194 supervisor_handle.unmark_configured_unless_rostered(module_id);
3195 }
3196 }
3197
3198 fn mark_first_process_running(
3201 &self,
3202 spec: &ModuleSpec,
3203 runtime: &SupervisorRuntimeConfig,
3204 snapshot: &SharedSnapshot,
3205 child: &mut SupervisedChild,
3206 ) -> Result<(), SuperviseError> {
3207 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3208 self.abandon_unrostered(&spec.module_id);
3209 return Err(err);
3210 }
3211 self.process_liveness
3212 .track(spec.module_id.clone(), Arc::clone(snapshot));
3213 Ok(())
3214 }
3215
3216 pub fn supervise_configured(
3222 &self,
3223 spec: ModuleSpec,
3224 enabled: bool,
3225 ) -> Result<SupervisedModule, SuperviseError> {
3226 self.supervise_configured_requested(
3227 spec,
3228 enabled,
3229 self.health.clone(),
3230 None,
3231 self.restart_policy,
3232 DEFAULT_REGISTER_TIMEOUT,
3233 None,
3234 )
3235 }
3236
3237 pub fn supervise_configured_with_health(
3243 &self,
3244 spec: ModuleSpec,
3245 enabled: bool,
3246 health: HealthConfig,
3247 drain_timeout_ms: Option<u64>,
3248 restart_policy: RestartPolicy,
3249 ) -> Result<SupervisedModule, SuperviseError> {
3250 self.supervise_configured_requested(
3251 spec,
3252 enabled,
3253 health,
3254 drain_timeout_ms,
3255 restart_policy,
3256 DEFAULT_REGISTER_TIMEOUT,
3257 None,
3258 )
3259 }
3260
3261 #[allow(clippy::too_many_arguments)]
3262 pub(crate) fn supervise_configured_requested(
3263 &self,
3264 spec: ModuleSpec,
3265 enabled: bool,
3266 health: HealthConfig,
3267 drain_timeout_ms: Option<u64>,
3268 restart_policy: RestartPolicy,
3269 register_timeout: Duration,
3270 requester: Option<Principal>,
3271 ) -> Result<SupervisedModule, SuperviseError> {
3272 let registered = self.register_configured_with_health(
3273 spec,
3274 enabled,
3275 health,
3276 drain_timeout_ms,
3277 restart_policy,
3278 register_timeout,
3279 )?;
3280 let daemon_initiated = requester.is_none();
3281 self.start_registered_configured(registered, requester, daemon_initiated)
3282 }
3283
3284 pub(crate) fn register_configured_with_health(
3289 &self,
3290 spec: ModuleSpec,
3291 enabled: bool,
3292 health: HealthConfig,
3293 drain_timeout_ms: Option<u64>,
3294 restart_policy: RestartPolicy,
3295 register_timeout: Duration,
3296 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3297 validate_spec(&spec)?;
3298 self.establish_identity(&spec);
3299
3300 let mut runtime = self.runtime_config();
3301 runtime.health = health;
3302 runtime.restart_policy = restart_policy;
3303 runtime.register_timeout = register_timeout;
3304 if let Some(ms) = drain_timeout_ms {
3305 runtime.drain_timeout = Duration::from_millis(ms);
3306 *runtime
3307 .effective_drain_timeout
3308 .lock()
3309 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3310 }
3311 let snapshot = if enabled {
3312 SupervisorSnapshot::starting()
3313 } else {
3314 SupervisorSnapshot::disabled()
3315 };
3316 let snapshot = {
3317 let mut snapshot = snapshot;
3318 if enabled {
3319 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3320 {
3321 snapshot.waiting_for_previous_pid = Some(pid);
3322 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3323 }
3324 }
3325 snapshot
3326 };
3327 let (module, commands) =
3328 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3329 Ok(RegisteredConfiguredModule {
3330 spec,
3331 runtime,
3332 module,
3333 commands,
3334 })
3335 }
3336
3337 pub(crate) fn start_registered_configured(
3340 &self,
3341 registered: RegisteredConfiguredModule,
3342 requester: Option<Principal>,
3343 daemon_initiated: bool,
3344 ) -> Result<SupervisedModule, SuperviseError> {
3345 let RegisteredConfiguredModule {
3346 spec,
3347 mut runtime,
3348 module,
3349 commands,
3350 } = registered;
3351 runtime.start_requester = requester;
3352 runtime.recovery_is_daemon = daemon_initiated;
3353 let snapshot = &module.inner.snapshot;
3354 let should_spawn = {
3355 let state = lock_snapshot(snapshot)?;
3356 state.enabled && state.waiting_for_previous_pid.is_none()
3357 };
3358 let child = if should_spawn {
3359 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3360 let registration_watch = watch_registration(&self.registry, &spec);
3361 let spawned = spawn_child(
3362 &spec,
3363 runtime.connection_file_path.as_deref(),
3364 self.supervisor_handle.as_ref(),
3365 &runtime.stderr_ring,
3366 runtime.capture_logs_dir.as_deref(),
3367 &runtime.child_roster,
3368 #[cfg(target_os = "linux")]
3369 runtime.cgroup_placement.as_ref(),
3370 );
3371 #[cfg(test)]
3372 self.test_after_first_spawn.run(&spec.module_id);
3373 match spawned {
3374 Ok(mut child) => {
3375 arm_registration_deadline(
3376 &mut child,
3377 runtime.register_timeout,
3378 registration_watch,
3379 );
3380 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3381 if let Some(spawn) = activity_spawn {
3382 spawn.complete(snapshot);
3383 }
3384 Some(child)
3385 }
3386 Err(err) => {
3387 if runtime.health.critical {
3388 error!(
3389 module_id = %spec.module_id,
3390 program = %spec.program.display(),
3391 error = %err,
3392 "critical configured module failed to spawn; marking failed and alerting"
3393 );
3394 } else {
3395 error!(
3396 module_id = %spec.module_id,
3397 program = %spec.program.display(),
3398 error = %err,
3399 "configured module failed to spawn; marking failed and continuing"
3400 );
3401 }
3402 fail_snapshot(
3403 snapshot,
3404 Some(&spec.module_id),
3405 None,
3406 &runtime.failure_context(daemon_initiated),
3407 );
3408 None
3409 }
3410 }
3411 } else {
3412 None
3413 };
3414 self.start_monitor(spec, runtime, &module, child, commands);
3415 Ok(module)
3416 }
3417
3418 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3419 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3420 SupervisorRuntimeConfig {
3421 registry: self.registry.clone(),
3422 recovery_is_daemon: true,
3423 start_requester: None,
3424 pending_activity_restart: false,
3425 provenance_probe: self.provenance_probe.clone(),
3426 scheduled_respawn: Arc::default(),
3427 deferred_reload_reply: Arc::default(),
3428 restart_policy: self.restart_policy,
3429 register_timeout: DEFAULT_REGISTER_TIMEOUT,
3430 drain_timeout: self.drain_timeout,
3431 child_roster: self
3434 .child_roster
3435 .for_module(Arc::clone(&effective_drain_timeout)),
3436 effective_drain_timeout,
3437 default_drain_timeout: self.drain_timeout,
3438 health: self.health.clone(),
3439 connection_file_path: self.connection_file_path.clone(),
3440 capture_logs_dir: self.capture_logs_dir.clone(),
3441 forwarding: self.forwarding.clone(),
3442 supervisor_handle: self.supervisor_handle.clone(),
3443 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3444 terminal_ring: Arc::new(Mutex::new(
3445 TerminalRing::new(
3446 TerminalRingConfig::default(),
3447 self.daemon_start_clock.started_at_ms(),
3448 )
3449 .with_start_clock(self.daemon_start_clock)
3450 .with_journal(self.terminal_journal.clone())
3451 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3452 )),
3453 spawn_events: self.spawn_events.clone(),
3454 #[cfg(target_os = "linux")]
3455 cgroup_placement: self.cgroup_placement.clone(),
3456 #[cfg(test)]
3457 test_seed_stale_facts_before_enable_spawn: false,
3458 #[cfg(test)]
3459 test_reload_exit_record_gate: None,
3460 }
3461 }
3462
3463 fn supervised_module(
3464 &self,
3465 spec: ModuleSpec,
3466 runtime: SupervisorRuntimeConfig,
3467 snapshot: SharedSnapshot,
3468 child: Option<SupervisedChild>,
3469 ) -> SupervisedModule {
3470 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3471 self.start_monitor(spec, runtime, &module, child, commands);
3472 module
3473 }
3474
3475 fn register_module(
3476 &self,
3477 spec: &ModuleSpec,
3478 runtime: &SupervisorRuntimeConfig,
3479 snapshot: SharedSnapshot,
3480 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3481 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3482 spec: spec.clone(),
3483 health: runtime.health.clone(),
3484 }));
3485 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3486 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3487 let restart_policy = runtime.restart_policy;
3491 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3492 let (tx, rx) = mpsc::channel(4);
3493 let module_id = spec.module_id.clone();
3494 let module = SupervisedModule {
3495 inner: Arc::new(SupervisedModuleInner {
3496 module_id: module_id.clone(),
3497 registry: Arc::clone(&self.registry),
3498 snapshot,
3499 configuration,
3500 stderr_ring,
3501 terminal_ring,
3502 commands: tx,
3503 monitor: Mutex::new(None),
3504 restart_policy,
3505 effective_drain_timeout,
3506 provenance_probe: self.provenance_probe.clone(),
3507 }),
3508 };
3509 if let Some(supervisor_handle) = &self.supervisor_handle {
3512 supervisor_handle.insert(module.clone());
3513 }
3514 (module, rx)
3515 }
3516
3517 fn start_monitor(
3518 &self,
3519 spec: ModuleSpec,
3520 runtime: SupervisorRuntimeConfig,
3521 module: &SupervisedModule,
3522 child: Option<SupervisedChild>,
3523 commands: mpsc::Receiver<SupervisorCommand>,
3524 ) {
3525 let monitor = tokio::spawn(supervise_loop(
3526 spec,
3527 runtime,
3528 Arc::clone(&self.registry),
3529 Arc::clone(&self.process_liveness),
3530 Arc::clone(&module.inner.snapshot),
3531 child,
3532 commands,
3533 ));
3534 *module
3535 .inner
3536 .monitor
3537 .lock()
3538 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3539 }
3540}
3541
3542impl Default for Supervisor {
3543 fn default() -> Self {
3544 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3545 }
3546}
3547
3548#[derive(Clone)]
3550pub struct SupervisedModule {
3551 inner: Arc<SupervisedModuleInner>,
3552}
3553
3554struct SupervisedModuleInner {
3555 module_id: String,
3556 registry: Arc<Registry>,
3557 snapshot: SharedSnapshot,
3558 configuration: Arc<Mutex<SupervisedConfiguration>>,
3559 stderr_ring: Arc<Mutex<StderrRing>>,
3560 terminal_ring: Arc<Mutex<TerminalRing>>,
3561 commands: mpsc::Sender<SupervisorCommand>,
3562 monitor: Mutex<Option<JoinHandle<()>>>,
3563 restart_policy: RestartPolicy,
3567 effective_drain_timeout: Arc<Mutex<Duration>>,
3568 provenance_probe: ExecutableIdentityProbe,
3569}
3570
3571impl fmt::Debug for SupervisedModule {
3572 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3573 f.debug_struct("SupervisedModule")
3574 .field("module_id", &self.inner.module_id)
3575 .field("status", &self.status())
3576 .finish_non_exhaustive()
3577 }
3578}
3579
3580impl SupervisedModule {
3581 pub fn module_id(&self) -> &str {
3582 &self.inner.module_id
3583 }
3584
3585 #[cfg(test)]
3589 pub(crate) fn record_health_probe_failure_for_test(
3590 &self,
3591 detail: &str,
3592 ) -> Result<(), SuperviseError> {
3593 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3594 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3595 state.health.detail = Some(detail.to_string());
3596 })
3597 }
3598
3599 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3600 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3601 }
3602
3603 pub(crate) fn registration_end_reason(
3604 &self,
3605 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3606 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3607 Ok(match snapshot.state {
3608 ModuleState::Draining if snapshot.draining_to_replace => {
3609 Some(RegistrationEndReason::SupervisorRestart)
3610 }
3611 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3612 ModuleState::Restarting if snapshot.draining_to_replace => {
3613 Some(RegistrationEndReason::SupervisorRestart)
3614 }
3615 _ => None,
3616 })
3617 }
3618
3619 pub fn stderr_tail(
3626 &self,
3627 max_lines: Option<usize>,
3628 max_bytes: Option<usize>,
3629 ) -> StderrTailSnapshot {
3630 self.inner
3631 .stderr_ring
3632 .lock()
3633 .unwrap_or_else(|poisoned| poisoned.into_inner())
3634 .snapshot(max_lines, max_bytes)
3635 }
3636
3637 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3642 self.inner
3643 .terminal_ring
3644 .lock()
3645 .unwrap_or_else(|poisoned| poisoned.into_inner())
3646 .snapshot()
3647 }
3648
3649 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3654 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3655 }
3656
3657 pub(crate) async fn read_durable_terminal_history(
3662 &self,
3663 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3664 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3665 let module_id = self.inner.module_id.clone();
3666 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3667 .await
3668 }
3669
3670 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3671 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3672 .map(|(status, _)| status)
3673 }
3674
3675 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3676 let state = lock_snapshot(&self.inner.snapshot)?;
3677 Ok(state
3678 .failed_retry
3679 .pending
3680 .filter(|_| state.state == ModuleState::Failed && state.enabled)
3681 .map(|retry| {
3682 unix_ms_now().saturating_add(
3683 u64::try_from(
3684 retry
3685 .deadline
3686 .saturating_duration_since(Instant::now())
3687 .as_millis(),
3688 )
3689 .unwrap_or(u64::MAX),
3690 )
3691 }))
3692 }
3693
3694 pub(crate) fn record_deliberate_severance(
3695 &self,
3696 identity: ProcessIdentity,
3697 ) -> Result<bool, SuperviseError> {
3698 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3699 if snapshot.pid != Some(identity.pid)
3700 || snapshot.process_start_time != Some(identity.start_time)
3701 {
3702 return Ok(false);
3703 }
3704 snapshot.deliberate_severance = Some(identity);
3705 Ok(true)
3706 }
3707
3708 pub(crate) fn status_for_control(
3713 &self,
3714 caller: &'static str,
3715 ) -> Result<ModuleStatus, SuperviseError> {
3716 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3717 .map(|(status, _)| status)
3718 }
3719
3720 fn status_with_snapshot_lock(
3721 &self,
3722 snapshot: &SharedSnapshot,
3723 caller: Option<&'static str>,
3724 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3725 let mut guard = match caller {
3726 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3727 None => lock_snapshot(snapshot)?,
3728 };
3729 let restart_count =
3732 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3733 let snapshot = guard.clone();
3734 drop(guard);
3735 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3736 SuperviseError::StatePoisoned {
3737 module_id: Some(self.inner.module_id.clone()),
3738 }
3739 })?;
3740 let registration_active = self
3741 .inner
3742 .registry
3743 .get_module(&self.inner.module_id)
3744 .map_err(SuperviseError::Registry)?
3745 .is_some();
3746 let protocol = snapshot
3747 .spawned_protocol
3748 .unwrap_or(self.declared_protocol()?);
3749 let running_process =
3750 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3751 let live = match protocol {
3757 ModuleProtocol::Subc => running_process && registration_active,
3758 ModuleProtocol::None => running_process,
3759 };
3760
3761 Ok((
3762 ModuleStatus {
3763 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3764 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3765 module_id: self.inner.module_id.clone(),
3766 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3767 state: snapshot.state,
3768 enabled: snapshot.enabled,
3769 process_alive: snapshot.process_alive,
3770 registration_active,
3771 protocol,
3772 live,
3773 restart_count,
3774 lifetime_restarts: snapshot.lifetime_restarts,
3775 spawn_generation: snapshot.spawn_generation,
3776 max_restarts: self.inner.restart_policy.max_restarts,
3777 restart_window: self.inner.restart_policy.window,
3778 drain_timeout,
3779 restart_backoff: self.inner.restart_policy.backoff,
3780 restart_max_backoff: self.inner.restart_policy.max_backoff,
3781 pid: snapshot.reported_pid(),
3782 spawned_at_ms: snapshot.spawned_at_ms,
3783 spawned_from: snapshot.spawned_from,
3784 launch_spec_digest: snapshot.launch_spec_digest,
3785 process_start_time: snapshot.process_start_time,
3786 last_exit: snapshot.last_exit,
3787 health: snapshot.health,
3788 },
3789 snapshot.spawned_file_identity.clone(),
3790 ))
3791 }
3792
3793 #[cfg(test)]
3794 pub(crate) fn hold_snapshot_for_test(
3795 &self,
3796 acquired: std::sync::mpsc::Sender<()>,
3797 hold: Duration,
3798 ) -> std::thread::JoinHandle<()> {
3799 let snapshot = Arc::clone(&self.inner.snapshot);
3800 std::thread::spawn(move || {
3801 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3802 acquired
3803 .send(())
3804 .expect("test receiver waits for snapshot lock");
3805 std::thread::sleep(hold);
3806 })
3807 }
3808
3809 pub(crate) async fn status_and_running_image_agreement(
3815 &self,
3816 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3817 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3818 let image = self
3819 .inner
3820 .provenance_probe
3821 .observe(
3822 status.pid,
3823 status.spawned_from.as_deref(),
3824 identity,
3825 status.process_start_time,
3826 )
3827 .await;
3828 Ok((status, image))
3829 }
3830
3831 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3832 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3833 Ok(snapshot) => snapshot.clone(),
3834 Err(_) => {
3835 return subc_control::RunningImageAgreement::Unavailable {
3836 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3837 };
3838 }
3839 };
3840 self.inner
3841 .provenance_probe
3842 .observe(
3843 snapshot.reported_pid(),
3844 snapshot.spawned_from.as_deref(),
3845 snapshot.spawned_file_identity.clone(),
3846 snapshot.process_start_time,
3847 )
3848 .await
3849 }
3850
3851 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3854 #[cfg(windows)]
3855 {
3856 let image = lock_snapshot(&self.inner.snapshot)
3857 .ok()
3858 .and_then(|snapshot| {
3859 snapshot
3860 .spawned_file_identity
3861 .as_ref()
3862 .and_then(|identity| identity.windows_image.clone())
3863 });
3864 match image {
3865 Some(image) => {
3866 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3867 }
3868 None => subc_control::ChildResourceUsage::Unavailable {
3869 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3870 },
3871 }
3872 }
3873 #[cfg(not(windows))]
3874 {
3875 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3876 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3877 Err(_) => {
3878 return subc_control::ChildResourceUsage::Unavailable {
3879 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3880 }
3881 }
3882 };
3883 crate::child_resources::read(pid, start_time)
3884 }
3885 }
3886
3887 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3888 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3889 Ok(match snapshot.state {
3890 ModuleState::Restarting => true,
3891 ModuleState::Failed | ModuleState::Disabled => false,
3892 _ if snapshot.enabled && snapshot.respawn_pending => true,
3893 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3894 })
3895 }
3896
3897 #[cfg(test)]
3898 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3899 self.is_warming_with_snapshot_lock(None)
3900 }
3901
3902 pub(crate) fn is_warming_for_control(
3903 &self,
3904 caller: &'static str,
3905 ) -> Result<bool, SuperviseError> {
3906 self.is_warming_with_snapshot_lock(Some(caller))
3907 }
3908
3909 fn is_warming_with_snapshot_lock(
3910 &self,
3911 caller: Option<&'static str>,
3912 ) -> Result<bool, SuperviseError> {
3913 let snapshot = match caller {
3914 Some(caller) => {
3915 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3916 }
3917 None => lock_snapshot(&self.inner.snapshot)?,
3918 }
3919 .clone();
3920 Ok(matches!(
3921 snapshot.state,
3922 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3923 ))
3924 }
3925
3926 pub async fn drain(&self) -> Result<(), SuperviseError> {
3928 self.stop().await
3929 }
3930
3931 pub(crate) async fn retire_requested(
3932 &self,
3933 requester: Option<Principal>,
3934 ) -> Result<(), SuperviseError> {
3935 match self.state()? {
3936 ModuleState::Stopped => return Ok(()),
3937 ModuleState::Failed
3938 if lock_snapshot(&self.inner.snapshot)?
3939 .failed_retry
3940 .pending
3941 .is_none() =>
3942 {
3943 return Ok(())
3944 }
3945 ModuleState::Failed
3946 | ModuleState::Starting
3947 | ModuleState::Running
3948 | ModuleState::Unresponsive
3949 | ModuleState::Restarting
3950 | ModuleState::Draining
3951 | ModuleState::Disabled => {}
3952 }
3953
3954 let (reply_tx, reply_rx) = oneshot::channel();
3955 self.inner
3956 .commands
3957 .send(SupervisorCommand::Retire {
3958 requester,
3959 reply: reply_tx,
3960 })
3961 .await
3962 .map_err(|_| SuperviseError::CommandClosed {
3963 module_id: self.inner.module_id.clone(),
3964 })?;
3965 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3966 module_id: self.inner.module_id.clone(),
3967 })?
3968 }
3969
3970 pub async fn stop(&self) -> Result<(), SuperviseError> {
3971 self.stop_requested(None).await
3972 }
3973
3974 pub(crate) async fn stop_requested(
3975 &self,
3976 requester: Option<Principal>,
3977 ) -> Result<(), SuperviseError> {
3978 match self.state()? {
3979 ModuleState::Stopped => return Ok(()),
3980 ModuleState::Failed
3981 if lock_snapshot(&self.inner.snapshot)?
3982 .failed_retry
3983 .pending
3984 .is_none() =>
3985 {
3986 return Ok(())
3987 }
3988 ModuleState::Failed
3989 | ModuleState::Starting
3990 | ModuleState::Running
3991 | ModuleState::Unresponsive
3992 | ModuleState::Restarting
3993 | ModuleState::Draining
3994 | ModuleState::Disabled => {}
3995 }
3996
3997 let (reply_tx, reply_rx) = oneshot::channel();
3998 self.inner
3999 .commands
4000 .send(SupervisorCommand::Drain {
4001 requester,
4002 reply: reply_tx,
4003 })
4004 .await
4005 .map_err(|_| SuperviseError::CommandClosed {
4006 module_id: self.inner.module_id.clone(),
4007 })?;
4008 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4009 module_id: self.inner.module_id.clone(),
4010 })?
4011 }
4012
4013 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4014 self.restart_requested(drain_timeout_ms, None).await
4015 }
4016
4017 pub(crate) async fn restart_requested(
4018 &self,
4019 drain_timeout_ms: Option<u64>,
4020 requester: Option<Principal>,
4021 ) -> Result<(), SuperviseError> {
4022 let queued_at = Instant::now();
4023 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4024 let (reply_tx, reply_rx) = oneshot::channel();
4025 self.inner
4026 .commands
4027 .send(SupervisorCommand::Restart {
4028 requester,
4029 drain_timeout_ms,
4030 received_at_generation,
4031 queued_at,
4032 reply: reply_tx,
4033 })
4034 .await
4035 .map_err(|_| SuperviseError::CommandClosed {
4036 module_id: self.inner.module_id.clone(),
4037 })?;
4038 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4039 module_id: self.inner.module_id.clone(),
4040 })?
4041 }
4042
4043 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4048 self.swap_requested(ready_timeout, None).await
4049 }
4050
4051 pub(crate) async fn swap_requested(
4052 &self,
4053 ready_timeout: Option<Duration>,
4054 requester: Option<Principal>,
4055 ) -> Result<(), SuperviseError> {
4056 let (reply_tx, reply_rx) = oneshot::channel();
4057 self.inner
4058 .commands
4059 .send(SupervisorCommand::Swap {
4060 requester,
4061 ready_timeout,
4062 reply: reply_tx,
4063 })
4064 .await
4065 .map_err(|_| SuperviseError::CommandClosed {
4066 module_id: self.inner.module_id.clone(),
4067 })?;
4068 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4069 module_id: self.inner.module_id.clone(),
4070 })?
4071 }
4072
4073 pub async fn reload(&self) -> Result<(), SuperviseError> {
4074 self.reload_requested(None).await
4075 }
4076
4077 pub(crate) async fn reload_requested(
4078 &self,
4079 requester: Option<Principal>,
4080 ) -> Result<(), SuperviseError> {
4081 let (reply_tx, reply_rx) = oneshot::channel();
4082 self.inner
4083 .commands
4084 .send(SupervisorCommand::Reload {
4085 requester,
4086 reply: reply_tx,
4087 })
4088 .await
4089 .map_err(|_| SuperviseError::CommandClosed {
4090 module_id: self.inner.module_id.clone(),
4091 })?;
4092 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4093 module_id: self.inner.module_id.clone(),
4094 })?
4095 }
4096
4097 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4098 self.set_enabled_requested(enabled, None).await
4099 }
4100
4101 pub(crate) async fn set_enabled_requested(
4102 &self,
4103 enabled: bool,
4104 requester: Option<Principal>,
4105 ) -> Result<bool, SuperviseError> {
4106 let (reply_tx, reply_rx) = oneshot::channel();
4107 self.inner
4108 .commands
4109 .send(SupervisorCommand::SetEnabled {
4110 requester,
4111 enabled,
4112 reply: reply_tx,
4113 })
4114 .await
4115 .map_err(|_| SuperviseError::CommandClosed {
4116 module_id: self.inner.module_id.clone(),
4117 })?;
4118 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4119 module_id: self.inner.module_id.clone(),
4120 })?
4121 }
4122
4123 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4127 let configured = self
4128 .inner
4129 .configuration
4130 .lock()
4131 .map_err(|_| SuperviseError::StatePoisoned {
4132 module_id: Some(self.inner.module_id.clone()),
4133 })?
4134 .spec
4135 .protocol;
4136 let state = lock_snapshot(&self.inner.snapshot)?;
4137 Ok(state.spawned_protocol.unwrap_or(configured))
4138 }
4139
4140 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4141 let configuration =
4142 self.inner
4143 .configuration
4144 .lock()
4145 .map_err(|_| SuperviseError::StatePoisoned {
4146 module_id: Some(self.inner.module_id.clone()),
4147 })?;
4148 Ok((configuration.spec.clone(), configuration.health.clone()))
4149 }
4150
4151 #[cfg(any(test, feature = "test-support"))]
4155 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4156 let (_, health) = self.configuration()?;
4157 let drain_timeout_ms = u64::try_from(
4158 self.inner
4159 .effective_drain_timeout
4160 .lock()
4161 .unwrap_or_else(|poisoned| poisoned.into_inner())
4162 .as_millis(),
4163 )
4164 .ok();
4165 self.update_configuration(spec, health, drain_timeout_ms)
4166 .await
4167 }
4168
4169 pub(crate) async fn update_configuration(
4170 &self,
4171 spec: ModuleSpec,
4172 health: HealthConfig,
4173 drain_timeout_ms: Option<u64>,
4174 ) -> Result<(), SuperviseError> {
4175 if spec.module_id != self.inner.module_id {
4176 return Err(SuperviseError::InvalidSpec {
4177 reason: "a supervised module's module_id cannot be changed".to_string(),
4178 });
4179 }
4180 validate_spec(&spec)?;
4181 let (reply_tx, reply_rx) = oneshot::channel();
4182 self.inner
4183 .commands
4184 .send(SupervisorCommand::UpdateConfiguration {
4185 spec: spec.clone(),
4186 health: health.clone(),
4187 drain_timeout_ms,
4188 reply: reply_tx,
4189 })
4190 .await
4191 .map_err(|_| SuperviseError::CommandClosed {
4192 module_id: self.inner.module_id.clone(),
4193 })?;
4194 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4195 module_id: self.inner.module_id.clone(),
4196 })?;
4197 let mut configuration =
4198 self.inner
4199 .configuration
4200 .lock()
4201 .map_err(|_| SuperviseError::StatePoisoned {
4202 module_id: Some(self.inner.module_id.clone()),
4203 })?;
4204 configuration.spec = spec;
4205 configuration.health = health;
4206 Ok(())
4207 }
4208}
4209
4210impl Drop for SupervisedModuleInner {
4211 fn drop(&mut self) {
4212 let Ok(mut monitor) = self.monitor.lock() else {
4213 return;
4214 };
4215 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4216 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4217 state.state = ModuleState::Stopped;
4218 clear_current_process_facts(state);
4219 });
4220 monitor.abort();
4221 }
4222 let _ = monitor.take();
4223 }
4224}
4225
4226#[derive(Debug)]
4227enum SupervisorCommand {
4228 Drain {
4229 requester: Option<Principal>,
4230 reply: oneshot::Sender<Result<(), SuperviseError>>,
4231 },
4232 Retire {
4233 requester: Option<Principal>,
4234 reply: oneshot::Sender<Result<(), SuperviseError>>,
4235 },
4236 Restart {
4237 requester: Option<Principal>,
4238 drain_timeout_ms: Option<u64>,
4243 received_at_generation: u64,
4247 queued_at: Instant,
4250 reply: oneshot::Sender<Result<(), SuperviseError>>,
4251 },
4252 Reload {
4253 requester: Option<Principal>,
4254 reply: oneshot::Sender<Result<(), SuperviseError>>,
4255 },
4256 SetEnabled {
4257 requester: Option<Principal>,
4258 enabled: bool,
4259 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4260 },
4261 UpdateConfiguration {
4262 spec: ModuleSpec,
4263 health: HealthConfig,
4264 drain_timeout_ms: Option<u64>,
4267 reply: oneshot::Sender<()>,
4268 },
4269 Swap {
4270 requester: Option<Principal>,
4271 ready_timeout: Option<Duration>,
4274 reply: oneshot::Sender<Result<(), SuperviseError>>,
4276 },
4277}
4278
4279#[derive(Debug)]
4280pub enum SuperviseError {
4281 InvalidSpec {
4282 reason: String,
4283 },
4284 Spawn {
4285 program: PathBuf,
4286 source: io::Error,
4287 cgroup_path: Option<PathBuf>,
4288 },
4289 Cgroup {
4290 module_id: String,
4291 source: io::Error,
4292 },
4293 LaunchNonce {
4296 reason: String,
4297 },
4298 Wait {
4299 module_id: String,
4300 source: io::Error,
4301 },
4302 Kill {
4303 module_id: String,
4304 source: io::Error,
4305 },
4306 Forwarding(ForwardingError),
4307 Registry(RegistryError),
4308 ReloadUnavailable {
4309 module_id: String,
4310 reason: String,
4311 },
4312 Disabled {
4317 module_id: String,
4318 },
4319 ReloadFailed {
4320 module_id: String,
4321 reason: String,
4322 },
4323 RegistrationStillActive {
4324 module_id: String,
4325 waited: Duration,
4326 },
4327 StatePoisoned {
4328 module_id: Option<String>,
4329 },
4330 CommandClosed {
4331 module_id: String,
4332 },
4333 SwapInProgress {
4337 module_id: String,
4338 },
4339 SwapRefused {
4341 module_id: String,
4342 reason: SwapRefusal,
4343 },
4344 SwapFailed {
4348 module_id: String,
4349 arm: SwapFailureArm,
4350 detail: String,
4351 candidate_exit: Option<ExitReport>,
4354 },
4355}
4356
4357#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4359pub enum SwapRefusal {
4360 OverlapExclusive,
4362 NotRegistered,
4365 ProtocolNone,
4368 NotConfigured,
4371 AlreadySwapping,
4373}
4374
4375impl SwapRefusal {
4376 pub fn as_str(self) -> &'static str {
4377 match self {
4378 Self::OverlapExclusive => "overlap_exclusive",
4379 Self::NotRegistered => "not_registered",
4380 Self::ProtocolNone => "protocol_none",
4381 Self::NotConfigured => "not_configured",
4382 Self::AlreadySwapping => "already_swapping",
4383 }
4384 }
4385}
4386
4387#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4390pub enum SwapFailureArm {
4391 SpawnFailed,
4393 NeverRegistered,
4395 NeverReady,
4397 CandidateExited,
4399 CandidateUnhealthy,
4401 Interrupted,
4405 CutoverLost,
4410}
4411
4412impl SwapFailureArm {
4413 pub fn as_str(self) -> &'static str {
4414 match self {
4415 Self::SpawnFailed => "spawn_failed",
4416 Self::NeverRegistered => "never_registered",
4417 Self::NeverReady => "never_ready",
4418 Self::CandidateExited => "candidate_exited",
4419 Self::CandidateUnhealthy => "candidate_unhealthy",
4420 Self::Interrupted => "interrupted",
4421 Self::CutoverLost => "cutover_lost",
4422 }
4423 }
4424}
4425
4426impl fmt::Display for SuperviseError {
4427 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4428 match self {
4429 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4430 Self::Spawn {
4431 program,
4432 source,
4433 cgroup_path: Some(cgroup_path),
4434 } => write!(
4435 f,
4436 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4437 cgroup_path.display(),
4438 program.display()
4439 ),
4440 Self::Spawn {
4441 program,
4442 source,
4443 cgroup_path: None,
4444 } => write!(
4445 f,
4446 "failed to spawn module '{}': {source}",
4447 program.display()
4448 ),
4449 Self::Cgroup { module_id, source } => {
4450 write!(
4451 f,
4452 "failed to prepare cgroup for module '{module_id}': {source}"
4453 )
4454 }
4455 Self::LaunchNonce { reason } => {
4456 write!(
4457 f,
4458 "failed to generate reserved-module launch nonce: {reason}"
4459 )
4460 }
4461 Self::Wait { module_id, source } => {
4462 write!(f, "failed to wait for module '{module_id}': {source}")
4463 }
4464 Self::Kill { module_id, source } => {
4465 write!(f, "failed to kill module '{module_id}': {source}")
4466 }
4467 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4468 Self::Registry(err) => write!(f, "registry error: {err}"),
4469 Self::ReloadUnavailable { module_id, reason } => {
4470 write!(f, "reload unavailable for module '{module_id}': {reason}")
4471 }
4472 Self::Disabled { module_id } => {
4473 write!(
4474 f,
4475 "module '{module_id}' is disabled; enable it before restart or reload"
4476 )
4477 }
4478 Self::ReloadFailed { module_id, reason } => {
4479 write!(f, "reload failed for module '{module_id}': {reason}")
4480 }
4481 Self::RegistrationStillActive { module_id, waited } => write!(
4482 f,
4483 "module '{module_id}' registration remained active after waiting {waited:?}"
4484 ),
4485 Self::StatePoisoned { module_id } => match module_id {
4486 Some(module_id) => {
4487 write!(f, "supervisor state for module '{module_id}' was poisoned")
4488 }
4489 None => write!(f, "supervisor state was poisoned"),
4490 },
4491 Self::CommandClosed { module_id } => {
4492 write!(
4493 f,
4494 "supervisor command channel for module '{module_id}' is closed"
4495 )
4496 }
4497 Self::SwapInProgress { module_id } => write!(
4498 f,
4499 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4500 ),
4501 Self::SwapRefused { module_id, reason } => match reason {
4502 SwapRefusal::OverlapExclusive => write!(
4503 f,
4504 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4505 ),
4506 SwapRefusal::NotRegistered => write!(
4507 f,
4508 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4509 ),
4510 SwapRefusal::ProtocolNone => write!(
4511 f,
4512 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4513 ),
4514 SwapRefusal::NotConfigured => write!(
4515 f,
4516 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4517 ),
4518 SwapRefusal::AlreadySwapping => {
4519 write!(f, "module '{module_id}' is already being swapped")
4520 }
4521 },
4522 Self::SwapFailed {
4523 module_id,
4524 arm,
4525 detail,
4526 ..
4527 } => write!(
4528 f,
4529 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4530 arm.as_str()
4531 ),
4532 }
4533 }
4534}
4535
4536impl Error for SuperviseError {
4537 fn source(&self) -> Option<&(dyn Error + 'static)> {
4538 match self {
4539 Self::Spawn { source, .. }
4540 | Self::Cgroup { source, .. }
4541 | Self::Wait { source, .. }
4542 | Self::Kill { source, .. } => Some(source),
4543 Self::Forwarding(err) => Some(err),
4544 Self::Registry(err) => Some(err),
4545 Self::LaunchNonce { .. }
4546 | Self::InvalidSpec { .. }
4547 | Self::ReloadUnavailable { .. }
4548 | Self::Disabled { .. }
4549 | Self::ReloadFailed { .. }
4550 | Self::RegistrationStillActive { .. }
4551 | Self::StatePoisoned { .. }
4552 | Self::CommandClosed { .. }
4553 | Self::SwapInProgress { .. }
4554 | Self::SwapRefused { .. }
4555 | Self::SwapFailed { .. } => None,
4556 }
4557 }
4558}
4559
4560pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4561 if spec.module_id.trim().is_empty() {
4562 return Err(SuperviseError::InvalidSpec {
4563 reason: "module_id must not be empty".to_string(),
4564 });
4565 }
4566
4567 Ok(())
4568}
4569
4570#[derive(Debug, Default)]
4571struct HealthProbeRuntime {
4572 spawn_generation: Option<u64>,
4573 configured_health: Option<HealthConfig>,
4574 registered_connection: Option<crate::ConnectionId>,
4575 advertised: bool,
4576 next_probe_at: Option<Instant>,
4577 probe_index: u64,
4578}
4579
4580fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4581 lock_snapshot(snapshot)
4582 .ok()
4583 .and_then(|state| state.spawned_protocol)
4584 .unwrap_or(spec.protocol)
4585}
4586
4587impl HealthProbeRuntime {
4588 fn refresh_registration(
4589 &mut self,
4590 spec: &ModuleSpec,
4591 runtime: &SupervisorRuntimeConfig,
4592 registry: &Registry,
4593 snapshot: &SharedSnapshot,
4594 ) {
4595 let generation = lock_snapshot(snapshot)
4596 .ok()
4597 .map(|state| state.spawn_generation);
4598 if self.spawn_generation != generation {
4599 *self = Self {
4603 spawn_generation: generation,
4604 ..Self::default()
4605 };
4606 }
4607 if self.configured_health.as_ref() != Some(&runtime.health) {
4608 self.configured_health = Some(runtime.health.clone());
4609 self.next_probe_at = None;
4610 self.registered_connection = None;
4611 self.probe_index = 0;
4612 }
4613 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4617 self.registered_connection = None;
4618 self.advertised = runtime.health.http.is_some();
4619 if !self.advertised {
4620 self.next_probe_at = None;
4621 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4622 let unknown = ModuleHealthStatus::default();
4623 if state.health != unknown {
4624 state.health = unknown;
4625 }
4626 });
4627 } else if self.next_probe_at.is_none() {
4628 self.next_probe_at = Some(
4629 Instant::now()
4630 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4631 );
4632 }
4633 return;
4634 }
4635
4636 let registration = match registry.get_module(&spec.module_id) {
4637 Ok(registration) => registration,
4638 Err(err) => {
4639 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4640 self.advertised = false;
4641 self.next_probe_at = None;
4642 return;
4643 }
4644 };
4645
4646 let Some(registration) = registration else {
4647 self.registered_connection = None;
4648 self.advertised = false;
4649 self.next_probe_at = None;
4650 return;
4651 };
4652
4653 let advertised = registration
4654 .control_ops
4655 .iter()
4656 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4657 if !advertised {
4658 self.registered_connection = Some(registration.connection_id);
4659 self.advertised = false;
4660 self.next_probe_at = None;
4661 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4662 if state.health.status != SupervisorHealthStatus::Unknown
4663 || state.health.consecutive_failures != 0
4664 || state.health.last_probe_ms.is_some()
4665 || state.health.detail.is_some()
4666 || state.health.metrics.is_some()
4667 {
4668 state.health.status = SupervisorHealthStatus::Unknown;
4669 state.health.consecutive_failures = 0;
4670 state.health.last_probe_ms = None;
4671 state.health.detail = None;
4672 state.health.metrics = None;
4673 }
4674 });
4675 return;
4676 }
4677
4678 let reregistered = self.registered_connection != Some(registration.connection_id);
4679 self.registered_connection = Some(registration.connection_id);
4680 self.advertised = true;
4681 if reregistered || self.next_probe_at.is_none() {
4682 self.probe_index = 0;
4683 self.next_probe_at = Some(
4684 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4685 );
4686 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4687 state.health.status = SupervisorHealthStatus::Unknown;
4688 state.health.consecutive_failures = 0;
4689 state.health.detail = None;
4690 state.health.metrics = None;
4691 });
4692 }
4693 }
4694
4695 fn wake_at(&self) -> Option<Instant> {
4696 if !self.advertised {
4697 return None;
4698 }
4699 self.next_probe_at
4700 }
4701
4702 fn due(&self) -> bool {
4703 self.advertised
4704 && self
4705 .next_probe_at
4706 .is_some_and(|next| Instant::now() >= next)
4707 }
4708
4709 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4710 self.probe_index = self.probe_index.wrapping_add(1);
4711 self.next_probe_at = Some(
4712 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4713 );
4714 }
4715}
4716
4717#[derive(Debug)]
4752enum HealthProbeEvidence {
4753 LaneDead,
4755 NoAnswer,
4757 BadAnswer,
4759 Misconfigured,
4761}
4762
4763#[derive(Debug)]
4764struct HealthProbeError {
4765 evidence: HealthProbeEvidence,
4766 message: String,
4767}
4768
4769impl HealthProbeError {
4770 fn lane_dead(message: impl Into<String>) -> Self {
4771 Self::with(HealthProbeEvidence::LaneDead, message)
4772 }
4773
4774 fn no_answer(message: impl Into<String>) -> Self {
4775 Self::with(HealthProbeEvidence::NoAnswer, message)
4776 }
4777
4778 fn bad_answer(message: impl Into<String>) -> Self {
4779 Self::with(HealthProbeEvidence::BadAnswer, message)
4780 }
4781
4782 fn misconfigured(message: impl Into<String>) -> Self {
4783 Self::with(HealthProbeEvidence::Misconfigured, message)
4784 }
4785
4786 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4787 Self {
4788 evidence,
4789 message: message.into(),
4790 }
4791 }
4792
4793 #[allow(dead_code)]
4807 fn is_proof_of_death(&self) -> bool {
4808 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4809 }
4810
4811 fn label(&self) -> &'static str {
4819 match self.evidence {
4820 HealthProbeEvidence::LaneDead => "lane-dead",
4821 HealthProbeEvidence::NoAnswer => "no-answer",
4822 HealthProbeEvidence::BadAnswer => "bad-answer",
4823 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4824 }
4825 }
4826}
4827
4828impl fmt::Display for HealthProbeError {
4829 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4830 f.write_str(&self.message)
4831 }
4832}
4833
4834async fn run_health_probe_cycle(
4835 spec: &ModuleSpec,
4836 runtime: &SupervisorRuntimeConfig,
4837 registry: &Registry,
4838 process_liveness: &SupervisorProcessLiveness,
4839 snapshot: &SharedSnapshot,
4840 child: &mut Option<SupervisedChild>,
4841) {
4842 let now_ms = unix_ms_now();
4843 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4844 .then_some(runtime.health.http.as_deref())
4845 .flatten();
4846 let result = match http {
4847 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4848 None => probe_module_health(&spec.module_id, runtime, None).await,
4849 };
4850 match result {
4851 Ok(report) => {
4852 handle_health_report(
4853 spec,
4854 runtime,
4855 registry,
4856 process_liveness,
4857 snapshot,
4858 child,
4859 report,
4860 now_ms,
4861 )
4862 .await;
4863 }
4864 Err(err) => {
4865 if http.is_some() {
4866 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4867 state.health.status = SupervisorHealthStatus::Failing;
4868 });
4869 }
4870 handle_health_probe_failure(
4871 spec,
4872 runtime,
4873 registry,
4874 process_liveness,
4875 snapshot,
4876 child,
4877 err,
4878 now_ms,
4879 )
4880 .await;
4881 }
4882 }
4883}
4884
4885pub(crate) struct HttpProbeTarget<'a> {
4886 address: std::net::SocketAddr,
4887 localhost: bool,
4888 authority: &'a str,
4889 path: String,
4890}
4891
4892pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4895 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4896 return Err("must not contain whitespace, controls, or a fragment".into());
4897 }
4898 let rest = url
4899 .strip_prefix("http://")
4900 .ok_or("must use plain http://")?;
4901 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4902 let (authority, suffix) = rest.split_at(split);
4903 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4904 ("::1", rest)
4905 } else {
4906 let split = authority.find(':').unwrap_or(authority.len());
4907 authority.split_at(split)
4908 };
4909 let ip: std::net::IpAddr = match host {
4910 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4911 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4912 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4913 };
4914 let port = if port.is_empty() {
4915 80
4916 } else {
4917 port.strip_prefix(':')
4918 .and_then(|p| p.parse::<u16>().ok())
4919 .filter(|p| *p > 0)
4920 .ok_or("must have a valid nonzero TCP port")?
4921 };
4922 let path = if suffix.is_empty() {
4923 "/".into()
4924 } else if suffix.starts_with('?') {
4925 format!("/{suffix}")
4926 } else {
4927 suffix.into()
4928 };
4929 Ok(HttpProbeTarget {
4930 address: std::net::SocketAddr::new(ip, port),
4931 localhost: host == "localhost",
4932 authority,
4933 path,
4934 })
4935}
4936
4937async fn probe_http_health(
4938 url: &str,
4939 deadline: Duration,
4940) -> Result<HealthReport, HealthProbeError> {
4941 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4942 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4943 let mut response_status = String::new();
4946 let mut body = Vec::new();
4947 let probe = async {
4948 let connection = match tokio::net::TcpStream::connect(target.address).await {
4951 Err(_) if target.localhost => {
4952 tokio::net::TcpStream::connect((
4953 std::net::Ipv6Addr::LOCALHOST,
4954 target.address.port(),
4955 ))
4956 .await
4957 }
4958 result => result,
4959 };
4960 let mut stream = connection.map_err(|error| {
4961 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4962 })?;
4963 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4964 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4965 let mut reader = BufReader::new(stream);
4966 let mut budget = 16 * 1024;
4967 let status = http_line(&mut reader, &mut budget).await?;
4968 let mut words = status.split_ascii_whitespace();
4969 let version = words.next();
4970 let code = words
4971 .next()
4972 .filter(|word| word.len() == 3)
4973 .and_then(|word| word.parse::<u16>().ok());
4974 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4975 || !code.is_some_and(|code| (100..600).contains(&code))
4976 {
4977 return Err(HealthProbeError::bad_answer(format!(
4978 "invalid HTTP status: {status}"
4979 )));
4980 }
4981 let code = code.expect("validated status code");
4982 response_status = status.clone();
4983 let mut length = None;
4984 let mut chunked = false;
4985 loop {
4986 let line = http_line(&mut reader, &mut budget).await?;
4987 if line.is_empty() {
4988 break;
4989 }
4990 if let Some((name, value)) = line.split_once(':') {
4991 if name.eq_ignore_ascii_case("content-length") {
4992 length = Some(value.trim().parse::<u64>().map_err(|_| {
4993 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4994 })?);
4995 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4996 chunked = value.trim().eq_ignore_ascii_case("chunked");
4997 }
4998 }
4999 }
5000 if chunked {
5001 while body.len() < 200 {
5002 let line = http_line(&mut reader, &mut budget).await?;
5003 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5004 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5005 if size == 0 {
5006 break;
5007 }
5008 let count = size.min((200 - body.len()) as u64) as usize;
5009 let start = body.len();
5010 (&mut reader)
5011 .take(count as u64)
5012 .read_to_end(&mut body)
5013 .await
5014 .map_err(|error| {
5015 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5016 })?;
5017 if body.len() - start != count {
5018 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5019 }
5020 if size > count as u64 || body.len() == 200 {
5021 break;
5022 }
5023 if !http_line(&mut reader, &mut budget).await?.is_empty() {
5024 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5025 }
5026 }
5027 } else {
5028 reader
5029 .take(length.unwrap_or(200).min(200))
5030 .read_to_end(&mut body)
5031 .await
5032 .map_err(|error| {
5033 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5034 })?;
5035 }
5036 if (200..300).contains(&code) {
5037 Ok(HealthReport::ok())
5038 } else {
5039 Err(HealthProbeError::bad_answer(
5040 "HTTP health endpoint returned non-2xx",
5041 ))
5042 }
5043 };
5044 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5045 Err(HealthProbeError::no_answer(format!(
5046 "HTTP probe timed out after {deadline:?}"
5047 )))
5048 });
5049 if let Err(error) = &mut result {
5050 if !response_status.is_empty() {
5051 error.message = format!(
5052 "{}; {response_status}: {}",
5053 error.message,
5054 String::from_utf8_lossy(&body)
5055 );
5056 }
5057 }
5058 result
5059}
5060
5061async fn http_line(
5062 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5063 remaining: &mut usize,
5064) -> Result<String, HealthProbeError> {
5065 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5066 let mut line = Vec::new();
5067 (&mut *reader)
5068 .take(*remaining as u64)
5069 .read_until(b'\n', &mut line)
5070 .await
5071 .map_err(|error| {
5072 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5073 })?;
5074 *remaining -= line.len();
5075 if !line.ends_with(b"\r\n") {
5076 return Err(HealthProbeError::bad_answer(
5077 "HTTP headers are incomplete or exceed 16 KiB",
5078 ));
5079 }
5080 line.truncate(line.len() - 2);
5081 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5082}
5083
5084async fn probe_module_health(
5085 module_id: &str,
5086 runtime: &SupervisorRuntimeConfig,
5087 drain_deadline: Option<Instant>,
5088) -> Result<HealthReport, HealthProbeError> {
5089 let Some(forwarding) = runtime.forwarding.as_ref() else {
5090 return Err(HealthProbeError::misconfigured(
5091 "supervisor was not configured with a forwarding table",
5092 ));
5093 };
5094 let probe_started_at = Instant::now();
5095 let mut deadline = probe_started_at + runtime.health.deadline;
5096 if let Some(drain_deadline) = drain_deadline {
5097 deadline = deadline.min(drain_deadline);
5098 }
5099 let pending = if drain_deadline.is_some() {
5100 forwarding.begin_drain_health_probe_rpc_for(
5101 module_id,
5102 MODULE_CONTROL_OP_HEALTH_CHECK,
5103 probe_started_at,
5104 deadline,
5105 )
5106 } else {
5107 forwarding.begin_health_probe_rpc_for(
5108 module_id,
5109 MODULE_CONTROL_OP_HEALTH_CHECK,
5110 probe_started_at,
5111 deadline,
5112 )
5113 }
5114 .map_err(|err| {
5115 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5118 })?;
5119 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5120}
5121
5122async fn probe_endpoint_health(
5129 endpoint: crate::ModuleEndpointId,
5130 runtime: &SupervisorRuntimeConfig,
5131 deadline_cap: Option<Instant>,
5132) -> Result<HealthReport, HealthProbeError> {
5133 let Some(forwarding) = runtime.forwarding.as_ref() else {
5134 return Err(HealthProbeError::misconfigured(
5135 "supervisor was not configured with a forwarding table",
5136 ));
5137 };
5138 let probe_started_at = Instant::now();
5139 let mut deadline = probe_started_at + runtime.health.deadline;
5140 if let Some(cap) = deadline_cap {
5141 deadline = deadline.min(cap);
5142 }
5143 let pending = forwarding
5144 .begin_endpoint_health_probe_rpc_for(
5145 endpoint,
5146 MODULE_CONTROL_OP_HEALTH_CHECK,
5147 probe_started_at,
5148 deadline,
5149 )
5150 .map_err(|err| {
5151 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5152 })?;
5153 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5154}
5155
5156async fn await_health_probe(
5158 forwarding: &ForwardingTable,
5159 pending: PendingModuleControlRpc,
5160 deadline: Instant,
5161 probe_budget: Duration,
5162) -> Result<HealthReport, HealthProbeError> {
5163 let PendingModuleControlRpc {
5164 endpoint,
5165 module_sink,
5166 negotiated_ver,
5167 corr,
5168 receiver,
5169 } = pending;
5170 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5171 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5172 })?;
5173 let frame = Frame::build_with_version(
5174 negotiated_ver,
5175 FrameType::Request,
5176 control_flags(),
5177 0,
5178 0,
5179 corr,
5180 body,
5181 )
5182 .map_err(|err| {
5183 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5184 })?;
5185
5186 match timeout_at(deadline, module_sink.send(frame)).await {
5192 Ok(Ok(())) => {}
5193 Ok(Err(err)) => {
5194 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5195 return Err(HealthProbeError::lane_dead(format!(
5198 "failed to send health.check: {err}"
5199 )));
5200 }
5201 Err(_elapsed) => {
5202 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5203 return Err(HealthProbeError::no_answer(
5207 "health.check send timed out before enqueue (module egress full)",
5208 ));
5209 }
5210 }
5211
5212 match timeout_at(deadline, receiver).await {
5213 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5217 response.health_report().ok_or_else(|| {
5218 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5219 })
5220 }
5221 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5222 format!("health.check rejected: {}", body.message),
5223 )),
5224 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5225 Err(HealthProbeError::lane_dead(message))
5226 }
5227 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5228 Err(HealthProbeError::bad_answer(message))
5229 }
5230 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5231 Err(HealthProbeError::bad_answer(format!(
5232 "expected module-control op '{expected}', got '{actual}'"
5233 )))
5234 }
5235 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5239 "module answered health.check after its daemon deadline",
5240 )),
5241 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5242 "health.check waiter was canceled before the module responded",
5243 )),
5244 Err(_) => {
5245 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5246 Err(HealthProbeError::no_answer(format!(
5247 "module did not answer health.check within {probe_budget:?}"
5248 )))
5249 }
5250 }
5251}
5252
5253#[allow(clippy::too_many_arguments)]
5254async fn handle_health_report(
5255 spec: &ModuleSpec,
5256 runtime: &SupervisorRuntimeConfig,
5257 registry: &Registry,
5258 process_liveness: &SupervisorProcessLiveness,
5259 snapshot: &SharedSnapshot,
5260 child: &mut Option<SupervisedChild>,
5261 report: HealthReport,
5262 now_ms: u64,
5263) {
5264 let status = supervisor_health_status(report.status);
5265 let detail = report.detail.clone();
5266 let metrics = truncate_health_metrics(report.metrics);
5267 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5268 state.health.status = status;
5269 state.health.last_probe_ms = Some(now_ms);
5270 state.health.detail = detail.clone();
5271 state.health.metrics = metrics.clone();
5272 state.health.consecutive_failures = 0;
5273 state.failed_retry.observe_health(
5274 report.status == HealthStatus::Ok,
5275 runtime.restart_policy.window,
5276 );
5277 });
5278
5279 let action = match report.status {
5280 HealthStatus::Ok => return,
5281 HealthStatus::Degraded => runtime.health.on_degraded,
5282 HealthStatus::Failing => runtime.health.on_failing,
5283 };
5284 apply_l3_health_action(
5285 spec,
5286 runtime,
5287 registry,
5288 process_liveness,
5289 snapshot,
5290 child,
5291 status,
5292 detail.as_deref(),
5293 action,
5294 now_ms,
5295 )
5296 .await;
5297}
5298
5299#[allow(clippy::too_many_arguments)]
5300async fn handle_health_probe_failure(
5301 spec: &ModuleSpec,
5302 runtime: &SupervisorRuntimeConfig,
5303 registry: &Registry,
5304 process_liveness: &SupervisorProcessLiveness,
5305 snapshot: &SharedSnapshot,
5306 child: &mut Option<SupervisedChild>,
5307 err: HealthProbeError,
5308 now_ms: u64,
5309) {
5310 let threshold = runtime.health.failure_threshold.max(1);
5311 let mut failures = 0;
5312 let detail = format!("[{}] {err}", err.label());
5317 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5318 state
5319 .failed_retry
5320 .observe_health(false, runtime.restart_policy.window);
5321 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5324 state.health.status = SupervisorHealthStatus::Unknown;
5325 }
5326 state.health.last_probe_ms = Some(now_ms);
5327 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5328 state.health.detail = Some(detail.clone());
5329 state.health.metrics = None;
5330 failures = state.health.consecutive_failures;
5331 });
5332
5333 if failures < threshold {
5334 warn!(
5335 module_id = %spec.module_id,
5336 consecutive_failures = failures,
5337 threshold,
5338 evidence = err.label(),
5339 detail = %detail,
5340 "health.check probe failed"
5341 );
5342 return;
5343 }
5344
5345 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5346 state.state = ModuleState::Unresponsive;
5347 state.health.status = SupervisorHealthStatus::Unresponsive;
5348 });
5349 if runtime.health.critical {
5353 error!(
5354 module_id = %spec.module_id,
5355 status = "unresponsive",
5356 evidence = err.label(),
5357 detail = %detail,
5358 "critical module health alert"
5359 );
5360 } else {
5361 warn!(
5362 module_id = %spec.module_id,
5363 status = "unresponsive",
5364 evidence = err.label(),
5365 detail = %detail,
5366 "module health threshold breached"
5367 );
5368 }
5369 if let Err(err) = health_restart_child(
5370 spec,
5371 runtime,
5372 registry,
5373 process_liveness,
5374 snapshot,
5375 child,
5376 SupervisorHealthStatus::Unresponsive,
5377 Some(&detail),
5378 now_ms,
5379 )
5380 .await
5381 {
5382 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5383 }
5384}
5385
5386#[allow(clippy::too_many_arguments)]
5387async fn apply_l3_health_action(
5388 spec: &ModuleSpec,
5389 runtime: &SupervisorRuntimeConfig,
5390 registry: &Registry,
5391 process_liveness: &SupervisorProcessLiveness,
5392 snapshot: &SharedSnapshot,
5393 child: &mut Option<SupervisedChild>,
5394 status: SupervisorHealthStatus,
5395 detail: Option<&str>,
5396 action: HealthAction,
5397 now_ms: u64,
5398) {
5399 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5400 match action {
5401 HealthAction::Report => {
5402 info!(
5403 module_id = %spec.module_id,
5404 status = ?status,
5405 detail,
5406 "module reported non-ok health"
5407 );
5408 }
5409 HealthAction::Alert => {
5410 error!(
5411 module_id = %spec.module_id,
5412 status = ?status,
5413 detail,
5414 "module health alert"
5415 );
5416 }
5417 HealthAction::Restart => {
5418 if let Err(err) = health_restart_child(
5419 spec,
5420 runtime,
5421 registry,
5422 process_liveness,
5423 snapshot,
5424 child,
5425 status,
5426 detail,
5427 now_ms,
5428 )
5429 .await
5430 {
5431 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5432 }
5433 }
5434 }
5435}
5436
5437#[allow(clippy::too_many_arguments)]
5438async fn health_restart_child(
5439 spec: &ModuleSpec,
5440 runtime: &SupervisorRuntimeConfig,
5441 registry: &Registry,
5442 process_liveness: &SupervisorProcessLiveness,
5443 snapshot: &SharedSnapshot,
5444 child: &mut Option<SupervisedChild>,
5445 status: SupervisorHealthStatus,
5446 detail: Option<&str>,
5447 now_ms: u64,
5448) -> Result<(), SuperviseError> {
5449 let (enabled, schedule) = {
5450 let mut state = lock_snapshot(snapshot)?;
5451 let enabled = state.enabled;
5452 let schedule = if enabled {
5453 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5454 } else {
5455 None
5456 };
5457 (enabled, schedule)
5458 };
5459
5460 if !enabled {
5461 return Err(SuperviseError::Disabled {
5462 module_id: spec.module_id.clone(),
5463 });
5464 }
5465
5466 if schedule.is_none() {
5467 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5468 error!(
5469 module_id = %spec.module_id,
5470 status = ?status,
5471 detail,
5472 max_restarts = runtime.restart_policy.max_restarts,
5473 window_secs = runtime.restart_policy.window.as_secs(),
5474 reason = %runtime.restart_policy.budget_exhausted_detail(),
5475 "health restart budget exhausted; marking module failed"
5476 );
5477 let stop_notice = begin_forwarding_drain_if_configured(
5478 spec,
5479 runtime,
5480 registry,
5481 snapshot,
5482 Some(true),
5483 RouteCloseReason::Disable,
5484 )
5485 .await?;
5486 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5487 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5488 })?;
5489 let result = drain_optional_child(
5490 &spec.module_id,
5491 spec.protocol,
5492 stop_notice,
5493 registry,
5494 runtime.forwarding.as_deref(),
5495 snapshot,
5496 &runtime.terminal_ring,
5497 &runtime.spawn_events,
5498 child,
5499 runtime.drain_timeout,
5500 ModuleState::Failed,
5501 Some(true),
5502 &runtime.failure_context(true),
5503 )
5504 .await;
5505 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5506 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5507 state.failed_retry.schedule(&spec.module_id);
5508 }
5509 })?;
5510 result?;
5511 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5512 return Ok(());
5513 }
5514
5515 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5516 let mut restart_count = 0;
5517 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5518 restart_count = state.crash_restarts.len();
5519 state.state = ModuleState::Unresponsive;
5520 state.health.status = status;
5521 state.health.last_action = Some(HealthAction::Restart.to_string());
5522 state.health.last_action_ms = Some(now_ms);
5523 })?;
5524 warn!(
5525 module_id = %spec.module_id,
5526 status = ?status,
5527 detail,
5528 restart_count,
5529 restart_in_window = schedule.restart_in_window,
5530 delay_ms = schedule.delay.as_millis() as u64,
5531 "health-triggered module restart"
5532 );
5533
5534 let stop_notice = begin_forwarding_drain_if_configured(
5535 spec,
5536 runtime,
5537 registry,
5538 snapshot,
5539 Some(true),
5540 RouteCloseReason::Restart,
5541 )
5542 .await?;
5543 drain_optional_child(
5544 &spec.module_id,
5545 spec.protocol,
5546 stop_notice,
5547 registry,
5548 runtime.forwarding.as_deref(),
5549 snapshot,
5550 &runtime.terminal_ring,
5551 &runtime.spawn_events,
5552 child,
5553 runtime.drain_timeout,
5554 ModuleState::Restarting,
5555 Some(true),
5556 &runtime.failure_context(true),
5557 )
5558 .await?;
5559 schedule_respawn(
5560 runtime,
5561 snapshot,
5562 &spec.module_id,
5563 schedule.delay,
5564 RespawnKind::Spawn,
5565 true,
5566 )
5567}
5568
5569fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5570 if let Some(reply) = runtime
5571 .deferred_reload_reply
5572 .lock()
5573 .unwrap_or_else(|p| p.into_inner())
5574 .take()
5575 {
5576 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5577 module_id: module_id.to_string(),
5578 reason: reason.to_string(),
5579 }));
5580 }
5581}
5582
5583fn schedule_respawn(
5584 runtime: &SupervisorRuntimeConfig,
5585 snapshot: &SharedSnapshot,
5586 module_id: &str,
5587 delay: Duration,
5588 kind: RespawnKind,
5589 daemon_initiated: bool,
5590) -> Result<(), SuperviseError> {
5591 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5592 update_snapshot(snapshot, Some(module_id), |state| {
5593 state.respawn_pending = true
5594 })?;
5595 *runtime
5596 .scheduled_respawn
5597 .lock()
5598 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5599 deadline: Instant::now() + delay,
5600 kind,
5601 daemon_initiated,
5602 });
5603 Ok(())
5604}
5605
5606fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5607 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5608 state.health.last_action = Some(action);
5609 state.health.last_action_ms = Some(now_ms);
5610 });
5611}
5612
5613fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5614 match status {
5615 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5616 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5617 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5618 }
5619}
5620
5621fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5633 let metrics = metrics?;
5634 match serde_json::to_vec(&metrics) {
5635 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5636 "truncated": true,
5637 "original_bytes": encoded.len(),
5638 })),
5639 Ok(_) | Err(_) => Some(metrics),
5640 }
5641}
5642
5643fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5649 if cadence.is_zero() {
5650 return Duration::ZERO;
5651 }
5652 let cadence_ms = cadence.as_millis() as u64;
5653 if cadence_ms == 0 {
5669 return cadence;
5670 }
5671 let jitter_span = (cadence_ms / 10).max(1);
5686 let hash = module_id.as_bytes().iter().fold(
5687 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5688 |acc, byte| {
5689 acc.wrapping_mul(1099511628211)
5690 .wrapping_add(u64::from(*byte))
5691 },
5692 );
5693 cadence + Duration::from_millis(hash % jitter_span)
5694}
5695
5696#[cfg(test)]
5697mod tests {
5698 use super::*;
5699
5700 #[test]
5701 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5702 let handle = SupervisorHandle::new();
5703 let module_id = "readded-tombstone";
5704 handle.record_rescan_removal(module_id);
5705 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5706
5707 handle.apply_identity_configuration(&ModuleSpec {
5708 module_id: module_id.to_string(),
5709 program: PathBuf::from("/test/module"),
5710 args: Vec::new(),
5711 env: Vec::new(),
5712 reserved: false,
5713 reserved_prefixes: Vec::new(),
5714 protocol: ModuleProtocol::Subc,
5715 overlap: Default::default(),
5716 });
5717
5718 assert!(
5719 handle.removal_tombstone_age_ms(module_id).is_none(),
5720 "a re-added module must not retain a stale removal tombstone"
5721 );
5722 }
5723
5724 #[derive(Debug, PartialEq, Eq)]
5727 struct OwnerInSpawnWindow {
5728 module_id: String,
5729 configured: bool,
5730 on_roster: bool,
5731 admission_refusal: Option<&'static str>,
5732 }
5733
5734 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5745 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5746 use crate::scopes::ScopeTable;
5747 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5748
5749 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5750 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5751 module_id: module_id.to_string(),
5752 program,
5753 args: Vec::new(),
5754 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5755 .into_iter()
5756 .map(|key| (key.to_string(), dir.path().display().to_string()))
5757 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5758 .collect(),
5759 reserved: false,
5760 reserved_prefixes: Vec::new(),
5761 protocol: ModuleProtocol::Subc,
5762 overlap: Default::default(),
5763 };
5764 let live = super::terminal_history_tests::fake_aft_stub_path();
5765 let missing = dir.path().join("definitely-missing-module");
5766
5767 let handle = SupervisorHandle::new();
5768 let mut supervisor =
5769 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5770 .with_handle(handle.clone());
5771 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5772 let hook_handle = handle.clone();
5773 let hook_observed = Arc::clone(&observed);
5774 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5775 let configured = hook_handle.is_configured(module_id);
5778 let selector = ScopeSelector::new(
5779 Principal::Reserved {
5780 module_id: module_id.to_string(),
5781 },
5782 "s".to_string(),
5783 Some(1),
5784 );
5785 let carrier = Principal::Reserved {
5786 module_id: "carrier".to_string(),
5787 };
5788 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5789 .admit(&carrier, module_id, &selector, configured)
5790 {
5791 Ok(_) => None,
5792 Err(refusal) => Some(refusal.code),
5793 };
5794 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5795 module_id: module_id.to_string(),
5796 configured,
5797 on_roster: hook_handle.get(module_id).is_some(),
5798 admission_refusal,
5799 });
5800 })));
5801
5802 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5803 let configured = supervisor
5804 .supervise_configured(stub("configured", live.clone()), true)
5805 .unwrap();
5806 let with_health = supervisor
5807 .supervise_configured_with_health(
5808 stub("with-health", live.clone()),
5809 true,
5810 HealthConfig::default(),
5811 None,
5812 RestartPolicy::default(),
5813 )
5814 .unwrap();
5815 let failed = supervisor
5818 .supervise_configured_with_health(
5819 stub("failed-spawn", missing.clone()),
5820 true,
5821 HealthConfig::default(),
5822 None,
5823 RestartPolicy::default(),
5824 )
5825 .unwrap();
5826 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5829
5830 let expected = [
5831 "plain",
5832 "configured",
5833 "with-health",
5834 "failed-spawn",
5835 "spawn-error",
5836 ]
5837 .into_iter()
5838 .map(|module_id| OwnerInSpawnWindow {
5839 module_id: module_id.to_string(),
5840 configured: true,
5841 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5842 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5843 })
5844 .collect::<Vec<_>>();
5845 assert_eq!(*observed.lock().unwrap(), expected);
5846
5847 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5848 assert!(
5849 handle.get(module_id).is_some(),
5850 "{module_id} is on the roster"
5851 );
5852 assert!(
5853 handle.is_configured(module_id),
5854 "{module_id} stays configured"
5855 );
5856 }
5857 assert!(handle.get("spawn-error").is_none());
5858 assert!(
5859 !handle.is_configured("spawn-error"),
5860 "a plain spawn that failed must not leave its module marked configured"
5861 );
5862
5863 handle.retire("failed-spawn");
5865 assert!(!handle.is_configured("failed-spawn"));
5866
5867 for module in [plain, configured, with_health] {
5868 module.stop().await.unwrap();
5869 }
5870 drop(failed);
5871 }
5872
5873 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5880 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5881 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5882 let spec = ModuleSpec {
5883 module_id: "spawned-not-admitted".to_string(),
5884 program: super::terminal_history_tests::fake_aft_stub_path(),
5885 args: Vec::new(),
5886 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5887 .into_iter()
5888 .map(|key| (key.to_string(), dir.path().display().to_string()))
5889 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5890 .collect(),
5891 reserved: false,
5892 reserved_prefixes: Vec::new(),
5893 protocol: ModuleProtocol::Subc,
5894 overlap: Default::default(),
5895 };
5896 let handle = SupervisorHandle::new();
5897 let mut supervisor =
5898 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5899 .with_handle(handle.clone());
5900 let observed = Arc::new(Mutex::new(None));
5901 let hook_handle = handle.clone();
5902 let hook_observed = Arc::clone(&observed);
5903 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5904 let state = hook_handle
5905 .get(module_id)
5906 .expect("a configured module is on the roster before it is spawned")
5907 .state()
5908 .unwrap();
5909 *hook_observed.lock().unwrap() = Some(state);
5910 })));
5911
5912 let module = supervisor.supervise_configured(spec, true).unwrap();
5913 assert_eq!(
5914 *observed.lock().unwrap(),
5915 Some(ModuleState::Starting),
5916 "between the spawn and its admission the supervisor still reports starting"
5917 );
5918 assert_eq!(module.state().unwrap(), ModuleState::Running);
5919 module.stop().await.unwrap();
5920 }
5921
5922 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5923 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5924 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5925 snapshot.process_alive = true;
5926 snapshot.pid = Some(41);
5927 snapshot.spawned_at_ms = Some(42);
5928 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5929 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5930 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5931 device: 43,
5932 inode: 44,
5933 #[cfg(windows)]
5934 windows_image: None,
5935 });
5936 })
5937 .unwrap();
5938 snapshot
5939 }
5940
5941 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5942 let snapshot = lock_snapshot(snapshot).unwrap();
5943 assert!(!snapshot.process_alive);
5944 assert_eq!(snapshot.pid, None);
5945 assert_eq!(snapshot.spawned_at_ms, None);
5946 assert_eq!(snapshot.spawned_from, None);
5947 assert_eq!(snapshot.launch_spec_digest, None);
5948 assert_eq!(snapshot.spawned_file_identity, None);
5949 }
5950
5951 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5952 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5953 let supervisor =
5954 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5955 let mut runtime = supervisor.runtime_config();
5956 runtime.test_seed_stale_facts_before_enable_spawn = true;
5957 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5958 let mut child = None;
5959 let spec = ModuleSpec {
5960 module_id: "failed-enable-clears-facts".to_string(),
5961 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5962 args: Vec::new(),
5963 env: Vec::new(),
5964 reserved: false,
5965 reserved_prefixes: Vec::new(),
5966 protocol: ModuleProtocol::Subc,
5967 overlap: Default::default(),
5968 };
5969
5970 let result = set_child_enabled(
5971 &spec,
5972 &runtime,
5973 &supervisor.registry,
5974 &supervisor.process_liveness,
5975 &snapshot,
5976 &mut child,
5977 true,
5978 &runtime.failure_context(false),
5979 )
5980 .await;
5981
5982 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5983 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5984 assert_snapshot_process_facts_cleared(&snapshot);
5985 }
5986
5987 #[tokio::test]
5988 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5989 let supervisor =
5990 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5991 let runtime = supervisor.runtime_config();
5992 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5993 ModuleState::Restarting,
5994 true,
5995 )));
5996 let spec = ModuleSpec {
5997 module_id: "start-stranded-restarting".to_string(),
5998 program: super::terminal_history_tests::fake_aft_stub_path(),
5999 args: Vec::new(),
6000 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6001 reserved: false,
6002 reserved_prefixes: Vec::new(),
6003 protocol: ModuleProtocol::None,
6004 overlap: Default::default(),
6005 };
6006 let mut child = None;
6007 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6008 assert!(!super::set_child_enabled(
6009 &spec,
6010 &runtime,
6011 &Registry::default(),
6012 &supervisor.process_liveness,
6013 &snapshot,
6014 &mut child,
6015 true,
6016 &runtime.failure_context(false),
6017 )
6018 .await
6019 .unwrap());
6020 assert!(child.is_none());
6021 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6022 assert!(super::set_child_enabled(
6023 &spec,
6024 &runtime,
6025 &Registry::default(),
6026 &supervisor.process_liveness,
6027 &snapshot,
6028 &mut child,
6029 true,
6030 &runtime.failure_context(false),
6031 )
6032 .await
6033 .unwrap());
6034 assert_eq!(
6035 lock_snapshot(&snapshot).unwrap().state,
6036 ModuleState::Running
6037 );
6038 let mut child = child.unwrap();
6039 child.start_kill().unwrap();
6040 child.wait().await.unwrap();
6041 }
6042
6043 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6044 async fn failed_reload_spawn_clears_current_process_facts() {
6045 let supervisor =
6046 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6047 let mut runtime = supervisor.runtime_config();
6048 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6049 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6050 let mut child = None;
6051 let spec = ModuleSpec {
6052 module_id: "failed-reload-clears-facts".to_string(),
6053 program: PathBuf::from("/unused/failed-reload-module"),
6054 args: Vec::new(),
6055 env: Vec::new(),
6056 reserved: false,
6057 reserved_prefixes: Vec::new(),
6058 protocol: ModuleProtocol::Subc,
6059 overlap: Default::default(),
6060 };
6061
6062 let result = handle_reload_spawn_failure(
6063 &spec,
6064 &runtime,
6065 &supervisor.process_liveness,
6066 &snapshot,
6067 &mut child,
6068 "forced reload spawn failure".to_string(),
6069 )
6070 .await;
6071
6072 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6073 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6074 assert_snapshot_process_facts_cleared(&snapshot);
6075 }
6076
6077 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6078 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6079 let supervisor =
6080 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6081 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6082 let module = supervisor.supervised_module(
6083 ModuleSpec {
6084 module_id: "drop-clears-facts".to_string(),
6085 program: PathBuf::from("/unused/drop-module"),
6086 args: Vec::new(),
6087 env: Vec::new(),
6088 reserved: false,
6089 reserved_prefixes: Vec::new(),
6090 protocol: ModuleProtocol::Subc,
6091 overlap: Default::default(),
6092 },
6093 supervisor.runtime_config(),
6094 Arc::clone(&snapshot),
6095 None,
6096 );
6097 assert!(!module
6098 .inner
6099 .monitor
6100 .lock()
6101 .unwrap()
6102 .as_ref()
6103 .unwrap()
6104 .is_finished());
6105
6106 drop(module);
6107
6108 assert_eq!(
6109 lock_snapshot(&snapshot).unwrap().state,
6110 ModuleState::Stopped
6111 );
6112 assert_snapshot_process_facts_cleared(&snapshot);
6113 }
6114
6115 #[cfg(unix)]
6116 #[tokio::test]
6117 async fn rescan_preserves_running_protocol_until_respawn() {
6118 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6119 let initial = ModuleSpec {
6120 module_id: "rescan-protocol".into(),
6121 program: PathBuf::from("/bin/sleep"),
6122 args: vec!["60".into()],
6123 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6124 .into_iter()
6125 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6126 .collect(),
6127 reserved: false,
6128 reserved_prefixes: vec![],
6129 protocol: ModuleProtocol::None,
6130 overlap: Default::default(),
6131 };
6132 let supervisor =
6133 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6134 let module = supervisor.spawn(initial.clone()).unwrap();
6135 assert!(module.status().unwrap().live);
6136 let mut next = initial;
6137 next.protocol = ModuleProtocol::Subc;
6138 module
6139 .update_configuration(next.clone(), HealthConfig::default(), None)
6140 .await
6141 .unwrap();
6142 assert!(
6143 module.status().unwrap().live,
6144 "rescan must not require HELLO from the old non-wire process"
6145 );
6146 let runtime = supervisor.runtime_config();
6147 let action = on_child_exit(
6148 &next,
6149 RestartPolicy::default(),
6150 &supervisor.registry,
6151 &module.inner.snapshot,
6152 &runtime.terminal_ring,
6153 &runtime.spawn_events,
6154 &runtime.child_roster,
6155 ExitReport {
6156 kind: ExitKind::Clean,
6157 code: Some(0),
6158 signal: None,
6159 at_ms: unix_ms_now(),
6160 },
6161 true,
6162 )
6163 .await;
6164 assert!(
6165 matches!(action, NextAction::Restart { .. }),
6166 "the old non-wire process's clean exit must restart"
6167 );
6168 module.drain().await.unwrap();
6169 }
6170
6171 #[cfg(unix)]
6172 #[tokio::test]
6173 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6174 use std::os::unix::fs::PermissionsExt;
6175 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6176 let script = dir.join("module.sh");
6177 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6178 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6179 let record_path = dir.join("live-children.json");
6180 let supervisor =
6181 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6182 .with_live_children_record(&record_path);
6183 for (program, args) in [
6184 (PathBuf::from("sleep"), vec!["60".into()]),
6185 (script, vec![]),
6186 ] {
6187 let spec = ModuleSpec {
6188 module_id: "image-identity".into(),
6189 program: program.clone(),
6190 args,
6191 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6192 .into_iter()
6193 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6194 .collect(),
6195 reserved: false,
6196 reserved_prefixes: vec![],
6197 protocol: ModuleProtocol::None,
6198 overlap: Default::default(),
6199 };
6200 let module = supervisor.spawn(spec).unwrap();
6201 #[cfg(target_os = "macos")]
6202 {
6203 let deadline = Instant::now() + Duration::from_secs(5);
6206 while crate::live_children::read_record(&record_path)
6207 .unwrap()
6208 .iter()
6209 .all(|entry| entry.executable.is_none())
6210 {
6211 assert!(Instant::now() < deadline, "module image was not confirmed");
6212 tokio::time::sleep(Duration::from_millis(5)).await;
6213 }
6214 }
6215 let entry = crate::live_children::read_record(&record_path)
6216 .unwrap()
6217 .pop()
6218 .unwrap();
6219 let observed = subc_os::Process::open(entry.pid)
6220 .unwrap()
6221 .unwrap()
6222 .observe()
6223 .unwrap();
6224 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6225 module.drain().await.unwrap();
6226 assert_eq!(
6227 verdict,
6228 crate::live_children::IdentityVerdict::Matches,
6229 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6230 );
6231 }
6232 }
6233
6234 #[cfg(unix)]
6235 fn http_fixture(
6236 dir: &std::path::Path,
6237 url: &str,
6238 threshold: u32,
6239 ) -> crate::daemon_config::ConfiguredModule {
6240 let path = dir.join("subc.jsonc");
6241 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6242 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6243 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6244 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6245 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6246 }}}).to_string()).unwrap();
6247 crate::daemon_config::load(&path)
6248 .unwrap()
6249 .unwrap()
6250 .modules
6251 .pop()
6252 .unwrap()
6253 }
6254
6255 #[cfg(unix)]
6256 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6257 timeout(Duration::from_secs(5), async {
6258 loop {
6259 if module.status().unwrap().health.status == status {
6260 break;
6261 }
6262 sleep(Duration::from_millis(5)).await;
6263 }
6264 })
6265 .await
6266 .unwrap_or_else(|_| {
6267 panic!(
6268 "expected {status:?}, got {:?}",
6269 module.status().unwrap().health
6270 )
6271 });
6272 }
6273
6274 #[cfg(unix)]
6275 #[tokio::test]
6276 async fn http_health_status_flips_ok_failing_ok() {
6277 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6278 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6279 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6280 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6281 let serving_status = status.clone();
6282 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6283 let server = tokio::spawn(async move {
6284 loop {
6285 let (mut stream, _) = listener.accept().await.unwrap();
6286 let mut request = [0u8; 2048];
6287 let count = stream.read(&mut request).await.unwrap();
6288 assert!(count > 0, "a probe must send an HTTP request");
6289 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6290 let body = if code == 200 {
6291 "ready"
6292 } else {
6293 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6294 };
6295 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6296 let _ = stream.write_all(response.as_bytes()).await;
6297 }
6298 });
6299 let configured = http_fixture(&dir, &url, 1000);
6300 let module =
6301 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6302 .supervise_configured_with_health(
6303 configured.module_spec(),
6304 true,
6305 configured.health,
6306 configured.drain_timeout_ms,
6307 configured.restart,
6308 )
6309 .unwrap();
6310 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6311 status.store(503, std::sync::atomic::Ordering::SeqCst);
6312 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6313 assert!(module
6314 .status()
6315 .unwrap()
6316 .health
6317 .detail
6318 .unwrap()
6319 .contains("scratch failure"));
6320 status.store(200, std::sync::atomic::Ordering::SeqCst);
6321 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6322 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6323 let before = module.status().unwrap();
6324 let (spec, mut health) = module.configuration().unwrap();
6325 health.http = None;
6326 module
6327 .update_configuration(spec.clone(), health.clone(), Some(10))
6328 .await
6329 .unwrap();
6330 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6331 health.http = Some(url);
6332 module
6333 .update_configuration(spec, health, Some(10))
6334 .await
6335 .unwrap();
6336 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6337 assert_eq!(
6338 module.status().unwrap().pid,
6339 before.pid,
6340 "changing a probe must apply live, not restart its process"
6341 );
6342 let (spec, mut health) = module.configuration().unwrap();
6343 health.failure_threshold = 2;
6344 module
6345 .update_configuration(spec, health, Some(10))
6346 .await
6347 .unwrap();
6348 status.store(503, std::sync::atomic::Ordering::SeqCst);
6349 timeout(Duration::from_secs(5), async {
6350 while module.status().unwrap().spawn_generation == before.spawn_generation {
6351 sleep(Duration::from_millis(5)).await;
6352 }
6353 })
6354 .await
6355 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6356 module.drain().await.unwrap();
6357 server.abort();
6358 }
6359
6360 #[cfg(unix)]
6361 #[tokio::test]
6362 async fn http_health_replacement_recovers_after_one_restart() {
6363 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6364 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6365 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6366 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6367 let mut configured = http_fixture(&dir, &url, 2);
6368 configured.restart.max_restarts = 2;
6369 let module =
6370 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6371 .supervise_configured_with_health(
6372 configured.module_spec(),
6373 true,
6374 configured.health,
6375 configured.drain_timeout_ms,
6376 configured.restart,
6377 )
6378 .unwrap();
6379 let first_generation = module.status().unwrap().spawn_generation;
6380 let observed = module.clone();
6381 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6382 let server = tokio::spawn(async move {
6383 let mut generation = first_generation;
6384 let mut probes = 0;
6385 loop {
6386 let (mut stream, _) = listener.accept().await.unwrap();
6387 let mut request = [0; 2048];
6388 assert!(stream.read(&mut request).await.unwrap() > 0);
6389 let current = observed.status().unwrap().spawn_generation;
6390 if current != generation {
6391 generation = current;
6392 probes = 0;
6393 }
6394 probes += 1;
6395 let healthy = generation != first_generation && probes > 1;
6398 let code = if healthy { 200 } else { 503 };
6399 let response = format!(
6400 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6401 );
6402 let _ = stream.write_all(response.as_bytes()).await;
6403 if healthy {
6404 healthy_tx.send(()).await.unwrap();
6405 }
6406 }
6407 });
6408 timeout(Duration::from_secs(5), async {
6409 let mut healthy_probes = 0;
6410 loop {
6411 if module.state().unwrap() == ModuleState::Failed {
6412 break;
6413 }
6414 while healthy_rx.try_recv().is_ok() {
6415 healthy_probes += 1;
6416 }
6417 if healthy_probes >= 3
6418 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6419 {
6420 break;
6421 }
6422 sleep(Duration::from_millis(5)).await;
6423 }
6424 })
6425 .await
6426 .expect("the replacement must settle healthy without exhausting its restart budget");
6427 let status = module.status().unwrap();
6428 module.drain().await.unwrap();
6429 server.abort();
6430 assert_eq!(status.state, ModuleState::Running);
6431 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6432 assert_eq!(status.health.consecutive_failures, 0);
6433 assert_eq!(status.lifetime_restarts, 1);
6434 assert_eq!(status.restart_count, 1);
6435 }
6436
6437 #[cfg(unix)]
6438 #[tokio::test]
6439 async fn http_health_sustained_failure_exhausts_restart_budget() {
6440 use std::collections::BTreeMap;
6441 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6442 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6443 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6444 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6445 let mut configured = http_fixture(&dir, &url, 2);
6446 configured.restart.max_restarts = 2;
6447 let module =
6448 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6449 .supervise_configured_with_health(
6450 configured.module_spec(),
6451 true,
6452 configured.health,
6453 configured.drain_timeout_ms,
6454 configured.restart,
6455 )
6456 .unwrap();
6457 let observed = module.clone();
6458 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6459 let served = probes.clone();
6460 let server = tokio::spawn(async move {
6461 loop {
6462 let (mut stream, _) = listener.accept().await.unwrap();
6463 let mut request = [0; 2048];
6464 assert!(stream.read(&mut request).await.unwrap() > 0);
6465 *served
6466 .lock()
6467 .unwrap()
6468 .entry(observed.status().unwrap().spawn_generation)
6469 .or_default() += 1;
6470 let _ = stream
6471 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6472 .await;
6473 }
6474 });
6475 timeout(Duration::from_secs(5), async {
6476 while module.state().unwrap() != ModuleState::Failed {
6477 sleep(Duration::from_millis(5)).await;
6478 }
6479 })
6480 .await
6481 .expect("sustained failures must still exhaust the health restart budget");
6482 let status = module.status().unwrap();
6483 server.abort();
6484 assert_eq!(status.restart_count, 2);
6485 assert_eq!(status.lifetime_restarts, 2);
6486 assert!(!status.process_alive);
6487 assert_eq!(
6488 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6489 vec![2, 2, 2]
6490 );
6491 }
6492
6493 #[cfg(unix)]
6494 #[tokio::test]
6495 async fn spawn_clears_the_previous_process_health() {
6496 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6497 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6498 configured.health.cadence = Duration::from_secs(3600);
6499 let module =
6500 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6501 .supervise_configured_with_health(
6502 configured.module_spec(),
6503 false,
6504 configured.health,
6505 configured.drain_timeout_ms,
6506 configured.restart,
6507 )
6508 .unwrap();
6509 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6510 state.health.status = SupervisorHealthStatus::Unresponsive;
6511 state.health.consecutive_failures = 3;
6512 state.health.detail = Some("old process refused connections".into());
6513 state.health.metrics = Some(serde_json::json!({"old": true}));
6514 state.health.last_probe_ms = Some(123);
6515 state.health.last_action = Some("restart".into());
6516 state.health.last_action_ms = Some(124);
6517 })
6518 .unwrap();
6519 module.set_enabled(true).await.unwrap();
6520 let health = module.status().unwrap().health;
6521 module.drain().await.unwrap();
6522 assert_eq!(health.consecutive_failures, 0);
6523 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6524 assert_eq!(health.detail, None);
6525 assert_eq!(health.metrics, None);
6526 assert_eq!(health.last_probe_ms, None);
6527 assert_eq!(health.last_action.as_deref(), Some("restart"));
6528 assert_eq!(health.last_action_ms, Some(124));
6529 }
6530
6531 #[cfg(unix)]
6532 #[tokio::test]
6533 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6534 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6535 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6536 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6537 drop(unused);
6538 let configured = http_fixture(&dir, &url, 2);
6539 let module =
6540 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6541 .supervise_configured_with_health(
6542 configured.module_spec(),
6543 true,
6544 configured.health,
6545 configured.drain_timeout_ms,
6546 configured.restart,
6547 )
6548 .unwrap();
6549 let before = module.status().unwrap().spawn_generation;
6550 timeout(Duration::from_secs(5), async {
6551 loop {
6552 let status = module.status().unwrap();
6553 if status.spawn_generation > before {
6554 assert!(status.lifetime_restarts > 0);
6555 break;
6556 }
6557 sleep(Duration::from_millis(5)).await;
6558 }
6559 })
6560 .await
6561 .expect("sustained HTTP refusal must trigger the health restart policy");
6562 module.drain().await.unwrap();
6563 }
6564
6565 #[cfg(unix)]
6566 #[tokio::test]
6567 async fn http_health_timeout_honours_deadline() {
6568 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6569 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6570 let server = tokio::spawn(async move {
6571 let _held = listener.accept().await.unwrap();
6572 std::future::pending::<()>().await;
6573 });
6574 let error = timeout(
6575 Duration::from_secs(1),
6576 probe_http_health(&url, Duration::from_millis(10)),
6577 )
6578 .await
6579 .expect("the probe must enforce its own deadline")
6580 .unwrap_err();
6581 server.abort();
6582 assert!(error.to_string().contains("timed out"));
6583 }
6584
6585 #[cfg(unix)]
6586 #[tokio::test]
6587 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6588 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6589 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6590 let url = format!(
6591 "http://localhost:{}/healthz",
6592 listener.local_addr().unwrap().port()
6593 );
6594 let server = tokio::spawn(async move {
6595 let (mut stream, _) = listener.accept().await.unwrap();
6596 let mut request = [0u8; 2048];
6597 assert!(stream.read(&mut request).await.unwrap() > 0);
6598 stream
6599 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6600 .await
6601 .unwrap();
6602 });
6603 assert_eq!(
6608 probe_http_health(&url, Duration::from_secs(10))
6609 .await
6610 .unwrap()
6611 .status,
6612 HealthStatus::Ok
6613 );
6614 server.await.unwrap();
6615 }
6616
6617 #[cfg(unix)]
6618 #[tokio::test]
6619 async fn http_health_timeout_keeps_partial_status_and_body() {
6620 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6621 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6622 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6623 let server = tokio::spawn(async move {
6624 let (mut stream, _) = listener.accept().await.unwrap();
6625 let mut request = [0u8; 2048];
6626 assert!(stream.read(&mut request).await.unwrap() > 0);
6627 stream
6628 .write_all(
6629 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6630 )
6631 .await
6632 .unwrap();
6633 std::future::pending::<()>().await;
6634 });
6635 let error = probe_http_health(&url, Duration::from_secs(1))
6636 .await
6637 .unwrap_err()
6638 .to_string();
6639 server.abort();
6640 assert!(
6641 error.contains("timed out")
6642 && error.contains("503 Unavailable")
6643 && error.contains("partial diagnostic"),
6644 "{error}"
6645 );
6646 }
6647
6648 #[cfg(unix)]
6649 #[tokio::test]
6650 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6651 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6652 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6653 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6654 let server = tokio::spawn(async move {
6655 let (mut stream, _) = listener.accept().await.unwrap();
6656 let mut request = [0u8; 2048];
6657 assert!(stream.read(&mut request).await.unwrap() > 0);
6658 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6659 let response = format!(
6660 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6661 body.len()
6662 );
6663 stream.write_all(response.as_bytes()).await.unwrap();
6664 });
6665 let error = probe_http_health(&url, Duration::from_secs(1))
6666 .await
6667 .unwrap_err()
6668 .to_string();
6669 server.await.unwrap();
6670 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6671 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6672 assert!(!error.contains("not-in-diagnostic"));
6673 }
6674
6675 #[cfg(unix)]
6676 #[tokio::test]
6677 async fn http_health_real_nats_server_monitoring() {
6678 if std::process::Command::new("nats-server")
6679 .arg("--version")
6680 .env("XDG_DATA_HOME", std::env::temp_dir())
6681 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6682 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6683 .output()
6684 .is_err()
6685 {
6686 eprintln!(
6687 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6688 );
6689 return;
6690 }
6691 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6692 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6693 let port = monitor.local_addr().unwrap().port();
6694 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6695 let client_port = client.local_addr().unwrap().port();
6696 let config = dir.join("server.conf");
6697 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6698 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6699 configured.program = PathBuf::from("nats-server");
6700 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6701 drop(monitor);
6702 drop(client);
6703 let module =
6704 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6705 .supervise_configured_with_health(
6706 configured.module_spec(),
6707 true,
6708 configured.health,
6709 configured.drain_timeout_ms,
6710 configured.restart,
6711 )
6712 .unwrap();
6713 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6714 module.drain().await.unwrap();
6715 }
6716
6717 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6718 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6719 let supervisor =
6720 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6721 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6722 let initial = ModuleSpec {
6723 module_id: "rescan-preserves-spawn-facts".to_string(),
6724 program: PathBuf::from("/spawned/module"),
6725 args: Vec::new(),
6726 env: Vec::new(),
6727 reserved: false,
6728 reserved_prefixes: Vec::new(),
6729 protocol: ModuleProtocol::Subc,
6730 overlap: Default::default(),
6731 };
6732 let module = supervisor.supervised_module(
6733 initial.clone(),
6734 supervisor.runtime_config(),
6735 snapshot,
6736 None,
6737 );
6738 let before = module.status().unwrap();
6739 let mut replacement = initial;
6740 replacement.program = PathBuf::from("/rescanned/replacement-module");
6741
6742 module
6743 .update_configuration(replacement, HealthConfig::default(), None)
6744 .await
6745 .unwrap();
6746
6747 let after = module.status().unwrap();
6748 assert_eq!(after.pid, before.pid);
6749 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6750 assert_eq!(after.spawned_from, before.spawned_from);
6751 drop(module);
6752 }
6753}
6754
6755fn unix_ms_now() -> u64 {
6756 SystemTime::now()
6757 .duration_since(UNIX_EPOCH)
6758 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6759 .unwrap_or(0)
6760}
6761
6762async fn supervise_loop(
6763 mut spec: ModuleSpec,
6764 mut runtime: SupervisorRuntimeConfig,
6765 registry: Arc<Registry>,
6766 process_liveness: Arc<SupervisorProcessLiveness>,
6767 snapshot: SharedSnapshot,
6768 mut child: Option<SupervisedChild>,
6769 mut commands: mpsc::Receiver<SupervisorCommand>,
6770) {
6771 let mut health_probe = HealthProbeRuntime::default();
6772 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6776 Ok(changes) => Some(changes),
6777 Err(err) => {
6778 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6781 None
6782 }
6783 };
6784 let mut pending_respawn: Option<PendingRespawn> = None;
6788 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6791 let mut previous_instance_recheck = tokio::time::interval_at(
6792 Instant::now() + Duration::from_secs(5),
6793 Duration::from_secs(5),
6794 );
6795 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6796 loop {
6797 #[cfg(test)]
6798 {
6799 lock_snapshot(&snapshot).unwrap().actor_select = None;
6800 }
6801 #[cfg(target_os = "macos")]
6802 if let Some(active) = child.as_mut() {
6803 active.confirm_privacy_exec().await;
6804 }
6805 if let Some(scheduled) = runtime
6806 .scheduled_respawn
6807 .lock()
6808 .unwrap_or_else(|p| p.into_inner())
6809 .take()
6810 {
6811 runtime.recovery_is_daemon = scheduled.daemon_initiated;
6812 pending_respawn = Some(scheduled);
6813 }
6814 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6815 pending_respawn = None;
6816 cancel_deferred_reload(
6817 &runtime,
6818 &spec.module_id,
6819 "respawn cancelled by a supervisor command",
6820 );
6821 }
6822 if child.is_none()
6823 && pending_respawn.is_none()
6824 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6825 {
6826 cancel_deferred_reload(
6827 &runtime,
6828 &spec.module_id,
6829 "respawn cancelled before a replacement was spawned",
6830 );
6831 let mut stranded = false;
6832 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6833 state.respawn_pending = false;
6834 state.coalesced_restart_pending = false;
6835 if matches!(
6836 state.state,
6837 ModuleState::Restarting
6838 | ModuleState::Starting
6839 | ModuleState::Draining
6840 | ModuleState::Unresponsive
6841 ) {
6842 error!(module_id = %spec.module_id, state = ?state.state,
6843 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6844 stranded = true;
6845 }
6846 });
6847 if stranded {
6848 fail_snapshot(
6849 &snapshot,
6850 Some(&spec.module_id),
6851 None,
6852 &runtime.failure_context(runtime.recovery_is_daemon),
6853 );
6854 }
6855 }
6856 if let Some(command) = requeued.pop_front() {
6857 if !handle_supervisor_command(
6858 command,
6859 &mut spec,
6860 &mut runtime,
6861 ®istry,
6862 &process_liveness,
6863 &snapshot,
6864 &mut child,
6865 &mut commands,
6866 &mut requeued,
6867 )
6868 .await
6869 {
6870 return;
6871 }
6872 if child.is_some() || !respawn_still_pending(&snapshot) {
6873 pending_respawn = None;
6874 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6875 state.respawn_pending = false
6876 });
6877 }
6878 continue;
6879 }
6880 if child.is_some() {
6881 runtime.recovery_is_daemon = true;
6882 #[cfg(test)]
6883 {
6884 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6885 }
6886 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6887 let active = child.as_mut().expect("child checked above");
6888 if runtime.child_roster.is_closed() || active.registered_since_spawn() {
6889 active.registration_deadline = None;
6890 }
6891 let registration_deadline = active.registration_deadline;
6892 let no_checks = runtime.health.http.is_none()
6893 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
6894 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
6895 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
6896 state
6897 .failed_retry
6898 .uptime_reset_at(no_checks, runtime.restart_policy.window)
6899 });
6900 let wake_at = match (health_probe.wake_at(), uptime_reset) {
6901 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
6902 (probe, None) => probe,
6903 };
6904 let wake_at = match (wake_at, registration_deadline) {
6905 (Some(wake), Some(register)) => Some(wake.min(register)),
6906 (wake, register) => wake.or(register),
6907 };
6908 #[cfg(test)]
6909 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
6910 let probe_sleep = supervisor_sleep(wake_at);
6911 tokio::pin!(probe_sleep);
6912 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6917 #[cfg(test)]
6918 {
6919 let registration_pending = wire_child
6920 && registration_changes
6921 .as_ref()
6922 .is_some_and(|changes| changes.has_changed().unwrap());
6923 let mut state = lock_snapshot(&snapshot).unwrap();
6924 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6925 generation: state.spawn_generation,
6926 turn: state.actor_turns,
6927 registered_connection: health_probe.registered_connection,
6928 next_probe_at: health_probe.next_probe_at,
6929 wake_after,
6930 });
6931 }
6932 let active_child = child.as_mut().expect("child checked above");
6933 tokio::select! {
6934 wait_result = active_child.wait() => {
6935 if let Ok(mut state) = lock_snapshot(&snapshot) {
6936 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
6937 }
6938 #[cfg(test)]
6939 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6940 let exit_report = match wait_result {
6949 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6950 Err(err) => {
6951 active_child.drain_stderr(&spec.module_id).await;
6952 fail_after_wait_error(
6953 &spec.module_id,
6954 &snapshot,
6955 &runtime.terminal_ring,
6956 &runtime.spawn_events,
6957 format!("failed to wait for supervised module: {err}"),
6958 &runtime.failure_context(true),
6959 );
6960 untrack_if_registration_released(
6961 &process_liveness,
6962 ®istry,
6963 &spec.module_id,
6964 &snapshot,
6965 );
6966 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6967 child = None;
6968 continue;
6969 }
6970 };
6971 active_child.drain_stderr(&spec.module_id).await;
6972
6973 let next = on_child_exit(
6974 &spec,
6975 runtime.restart_policy,
6976 ®istry,
6977 &snapshot,
6978 &runtime.terminal_ring,
6979 &runtime.spawn_events,
6980 &runtime.child_roster,
6981 exit_report,
6982 true,
6983 ).await;
6984 runtime.start_requester = None;
6985 runtime.pending_activity_restart = false;
6986 active_child.release_roster();
6989 match next {
6990 NextAction::Stop { registration_released } => {
6991 if registration_released {
6992 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6993 }
6994 child = None;
6995 }
6996 NextAction::Restart { schedule } => {
6997 let delay = schedule.map_or(
6998 runtime.restart_policy.delay_for_restart(0),
6999 |schedule| schedule.delay,
7000 );
7001 if let Some(schedule) = schedule {
7002 log_crash_respawn(&spec.module_id, schedule);
7003 }
7004 child = None;
7012 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7013 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7014 }
7015 }
7016 }
7017 command = commands.recv() => {
7018 #[cfg(test)]
7019 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7020 let Some(command) = command else {
7021 return;
7022 };
7023 if !handle_supervisor_command(
7024 command,
7025 &mut spec,
7026 &mut runtime,
7027 ®istry,
7028 &process_liveness,
7029 &snapshot,
7030 &mut child,
7031 &mut commands,
7032 &mut requeued,
7033 ).await {
7034 return;
7035 }
7036 }
7037 _ = async {
7038 match registration_changes.as_mut() {
7039 Some(changes) => { let _ = changes.changed().await; }
7040 None => std::future::pending().await,
7041 }
7042 }, if wire_child => {
7043 #[cfg(test)]
7044 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7045 }
7046 _ = &mut probe_sleep => {
7047 #[cfg(test)]
7048 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7049 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7050 active_child.registration_deadline = None;
7053 if !runtime.child_roster.is_closed()
7054 && !active_child.registered_since_spawn()
7055 && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7056 {
7057 warn!(module_id = %spec.module_id, pid = active_child.pid,
7058 wait_secs = runtime.register_timeout.as_secs(),
7059 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7060 active_child.spawn_failure = Some("registration_timeout".into());
7061 if let Err(error) = stop_unregistered_child(active_child).await {
7062 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7063 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7064 &runtime.spawn_events, format!("registration_timeout: {error}"),
7065 &runtime.failure_context(true));
7066 child = None;
7067 }
7068 continue;
7071 }
7072 }
7073 if health_probe.due() {
7074 run_health_probe_cycle(
7075 &spec,
7076 &runtime,
7077 ®istry,
7078 &process_liveness,
7079 &snapshot,
7080 &mut child,
7081 ).await;
7082 if child.is_none() {
7083 runtime.start_requester = None;
7084 runtime.pending_activity_restart = false;
7085 }
7086 if child.is_some() {
7087 health_probe.schedule_next(&spec, runtime.health.cadence);
7088 }
7089 }
7090 }
7091 }
7092 } else if let Some(pending) = pending_respawn {
7093 tokio::select! {
7094 _ = sleep_until(pending.deadline) => {
7095 pending_respawn = None;
7096 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7097 if !respawn_still_pending(&snapshot) {
7101 continue;
7102 }
7103 if runtime.child_roster.is_closed() {
7108 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7109 state.state = ModuleState::Stopped;
7110 });
7111 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7112 continue;
7113 }
7114 if let Err(err) = release_dead_registration(
7115 ®istry,
7116 runtime.forwarding.as_deref(),
7117 &snapshot,
7118 &spec.module_id,
7119 &runtime.failure_context(runtime.recovery_is_daemon),
7120 ).await {
7121 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7122 continue;
7123 }
7124
7125 if matches!(pending.kind, RespawnKind::Reload) {
7126 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7127 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
7128 if result.is_ok() && runtime.pending_activity_restart {
7129 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7130 }
7131 runtime.pending_activity_restart = false;
7132 if let Some(reply) = reply { let _ = reply.send(result); }
7133 continue;
7134 }
7135 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7136 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7137 Ok(next_child) => {
7138 child = Some(next_child);
7139 if runtime.pending_activity_restart {
7140 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7141 runtime.pending_activity_restart = false;
7142 }
7143 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7144 }
7145 Err(err) => {
7146 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7147 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7148 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7149 }
7150 }
7151 }
7152 command = commands.recv() => {
7153 let Some(command) = command else {
7154 return;
7155 };
7156 if !handle_supervisor_command(
7157 command,
7158 &mut spec,
7159 &mut runtime,
7160 ®istry,
7161 &process_liveness,
7162 &snapshot,
7163 &mut child,
7164 &mut commands,
7165 &mut requeued,
7166 ).await {
7167 return;
7168 }
7169 if child.is_some() || !respawn_still_pending(&snapshot) {
7174 pending_respawn = None;
7175 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7176 }
7177 }
7178 }
7179 } else {
7180 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7181 (state.enabled && state.state == ModuleState::Failed)
7182 .then_some(state.failed_retry.pending)
7183 .flatten()
7184 });
7185 let command = if lock_snapshot(&snapshot)
7186 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7187 {
7188 tokio::select! {
7189 command = commands.recv() => command,
7190 _ = previous_instance_recheck.tick() => {
7191 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7192 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7193 if enabled && !runtime.child_roster.is_closed() {
7194 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7195 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7196 Ok(next_child) => child = Some(next_child),
7197 Err(error) => {
7198 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7199 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7200 }
7201 }
7202 } else {
7203 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7204 state.waiting_for_previous_pid = None;
7205 state.waiting_for_previous_detail = None;
7206 });
7207 }
7208 }
7209 continue;
7210 }
7211 }
7212 } else {
7213 tokio::select! {
7214 biased;
7215 command = commands.recv() => command,
7216 _ = async {
7217 match slow_retry {
7218 Some(retry) => sleep_until(retry.deadline).await,
7219 None => std::future::pending().await,
7220 }
7221 } => {
7222 let retry = slow_retry.expect("only a scheduled slow retry can wake");
7223 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7224 state.failed_retry.pending = None;
7225 });
7226 if runtime.child_roster.is_closed() {
7227 continue;
7228 }
7229 info!(module_id = %spec.module_id, attempt = retry.attempt,
7230 delay_secs = retry.delay.as_secs(), "starting failed module retry");
7231 if let Err(error) = set_child_enabled(
7232 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
7233 &runtime.failure_context(true),
7234 ).await {
7235 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7236 }
7237 continue;
7238 }
7239 }
7240 };
7241 let Some(command) = command else {
7242 return;
7243 };
7244 if !handle_supervisor_command(
7245 command,
7246 &mut spec,
7247 &mut runtime,
7248 ®istry,
7249 &process_liveness,
7250 &snapshot,
7251 &mut child,
7252 &mut commands,
7253 &mut requeued,
7254 )
7255 .await
7256 {
7257 return;
7258 }
7259 }
7260 }
7261}
7262
7263async fn supervisor_sleep(wake_at: Option<Instant>) {
7266 match wake_at {
7267 Some(deadline) => sleep_until(deadline).await,
7268 None => std::future::pending().await,
7269 }
7270}
7271
7272fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7273 info!(
7274 module_id,
7275 restart_in_window = schedule.restart_in_window,
7276 delay_ms = schedule.delay.as_millis() as u64,
7277 "respawning after crash"
7278 );
7279}
7280
7281fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7287 matches!(
7288 lock_snapshot(snapshot),
7289 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7290 )
7291}
7292
7293enum NextAction {
7294 Stop {
7295 registration_released: bool,
7296 },
7297 Restart {
7298 schedule: Option<CrashRestartSchedule>,
7299 },
7300}
7301
7302#[allow(clippy::too_many_arguments)]
7303async fn handle_supervisor_command(
7304 command: SupervisorCommand,
7305 spec: &mut ModuleSpec,
7306 runtime: &mut SupervisorRuntimeConfig,
7307 registry: &Arc<Registry>,
7308 process_liveness: &SupervisorProcessLiveness,
7309 snapshot: &SharedSnapshot,
7310 child: &mut Option<SupervisedChild>,
7311 commands: &mut mpsc::Receiver<SupervisorCommand>,
7312 requeued: &mut VecDeque<SupervisorCommand>,
7313) -> bool {
7314 if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7315 runtime.recovery_is_daemon = false;
7316 }
7317 if matches!(
7318 command,
7319 SupervisorCommand::Drain { .. }
7320 | SupervisorCommand::Retire { .. }
7321 | SupervisorCommand::SetEnabled { .. }
7322 | SupervisorCommand::Restart { .. }
7323 ) {
7324 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7325 state.failed_retry.cancel()
7326 });
7327 }
7328 let command =
7329 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7330 match command {
7331 SupervisorCommand::Restart { reply, .. }
7332 | SupervisorCommand::Reload { reply, .. }
7333 | SupervisorCommand::Swap { reply, .. } => {
7334 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7335 return true;
7336 }
7337 SupervisorCommand::SetEnabled {
7338 enabled: true,
7339 reply,
7340 ..
7341 } => {
7342 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7343 return true;
7344 }
7345 other => other,
7346 }
7347 } else {
7348 command
7349 };
7350 match command {
7351 SupervisorCommand::Drain { requester, reply } => {
7352 let result = drain_optional_child(
7355 &spec.module_id,
7356 spec.protocol,
7357 StopNotice::NotSent,
7358 registry,
7359 runtime.forwarding.as_deref(),
7360 snapshot,
7361 &runtime.terminal_ring,
7362 &runtime.spawn_events,
7363 child,
7364 runtime.drain_timeout,
7365 ModuleState::Stopped,
7366 None,
7367 &FailureContext::operator(),
7368 )
7369 .await;
7370 let registration_released = result.is_ok();
7371 if registration_released {
7372 runtime.record_activity_transition(
7373 "subc.module_stopped",
7374 &spec.module_id,
7375 requester,
7376 );
7377 }
7378 let _ = reply.send(result);
7379 if registration_released {
7380 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7381 }
7382 false
7383 }
7384 SupervisorCommand::Retire { requester, reply } => {
7385 let result = async {
7386 let stop_notice = begin_forwarding_drain_if_configured(
7387 spec,
7388 runtime,
7389 registry,
7390 snapshot,
7391 None,
7392 RouteCloseReason::Disable,
7393 )
7394 .await?;
7395 drain_optional_child(
7396 &spec.module_id,
7397 spec.protocol,
7398 stop_notice,
7399 registry,
7400 runtime.forwarding.as_deref(),
7401 snapshot,
7402 &runtime.terminal_ring,
7403 &runtime.spawn_events,
7404 child,
7405 runtime.drain_timeout,
7406 ModuleState::Stopped,
7407 None,
7408 &FailureContext::operator(),
7409 )
7410 .await
7411 }
7412 .await;
7413 let registration_released = result.is_ok();
7414 if registration_released {
7415 runtime.record_activity_transition(
7416 "subc.module_stopped",
7417 &spec.module_id,
7418 requester,
7419 );
7420 }
7421 let _ = reply.send(result);
7422 if registration_released {
7423 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7424 }
7425 false
7426 }
7427 SupervisorCommand::Restart {
7428 requester,
7429 drain_timeout_ms,
7430 received_at_generation,
7431 queued_at,
7432 reply,
7433 } => {
7434 info!(
7438 module_id = %spec.module_id,
7439 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7440 "restart command dequeued"
7441 );
7442 let validation = match lock_snapshot(snapshot) {
7454 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7455 module_id: spec.module_id.clone(),
7456 }),
7457 Ok(_) => Ok(()),
7458 Err(err) => Err(err),
7459 };
7460 let initiated = validation.is_ok();
7461 let _ = reply.send(validation);
7462 let satisfied_by_generation = if initiated
7476 && child
7477 .as_ref()
7478 .is_some_and(|child| child.spawned_at > queued_at)
7479 {
7480 lock_snapshot(snapshot).ok().and_then(|state| {
7481 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7482 })
7483 } else {
7484 None
7485 };
7486 let satisfied_by_pending = initiated
7487 && child.is_none()
7488 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7489 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7493 if pending {
7494 state.coalesced_restart_pending = true;
7495 }
7496 pending
7497 });
7498 if satisfied_by_pending {
7499 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7500 } else if let Some(generation) = satisfied_by_generation {
7501 info!(
7502 module_id = %spec.module_id,
7503 received_at_generation,
7504 "restart already satisfied by generation {generation}; not restarting again"
7505 );
7506 } else if initiated {
7507 let drain_timeout = drain_timeout_ms
7510 .map(Duration::from_millis)
7511 .unwrap_or(runtime.drain_timeout);
7512 if let Err(err) = restart_child(
7513 spec,
7514 runtime,
7515 registry,
7516 process_liveness,
7517 snapshot,
7518 child,
7519 drain_timeout,
7520 )
7521 .await
7522 {
7523 warn!(
7524 module_id = %spec.module_id,
7525 error = %err,
7526 "operator restart failed after initiation ack; module state carries the outcome"
7527 );
7528 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7529 state.state = ModuleState::Failed;
7530 clear_current_process_facts(state);
7531 });
7532 } else {
7533 runtime.start_requester = requester;
7534 runtime.pending_activity_restart = true;
7535 }
7536 }
7537 true
7538 }
7539 SupervisorCommand::Reload { requester, reply } => {
7540 let result =
7541 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7542 if result.is_ok() {
7543 runtime.start_requester = requester;
7544 runtime.pending_activity_restart = true;
7545 }
7546 if result.is_ok()
7547 && runtime
7548 .scheduled_respawn
7549 .lock()
7550 .unwrap_or_else(|p| p.into_inner())
7551 .as_ref()
7552 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7553 {
7554 *runtime
7555 .deferred_reload_reply
7556 .lock()
7557 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7558 } else {
7559 let _ = reply.send(result);
7560 }
7561 true
7562 }
7563 SupervisorCommand::SetEnabled {
7564 enabled,
7565 requester,
7566 reply,
7567 } => {
7568 let previous_requester = runtime.start_requester.clone();
7569 if enabled {
7570 runtime.start_requester = requester.clone();
7571 }
7572 let result = set_child_enabled(
7573 spec,
7574 runtime,
7575 registry,
7576 process_liveness,
7577 snapshot,
7578 child,
7579 enabled,
7580 &runtime.failure_context(false),
7581 )
7582 .await;
7583 if matches!(result, Ok(true)) {
7584 runtime.pending_activity_restart = false;
7585 if enabled {
7586 runtime.start_requester = requester;
7587 } else {
7588 runtime.start_requester = None;
7589 runtime.record_activity_transition(
7590 "subc.module_disabled",
7591 &spec.module_id,
7592 requester,
7593 );
7594 }
7595 } else {
7596 runtime.start_requester = previous_requester;
7597 }
7598 let _ = reply.send(result);
7599 true
7600 }
7601 SupervisorCommand::UpdateConfiguration {
7602 spec: next_spec,
7603 health,
7604 drain_timeout_ms,
7605 reply,
7606 } => {
7607 if let Some(handle) = &runtime.supervisor_handle {
7608 handle.apply_identity_configuration(&next_spec);
7609 }
7610 *spec = next_spec;
7611 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7612 state.configuration_updated_since_spawn = true;
7613 });
7614 let health_changed = runtime.health != health;
7615 runtime.health = health;
7616 if health_changed {
7619 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7620 state.health = ModuleHealthStatus::default();
7621 });
7622 }
7623 runtime.drain_timeout = drain_timeout_ms
7624 .map(Duration::from_millis)
7625 .unwrap_or(runtime.default_drain_timeout);
7626 *runtime
7627 .effective_drain_timeout
7628 .lock()
7629 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7630 let _ = reply.send(());
7631 true
7632 }
7633 SupervisorCommand::Swap {
7634 requester,
7635 ready_timeout,
7636 reply,
7637 } => {
7638 let mut swap_runtime = runtime.clone();
7639 swap_runtime.start_requester = requester;
7640 let before = lock_snapshot(snapshot)
7641 .ok()
7642 .map(|state| state.spawn_generation);
7643 let end = swap::run_swap(
7644 spec,
7645 &swap_runtime,
7646 registry,
7647 process_liveness,
7648 snapshot,
7649 child,
7650 commands,
7651 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7652 reply,
7653 )
7654 .await;
7655 if before
7656 != lock_snapshot(snapshot)
7657 .ok()
7658 .map(|state| state.spawn_generation)
7659 {
7660 runtime.start_requester = swap_runtime.start_requester;
7661 }
7662 requeued.extend(end.requeue);
7663 true
7664 }
7665 }
7666}
7667
7668async fn restart_child(
7669 spec: &ModuleSpec,
7670 runtime: &SupervisorRuntimeConfig,
7671 registry: &Registry,
7672 process_liveness: &SupervisorProcessLiveness,
7673 snapshot: &SharedSnapshot,
7674 child: &mut Option<SupervisedChild>,
7675 drain_timeout: Duration,
7676) -> Result<(), SuperviseError> {
7677 if !lock_snapshot(snapshot)?.enabled {
7679 return Err(SuperviseError::Disabled {
7680 module_id: spec.module_id.clone(),
7681 });
7682 }
7683 let stop_notice = begin_forwarding_drain_with_timeout(
7684 spec,
7685 runtime,
7686 registry,
7687 snapshot,
7688 None,
7689 RouteCloseReason::Restart,
7690 drain_timeout,
7691 )
7692 .await?;
7693
7694 if child.is_some() {
7695 drain_optional_child(
7696 &spec.module_id,
7697 spec.protocol,
7698 stop_notice,
7699 registry,
7700 runtime.forwarding.as_deref(),
7701 snapshot,
7702 &runtime.terminal_ring,
7703 &runtime.spawn_events,
7704 child,
7705 drain_timeout,
7706 ModuleState::Restarting,
7707 Some(true),
7708 &FailureContext::operator(),
7709 )
7710 .await?;
7711 } else {
7712 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7713 state.enabled = true;
7714 state.state = ModuleState::Restarting;
7715 clear_current_process_facts(state);
7716 })?;
7717 release_dead_registration(
7718 registry,
7719 runtime.forwarding.as_deref(),
7720 snapshot,
7721 &spec.module_id,
7722 &runtime.failure_context(false),
7723 )
7724 .await?;
7725 }
7726
7727 reset_restart_count(snapshot, &spec.module_id)?;
7728 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7729 schedule_respawn(
7730 runtime,
7731 snapshot,
7732 &spec.module_id,
7733 runtime.restart_policy.backoff,
7734 RespawnKind::Spawn,
7735 false,
7736 )
7737}
7738
7739async fn reload_child(
7740 spec: &ModuleSpec,
7741 runtime: &SupervisorRuntimeConfig,
7742 registry: &Registry,
7743 process_liveness: &SupervisorProcessLiveness,
7744 snapshot: &SharedSnapshot,
7745 child: &mut Option<SupervisedChild>,
7746) -> Result<(), SuperviseError> {
7747 if !lock_snapshot(snapshot)?.enabled {
7749 return Err(SuperviseError::Disabled {
7750 module_id: spec.module_id.clone(),
7751 });
7752 }
7753 let stop_notice = begin_forwarding_drain(
7754 spec,
7755 runtime,
7756 registry,
7757 snapshot,
7758 Some(true),
7759 RouteCloseReason::Reload,
7760 )
7761 .await?;
7762
7763 if child.is_some() {
7764 drain_optional_child(
7765 &spec.module_id,
7766 spec.protocol,
7767 stop_notice,
7768 registry,
7769 runtime.forwarding.as_deref(),
7770 snapshot,
7771 &runtime.terminal_ring,
7772 &runtime.spawn_events,
7773 child,
7774 runtime.drain_timeout,
7775 ModuleState::Restarting,
7776 Some(true),
7777 &FailureContext::operator(),
7778 )
7779 .await?;
7780 } else {
7781 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7782 state.enabled = true;
7783 state.state = ModuleState::Restarting;
7784 clear_current_process_facts(state);
7785 })?;
7786 release_dead_registration(
7787 registry,
7788 runtime.forwarding.as_deref(),
7789 snapshot,
7790 &spec.module_id,
7791 &runtime.failure_context(false),
7792 )
7793 .await?;
7794 }
7795
7796 reset_restart_count(snapshot, &spec.module_id)?;
7797 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7798 schedule_respawn(
7799 runtime,
7800 snapshot,
7801 &spec.module_id,
7802 runtime.restart_policy.backoff,
7803 RespawnKind::Reload,
7804 false,
7805 )
7806}
7807
7808async fn finish_reload_child(
7809 spec: &ModuleSpec,
7810 runtime: &SupervisorRuntimeConfig,
7811 registry: &Registry,
7812 process_liveness: &SupervisorProcessLiveness,
7813 snapshot: &SharedSnapshot,
7814 child: &mut Option<SupervisedChild>,
7815) -> Result<(), SuperviseError> {
7816 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7817 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7818 Ok(next_child) => next_child,
7819 Err(err) => {
7820 return handle_reload_spawn_failure(
7821 spec,
7822 runtime,
7823 process_liveness,
7824 snapshot,
7825 child,
7826 format!("new child failed to spawn: {err}"),
7827 )
7828 .await;
7829 }
7830 };
7831 *child = Some(next_child);
7832
7833 let wait_outcome = {
7834 let active_child = child.as_mut().expect("new reload child was just stored");
7835 wait_for_registration_after_reload(
7836 registry,
7837 &spec.module_id,
7838 snapshot,
7839 active_child,
7840 REGISTRY_RELEASE_TIMEOUT,
7841 )
7842 .await?
7843 };
7844
7845 match wait_outcome {
7846 RegistrationWaitOutcome::Registered => {
7847 child
7848 .as_mut()
7849 .expect("registered reload child")
7850 .registration_deadline = None;
7851 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7852 Ok(())
7853 }
7854 RegistrationWaitOutcome::Exited(exit_report) => {
7855 if let Some(active_child) = child.as_mut() {
7856 active_child.drain_stderr(&spec.module_id).await;
7857 }
7858 let mut exited_child = child.take().expect("exited reload child is still stored");
7861 #[cfg(test)]
7862 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7863 gate.reached.notify_one();
7864 gate.resume.notified().await;
7865 }
7866 let result = handle_reload_child_registration_failure(
7867 spec,
7868 runtime,
7869 registry,
7870 process_liveness,
7871 snapshot,
7872 child,
7873 ReloadRegistrationFailure {
7874 exit_report: registration_failure_exit_report(exit_report),
7875 reason: exited_child
7876 .spawn_failure
7877 .clone()
7878 .unwrap_or_else(|| "new child exited before registering".to_string()),
7879 },
7880 )
7881 .await;
7882 exited_child.release_roster();
7883 result
7884 }
7885 RegistrationWaitOutcome::TimedOut => {
7886 let mut timed_out_child = child
7887 .take()
7888 .expect("timed-out reload child is still running");
7889 timed_out_child
7890 .start_kill()
7891 .map_err(|source| SuperviseError::Kill {
7892 module_id: spec.module_id.clone(),
7893 source,
7894 })?;
7895 let status = timed_out_child
7896 .wait()
7897 .await
7898 .map_err(|source| SuperviseError::Wait {
7899 module_id: spec.module_id.clone(),
7900 source,
7901 })?;
7902 timed_out_child.drain_stderr(&spec.module_id).await;
7903 handle_reload_child_registration_failure(
7904 spec,
7905 runtime,
7906 registry,
7907 process_liveness,
7908 snapshot,
7909 child,
7910 ReloadRegistrationFailure {
7911 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7912 snapshot,
7913 &timed_out_child,
7914 &status,
7915 )),
7916 reason: format!(
7917 "new child did not register within {:?}",
7918 REGISTRY_RELEASE_TIMEOUT
7919 ),
7920 },
7921 )
7922 .await
7923 }
7924 }
7925}
7926
7927#[allow(clippy::too_many_arguments)]
7928async fn set_child_enabled(
7929 spec: &ModuleSpec,
7930 runtime: &SupervisorRuntimeConfig,
7931 registry: &Registry,
7932 process_liveness: &SupervisorProcessLiveness,
7933 snapshot: &SharedSnapshot,
7934 child: &mut Option<SupervisedChild>,
7935 enabled: bool,
7936 failure: &FailureContext,
7937) -> Result<bool, SuperviseError> {
7938 let (current_enabled, current_state, respawn_pending) = {
7939 let state = lock_snapshot(snapshot)?;
7940 (state.enabled, state.state, state.respawn_pending)
7941 };
7942 let revive_terminal = enabled
7950 && current_enabled
7951 && child.is_none()
7952 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7953 || (current_state == ModuleState::Restarting && !respawn_pending));
7954 if current_enabled == enabled && !revive_terminal {
7955 return Ok(false);
7956 }
7957
7958 if enabled {
7959 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7960 state.enabled = true;
7961 state.state = ModuleState::Starting;
7962 clear_current_process_facts(state);
7963 })?;
7964 #[cfg(test)]
7965 if runtime.test_seed_stale_facts_before_enable_spawn {
7966 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7967 state.process_alive = true;
7968 state.pid = Some(41);
7969 state.spawned_at_ms = Some(42);
7970 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7971 state.spawned_file_identity = Some(SpawnedFileIdentity {
7972 device: 43,
7973 inode: 44,
7974 #[cfg(windows)]
7975 windows_image: None,
7976 });
7977 })?;
7978 }
7979 release_dead_registration(
7980 registry,
7981 runtime.forwarding.as_deref(),
7982 snapshot,
7983 &spec.module_id,
7984 failure,
7985 )
7986 .await?;
7987 reset_restart_count(snapshot, &spec.module_id)?;
7988 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7989 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7990 Ok(next_child) => next_child,
7991 Err(err) => {
7992 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7993 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7994 return Err(err);
7995 }
7996 };
7997 *child = Some(next_child);
7998 debug!(module_id = %spec.module_id, "supervised module enabled");
7999 Ok(true)
8000 } else {
8001 let stop_notice = begin_forwarding_drain_if_configured(
8002 spec,
8003 runtime,
8004 registry,
8005 snapshot,
8006 Some(false),
8007 RouteCloseReason::Disable,
8008 )
8009 .await?;
8010 drain_optional_child(
8011 &spec.module_id,
8012 spec.protocol,
8013 stop_notice,
8014 registry,
8015 runtime.forwarding.as_deref(),
8016 snapshot,
8017 &runtime.terminal_ring,
8018 &runtime.spawn_events,
8019 child,
8020 runtime.drain_timeout,
8021 ModuleState::Disabled,
8022 Some(false),
8023 failure,
8024 )
8025 .await?;
8026 debug!(module_id = %spec.module_id, "supervised module disabled");
8027 Ok(true)
8028 }
8029}
8030
8031#[allow(clippy::too_many_arguments)]
8032async fn on_child_exit(
8033 spec: &ModuleSpec,
8034 policy: RestartPolicy,
8035 registry: &Registry,
8036 snapshot: &SharedSnapshot,
8037 terminal_ring: &Arc<Mutex<TerminalRing>>,
8038 spawn_events: &SpawnEventFeed,
8039 roster: &ChildRoster,
8040 exit_report: ExitReport,
8041 daemon_initiated: bool,
8042) -> NextAction {
8043 if roster.is_closed() {
8049 return on_child_exit_during_daemon_shutdown(
8050 spec,
8051 registry,
8052 snapshot,
8053 terminal_ring,
8054 spawn_events,
8055 exit_report,
8056 )
8057 .await;
8058 }
8059 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8075 && running_protocol(spec, snapshot) == ModuleProtocol::None;
8076 match exit_report.kind {
8077 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8078 info!(
8079 module_id = %spec.module_id,
8080 exit_code = ?exit_report.code,
8081 exit_signal = ?exit_report.signal,
8082 "supervised module exited cleanly"
8083 );
8084 if let Err(err) = publish_after_terminal_record(
8085 &spec.module_id,
8086 || {
8087 record_terminal(
8088 &spec.module_id,
8089 terminal_ring,
8090 spawn_events,
8091 &exit_report,
8092 TerminalDisposition::Stopped,
8093 )
8094 },
8095 || {
8096 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8097 state.state = ModuleState::Stopped;
8098 clear_current_process_facts(state);
8099 state.last_exit = Some(exit_report.clone());
8100 })
8101 },
8102 ) {
8103 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8104 }
8105 let registration_released = match wait_for_registration_release(
8106 registry,
8107 &spec.module_id,
8108 REGISTRY_RELEASE_TIMEOUT,
8109 )
8110 .await
8111 {
8112 Ok(()) => true,
8113 Err(err) => {
8114 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8115 false
8116 }
8117 };
8118 NextAction::Stop {
8119 registration_released,
8120 }
8121 }
8122 ExitKind::Clean | ExitKind::Crash => {
8123 if unrequested_clean_exit_of_protocol_none {
8124 warn!(
8125 module_id = %spec.module_id,
8126 exit_code = ?exit_report.code,
8127 exit_signal = ?exit_report.signal,
8128 "protocol-none module exited cleanly without a stop request; handling it as a crash"
8129 );
8130 } else {
8131 warn!(
8132 module_id = %spec.module_id,
8133 exit_code = ?exit_report.code,
8134 exit_signal = ?exit_report.signal,
8135 "supervised module exited abnormally (crash)"
8136 );
8137 }
8138 let mut restart_schedule = None;
8139 let mut disposition = TerminalDisposition::Disabled;
8140 let mut final_state = ModuleState::Disabled;
8141 let mut disposition_detail = lock_snapshot(snapshot)
8145 .ok()
8146 .and_then(|mut state| state.spawn_failure.take());
8147 let now = Instant::now();
8148 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8149 if state.enabled {
8150 if let Some(schedule) = state.next_crash_restart(&policy, now) {
8151 state.respawn_pending = true;
8157 final_state = ModuleState::Restarting;
8158 restart_schedule = Some(schedule);
8159 disposition = TerminalDisposition::Restarting;
8160 } else {
8161 final_state = ModuleState::Failed;
8162 disposition = TerminalDisposition::Failed;
8163 let budget = policy.budget_exhausted_detail();
8164 disposition_detail =
8165 Some(disposition_detail.take().map_or_else(
8166 || budget.clone(),
8167 |cause| format!("{cause}; {budget}"),
8168 ));
8169 }
8170 } else {
8171 disposition = TerminalDisposition::Disabled;
8172 }
8173 }) {
8174 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8175 return NextAction::Stop {
8176 registration_released: false,
8177 };
8178 }
8179 if disposition == TerminalDisposition::Failed {
8180 error!(
8185 module_id = %spec.module_id,
8186 max_restarts = policy.max_restarts,
8187 window_secs = policy.window.as_secs(),
8188 "module stopped: {}",
8189 policy.budget_exhausted_detail()
8190 );
8191 }
8192 if let Err(err) = publish_after_terminal_record(
8193 &spec.module_id,
8194 || {
8195 record_terminal_with_detail(
8196 &spec.module_id,
8197 terminal_ring,
8198 spawn_events,
8199 &exit_report,
8200 disposition,
8201 disposition_detail,
8202 )
8203 },
8204 || {
8205 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8206 state.state = final_state;
8207 clear_current_process_facts(state);
8208 if final_state == ModuleState::Failed
8209 && daemon_initiated
8210 && !roster.is_closed()
8211 {
8212 state.failed_retry.schedule(&spec.module_id);
8213 }
8214 state.last_exit = Some(exit_report.clone());
8215 })
8216 },
8217 ) {
8218 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8219 }
8220
8221 if let Some(schedule) = restart_schedule {
8222 NextAction::Restart {
8223 schedule: Some(schedule),
8224 }
8225 } else {
8226 let registration_released = match wait_for_registration_release(
8227 registry,
8228 &spec.module_id,
8229 REGISTRY_RELEASE_TIMEOUT,
8230 )
8231 .await
8232 {
8233 Ok(()) => true,
8234 Err(err) => {
8235 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8236 false
8237 }
8238 };
8239 NextAction::Stop {
8240 registration_released,
8241 }
8242 }
8243 }
8244 ExitKind::DeliberateSeverance => {
8245 warn!(
8246 module_id = %spec.module_id,
8247 exit_code = ?exit_report.code,
8248 exit_signal = ?exit_report.signal,
8249 "supervised module exited after deliberate connection severance"
8250 );
8251 let should_restart = match lock_snapshot(snapshot) {
8252 Ok(state) => state.enabled,
8253 Err(err) => {
8254 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8255 return NextAction::Stop {
8256 registration_released: false,
8257 };
8258 }
8259 };
8260 let final_state = if should_restart {
8261 ModuleState::Restarting
8262 } else {
8263 ModuleState::Disabled
8264 };
8265 if let Err(err) = publish_after_terminal_record(
8266 &spec.module_id,
8267 || {
8268 record_terminal(
8269 &spec.module_id,
8270 terminal_ring,
8271 spawn_events,
8272 &exit_report,
8273 terminal_disposition(final_state),
8274 )
8275 },
8276 || {
8277 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8278 state.state = final_state;
8279 clear_current_process_facts(state);
8280 state.last_exit = Some(exit_report.clone());
8281 state.lifetime_restarts += 1;
8282 })
8283 },
8284 ) {
8285 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8286 return NextAction::Stop {
8287 registration_released: false,
8288 };
8289 }
8290
8291 if should_restart {
8292 NextAction::Restart { schedule: None }
8293 } else {
8294 let registration_released = match wait_for_registration_release(
8295 registry,
8296 &spec.module_id,
8297 REGISTRY_RELEASE_TIMEOUT,
8298 )
8299 .await
8300 {
8301 Ok(()) => true,
8302 Err(err) => {
8303 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8304 false
8305 }
8306 };
8307 NextAction::Stop {
8308 registration_released,
8309 }
8310 }
8311 }
8312 }
8313}
8314
8315async fn on_child_exit_during_daemon_shutdown(
8316 spec: &ModuleSpec,
8317 registry: &Registry,
8318 snapshot: &SharedSnapshot,
8319 terminal_ring: &Arc<Mutex<TerminalRing>>,
8320 spawn_events: &SpawnEventFeed,
8321 exit_report: ExitReport,
8322) -> NextAction {
8323 info!(
8324 module_id = %spec.module_id,
8325 exit_code = ?exit_report.code,
8326 exit_signal = ?exit_report.signal,
8327 exit_kind = ?exit_report.kind,
8328 "supervised module exited during daemon shutdown; not restarting it"
8329 );
8330 if let Err(err) = publish_after_terminal_record(
8331 &spec.module_id,
8332 || {
8333 record_terminal(
8334 &spec.module_id,
8335 terminal_ring,
8336 spawn_events,
8337 &exit_report,
8338 TerminalDisposition::DaemonShutdown,
8339 )
8340 },
8341 || {
8342 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8343 state.state = ModuleState::Stopped;
8344 clear_current_process_facts(state);
8345 state.last_exit = Some(exit_report.clone());
8346 })
8347 },
8348 ) {
8349 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8350 }
8351 let registration_released =
8352 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8353 .await
8354 .is_ok();
8355 NextAction::Stop {
8356 registration_released,
8357 }
8358}
8359
8360fn publish_after_terminal_record<T>(
8365 module_id: &str,
8366 record: impl FnOnce(),
8367 publish: impl FnOnce() -> T,
8368) -> T {
8369 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8370 let published = publish();
8371 if let Err(panic) = recorded {
8372 error!(
8373 module_id,
8374 "terminal recording panicked; published final state before resuming unwind"
8375 );
8376 std::panic::resume_unwind(panic);
8377 }
8378 published
8379}
8380
8381fn fail_after_wait_error(
8382 module_id: &str,
8383 snapshot: &SharedSnapshot,
8384 terminal_ring: &Arc<Mutex<TerminalRing>>,
8385 spawn_events: &SpawnEventFeed,
8386 detail: String,
8387 failure: &FailureContext,
8388) {
8389 let report = wait_error_exit_report();
8390 publish_after_terminal_record(
8391 module_id,
8392 || {
8393 record_terminal_with_detail(
8394 module_id,
8395 terminal_ring,
8396 spawn_events,
8397 &report,
8398 TerminalDisposition::Failed,
8399 Some(detail),
8400 )
8401 },
8402 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8403 );
8404}
8405
8406#[cfg(test)]
8407fn record_wait_error_terminal(
8408 module_id: &str,
8409 terminal_ring: &Arc<Mutex<TerminalRing>>,
8410 spawn_events: &SpawnEventFeed,
8411) {
8412 record_terminal(
8413 module_id,
8414 terminal_ring,
8415 spawn_events,
8416 &wait_error_exit_report(),
8417 TerminalDisposition::Failed,
8418 );
8419}
8420
8421fn record_terminal(
8422 module_id: &str,
8423 terminal_ring: &Arc<Mutex<TerminalRing>>,
8424 spawn_events: &SpawnEventFeed,
8425 exit_report: &ExitReport,
8426 disposition: TerminalDisposition,
8427) {
8428 record_terminal_with_detail(
8429 module_id,
8430 terminal_ring,
8431 spawn_events,
8432 exit_report,
8433 disposition,
8434 None,
8435 );
8436}
8437
8438fn durable_terminal_history_of(
8442 terminal_ring: &Mutex<TerminalRing>,
8443 module_id: &str,
8444) -> subc_control::TerminalHistory {
8445 let read = terminal_ring
8446 .lock()
8447 .unwrap_or_else(|p| p.into_inner())
8448 .capture_durable_history();
8449 read.read(module_id)
8450}
8451
8452fn record_terminal_with_detail(
8453 module_id: &str,
8454 terminal_ring: &Arc<Mutex<TerminalRing>>,
8455 spawn_events: &SpawnEventFeed,
8456 exit_report: &ExitReport,
8457 disposition: TerminalDisposition,
8458 disposition_detail: Option<String>,
8459) {
8460 #[cfg(test)]
8461 terminal_history_tests::before_terminal_record(spawn_events);
8462 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8463 let record = TerminalRecord {
8464 exit_code: exit_report.code,
8465 exit_signal: exit_report.signal,
8466 at_ms: exit_report.at_ms,
8467 disposition,
8468 exit_kind: exit_report.kind.into(),
8469 disposition_detail,
8470 };
8471 terminal_ring
8472 .lock()
8473 .unwrap_or_else(|poisoned| poisoned.into_inner())
8474 .record_exit(module_id, record);
8475}
8476
8477fn untrack_if_registration_released(
8478 process_liveness: &SupervisorProcessLiveness,
8479 registry: &Registry,
8480 module_id: &str,
8481 snapshot: &SharedSnapshot,
8482) {
8483 match registry.get_module(module_id) {
8484 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8485 Ok(Some(_)) => {}
8486 Err(err) => {
8487 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8488 }
8489 }
8490}
8491
8492#[cfg(test)]
8506fn apply_wire_spawn_args(
8507 command: &mut Command,
8508 spec: &ModuleSpec,
8509 connection_file_path: Option<&std::path::Path>,
8510 handle: Option<&SupervisorHandle>,
8511) -> Result<Option<NonceHandoff>, SuperviseError> {
8512 apply_wire_spawn_args_for_role(
8513 command,
8514 spec,
8515 connection_file_path,
8516 handle,
8517 SpawnRole::Plain,
8518 )
8519}
8520
8521#[cfg(unix)]
8526type NonceHandoff = subc_os::LaunchNonceHandoff;
8527#[cfg(windows)]
8528type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8529#[cfg(not(any(unix, windows)))]
8530type NonceHandoff = std::convert::Infallible;
8531
8532fn apply_wire_spawn_args_for_role(
8553 command: &mut Command,
8554 spec: &ModuleSpec,
8555 connection_file_path: Option<&std::path::Path>,
8556 handle: Option<&SupervisorHandle>,
8557 role: SpawnRole,
8558) -> Result<Option<NonceHandoff>, SuperviseError> {
8559 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8560 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8565 #[cfg(windows)]
8566 {
8567 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8568 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8569 }
8570 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8572 if spec.protocol == ModuleProtocol::None {
8573 return Ok(None);
8574 }
8575 if let Some(connection_file_path) = connection_file_path {
8576 command.arg(SUBC_ARG).arg(connection_file_path);
8577 }
8578
8579 let nonce = generate_launch_nonce()?;
8583 if let Some(handle) = handle {
8584 match role {
8585 SpawnRole::Plain => {
8586 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8587 if spec.reserved {
8588 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8589 }
8590 }
8591 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8592 }
8593 }
8594 #[cfg(unix)]
8595 let handoff = {
8596 let handoff =
8597 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8598 program: spec.program.clone(),
8599 source,
8600 cgroup_path: None,
8601 })?;
8602 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8603 Some(handoff)
8604 };
8605 #[cfg(windows)]
8606 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8607 #[cfg(not(any(unix, windows)))]
8608 let handoff = None;
8609 #[cfg(not(unix))]
8613 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8614 Ok(handoff)
8615}
8616
8617fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8618 command.env_remove(CK_LOG_ENV);
8619 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8626 for (key, value) in &spec.env {
8627 if matches!(
8631 key.as_str(),
8632 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8633 ) || key == SUBC_SPAWN_ROLE_ENV
8634 {
8635 continue;
8636 }
8637 command.env(key, value);
8638 }
8639}
8640
8641#[cfg(windows)]
8642fn prepare_windows_nonce_pipe(
8643 command: &mut Command,
8644 spec: &ModuleSpec,
8645 nonce: &str,
8646) -> Option<NonceHandoff> {
8647 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8648 Ok(handoff) => {
8649 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8650 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8655 Some(handoff)
8656 }
8657 Err(error) => {
8658 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8659 None
8660 }
8661 }
8662}
8663
8664#[cfg(windows)]
8665fn start_windows_nonce_pipe(
8666 handoff: Option<NonceHandoff>,
8667 pid: u32,
8668 spec: &ModuleSpec,
8669) -> Option<subc_os::LaunchNoncePipeDelivery> {
8670 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8674 Ok(delivery) => Some(delivery),
8675 Err(error) => {
8676 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8677 None
8678 }
8679 })
8680}
8681
8682fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8683 #[cfg(windows)]
8684 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8685 return Some(
8686 if snapshot
8687 .pipe_consumed
8688 .as_ref()
8689 .is_some_and(|consumed| consumed.get().is_some())
8690 {
8691 "pipe"
8692 } else {
8693 "env"
8694 }
8695 .to_owned(),
8696 );
8697 }
8698 let _ = snapshot;
8699 None
8700}
8701
8702#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8705enum SpawnRole {
8706 Plain,
8707 SwapCandidate,
8708}
8709
8710fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8713 if role == SpawnRole::SwapCandidate {
8714 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8715 }
8716}
8717
8718fn previous_instance_error(
8719 spec: &ModuleSpec,
8720 pid: u32,
8721 reason: crate::live_children::RetainedReason,
8722) -> SuperviseError {
8723 SuperviseError::Spawn {
8724 program: spec.program.clone(),
8725 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8726 cgroup_path: None,
8727 }
8728}
8729
8730fn spawn_child(
8731 spec: &ModuleSpec,
8732 connection_file_path: Option<&std::path::Path>,
8733 handle: Option<&SupervisorHandle>,
8734 ring: &Arc<Mutex<StderrRing>>,
8735 capture_logs_dir: Option<&std::path::Path>,
8736 roster: &ChildRoster,
8737 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8738) -> Result<SupervisedChild, SuperviseError> {
8739 spawn_child_in_slot(
8740 spec,
8741 connection_file_path,
8742 handle,
8743 ring,
8744 capture_logs_dir,
8745 roster,
8746 #[cfg(target_os = "linux")]
8747 cgroup_placement,
8748 SpawnRole::Plain,
8749 false,
8750 )
8751}
8752
8753#[allow(clippy::too_many_arguments)]
8766fn spawn_child_in_slot(
8767 spec: &ModuleSpec,
8768 connection_file_path: Option<&std::path::Path>,
8769 handle: Option<&SupervisorHandle>,
8770 ring: &Arc<Mutex<StderrRing>>,
8771 capture_logs_dir: Option<&std::path::Path>,
8772 roster: &ChildRoster,
8773 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8774 role: SpawnRole,
8775 alternate_slot: bool,
8776) -> Result<SupervisedChild, SuperviseError> {
8777 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8778 return Err(previous_instance_error(spec, pid, reason));
8779 }
8780 if roster.is_closed() {
8781 return Err(SuperviseError::Spawn {
8782 program: spec.program.clone(),
8783 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8784 cgroup_path: None,
8785 });
8786 }
8787 #[cfg(target_os = "linux")]
8788 let cgroup_name = {
8789 if cgroup_placement.is_none() {
8794 swap::cgroup_name(&spec.module_id, alternate_slot)
8795 } else {
8796 let nonce = generate_launch_nonce()?;
8797 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8798 let mut end = spec.module_id.len().min(64);
8801 while !spec.module_id.is_char_boundary(end) {
8802 end -= 1;
8803 }
8804 format!(
8805 "{}_{suffix}",
8806 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8807 )
8808 }
8809 };
8810 #[cfg(not(target_os = "linux"))]
8811 let _ = alternate_slot;
8812 #[cfg(target_os = "macos")]
8813 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8814 #[cfg(windows)]
8815 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8816 program: spec.program.clone(),
8817 source,
8818 cgroup_path: None,
8819 })?;
8820 #[cfg(windows)]
8821 let mut command = Command::new(image_capture.path());
8822 #[cfg(not(any(target_os = "macos", windows)))]
8823 let mut command = Command::new(&spec.program);
8824 command.args(&spec.args);
8825 apply_child_env(&mut command, spec);
8855 apply_spawn_role(&mut command, role);
8856 let nonce_handoff =
8857 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8858
8859 #[cfg(target_os = "linux")]
8860 let cgroup_path = cgroup_placement
8861 .map(|placement| placement.module_path(&cgroup_name))
8862 .transpose()
8863 .map_err(|source| SuperviseError::Cgroup {
8864 module_id: spec.module_id.clone(),
8865 source,
8866 })?;
8867 #[cfg(not(target_os = "linux"))]
8868 let cgroup_path: Option<PathBuf> = None;
8869 #[cfg(target_os = "linux")]
8870 if let Some(path) = &cgroup_path {
8871 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8872 if let Some(placement) = cgroup_placement {
8873 remove_module_cgroup(placement, &cgroup_name);
8874 }
8875 return Err(error);
8876 }
8877 }
8878
8879 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8880 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8881 match ChildOutputSink::open(&path, capture_retention(spec)) {
8882 Ok(sink) => sink,
8883 Err(error) => {
8884 warn!(
8885 module_id = %spec.module_id,
8886 path = %path.display(),
8887 error = %error,
8888 "could not open child output capture file; forwarding to stderr"
8889 );
8890 ChildOutputSink::Stderr
8891 }
8892 }
8893 } else {
8894 ChildOutputSink::Stderr
8895 };
8896
8897 command.stdout(Stdio::piped());
8898 command.stderr(Stdio::piped());
8899 command.kill_on_drop(true);
8900 #[cfg(unix)]
8917 command.process_group(0);
8918 command.stdin(Stdio::null());
8919 #[cfg(unix)]
8923 if let Some(handoff) = nonce_handoff {
8924 handoff.install_last(command.as_std_mut());
8925 }
8926 #[cfg(not(any(unix, windows)))]
8927 let _ = nonce_handoff;
8928
8929 #[cfg(windows)]
8934 {
8935 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8936 program: spec.program.clone(),
8937 source,
8938 cgroup_path: None,
8939 })?;
8940 subc_jobobject::suspend_on_create_async(&mut command);
8941 }
8942 let spawned_at = Instant::now();
8946 let mut child = match command.spawn() {
8947 Ok(child) => child,
8948 Err(source) => {
8949 #[cfg(target_os = "linux")]
8950 if let Some(placement) = cgroup_placement {
8951 remove_module_cgroup(placement, &cgroup_name);
8952 }
8953 return Err(SuperviseError::Spawn {
8954 program: spec.program.clone(),
8955 source,
8956 cgroup_path,
8957 });
8958 }
8959 };
8960 #[cfg(test)]
8961 spawn_record_tests::after_create(&child, spec);
8962 #[cfg(target_os = "macos")]
8967 drop(exec_ack);
8968
8969 #[cfg(windows)]
8974 let windows_image = match image_capture.bind_tokio(&child) {
8975 Ok(image) => Arc::new(image),
8976 Err(source) => {
8977 kill_and_reap_rejected_child(&mut child);
8978 return Err(SuperviseError::Spawn {
8979 program: spec.program.clone(),
8980 source,
8981 cgroup_path: None,
8982 });
8983 }
8984 };
8985 #[cfg(windows)]
8988 let job = contain_spawned_child(&mut child, spec)?;
8989 let spawned_at_ms = unix_ms_now();
8990 #[cfg(not(windows))]
8991 let spawned_from = spec.program.clone();
8992 #[cfg(windows)]
8993 let spawned_from = windows_image.path().to_path_buf();
8994 #[cfg(not(windows))]
8995 let spawned_file_identity = spawned_file_identity(&spawned_from);
8996 #[cfg(windows)]
8997 let spawned_file_identity = Some(SpawnedFileIdentity {
8998 device: 0,
8999 inode: 0,
9000 windows_image: Some(Arc::clone(&windows_image)),
9001 });
9002 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9003 program: spec.program.clone(),
9004 source: io::Error::other("spawned child exposed no live pid"),
9005 cgroup_path: cgroup_path.clone(),
9006 })?;
9007 #[cfg(windows)]
9008 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9009 #[cfg(not(windows))]
9010 let process_start_time = crate::provenance::process_start_time(pid);
9011 #[cfg(windows)]
9012 let process_start_time = Some(windows_image.start_time());
9013 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9014 #[cfg(all(test, target_os = "macos"))]
9015 privacy_exec_boundary_tests::before_image_sample(spec, pid);
9016 #[cfg(not(windows))]
9021 let recorded_image = observe_spawned_image(pid);
9022 #[cfg(target_os = "macos")]
9026 let recorded_image = if privacy_exec.is_some() {
9027 None
9028 } else {
9029 recorded_image
9030 };
9031 #[cfg(target_os = "linux")]
9032 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9033 #[cfg(not(target_os = "linux"))]
9034 let recorded_cgroup_name = None;
9035 let roster_guard = roster.admit(
9036 spec.module_id.clone(),
9037 pid,
9038 spec.protocol,
9039 process_start_time,
9040 crate::child_roster::RecordedIdentity {
9041 #[cfg(not(windows))]
9042 start_time: recorded_image.map(|image| image.start_time),
9043 #[cfg(windows)]
9044 start_time: process_start_time,
9045 #[cfg(not(windows))]
9046 executable: recorded_image
9047 .and_then(|image| image.executable)
9048 .map(crate::live_children::ExecutableIdentity::from),
9049 #[cfg(windows)]
9050 executable: Some(crate::live_children::ExecutableIdentity::from(
9051 windows_image.identity(),
9052 )),
9053 cgroup_name: recorded_cgroup_name,
9054 #[cfg(target_os = "linux")]
9055 cgroup_placement: cgroup_placement.cloned(),
9056 },
9057 );
9058 let roster_guard = match roster_guard {
9059 Ok(guard) => guard,
9060 Err(failure) => {
9061 return failure.with_cleanup(|source| {
9062 #[cfg(windows)]
9063 if let Some(job) = &job {
9064 let _ = job.terminate();
9065 }
9066 #[cfg(target_os = "linux")]
9067 kill_module_cgroup(cgroup_placement, &cgroup_name);
9068 kill_and_reap_rejected_child(&mut child);
9069 #[cfg(target_os = "linux")]
9070 if let Some(placement) = cgroup_placement {
9071 remove_module_cgroup(placement, &cgroup_name);
9072 }
9073 Err(SuperviseError::Spawn {
9074 program: spec.program.clone(),
9075 source: io::Error::other(format!(
9076 "could not durably record module child: {source}"
9077 )),
9078 cgroup_path,
9079 })
9080 });
9081 }
9082 };
9083 if roster.is_closed() {
9088 #[cfg(target_os = "linux")]
9090 kill_module_cgroup(cgroup_placement, &cgroup_name);
9091 if let Err(error) = child.start_kill() {
9092 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9093 }
9094 #[cfg(target_os = "linux")]
9095 if let Some(placement) = cgroup_placement {
9096 while matches!(child.try_wait(), Ok(None)) {
9100 std::thread::yield_now();
9101 }
9102 if matches!(
9103 subc_cgroup::kill_module(Some(placement), &cgroup_name),
9104 subc_cgroup::KillOutcome::Killed
9105 ) {
9106 if let Ok(path) = placement.module_path(&cgroup_name) {
9107 while std::fs::read_to_string(path.join("cgroup.events"))
9108 .ok()
9109 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9110 {
9111 std::thread::yield_now();
9112 }
9113 }
9114 }
9115 remove_module_cgroup(placement, &cgroup_name);
9116 }
9117 drop(roster_guard);
9118 return Err(SuperviseError::Spawn {
9119 program: spec.program.clone(),
9120 source: io::Error::other(
9121 "the daemon began shutting down while this process was starting; ended it",
9122 ),
9123 cgroup_path,
9124 });
9125 }
9126
9127 #[cfg(windows)]
9128 if let Err(error) = resume_suspended_child(pid, spec) {
9129 kill_and_reap_rejected_child(&mut child);
9130 return Err(error);
9131 }
9132
9133 let stdout_pump = match child.stdout.take() {
9134 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9135 None => {
9136 warn!(
9137 module_id = %spec.module_id,
9138 "spawned child exposed no stdout pipe; file capture will be incomplete"
9139 );
9140 None
9141 }
9142 };
9143 let stderr_pump = match child.stderr.take() {
9144 Some(stderr) => {
9145 let generation = ring
9146 .lock()
9147 .unwrap_or_else(|poisoned| poisoned.into_inner())
9148 .begin_process();
9149 Some(StderrPump {
9150 task: tokio::spawn(pump_stderr_to(
9151 stderr,
9152 Arc::clone(ring),
9153 generation,
9154 output_sink,
9155 )),
9156 generation,
9157 })
9158 }
9159 None => {
9160 ring.lock()
9164 .unwrap_or_else(|poisoned| poisoned.into_inner())
9165 .mark_not_captured("stderr pipe was not available on spawn");
9166 warn!(
9167 module_id = %spec.module_id,
9168 "spawned child exposed no stderr pipe; tail will be unavailable"
9169 );
9170 None
9171 }
9172 };
9173
9174 let exec = Arc::new(crate::activity::ExecOutcome::new());
9175 #[cfg(target_os = "macos")]
9176 let pending_exec = privacy_exec.is_some();
9177 #[cfg(not(target_os = "macos"))]
9178 let pending_exec = false;
9179 if !pending_exec {
9180 exec.finish(crate::activity::ExecState::CommittedExec);
9181 }
9182 Ok(SupervisedChild {
9183 child,
9184 exec: crate::activity::ExecOwner(exec),
9185 #[cfg(windows)]
9186 nonce_delivery,
9187 protocol: spec.protocol,
9188 #[cfg(target_os = "linux")]
9189 module_id: cgroup_name,
9190 #[cfg(target_os = "linux")]
9191 cgroup_placement: cgroup_placement.cloned(),
9192 #[cfg(windows)]
9193 job,
9194 stdout_pump,
9195 stderr_pump,
9196 stderr_ring: Arc::clone(ring),
9197 spawned_at,
9198 registration_deadline: None,
9199 registration_watch: None,
9200 spawned_at_ms,
9201 spawned_from,
9202 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9203 spawned_file_identity,
9204 process_start_time,
9205 process_identity,
9206 pid,
9207 roster_guard: Some(roster_guard),
9208 #[cfg(target_os = "macos")]
9209 privacy_exec,
9210 #[cfg(target_os = "macos")]
9211 report_ready: Arc::new(OnceLock::new()),
9212 spawn_failure: None,
9213 })
9214}
9215
9216#[cfg(test)]
9217mod spawn_record_tests {
9218 use super::*;
9219
9220 thread_local! {
9221 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9222 }
9223
9224 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9225 LAST_PID.set(child.id());
9226 #[cfg(windows)]
9227 if let Some((_, address)) = _spec
9228 .env
9229 .iter()
9230 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9231 {
9232 use std::io::{Read, Write};
9233 let mut stream = std::net::TcpStream::connect(address).unwrap();
9234 stream
9235 .write_all(&child.id().unwrap().to_ne_bytes())
9236 .unwrap();
9237 let mut release = [0];
9238 stream.read_exact(&mut release).unwrap();
9239 }
9240 }
9241
9242 #[cfg(windows)]
9243 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9244 if spec
9245 .env
9246 .iter()
9247 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9248 {
9249 Err(io::Error::other("injected module job assignment failure"))
9250 } else {
9251 Ok(())
9252 }
9253 }
9254
9255 fn spec() -> ModuleSpec {
9256 ModuleSpec {
9257 module_id: "spawn-record-worker".into(),
9258 #[cfg(unix)]
9259 program: "/bin/sleep".into(),
9260 #[cfg(windows)]
9261 program: std::env::current_exe().unwrap(),
9262 #[cfg(unix)]
9263 args: vec!["60".into()],
9264 #[cfg(windows)]
9265 args: vec![
9266 "--exact".into(),
9267 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9268 "--nocapture".into(),
9269 ],
9270 env: vec![],
9271 reserved: false,
9272 reserved_prefixes: vec![],
9273 protocol: ModuleProtocol::None,
9274 overlap: Default::default(),
9275 }
9276 }
9277
9278 #[tokio::test]
9279 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9280 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9281 let supervisor =
9282 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9283 .with_live_children_record(home.join("live-children.json"));
9284 crate::live_children::fail_next_file_sync();
9285 let result = supervisor.spawn(spec());
9286 let failed = result.is_err();
9287 if let Ok(module) = result {
9288 module.stop().await.unwrap();
9289 }
9290 assert!(failed, "record failure must fail spawn");
9291 let pid = LAST_PID.get().expect("the real OS child was created");
9292 assert!(
9293 subc_os::Process::open(pid).unwrap().is_none(),
9294 "child {pid} survived rejected admission or was not reaped"
9295 );
9296 }
9297
9298 #[cfg(target_os = "macos")]
9299 #[test]
9300 #[ignore = "manual on-device latency measurement"]
9301 fn durable_record_write_latency_50() {
9302 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9303 let path = home.join("live-children.json");
9304 let children = vec![crate::live_children::LiveChild {
9305 module_id: "latency-worker".into(),
9306 pid: std::process::id(),
9307 protocol: ModuleProtocol::None,
9308 start_time: Some(1),
9309 executable: None,
9310 cgroup_name: None,
9311 }];
9312 let mut samples = Vec::new();
9313 for _ in 0..50 {
9314 let start = std::time::Instant::now();
9315 crate::live_children::write_record(&path, &children).unwrap();
9316 samples.push(start.elapsed());
9317 }
9318 samples.sort();
9319 eprintln!(
9320 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9321 (samples[24] + samples[25]) / 2,
9322 samples[47]
9323 );
9324 }
9325
9326 #[cfg(windows)]
9327 #[test]
9328 fn windows_store_touch_fixture() {
9329 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9330 std::fs::write(path, b"module ran").unwrap();
9331 }
9332 }
9333
9334 #[cfg(windows)]
9335 #[tokio::test]
9336 async fn windows_daemon_fixture() {
9337 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9338 return;
9339 };
9340 let mut spec = spec();
9341 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9342 let supervisor =
9343 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9344 let _module = supervisor.spawn(spec).unwrap();
9345 }
9346
9347 #[cfg(windows)]
9348 #[test]
9349 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9350 use std::io::Read;
9351 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9352 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9355 command
9356 .args([
9357 "--exact",
9358 "supervise::spawn_record_tests::windows_daemon_fixture",
9359 "--nocapture",
9360 ])
9361 .env(
9362 "SUBC_TEST_DAEMON_BARRIER",
9363 listener.local_addr().unwrap().to_string(),
9364 );
9365 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9366 listener.set_nonblocking(true).unwrap();
9367 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9368 let mut stream = loop {
9369 if let Ok((stream, _)) = listener.accept() {
9370 break stream;
9371 }
9372 assert!(
9373 daemon.child.try_wait().unwrap().is_none(),
9374 "daemon failed before rendezvous"
9375 );
9376 assert!(
9377 std::time::Instant::now() < deadline,
9378 "daemon did not reach CreateProcess failpoint"
9379 );
9380 std::thread::yield_now();
9381 };
9382 stream
9383 .set_read_timeout(Some(Duration::from_secs(30)))
9384 .unwrap();
9385 let mut pid = [0; 4];
9386 stream.read_exact(&mut pid).unwrap();
9387 let pid = u32::from_ne_bytes(pid);
9388 assert!(subc_jobobject::process_exists(pid));
9389 daemon.child.kill().unwrap();
9392 daemon.child.wait().unwrap();
9393 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9394 if !exited {
9395 daemon.job.terminate().unwrap();
9396 }
9397 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9398 }
9399
9400 #[cfg(windows)]
9401 #[tokio::test]
9402 async fn windows_containment_failure_never_resumes() {
9403 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9404 let sentinel = home.join("store-touched");
9405 let mut spec = spec();
9406 spec.env
9407 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9408 spec.env.push((
9409 "SUBC_TEST_STORE_TOUCH".into(),
9410 sentinel.display().to_string(),
9411 ));
9412 let supervisor =
9413 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9414 let result = supervisor.spawn(spec);
9415 if let Ok(module) = result {
9418 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9419 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9420 tokio::task::yield_now().await;
9421 }
9422 })
9423 .await;
9424 let _ = module.stop().await;
9425 panic!(
9426 "assignment failure resumed a module; store touched={}",
9427 sentinel.exists()
9428 );
9429 }
9430 assert!(
9431 !sentinel.exists(),
9432 "refused module executed store-touch sentinel"
9433 );
9434 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9435 }
9436
9437 #[cfg(windows)]
9438 #[tokio::test]
9439 async fn windows_nested_daemon_fixture() {
9440 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9441 return;
9442 };
9443 let mut spec = spec();
9444 spec.env.push((
9445 "SUBC_TEST_STORE_TOUCH".into(),
9446 sentinel.to_str().unwrap().into(),
9447 ));
9448 let supervisor =
9449 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9450 let module = supervisor
9451 .spawn(spec)
9452 .expect("per-module job must nest below daemon and host jobs");
9453 tokio::time::timeout(Duration::from_secs(10), async {
9454 while !PathBuf::from(&sentinel).exists() {
9455 tokio::task::yield_now().await;
9456 }
9457 })
9458 .await
9459 .unwrap();
9460 module.stop().await.unwrap();
9461 }
9462
9463 #[cfg(windows)]
9464 #[test]
9465 fn windows_module_job_nests_under_host_job() {
9466 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9467 let sentinel = home.join("nested-store-touched");
9468 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9469 command
9470 .args([
9471 "--exact",
9472 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9473 "--nocapture",
9474 ])
9475 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9476 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9477 assert!(daemon.child.wait().unwrap().success());
9478 assert!(sentinel.exists(), "nested protocol:none module never ran");
9479 }
9480}
9481
9482#[cfg(target_os = "linux")]
9483pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9484 use subc_cgroup::KillOutcome;
9485 match subc_cgroup::kill_module(placement, module_id) {
9486 KillOutcome::Killed => {}
9487 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9488 debug!(
9489 module_id,
9490 "cgroup tree kill unavailable; using direct-child kill"
9491 );
9492 }
9493 KillOutcome::IoError { path, error } => {
9494 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9495 }
9496 }
9497}
9498
9499#[cfg(windows)]
9507fn contain_spawned_child(
9508 child: &mut Child,
9509 spec: &ModuleSpec,
9510) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9511 let assignment = || {
9512 let job = subc_jobobject::JobObject::new()?;
9513 #[cfg(test)]
9514 spawn_record_tests::assignment_failpoint(spec)?;
9515 job.assign(child)?;
9516 Ok::<_, io::Error>(job)
9517 };
9518 match assignment() {
9519 Ok(job) => Ok(Some(job)),
9520 Err(source) => {
9521 kill_and_reap_rejected_child(child);
9522 Err(SuperviseError::Spawn {
9523 program: spec.program.clone(),
9524 source: io::Error::other(format!(
9525 "Windows module job containment failed: {source}"
9526 )),
9527 cgroup_path: None,
9528 })
9529 }
9530 }
9531}
9532
9533fn kill_and_reap_rejected_child(child: &mut Child) {
9536 #[cfg(unix)]
9537 if let Some(pid) = child
9538 .id()
9539 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9540 {
9541 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9542 }
9543 let _ = child.start_kill();
9544 while matches!(child.try_wait(), Ok(None)) {
9545 std::thread::yield_now();
9546 }
9547}
9548
9549#[cfg(windows)]
9554fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9555 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9556 let _ = std::process::Command::new("taskkill.exe")
9560 .args(["/PID", &pid.to_string(), "/T", "/F"])
9561 .stdin(Stdio::null())
9562 .stdout(Stdio::null())
9563 .stderr(Stdio::null())
9564 .status();
9565 return Err(SuperviseError::Spawn {
9566 program: spec.program.clone(),
9567 source,
9568 cgroup_path: None,
9569 });
9570 }
9571 Ok(())
9572}
9573
9574#[cfg(target_os = "linux")]
9575fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9576 match placement.remove_module(module_id) {
9577 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9578 Err(error) => warn!(
9579 module_id,
9580 error = %error,
9581 "could not remove module cgroup after process exit; continuing teardown"
9582 ),
9583 }
9584}
9585
9586#[cfg(target_os = "linux")]
9587const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
9588#[cfg(target_os = "linux")]
9589const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
9590
9591#[cfg(target_os = "linux")]
9592async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9593 let path = placement.module_path(module_id).ok();
9594 if path.as_ref().is_some_and(|path| {
9595 std::fs::read_to_string(path.join("cgroup.events"))
9596 .ok()
9597 .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
9598 }) {
9599 remove_module_cgroup(placement, module_id);
9600 return;
9601 }
9602
9603 if terminate_reaped_cgroup(placement, module_id).await {
9606 if let Some(path) = &path {
9607 let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
9608 while cgroup_is_populated(path) && Instant::now() < deadline {
9609 sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
9610 }
9611 }
9612 }
9613
9614 if matches!(
9618 subc_cgroup::kill_module(Some(placement), module_id),
9619 subc_cgroup::KillOutcome::Killed
9620 ) {
9621 if let Some(path) = &path {
9622 while cgroup_is_populated(path) {
9623 sleep(Duration::from_millis(1)).await;
9624 }
9625 }
9626 }
9627 remove_module_cgroup(placement, module_id);
9628}
9629
9630#[cfg(target_os = "linux")]
9631fn cgroup_is_populated(path: &std::path::Path) -> bool {
9632 std::fs::read_to_string(path.join("cgroup.events"))
9633 .ok()
9634 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9635}
9636
9637#[cfg(target_os = "linux")]
9638async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
9639 let frozen =
9640 match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
9641 .await
9642 {
9643 subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
9644 outcome => {
9645 tracing::debug!(
9646 module_id,
9647 ?outcome,
9648 "cgroup freeze failed; skipping SIGTERM grace"
9649 );
9650 return false;
9651 }
9652 };
9653 let listed = match frozen.processes() {
9654 subc_cgroup::ProcsOutcome::Listed(pids) => {
9655 for pid in pids.into_iter().filter(|pid| *pid > 1) {
9656 if let Some(pid) = rustix::process::Pid::from_raw(pid) {
9657 if let Err(error) =
9658 rustix::process::kill_process(pid, rustix::process::Signal::TERM)
9659 {
9660 tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
9661 }
9662 }
9663 }
9664 true
9665 }
9666 outcome => {
9667 tracing::debug!(
9668 module_id,
9669 ?outcome,
9670 "cgroup process listing failed; skipping SIGTERM grace"
9671 );
9672 false
9673 }
9674 };
9675 match frozen.thaw() {
9676 subc_cgroup::ThawOutcome::Thawed => listed,
9677 outcome => {
9678 tracing::debug!(
9679 module_id,
9680 ?outcome,
9681 "cgroup thaw failed; proceeding to cgroup.kill"
9682 );
9683 false
9684 }
9685 }
9686}
9687
9688#[cfg(target_os = "linux")]
9689fn apply_cgroup_placement(
9690 command: &mut Command,
9691 spec: &ModuleSpec,
9692 path: &std::path::Path,
9693) -> Result<(), SuperviseError> {
9694 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9695 module_id: spec.module_id.clone(),
9696 source,
9697 })
9698}
9699
9700fn capture_retention(spec: &ModuleSpec) -> Retention {
9701 let defaults = Retention::default();
9702 let value = |name: &str| {
9703 spec.env
9704 .iter()
9705 .rev()
9706 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9707 };
9708 Retention {
9709 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9710 .and_then(|value| value.parse().ok())
9711 .unwrap_or(defaults.max_file_mb),
9712 keep: value(CAPTURE_KEEP_ENV)
9713 .and_then(|value| value.parse().ok())
9714 .unwrap_or(defaults.keep),
9715 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9716 .and_then(|value| value.parse().ok())
9717 .unwrap_or(defaults.max_age_days),
9718 }
9719}
9720
9721fn generate_launch_nonce() -> Result<String, SuperviseError> {
9724 let mut bytes = [0u8; 32];
9725 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9726 reason: source.to_string(),
9727 })?;
9728 let mut hex = String::with_capacity(64);
9729 for b in bytes {
9730 use std::fmt::Write;
9731 let _ = write!(hex, "{b:02x}");
9732 }
9733 Ok(hex)
9734}
9735
9736fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9739 if a.len() != b.len() {
9740 return false;
9741 }
9742 let mut diff = 0u8;
9743 for (x, y) in a.iter().zip(b.iter()) {
9744 diff |= x ^ y;
9745 }
9746 diff == 0
9747}
9748
9749#[cfg(not(windows))]
9753fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9754 let process = subc_os::Process::open(pid).ok().flatten()?;
9755 #[cfg(target_os = "linux")]
9756 return sample_after_linux_exec(
9757 || {
9758 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9759 let committed = linux_exec_committed(&stat);
9760 #[cfg(test)]
9761 spawn_image_tests::report_exec_state(committed);
9762 committed
9763 },
9764 || process.observe(),
9765 );
9766 #[cfg(not(target_os = "linux"))]
9767 process.observe()
9768}
9769
9770#[cfg(target_os = "linux")]
9774fn linux_exec_committed(stat: &str) -> Option<bool> {
9775 const PF_FORKNOEXEC: u64 = 0x40;
9776 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9777 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9778}
9779
9780#[cfg(target_os = "linux")]
9781fn sample_after_linux_exec(
9782 mut exec_committed: impl FnMut() -> Option<bool>,
9783 observe: impl FnOnce() -> Option<subc_os::Observation>,
9784) -> Option<subc_os::Observation> {
9785 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9786 while !exec_committed()? {
9787 if std::time::Instant::now() >= deadline {
9791 return None;
9792 }
9793 std::thread::sleep(Duration::from_micros(50));
9794 }
9795 observe()
9796}
9797
9798#[cfg(all(test, target_os = "linux"))]
9799mod spawn_image_tests {
9800 use super::*;
9801 use std::cell::{Cell, RefCell};
9802
9803 enum ExecEvent {
9804 Uncommitted,
9805 Image(Option<subc_os::Observation>),
9806 }
9807
9808 thread_local! {
9809 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9810 }
9811
9812 pub(super) fn report_exec_state(committed: Option<bool>) {
9813 if committed == Some(false) {
9814 EXEC_EVENTS.with(|events| {
9815 if let Some(events) = events.borrow_mut().take() {
9816 let _ = events.send(ExecEvent::Uncommitted);
9817 }
9818 });
9819 }
9820 }
9821
9822 #[test]
9823 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9824 use std::io::{Read, Write};
9825 use std::os::fd::AsRawFd;
9826
9827 let descriptor = std::fs::File::open("/dev/null").unwrap();
9828 let mut command = std::process::Command::new("/bin/sleep");
9829 command.arg("30");
9830 let (mut ready, mut release) =
9831 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9832 .unwrap();
9833 let spawn = std::thread::spawn(move || command.spawn());
9834 let mut message = [0u8; 8];
9835 ready.read_exact(&mut message).unwrap();
9836 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9837 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9838 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9839 let (events, received) = std::sync::mpsc::channel();
9840 let observer = std::thread::spawn(move || {
9841 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9842 let image = observe_spawned_image(pid);
9843 let _ = events.send(ExecEvent::Image(image));
9844 });
9845 let first = received.recv_timeout(Duration::from_secs(5));
9849 release.write_all(&[1]).unwrap();
9850 let mut child = spawn.join().unwrap().unwrap();
9851 let after = match first {
9852 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9853 Ok(ExecEvent::Image(image)) => image,
9854 _ => None,
9855 },
9856 Ok(ExecEvent::Image(image)) => image,
9857 Err(_) => None,
9858 };
9859 observer.join().unwrap();
9860 child.kill().unwrap();
9861 child.wait().unwrap();
9862
9863 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9864 assert_eq!(
9865 inherited.unwrap().executable,
9866 subc_os::file_identity(&std::env::current_exe().unwrap())
9867 );
9868 assert_eq!(
9869 after.unwrap().executable,
9870 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9871 );
9872 }
9873
9874 #[test]
9875 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9876 let committed = Cell::new(false);
9877 let inherited = subc_os::Observation {
9878 start_time: 23,
9879 executable: Some(subc_os::FileIdentity {
9880 device: 29,
9881 inode: 31,
9882 }),
9883 };
9884 let module = subc_os::Observation {
9885 start_time: 23,
9886 executable: Some(subc_os::FileIdentity {
9887 device: 29,
9888 inode: 37,
9889 }),
9890 };
9891 let polls = Cell::new(0);
9892 let image = sample_after_linux_exec(
9893 || {
9894 polls.set(polls.get() + 1);
9895 committed.set(polls.get() == 2);
9898 Some(committed.get())
9899 },
9900 || Some(if committed.get() { module } else { inherited }),
9901 );
9902 assert_eq!(image, Some(module));
9903 assert_eq!(polls.get(), 2);
9904 }
9905
9906 #[test]
9907 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9908 assert_eq!(
9909 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9910 Some(false)
9911 );
9912 assert_eq!(
9913 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9914 Some(true)
9915 );
9916 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9917 assert_eq!(linux_exec_committed("17 (a) R"), None);
9918 }
9919
9920 #[test]
9921 fn unreadable_exec_state_never_samples_an_inherited_image() {
9922 assert_eq!(
9923 sample_after_linux_exec(
9924 || None,
9925 || panic!("must not sample before exec confirmation")
9926 ),
9927 None
9928 );
9929 }
9930
9931 #[test]
9932 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9933 assert_eq!(
9934 sample_after_linux_exec(
9935 || Some(false),
9936 || panic!("must not sample after an exec timeout")
9937 ),
9938 None
9939 );
9940 }
9941}
9942
9943#[cfg(windows)]
9946fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9947 let mut program = spec.program.clone();
9948 if program.is_absolute() || program.components().count() > 1 {
9949 match subc_os::ExecutableCapture::open(&program) {
9952 Ok(capture) => return Ok(capture),
9953 Err(error)
9954 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9955 Err(error) => return Err(error),
9956 }
9957 program.set_extension("exe");
9958 return subc_os::ExecutableCapture::open(&program);
9959 }
9960 if program.extension().is_none() {
9961 program.set_extension("exe");
9962 }
9963 let custom_path = spec
9964 .env
9965 .iter()
9966 .rev()
9967 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9968 let mut directories = Vec::new();
9969 if let Some(parent) = std::env::current_exe()?.parent() {
9972 directories.push(parent.to_path_buf());
9973 }
9974 directories.push(std::env::current_dir()?);
9975 if let Some(windows) = std::env::var_os("SystemRoot") {
9976 let windows = PathBuf::from(windows);
9977 directories.push(windows.join("System32"));
9978 directories.push(windows.clone());
9979 }
9980 let path = custom_path
9981 .map(|(_, value)| std::ffi::OsString::from(value))
9982 .or_else(|| std::env::var_os("PATH"));
9983 if let Some(path) = path {
9984 directories.extend(std::env::split_paths(&path));
9985 }
9986 for directory in directories {
9987 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9988 Ok(capture) => return Ok(capture),
9989 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9990 Err(error) => return Err(error),
9991 }
9992 }
9993 Err(io::Error::new(
9994 io::ErrorKind::NotFound,
9995 "resolved executable not found",
9996 ))
9997}
9998
9999fn spawn_and_mark_running(
10000 spec: &ModuleSpec,
10001 runtime: &SupervisorRuntimeConfig,
10002 snapshot: &SharedSnapshot,
10003) -> Result<SupervisedChild, SuperviseError> {
10004 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10005 let registration_watch = watch_registration(&runtime.registry, spec);
10006 let child = spawn_child(
10007 spec,
10008 runtime.connection_file_path.as_deref(),
10009 runtime.supervisor_handle.as_ref(),
10010 &runtime.stderr_ring,
10011 runtime.capture_logs_dir.as_deref(),
10012 &runtime.child_roster,
10013 #[cfg(target_os = "linux")]
10014 runtime.cgroup_placement.as_ref(),
10015 )?;
10016 let mut child = child;
10017 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10018 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
10019 if let Some(spawn) = activity_spawn {
10020 spawn.complete(snapshot);
10021 }
10022 Ok(child)
10023}
10024
10025struct RegistrationWatch {
10026 changes: watch::Receiver<u64>,
10027 baseline: u64,
10028}
10029
10030fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10034 if spec.protocol != ModuleProtocol::Subc {
10035 return None;
10036 }
10037 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10038 let baseline = *changes.borrow();
10039 Some(RegistrationWatch { changes, baseline })
10040}
10041
10042fn arm_registration_deadline(
10043 child: &mut SupervisedChild,
10044 budget: Duration,
10045 registration_watch: Option<RegistrationWatch>,
10046) {
10047 child.registration_deadline =
10048 (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10049 child.registration_watch = registration_watch;
10050}
10051
10052async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10053 #[cfg(unix)]
10054 {
10055 let group = unreaped_registration_group(child)?;
10058 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10059 sleep(crate::child_roster::TERM_TO_KILL).await;
10064 let group = unreaped_registration_group(child)?;
10065 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10066 }
10067 child.start_kill()?;
10070 child.wait().await?;
10071 Ok(())
10072}
10073
10074#[cfg(unix)]
10075fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10076 let group = i32::try_from(child.pid)
10079 .ok()
10080 .filter(|pid| *pid > 1)
10081 .and_then(rustix::process::Pid::from_raw)
10082 .ok_or_else(|| {
10083 io::Error::new(
10084 io::ErrorKind::InvalidInput,
10085 "invalid supervised process group",
10086 )
10087 })?;
10088 if child.child.id() != Some(child.pid) {
10089 return Err(io::Error::new(
10090 io::ErrorKind::InvalidInput,
10091 "refusing process-group signal after supervised child was reaped",
10092 ));
10093 }
10094 Ok(group)
10095}
10096
10097enum RegistrationWaitOutcome {
10098 Registered,
10099 Exited(ExitReport),
10100 TimedOut,
10101}
10102
10103struct ReloadRegistrationFailure {
10104 exit_report: ExitReport,
10105 reason: String,
10106}
10107
10108#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10109enum BusyGaugeObservation {
10110 Quiescent,
10111 Busy,
10112 Omitted,
10113}
10114
10115#[derive(Debug, Clone, PartialEq, Eq)]
10117enum GaugeVerdict {
10118 Zero,
10120 NonZero(u64),
10122 NotACount(String),
10126 Missing,
10128 ProbeFailed(&'static str),
10131}
10132
10133impl GaugeVerdict {
10134 fn reads_quiet(&self) -> bool {
10135 matches!(self, Self::Zero)
10136 }
10137}
10138
10139impl fmt::Display for GaugeVerdict {
10140 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10141 match self {
10142 Self::Zero => f.write_str("zero"),
10143 Self::NonZero(value) => write!(f, "non_zero({value})"),
10144 Self::NotACount(value) => write!(f, "not_a_count({value})"),
10145 Self::Missing => f.write_str("missing_from_health_body"),
10146 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10147 }
10148 }
10149}
10150
10151const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10154
10155fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10157 let metrics = metrics.and_then(Value::as_object);
10158 gauges
10159 .iter()
10160 .map(
10161 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10162 None => GaugeVerdict::Missing,
10163 Some(value) => match value.as_u64() {
10164 Some(0) => GaugeVerdict::Zero,
10165 Some(count) => GaugeVerdict::NonZero(count),
10166 None => {
10167 let mut text = value.to_string();
10168 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10169 let mut end = GAUGE_VALUE_TEXT_LIMIT;
10170 while !text.is_char_boundary(end) {
10171 end -= 1;
10172 }
10173 text.truncate(end);
10174 text.push_str("...");
10175 }
10176 GaugeVerdict::NotACount(text)
10177 }
10178 },
10179 },
10180 )
10181 .collect()
10182}
10183
10184fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10187 if verdicts
10188 .iter()
10189 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10190 {
10191 BusyGaugeObservation::Omitted
10192 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10193 BusyGaugeObservation::Quiescent
10194 } else {
10195 BusyGaugeObservation::Busy
10196 }
10197}
10198
10199#[derive(Debug, Clone, Default, PartialEq, Eq)]
10204struct GaugeTrace {
10205 latest: Option<(GaugeVerdict, Duration)>,
10206 last_value: Option<(GaugeVerdict, Duration)>,
10207}
10208
10209#[derive(Debug, Clone, Default, PartialEq, Eq)]
10215struct DrainGaugeTrace {
10216 probes: u32,
10218 gauges: Vec<(String, GaugeTrace)>,
10220}
10221
10222impl DrainGaugeTrace {
10223 fn new(gauges: &[String]) -> Self {
10224 Self {
10225 probes: 0,
10226 gauges: gauges
10227 .iter()
10228 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10229 .collect(),
10230 }
10231 }
10232
10233 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10235 self.probes = self.probes.saturating_add(1);
10236 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10237 trace.latest = Some((verdict.clone(), at));
10238 if matches!(
10239 verdict,
10240 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10241 ) {
10242 trace.last_value = Some((verdict.clone(), at));
10243 }
10244 }
10245 }
10246}
10247
10248impl fmt::Display for DrainGaugeTrace {
10253 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10254 if self.gauges.is_empty() {
10255 return f.write_str("none declared");
10256 }
10257 for (index, (name, trace)) in self.gauges.iter().enumerate() {
10258 if index > 0 {
10259 f.write_str(" | ")?;
10260 }
10261 write!(f, "{name}: ")?;
10262 match &trace.latest {
10263 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10264 None => f.write_str("never probed")?,
10265 }
10266 match &trace.last_value {
10267 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10268 None => f.write_str("; no value ever read")?,
10269 }
10270 }
10271 Ok(())
10272 }
10273}
10274
10275pub(crate) fn declared_busy_gauges(
10276 registry: &Registry,
10277 module_id: &str,
10278) -> Result<Vec<String>, SuperviseError> {
10279 busy_gauges_of(
10280 registry
10281 .get_module(module_id)
10282 .map_err(SuperviseError::Registry)?,
10283 )
10284}
10285
10286fn declared_busy_gauges_for_connection(
10290 registry: &Registry,
10291 connection_id: ConnectionId,
10292) -> Result<Vec<String>, SuperviseError> {
10293 busy_gauges_of(
10294 registry
10295 .get_module_by_connection(connection_id)
10296 .map_err(SuperviseError::Registry)?,
10297 )
10298}
10299
10300fn busy_gauges_of(
10301 registration: Option<crate::registry::ModuleRegistration>,
10302) -> Result<Vec<String>, SuperviseError> {
10303 let Some(registration) = registration else {
10304 return Ok(Vec::new());
10305 };
10306 let Some(self_signals) = registration.manifest.self_signals else {
10307 return Ok(Vec::new());
10308 };
10309
10310 let mut gauges = Vec::new();
10311 for declaration in self_signals {
10312 if declaration.kind != SelfSignalKind::Busy {
10313 continue;
10314 }
10315 match declaration.anchored_to {
10316 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10317 gauges.extend(declared)
10318 }
10319 _ => {
10320 gauges.push(String::new());
10323 }
10324 }
10325 }
10326 Ok(gauges)
10327}
10328
10329#[allow(clippy::too_many_arguments)]
10338async fn wait_for_forwarding_quiescence(
10339 forwarding: &ForwardingTable,
10340 module_id: &str,
10341 runtime: &SupervisorRuntimeConfig,
10342 endpoint: crate::ModuleEndpointId,
10343 deadline: Instant,
10344 busy_gauges: &[String],
10345 scope: DrainScope,
10346 trace: &mut DrainGaugeTrace,
10347) -> Result<bool, SuperviseError> {
10348 let wait_started = Instant::now();
10349 let mut gauges_quiescent = busy_gauges.is_empty();
10350 let mut next_probe_at = wait_started;
10351 let mut omission_reported = false;
10352
10353 loop {
10354 let now = Instant::now();
10355 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10356 let report = match scope {
10357 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10358 DrainScope::Endpoint(endpoint) => {
10359 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10360 }
10361 };
10362 let verdicts = match &report {
10363 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10364 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10365 };
10366 trace.record(&verdicts, wait_started.elapsed());
10367 gauges_quiescent = match report {
10368 Ok(_) => match busy_gauge_observation(&verdicts) {
10369 BusyGaugeObservation::Quiescent => true,
10370 BusyGaugeObservation::Busy => false,
10371 BusyGaugeObservation::Omitted => {
10372 if !omission_reported {
10375 forwarding
10376 .counters()
10377 .increment_drains_with_undeclared_gauge();
10378 let missing = busy_gauges
10379 .iter()
10380 .zip(&verdicts)
10381 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10382 .map(|(gauge, _)| gauge.as_str())
10383 .collect::<Vec<_>>()
10384 .join(",");
10385 warn!(
10386 module_id,
10387 missing_gauges = %missing,
10388 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10389 );
10390 omission_reported = true;
10391 }
10392 false
10393 }
10394 },
10395 Err(err) => {
10396 warn!(
10397 module_id,
10398 error = %err,
10399 "drain health.check did not produce declared busy gauges; treating module as busy"
10400 );
10401 false
10402 }
10403 };
10404 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10407 }
10408
10409 let in_flight = forwarding
10410 .endpoint_in_flight_count(endpoint)
10411 .map_err(SuperviseError::Forwarding)?;
10412 if in_flight == 0 && gauges_quiescent {
10413 return Ok(true);
10414 }
10415
10416 let now = Instant::now();
10417 if now >= deadline {
10418 return Ok(false);
10419 }
10420 let mut wait = deadline
10421 .saturating_duration_since(now)
10422 .min(REGISTRY_RELEASE_POLL);
10423 if !busy_gauges.is_empty() {
10424 wait = wait.min(next_probe_at.saturating_duration_since(now));
10425 }
10426 sleep(wait).await;
10427 }
10428}
10429
10430fn drain_busy_reprobe_interval() -> Duration {
10434 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10435}
10436
10437fn warn_drain_timed_out(
10452 module_id: &str,
10453 drain_timeout: Duration,
10454 reason: RouteCloseReason,
10455 holdouts: &crate::forwarding::DrainHoldouts,
10456 gauges: &DrainGaugeTrace,
10457) {
10458 warn!(
10459 module_id = %module_id,
10460 waited = ?drain_timeout,
10461 ?reason,
10462 held_requests = holdouts.requests,
10463 held_routes = holdouts.routes,
10464 total_routes = holdouts.total_routes,
10465 top_connections = ?holdouts.top_connections,
10466 held = %holdouts
10469 .held
10470 .iter()
10471 .map(|(channel, corr)| format!("{channel}:{corr}"))
10472 .collect::<Vec<_>>()
10473 .join(","),
10474 gauge_probes = gauges.probes,
10475 busy_gauges = %gauges,
10476 "route drain timed out before request quiescence; forcing teardown"
10477 );
10478}
10479
10480fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10488 match wait_result {
10489 Ok(drained) => *drained,
10490 Err(_) => false,
10491 }
10492}
10493
10494fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10495 for released in released_routes {
10496 let frame = match Frame::build_with_version(
10497 released.negotiated_ver,
10498 FrameType::Goodbye,
10499 control_flags(),
10500 released.channel,
10501 released.epoch,
10502 0,
10503 Vec::new(),
10504 ) {
10505 Ok(frame) => frame,
10506 Err(err) => {
10507 warn!(
10508 route_channel = released.channel,
10509 error = %err,
10510 "failed to build supervisor drain route GOODBYE frame"
10511 );
10512 continue;
10513 }
10514 };
10515 if !released.close_on_delivery_failure() {
10516 crate::forwarding::send_module_route_goodbye(
10517 &forwarding.counters(),
10518 &released.sink,
10519 frame,
10520 released.module_id.as_deref(),
10521 "supervisor drain",
10522 );
10523 continue;
10524 }
10525 if let Err(err) = released.sink.try_send(frame) {
10526 warn!(
10527 target_connection_id = released.connection_id.get(),
10528 route_channel = released.channel,
10529 error = %err,
10530 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10531 );
10532 let _ = forwarding.escalate_client_delivery_failure(
10533 released.connection_id,
10534 released.channel,
10535 released.epoch,
10536 CloseReason::new(
10537 "route_goodbye_delivery_failed",
10538 format!(
10539 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10540 released.channel
10541 ),
10542 ),
10543 crate::forwarding::UndeliveredFrame {
10544 module_id: released.module_id.as_deref(),
10545 sink: &released.sink,
10546 },
10547 );
10548 }
10549 }
10550}
10551
10552fn send_module_draining(
10553 module_id: &str,
10554 reason: RouteCloseReason,
10555 deadline_ms: u64,
10556 target: &ModuleDrainTarget,
10557) {
10558 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10559 reason,
10560 deadline_ms,
10561 }) {
10562 Ok(body) => body,
10563 Err(err) => {
10564 warn!(
10565 module_id,
10566 error = %err,
10567 "failed to encode module draining command"
10568 );
10569 return;
10570 }
10571 };
10572 let frame = match Frame::build_with_version(
10573 target.negotiated_ver,
10574 FrameType::Push,
10575 control_flags(),
10576 0,
10577 0,
10578 0,
10579 body,
10580 ) {
10581 Ok(frame) => frame,
10582 Err(err) => {
10583 warn!(
10584 module_id,
10585 error = %err,
10586 "failed to build module draining command frame"
10587 );
10588 return;
10589 }
10590 };
10591 if let Err(err) = target.sink.try_send(frame) {
10592 warn!(
10593 module_id,
10594 target_connection_id = target.endpoint.connection_id.get(),
10595 error = %err,
10596 "module draining command was not delivered to peer"
10597 );
10598 }
10599}
10600
10601fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10603 match Frame::build_with_version(
10604 negotiated_ver,
10605 FrameType::Goodbye,
10606 control_flags(),
10607 0,
10608 0,
10609 0,
10610 Vec::new(),
10611 ) {
10612 Ok(frame) => Some(frame),
10613 Err(err) => {
10614 warn!(
10615 module_id,
10616 error = %err,
10617 "failed to build module GOODBYE frame"
10618 );
10619 None
10620 }
10621 }
10622}
10623
10624#[cfg(unix)]
10638async fn send_module_goodbyes_for_daemon_shutdown(
10639 forwarding: &Arc<ForwardingTable>,
10640 reason: &CloseReason,
10641 wait_for_flush: bool,
10642) {
10643 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10644 let targets = match forwarding.module_connections() {
10645 Ok(targets) => targets,
10646 Err(err) => {
10647 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10648 return;
10649 }
10650 };
10651 let deadline = Instant::now() + GOODBYE_BUDGET;
10652 let mut sends = tokio::task::JoinSet::new();
10653 for target in targets {
10654 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10655 continue;
10656 };
10657 if !wait_for_flush {
10658 if let Err(err) = target.sink.try_send(frame) {
10659 debug!(
10660 module_id = %target.module_id,
10661 error = %err,
10662 "shutdown module GOODBYE was not queued"
10663 );
10664 }
10665 continue;
10666 }
10667 let forwarding = Arc::clone(forwarding);
10668 let reason = reason.clone();
10669 sends.spawn(async move {
10670 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10671 Ok(Ok(())) => {}
10672 Ok(Err(err)) => debug!(
10673 module_id = %target.module_id,
10674 error = %err,
10675 "module connection closed before its shutdown GOODBYE was written"
10676 ),
10677 Err(_) => warn!(
10678 module_id = %target.module_id,
10679 budget = ?GOODBYE_BUDGET,
10680 "shutdown module GOODBYE was not written within its budget; closing anyway"
10681 ),
10682 }
10683 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10684 });
10685 }
10686 while sends.join_next().await.is_some() {}
10688}
10689
10690fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10691 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10692 return;
10693 };
10694 if let Err(err) = target.sink.try_send(frame) {
10695 warn!(
10696 module_id,
10697 target_connection_id = target.endpoint.connection_id.get(),
10698 error = %err,
10699 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10700 );
10701 forwarding.request_connection_close(
10702 target.endpoint.connection_id,
10703 CloseReason::new(
10704 "module_goodbye_delivery_failed",
10705 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10706 ),
10707 );
10708 }
10709}
10710
10711#[derive(Clone, Copy)]
10712struct ForwardingDrainContext<'a> {
10713 spec: &'a ModuleSpec,
10714 runtime: &'a SupervisorRuntimeConfig,
10715 registry: &'a Registry,
10716 scope: DrainScope,
10717}
10718
10719#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10721enum DrainScope {
10722 Active,
10725 Endpoint(crate::ModuleEndpointId),
10730}
10731
10732#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10740enum StopNotice {
10741 SentOverConnection,
10744 NoConnection,
10748 NotSent,
10752}
10753
10754async fn begin_forwarding_drain(
10755 spec: &ModuleSpec,
10756 runtime: &SupervisorRuntimeConfig,
10757 registry: &Registry,
10758 snapshot: &SharedSnapshot,
10759 enabled: Option<bool>,
10760 reason: RouteCloseReason,
10761) -> Result<StopNotice, SuperviseError> {
10762 let Some(forwarding) = runtime.forwarding.as_ref() else {
10763 return Err(SuperviseError::ReloadUnavailable {
10764 module_id: spec.module_id.clone(),
10765 reason: "supervisor was not configured with a forwarding table".to_string(),
10766 });
10767 };
10768
10769 begin_forwarding_drain_with(
10770 forwarding,
10771 ForwardingDrainContext {
10772 spec,
10773 runtime,
10774 registry,
10775 scope: DrainScope::Active,
10776 },
10777 snapshot,
10778 enabled,
10779 reason,
10780 runtime.drain_timeout,
10781 )
10782 .await
10783}
10784
10785async fn begin_forwarding_drain_if_configured(
10786 spec: &ModuleSpec,
10787 runtime: &SupervisorRuntimeConfig,
10788 registry: &Registry,
10789 snapshot: &SharedSnapshot,
10790 enabled: Option<bool>,
10791 reason: RouteCloseReason,
10792) -> Result<StopNotice, SuperviseError> {
10793 begin_forwarding_drain_with_timeout(
10794 spec,
10795 runtime,
10796 registry,
10797 snapshot,
10798 enabled,
10799 reason,
10800 runtime.drain_timeout,
10801 )
10802 .await
10803}
10804
10805async fn begin_forwarding_drain_with_timeout(
10809 spec: &ModuleSpec,
10810 runtime: &SupervisorRuntimeConfig,
10811 registry: &Registry,
10812 snapshot: &SharedSnapshot,
10813 enabled: Option<bool>,
10814 reason: RouteCloseReason,
10815 drain_timeout: Duration,
10816) -> Result<StopNotice, SuperviseError> {
10817 let Some(forwarding) = runtime.forwarding.as_ref() else {
10818 return Ok(StopNotice::NotSent);
10819 };
10820
10821 begin_forwarding_drain_with(
10822 forwarding,
10823 ForwardingDrainContext {
10824 spec,
10825 runtime,
10826 registry,
10827 scope: DrainScope::Active,
10828 },
10829 snapshot,
10830 enabled,
10831 reason,
10832 drain_timeout,
10833 )
10834 .await
10835}
10836
10837async fn begin_forwarding_drain_with(
10838 forwarding: &ForwardingTable,
10839 context: ForwardingDrainContext<'_>,
10840 snapshot: &SharedSnapshot,
10841 enabled: Option<bool>,
10842 reason: RouteCloseReason,
10843 drain_timeout: Duration,
10844) -> Result<StopNotice, SuperviseError> {
10845 let ForwardingDrainContext {
10846 spec,
10847 runtime,
10848 registry,
10849 scope,
10850 } = context;
10851 debug_assert_ne!(reason, RouteCloseReason::Crash);
10852 let terminal = matches!(reason, RouteCloseReason::Disable);
10853 let drain_started_at = Instant::now();
10854 let drain_deadline = drain_started_at + drain_timeout;
10855 let deadline_ms =
10856 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10857 let busy_gauges = match scope {
10858 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10859 DrainScope::Endpoint(endpoint) => {
10860 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10861 }
10862 };
10863
10864 let gate_started = Instant::now();
10867 let drain_target = match scope {
10868 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10869 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10870 }
10871 .map_err(SuperviseError::Forwarding)?;
10872 info!(
10877 module_id = %spec.module_id,
10878 ?reason,
10879 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10880 connected = drain_target.is_some(),
10881 "module drain began; route admission closed"
10882 );
10883 if scope == DrainScope::Active {
10884 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10885 state.state = ModuleState::Draining;
10886 state.draining_to_replace =
10887 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10888 if let Some(enabled) = enabled {
10889 state.enabled = enabled;
10890 }
10891 })?;
10892 }
10893
10894 let Some(target) = drain_target.as_ref() else {
10895 return Ok(StopNotice::NoConnection);
10899 };
10900 {
10901 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10902 #[cfg(feature = "test-support")]
10903 forwarding.wait_before_route_closing_for_test().await;
10904 let routes = forwarding
10905 .endpoint_routes(target.endpoint)
10906 .map_err(SuperviseError::Forwarding)?;
10907 let routes_notified = routes.len();
10908 crate::control::send_route_control_pushes(
10909 forwarding,
10910 routes.clone(),
10911 ClientControlPush::RouteClosing {
10912 module_id: spec.module_id.clone(),
10913 channels: Vec::new(),
10914 reason,
10915 },
10916 );
10917 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10918
10919 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10925 let wait_result = wait_for_forwarding_quiescence(
10926 forwarding,
10927 &spec.module_id,
10928 runtime,
10929 target.endpoint,
10930 drain_deadline,
10931 &busy_gauges,
10932 scope,
10933 &mut gauge_trace,
10934 )
10935 .await;
10936 let drained = drained_after_quiescence_wait(&wait_result);
10937 if let Err(err) = &wait_result {
10938 error!(
10939 module_id = %spec.module_id,
10940 ?reason,
10941 error = %err,
10942 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10943 );
10944 } else if !drained {
10945 let holdouts = forwarding
10947 .endpoint_drain_holdouts(target.endpoint)
10948 .unwrap_or_default();
10949 warn_drain_timed_out(
10950 &spec.module_id,
10951 drain_timeout,
10952 reason,
10953 &holdouts,
10954 &gauge_trace,
10955 );
10956 }
10957 crate::control::send_route_control_pushes(
10958 forwarding,
10959 routes,
10960 ClientControlPush::RouteClosed {
10961 module_id: spec.module_id.clone(),
10962 channels: Vec::new(),
10963 reason,
10964 drained,
10965 abandoned: target.abandoned_bindings.len() as u32,
10966 excluded_subscriptions: target.excluded_subscriptions,
10967 terminal: Some(terminal),
10968 },
10969 );
10970 wait_result?;
10971
10972 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10978 Ok(routes) => routes,
10979 Err(err) => {
10980 warn!(
10981 module_id = %spec.module_id,
10982 ?reason,
10983 error = %err,
10984 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10985 );
10986 send_module_goodbye(&spec.module_id, forwarding, target);
10987 return Err(SuperviseError::Forwarding(err));
10988 }
10989 };
10990 let route_goodbye_count = released_routes.len();
10991 send_route_goodbyes(forwarding, released_routes);
10992 send_module_goodbye(&spec.module_id, forwarding, target);
10993
10994 info!(
11000 module_id = %spec.module_id,
11001 ?reason,
11002 routes_notified,
11003 route_goodbyes = route_goodbye_count,
11004 abandoned_reservations = target.abandoned_bindings.len(),
11005 excluded_subscriptions = target.excluded_subscriptions,
11006 drained,
11007 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11008 );
11009 }
11010
11011 Ok(StopNotice::SentOverConnection)
11012}
11013
11014async fn wait_for_registration_after_reload(
11017 registry: &Registry,
11018 module_id: &str,
11019 snapshot: &SharedSnapshot,
11020 child: &mut SupervisedChild,
11021 wait: Duration,
11022) -> Result<RegistrationWaitOutcome, SuperviseError> {
11023 wait_for_slot_registration(
11024 registry,
11025 crate::registry::RegistrationSlot::Active(module_id),
11026 module_id,
11027 snapshot,
11028 child,
11029 wait,
11030 )
11031 .await
11032}
11033
11034async fn wait_for_slot_registration(
11042 registry: &Registry,
11043 slot: crate::registry::RegistrationSlot<'_>,
11044 module_id: &str,
11045 snapshot: &SharedSnapshot,
11046 child: &mut SupervisedChild,
11047 wait: Duration,
11048) -> Result<RegistrationWaitOutcome, SuperviseError> {
11049 let deadline = Instant::now() + wait;
11050 loop {
11051 if registry
11052 .registration(slot)
11053 .map_err(SuperviseError::Registry)?
11054 .is_some()
11055 {
11056 return Ok(RegistrationWaitOutcome::Registered);
11057 }
11058
11059 let now = Instant::now();
11060 if now >= deadline {
11061 return Ok(RegistrationWaitOutcome::TimedOut);
11062 }
11063 let remaining = deadline.saturating_duration_since(now);
11064 let poll = remaining.min(REGISTRY_RELEASE_POLL);
11065
11066 tokio::select! {
11067 wait_result = child.wait() => {
11068 let status = wait_result.map_err(|source| SuperviseError::Wait {
11069 module_id: module_id.to_string(),
11070 source,
11071 })?;
11072 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11073 snapshot,
11074 child,
11075 &status,
11076 )));
11077 }
11078 _ = sleep(poll) => {}
11079 }
11080 }
11081}
11082
11083fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11084 if exit_report.kind != ExitKind::DeliberateSeverance {
11088 exit_report.kind = ExitKind::Crash;
11089 }
11090 exit_report
11091}
11092
11093async fn handle_reload_child_registration_failure(
11094 spec: &ModuleSpec,
11095 runtime: &SupervisorRuntimeConfig,
11096 registry: &Registry,
11097 process_liveness: &SupervisorProcessLiveness,
11098 snapshot: &SharedSnapshot,
11099 _child: &mut Option<SupervisedChild>,
11100 failure: ReloadRegistrationFailure,
11101) -> Result<(), SuperviseError> {
11102 let ReloadRegistrationFailure {
11103 exit_report,
11104 reason,
11105 } = failure;
11106 match on_child_exit(
11107 spec,
11108 runtime.restart_policy,
11109 registry,
11110 snapshot,
11111 &runtime.terminal_ring,
11112 &runtime.spawn_events,
11113 &runtime.child_roster,
11114 exit_report,
11115 runtime.recovery_is_daemon,
11116 )
11117 .await
11118 {
11119 NextAction::Stop {
11120 registration_released,
11121 } => {
11122 if registration_released {
11123 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11124 }
11125 }
11126 NextAction::Restart { schedule } => {
11127 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11128 schedule.delay
11129 });
11130 if let Some(schedule) = schedule {
11131 log_crash_respawn(&spec.module_id, schedule);
11132 }
11133 schedule_respawn(
11134 runtime,
11135 snapshot,
11136 &spec.module_id,
11137 delay,
11138 RespawnKind::Spawn,
11139 true,
11140 )?;
11141 }
11142 }
11143 Err(SuperviseError::ReloadFailed {
11144 module_id: spec.module_id.clone(),
11145 reason,
11146 })
11147}
11148
11149async fn handle_reload_spawn_failure(
11150 spec: &ModuleSpec,
11151 runtime: &SupervisorRuntimeConfig,
11152 process_liveness: &SupervisorProcessLiveness,
11153 snapshot: &SharedSnapshot,
11154 _child: &mut Option<SupervisedChild>,
11155 reason: String,
11156) -> Result<(), SuperviseError> {
11157 let now = Instant::now();
11158 let mut schedule = None;
11159 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11160 clear_current_process_facts(state);
11161 if state.enabled {
11162 schedule = state.next_crash_restart(&runtime.restart_policy, now);
11163 state.state = if schedule.is_some() {
11164 ModuleState::Restarting
11165 } else {
11166 if runtime
11167 .failure_context(runtime.recovery_is_daemon)
11168 .may_retry()
11169 {
11170 state.failed_retry.schedule(&spec.module_id);
11171 }
11172 ModuleState::Failed
11173 };
11174 } else {
11175 state.state = ModuleState::Disabled;
11176 }
11177 })?;
11178 if let Some(schedule) = schedule {
11179 schedule_respawn(
11180 runtime,
11181 snapshot,
11182 &spec.module_id,
11183 schedule.delay,
11184 RespawnKind::Spawn,
11185 true,
11186 )?;
11187 } else {
11188 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11189 }
11190 Err(SuperviseError::ReloadFailed {
11191 module_id: spec.module_id.clone(),
11192 reason,
11193 })
11194}
11195
11196fn control_flags() -> Flags {
11197 Flags::new(false, Priority::Passive, false)
11198}
11199
11200#[allow(clippy::too_many_arguments)]
11201async fn drain_optional_child(
11202 module_id: &str,
11203 protocol: ModuleProtocol,
11204 stop_notice: StopNotice,
11205 registry: &Registry,
11206 forwarding: Option<&ForwardingTable>,
11207 snapshot: &SharedSnapshot,
11208 terminal_ring: &Arc<Mutex<TerminalRing>>,
11209 spawn_events: &SpawnEventFeed,
11210 child: &mut Option<SupervisedChild>,
11211 drain_timeout: Duration,
11212 final_state: ModuleState,
11213 enabled: Option<bool>,
11214 failure: &FailureContext,
11215) -> Result<(), SuperviseError> {
11216 if let Some(child) = child.take() {
11217 drain_child_to_state(
11218 module_id,
11219 protocol,
11220 stop_notice,
11221 registry,
11222 forwarding,
11223 snapshot,
11224 terminal_ring,
11225 spawn_events,
11226 child,
11227 drain_timeout,
11228 final_state,
11229 enabled,
11230 failure,
11231 )
11232 .await
11233 } else {
11234 update_snapshot(snapshot, Some(module_id), |state| {
11235 state.state = final_state;
11236 if final_state == ModuleState::Failed {
11237 state.failed_retry.failed_at = Some(Instant::now());
11238 }
11239 if let Some(enabled) = enabled {
11240 state.enabled = enabled;
11241 }
11242 clear_current_process_facts(state);
11243 })?;
11244 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11245 }
11246}
11247
11248#[allow(clippy::too_many_arguments)]
11249async fn drain_child_to_state(
11250 module_id: &str,
11251 _protocol: ModuleProtocol,
11252 stop_notice: StopNotice,
11253 registry: &Registry,
11254 forwarding: Option<&ForwardingTable>,
11255 snapshot: &SharedSnapshot,
11256 terminal_ring: &Arc<Mutex<TerminalRing>>,
11257 spawn_events: &SpawnEventFeed,
11258 mut child: SupervisedChild,
11259 drain_timeout: Duration,
11260 final_state: ModuleState,
11261 enabled: Option<bool>,
11262 failure: &FailureContext,
11263) -> Result<(), SuperviseError> {
11264 let protocol = child.protocol;
11265 update_snapshot(snapshot, Some(module_id), |state| {
11266 state.state = ModuleState::Draining;
11267 state.draining_to_replace = final_state == ModuleState::Restarting;
11268 if let Some(enabled) = enabled {
11269 state.enabled = enabled;
11270 }
11271 })?;
11272
11273 if stop_notice != StopNotice::SentOverConnection {
11284 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11285 info!(
11286 module_id,
11287 pid = child.pid,
11288 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11289 "module has no connection yet; requesting stop by signal"
11290 );
11291 }
11292 request_graceful_stop(module_id, &child);
11293 }
11294
11295 let exit_report = match timeout(drain_timeout, child.wait()).await {
11296 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11297 Ok(Err(source)) => {
11298 fail_after_wait_error(
11299 module_id,
11300 snapshot,
11301 terminal_ring,
11302 spawn_events,
11303 format!("failed to wait during drain: {source}"),
11304 failure,
11305 );
11306 return Err(SuperviseError::Wait {
11307 module_id: module_id.to_string(),
11308 source,
11309 });
11310 }
11311 Err(_) => {
11312 warn!(
11325 module_id,
11326 pid = child.pid,
11327 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11328 reason = ?final_state,
11329 ?stop_notice,
11330 "drain budget expired before the module exited; killing it"
11331 );
11332 child.start_kill().map_err(|source| {
11333 fail_after_wait_error(
11334 module_id,
11335 snapshot,
11336 terminal_ring,
11337 spawn_events,
11338 format!("failed to kill during drain: {source}"),
11339 failure,
11340 );
11341 SuperviseError::Kill {
11342 module_id: module_id.to_string(),
11343 source,
11344 }
11345 })?;
11346 let status = child.wait().await.map_err(|source| {
11347 fail_after_wait_error(
11348 module_id,
11349 snapshot,
11350 terminal_ring,
11351 spawn_events,
11352 format!("failed to wait after drain kill: {source}"),
11353 failure,
11354 );
11355 SuperviseError::Wait {
11356 module_id: module_id.to_string(),
11357 source,
11358 }
11359 })?;
11360 classify_reaped_child_exit(snapshot, &child, &status)
11361 }
11362 };
11363
11364 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11365 publish_after_terminal_record(
11366 module_id,
11367 || {
11368 record_terminal_with_detail(
11369 module_id,
11370 terminal_ring,
11371 spawn_events,
11372 &exit_report,
11373 terminal_disposition(final_state),
11374 detail,
11375 )
11376 },
11377 || {
11378 update_snapshot(snapshot, Some(module_id), |state| {
11379 state.state = final_state;
11380 if final_state == ModuleState::Failed {
11381 state.failed_retry.failed_at = Some(Instant::now());
11382 }
11383 if let Some(enabled) = enabled {
11384 state.enabled = enabled;
11385 }
11386 clear_current_process_facts(state);
11387 state.last_exit = Some(exit_report.clone());
11388 if exit_report.kind == ExitKind::DeliberateSeverance {
11389 state.lifetime_restarts += 1;
11390 }
11391 })
11392 },
11393 )?;
11394 child.drain_stderr(module_id).await;
11395
11396 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11397}
11398
11399#[cfg(unix)]
11419fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
11420 let Some(pid) = child
11421 .id()
11422 .and_then(|pid| i32::try_from(pid).ok())
11423 .and_then(rustix::process::Pid::from_raw)
11424 else {
11425 debug!(
11426 module_id,
11427 "no pid to signal for teardown; falling through to the drain wait"
11428 );
11429 return;
11430 };
11431 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
11432 Ok(()) => debug!(
11433 module_id,
11434 "sent SIGTERM to a module nothing else asked to stop"
11435 ),
11436 Err(err) => debug!(
11437 module_id,
11438 error = %err,
11439 "SIGTERM to module failed; the drain wait and kill still apply"
11440 ),
11441 }
11442}
11443
11444#[cfg(not(unix))]
11452fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11453 debug!(
11454 module_id,
11455 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11456 );
11457}
11458
11459fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11460 match final_state {
11461 ModuleState::Stopped => TerminalDisposition::Stopped,
11462 ModuleState::Disabled => TerminalDisposition::Disabled,
11463 ModuleState::Restarting => TerminalDisposition::Restarting,
11464 ModuleState::Failed => TerminalDisposition::Failed,
11465 ModuleState::Starting
11466 | ModuleState::Running
11467 | ModuleState::Unresponsive
11468 | ModuleState::Draining => {
11469 unreachable!("terminal exits only finish in terminal or restarting states")
11470 }
11471 }
11472}
11473
11474async fn release_dead_registration(
11482 registry: &Registry,
11483 forwarding: Option<&ForwardingTable>,
11484 snapshot: &SharedSnapshot,
11485 module_id: &str,
11486 failure: &FailureContext,
11487) -> Result<(), SuperviseError> {
11488 let result = async {
11489 let registration = registry
11490 .get_module(module_id)
11491 .map_err(SuperviseError::Registry)?;
11492 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11493 Ok(()) => return Ok(()),
11494 Err(SuperviseError::RegistrationStillActive { .. }) => {}
11495 Err(err) => return Err(err),
11496 }
11497 let pid = lock_snapshot(snapshot)?.reaped_pid;
11498 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11499 warn!(
11500 module_id,
11501 pid,
11502 connection_id = registration.connection_id.get(),
11503 "reaped module registration outlived release grace; closing dead connection"
11504 );
11505 forwarding.request_connection_close(
11506 registration.connection_id,
11507 CloseReason::new(
11508 "supervised_process_reaped",
11509 format!("module '{module_id}' pid {pid} exited"),
11510 ),
11511 );
11512 wait_for_slot_registration_release(
11513 registry,
11514 crate::registry::RegistrationSlot::Connection(registration.connection_id),
11515 REGISTRY_RELEASE_TIMEOUT,
11516 )
11517 .await?;
11518 }
11519 wait_for_registration_release(registry, module_id, Duration::ZERO).await
11520 }
11521 .await;
11522 if let Err(err) = &result {
11523 fail_snapshot(snapshot, Some(module_id), None, failure);
11524 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11525 }
11526 result
11527}
11528
11529async fn wait_for_registration_release(
11532 registry: &Registry,
11533 module_id: &str,
11534 wait: Duration,
11535) -> Result<(), SuperviseError> {
11536 wait_for_slot_registration_release(
11537 registry,
11538 crate::registry::RegistrationSlot::Active(module_id),
11539 wait,
11540 )
11541 .await
11542}
11543
11544async fn wait_for_slot_registration_release(
11552 registry: &Registry,
11553 slot: crate::registry::RegistrationSlot<'_>,
11554 wait: Duration,
11555) -> Result<(), SuperviseError> {
11556 let deadline = Instant::now() + wait;
11557 let mut release_events = registration_release_events().subscribe();
11558 let still_active = |registration: &crate::registry::ModuleRegistration| {
11559 SuperviseError::RegistrationStillActive {
11560 module_id: registration.manifest.module_id.clone(),
11561 waited: wait,
11562 }
11563 };
11564 loop {
11565 let _observed_generation = *release_events.borrow_and_update();
11566 let Some(registration) = registry
11567 .registration(slot)
11568 .map_err(SuperviseError::Registry)?
11569 else {
11570 return Ok(());
11571 };
11572
11573 let now = Instant::now();
11574 if now >= deadline {
11575 return Err(still_active(®istration));
11576 }
11577
11578 let remaining = deadline.saturating_duration_since(now);
11579 match timeout(remaining, release_events.changed()).await {
11580 Ok(Ok(())) | Ok(Err(_)) => {}
11581 Err(_) => return Err(still_active(®istration)),
11582 }
11583 }
11584}
11585
11586#[cfg(test)]
11587mod slot_registration_wait_tests {
11588 use super::*;
11589 use crate::registry::{ConnectionId, RegistrationSlot};
11590 use subc_protocol::manifest::ModuleManifest;
11591
11592 #[tokio::test]
11593 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11594 let registry = Arc::new(Registry::default());
11595 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
11596 let runtime = supervisor.runtime_config();
11597 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11598 let spec = ModuleSpec {
11599 module_id: "enable-stale-registration".to_string(),
11600 program: PathBuf::from("/missing/enable-retry-test"),
11601 args: Vec::new(),
11602 env: Vec::new(),
11603 reserved: false,
11604 reserved_prefixes: Vec::new(),
11605 protocol: ModuleProtocol::Subc,
11606 overlap: Default::default(),
11607 };
11608 let connection = ConnectionId::new(90);
11609 registry
11610 .register_with_control_ops(
11611 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11612 1,
11613 connection,
11614 Vec::new(),
11615 )
11616 .unwrap();
11617 let mut child = None;
11618 let err = set_child_enabled(
11619 &spec,
11620 &runtime,
11621 ®istry,
11622 &supervisor.process_liveness,
11623 &snapshot,
11624 &mut child,
11625 true,
11626 &runtime.failure_context(false),
11627 )
11628 .await
11629 .unwrap_err();
11630 assert!(matches!(
11631 err,
11632 SuperviseError::RegistrationStillActive { .. }
11633 ));
11634 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11635 assert!(child.is_none());
11636 registry.deregister_connection(connection).unwrap();
11637 let err = set_child_enabled(
11638 &spec,
11639 &runtime,
11640 ®istry,
11641 &supervisor.process_liveness,
11642 &snapshot,
11643 &mut child,
11644 true,
11645 &runtime.failure_context(false),
11646 )
11647 .await
11648 .unwrap_err();
11649 assert!(
11650 matches!(err, SuperviseError::Spawn { .. }),
11651 "second enable must attempt a spawn: {err}"
11652 );
11653 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11654 }
11655
11656 const INCUMBENT: u64 = 1;
11657 const CANDIDATE: u64 = 2;
11658
11659 fn swapped_registry() -> Arc<Registry> {
11660 let registry = Arc::new(Registry::default());
11661 let manifest = ModuleManifest::builder("m", "0.1.0").build();
11662 registry
11663 .register_with_control_ops(
11664 manifest.clone(),
11665 1,
11666 ConnectionId::new(INCUMBENT),
11667 Vec::new(),
11668 )
11669 .unwrap();
11670 registry
11671 .register_candidate_with_control_ops(
11672 manifest,
11673 1,
11674 ConnectionId::new(CANDIDATE),
11675 Vec::new(),
11676 )
11677 .unwrap();
11678 registry
11679 }
11680
11681 #[tokio::test]
11685 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11686 let registry = swapped_registry();
11687 registry.promote_candidate("m").unwrap().unwrap();
11688
11689 assert!(matches!(
11690 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
11691 Err(SuperviseError::RegistrationStillActive { .. })
11692 ));
11693
11694 assert!(matches!(
11696 wait_for_slot_registration_release(
11697 ®istry,
11698 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11699 Duration::from_millis(50),
11700 )
11701 .await,
11702 Err(SuperviseError::RegistrationStillActive { .. })
11703 ));
11704
11705 let releaser = Arc::clone(®istry);
11706 let release = tokio::spawn(async move {
11707 sleep(Duration::from_millis(20)).await;
11708 releaser
11709 .deregister_connection(ConnectionId::new(INCUMBENT))
11710 .unwrap();
11711 notify_registration_release();
11712 });
11713 wait_for_slot_registration_release(
11714 ®istry,
11715 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11716 Duration::from_secs(5),
11717 )
11718 .await
11719 .expect("the incumbent's own registration is released");
11720 release.await.unwrap();
11721 assert!(registry.get_module("m").unwrap().is_some());
11722 }
11723
11724 #[tokio::test]
11727 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11728 let registry = swapped_registry();
11729 assert!(matches!(
11730 wait_for_slot_registration_release(
11731 ®istry,
11732 RegistrationSlot::Candidate("m"),
11733 Duration::from_millis(50),
11734 )
11735 .await,
11736 Err(SuperviseError::RegistrationStillActive { .. })
11737 ));
11738 registry
11739 .deregister_connection(ConnectionId::new(CANDIDATE))
11740 .unwrap();
11741 wait_for_slot_registration_release(
11742 ®istry,
11743 RegistrationSlot::Candidate("m"),
11744 Duration::from_millis(50),
11745 )
11746 .await
11747 .expect("a candidate slot with no candidate is released");
11748 assert!(registry
11749 .registration(RegistrationSlot::Active("m"))
11750 .unwrap()
11751 .is_some());
11752 }
11753}
11754
11755fn classify_exit(status: &ExitStatus) -> ExitReport {
11756 ExitReport {
11757 kind: if status.success() {
11758 ExitKind::Clean
11759 } else {
11760 ExitKind::Crash
11761 },
11762 code: status.code(),
11763 signal: exit_signal(status),
11764 at_ms: unix_ms_now(),
11765 }
11766}
11767
11768fn wait_error_exit_report() -> ExitReport {
11774 ExitReport {
11775 kind: ExitKind::Crash,
11776 code: None,
11777 signal: None,
11778 at_ms: unix_ms_now(),
11779 }
11780}
11781
11782#[cfg(unix)]
11783fn exit_signal(status: &ExitStatus) -> Option<i32> {
11784 use std::os::unix::process::ExitStatusExt;
11785
11786 status.signal()
11787}
11788
11789#[cfg(not(unix))]
11790fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11791 None
11792}
11793
11794fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11800 update_snapshot(snapshot, Some(module_id), |state| {
11801 state.clear_crash_restarts();
11802 })
11803}
11804
11805fn set_running(
11806 snapshot: &SharedSnapshot,
11807 child: &SupervisedChild,
11808 module_id: &str,
11809 spawn_events: &SpawnEventFeed,
11810) -> Result<(), SuperviseError> {
11811 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11812 module_id: Some(module_id.to_string()),
11813 })?;
11814 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11815 if std::mem::take(&mut state.coalesced_restart_pending) {
11816 let generation = state.spawn_generation;
11817 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11818 }
11819 state.drain_disposition_detail = None;
11820 state.spawn_failure = None;
11821 state.waiting_for_previous_pid = None;
11822 state.waiting_for_previous_detail = None;
11823 state.in_alternate_slot = false;
11826 state.configuration_updated_since_spawn = false;
11827 state.spawned_protocol = Some(child.protocol);
11828 state.state = ModuleState::Running;
11829 state.enabled = true;
11830 state.process_alive = true;
11831 state.pid = child.id();
11832 state.health.status = SupervisorHealthStatus::Unknown;
11835 state.health.consecutive_failures = 0;
11836 state.health.detail = None;
11837 state.health.metrics = None;
11838 state.health.last_probe_ms = None;
11839 state.failed_retry.healthy_since = None;
11840 state.failed_retry.failed_at = None;
11841 state.failed_retry.running_since = Some(child.spawned_at);
11842 #[cfg(windows)]
11843 {
11844 state.pipe_consumed = child
11845 .nonce_delivery
11846 .as_ref()
11847 .map(|delivery| delivery.consumed());
11848 }
11849 #[cfg(target_os = "macos")]
11850 {
11851 state.report_ready = Some(Arc::clone(&child.report_ready));
11852 }
11853 state.spawned_at_ms = Some(child.spawned_at_ms);
11854 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11855 state.spawned_from = Some(child.spawned_from.clone());
11856 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11857 state.spawned_file_identity = child.spawned_file_identity.clone();
11858 state.process_start_time = child.process_start_time;
11859 Ok(())
11860}
11861
11862fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11863 state.failed_retry.healthy_since = None;
11864 state.failed_retry.running_since = None;
11865 state.waiting_for_previous_pid = None;
11866 state.waiting_for_previous_detail = None;
11867 state.process_alive = false;
11868 state.spawned_protocol = None;
11869 state.pid = None;
11870 #[cfg(windows)]
11871 {
11872 state.pipe_consumed = None;
11873 }
11874 #[cfg(target_os = "macos")]
11875 {
11876 state.report_ready = None;
11877 }
11878 state.spawned_at_ms = None;
11879 state.exec_outcome = None;
11880 state.spawned_from = None;
11881 state.launch_spec_digest = None;
11882 state.spawned_file_identity = None;
11883 state.process_start_time = None;
11884 state.deliberate_severance = None;
11885}
11886
11887#[cfg(test)]
11888fn record_deliberate_severance(
11889 snapshot: &SharedSnapshot,
11890 identity: ProcessIdentity,
11891) -> Result<(), SuperviseError> {
11892 update_snapshot(snapshot, None, |state| {
11893 state.deliberate_severance = Some(identity);
11894 })
11895}
11896
11897fn apply_deliberate_severance_marker(
11898 snapshot: &SharedSnapshot,
11899 exited_identity: Option<ProcessIdentity>,
11900 mut exit_report: ExitReport,
11901) -> ExitReport {
11902 let marker = lock_snapshot(snapshot)
11903 .ok()
11904 .and_then(|mut state| state.deliberate_severance.take());
11905 if marker.is_some() && marker == exited_identity {
11906 exit_report.kind = ExitKind::DeliberateSeverance;
11907 }
11908 exit_report
11909}
11910
11911fn classify_reaped_child_exit(
11912 snapshot: &SharedSnapshot,
11913 child: &SupervisedChild,
11914 status: &ExitStatus,
11915) -> ExitReport {
11916 let _ = update_snapshot(snapshot, None, |state| {
11917 state.reaped_pid = Some(child.pid);
11918 state.spawn_failure = child.spawn_failure.clone();
11919 });
11920 if child.spawn_failure.as_deref() == Some("registration_timeout") {
11921 registration_failure_exit_report(classify_exit(status))
11922 } else {
11923 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11924 }
11925}
11926
11927fn fail_snapshot(
11928 snapshot: &SharedSnapshot,
11929 module_id: Option<&str>,
11930 last_exit: Option<ExitReport>,
11931 failure: &FailureContext,
11932) {
11933 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11934 state.state = ModuleState::Failed;
11935 clear_current_process_facts(state);
11936 if failure.may_retry() && state.enabled {
11937 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
11938 } else if failure.shutdown.is_set() {
11939 state.failed_retry.cancel();
11940 }
11941 if let Some(last_exit) = last_exit {
11942 state.last_exit = Some(last_exit);
11943 }
11944 }) {
11945 error!(error = %err, "failed to mark supervisor state failed");
11946 }
11947}
11948
11949fn update_snapshot(
11950 snapshot: &SharedSnapshot,
11951 module_id: Option<&str>,
11952 update: impl FnOnce(&mut SupervisorSnapshot),
11953) -> Result<(), SuperviseError> {
11954 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11955 module_id: module_id.map(ToOwned::to_owned),
11956 })?;
11957 update(&mut state);
11958 Ok(())
11959}
11960
11961const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11962
11963fn lock_snapshot_for_control<'a>(
11964 snapshot: &'a SharedSnapshot,
11965 module_id: &str,
11966 caller: &'static str,
11967) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11968 let started_at = Instant::now();
11969 let guard = lock_snapshot(snapshot)?;
11970 let waited = started_at.elapsed();
11971 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11972 warn!(
11973 module_id = %module_id,
11974 waited_ms = waited.as_millis() as u64,
11975 caller = %caller,
11976 "slow snapshot lock"
11977 );
11978 }
11979 Ok(guard)
11980}
11981
11982fn lock_snapshot(
11983 snapshot: &SharedSnapshot,
11984) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11985 snapshot
11986 .lock()
11987 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11988}
11989
11990#[cfg(test)]
11991mod terminal_history_tests {
11992 use std::{
11993 path::PathBuf,
11994 sync::Arc,
11995 time::{Duration, Instant},
11996 };
11997
11998 use tokio::{sync::mpsc, time::sleep};
11999
12000 use super::{
12001 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12002 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12003 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12004 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12005 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12006 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12007 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12008 SupervisorHealthStatus, SupervisorSnapshot,
12009 };
12010 use super::Instant as ClockInstant;
12015 use crate::{
12016 registry::Registry,
12017 terminal_ring::{TerminalRing, TerminalRingConfig},
12018 };
12019 use std::sync::Mutex;
12020 use subc_control::TerminalDisposition;
12021
12022 pub(super) fn fake_aft_stub_path() -> PathBuf {
12027 let mut path = std::env::current_exe().expect("current_exe available in tests");
12028 path.pop();
12029 path.pop();
12030 path.push(if cfg!(windows) {
12031 "fake-aft-stub.exe"
12032 } else {
12033 "fake-aft-stub"
12034 });
12035 assert!(
12036 path.exists(),
12037 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12038 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12039 path.display()
12040 );
12041 path
12042 }
12043
12044 #[test]
12045 fn reserved_never_spawned_refuses_every_hello() {
12046 let supervisor = SupervisorHandle::default();
12051 supervisor.apply_identity_configuration(&ModuleSpec {
12052 module_id: "never-spawned".to_string(),
12053 program: PathBuf::from("/usr/bin/false"),
12054 args: Vec::new(),
12055 env: Vec::new(),
12056 reserved: true,
12057 reserved_prefixes: Vec::new(),
12058 protocol: ModuleProtocol::Subc,
12059 overlap: Default::default(),
12060 });
12061 assert!(
12062 supervisor
12063 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12064 .is_err(),
12065 "forged nonce must refuse on a reserved never-spawned id"
12066 );
12067 assert!(
12068 supervisor
12069 .reserved_hello_verification("never-spawned", None)
12070 .is_err(),
12071 "absent nonce must refuse on a reserved never-spawned id"
12072 );
12073 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12075 supervisor.apply_identity_configuration(&ModuleSpec {
12076 module_id: "never-spawned".to_string(),
12077 program: PathBuf::from("/usr/bin/false"),
12078 args: Vec::new(),
12079 env: Vec::new(),
12080 reserved: true,
12081 reserved_prefixes: Vec::new(),
12082 protocol: ModuleProtocol::Subc,
12083 overlap: Default::default(),
12084 });
12085 assert!(supervisor
12086 .reserved_hello_verification("never-spawned", Some("minted"))
12087 .is_ok());
12088 assert!(supervisor
12089 .reserved_hello_verification("never-spawned", Some("forged"))
12090 .is_err());
12091 }
12092
12093 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12096 let now = ClockInstant::now();
12097 for _ in 0..count {
12098 state.crash_restarts.push_back(now);
12099 }
12100 }
12101
12102 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12106 let aged = state
12107 .crash_restarts
12108 .front()
12109 .expect("a crash restart must be recorded before it can be aged")
12110 .checked_sub(window + Duration::from_secs(1))
12111 .expect("the test clock is far enough from its origin to age an instant");
12112 state.crash_restarts[0] = aged;
12113 }
12114
12115 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12116 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12117 seed_crash_restarts(&mut state, count);
12118 state
12119 }
12120
12121 #[test]
12122 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12123 let policy = RestartPolicy::new(3, Duration::ZERO);
12124 let now = ClockInstant::now();
12125 assert!(daemon_will_restart(
12126 &mut snapshot_with_restarts(true, 2),
12127 &policy,
12128 now
12129 ));
12130 assert!(!daemon_will_restart(
12131 &mut snapshot_with_restarts(true, 3),
12132 &policy,
12133 now
12134 ));
12135 assert!(!daemon_will_restart(
12136 &mut snapshot_with_restarts(false, 0),
12137 &policy,
12138 now
12139 ));
12140 }
12141
12142 #[test]
12143 fn crash_restart_backoff_escalates_with_in_window_count() {
12144 let policy = RestartPolicy::new(4, Duration::from_millis(100))
12145 .with_max_backoff(Duration::from_secs(30));
12146 let now = ClockInstant::now();
12147 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12148 let schedules = (0..4)
12149 .map(|_| {
12150 state
12151 .next_crash_restart(&policy, now)
12152 .expect("the test policy allows four crash restarts")
12153 })
12154 .collect::<Vec<_>>();
12155
12156 assert_eq!(
12157 schedules
12158 .iter()
12159 .map(|schedule| schedule.restart_in_window)
12160 .collect::<Vec<_>>(),
12161 vec![0, 1, 2, 3]
12162 );
12163 assert_eq!(
12164 schedules
12165 .iter()
12166 .map(|schedule| schedule.delay)
12167 .collect::<Vec<_>>(),
12168 vec![
12169 Duration::from_millis(100),
12170 Duration::from_secs(1),
12171 Duration::from_secs(10),
12172 Duration::from_secs(30),
12173 ]
12174 );
12175 }
12176
12177 #[test]
12178 fn crash_restart_backoff_resets_after_ring_clear() {
12179 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12180 let now = ClockInstant::now();
12181 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12182 assert_eq!(
12183 state.next_crash_restart(&policy, now).unwrap().delay,
12184 Duration::from_millis(100)
12185 );
12186 assert_eq!(
12187 state.next_crash_restart(&policy, now).unwrap().delay,
12188 Duration::from_secs(1)
12189 );
12190
12191 state.clear_crash_restarts();
12192 let schedule = state
12193 .next_crash_restart(&policy, now)
12194 .expect("a cleared ring must allow another restart");
12195 assert_eq!(schedule.restart_in_window, 0);
12196 assert_eq!(schedule.delay, Duration::from_millis(100));
12197 }
12198
12199 #[test]
12200 fn crash_restart_backoff_ignores_aged_restarts() {
12201 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12202 let now = ClockInstant::now();
12203 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12204 state
12205 .next_crash_restart(&policy, now)
12206 .expect("the first restart is allowed");
12207 state
12208 .next_crash_restart(&policy, now)
12209 .expect("the second restart is allowed");
12210 state.crash_restarts[0] = now
12211 .checked_sub(policy.window + Duration::from_secs(1))
12212 .expect("the fake clock can age a restart past the window");
12213
12214 let schedule = state
12215 .next_crash_restart(&policy, now)
12216 .expect("an aged restart must release its slot");
12217 assert_eq!(schedule.restart_in_window, 1);
12218 assert_eq!(schedule.delay, Duration::from_secs(1));
12219 assert_eq!(state.crash_restarts.len(), 2);
12220 }
12221
12222 #[test]
12226 fn a_budget_spent_before_the_window_no_longer_refuses() {
12227 let policy = RestartPolicy::new(3, Duration::ZERO);
12228 let mut state = snapshot_with_restarts(true, 3);
12229 let now = ClockInstant::now();
12230 assert!(!daemon_will_restart(&mut state, &policy, now));
12231
12232 assert!(daemon_will_restart(
12233 &mut state,
12234 &policy,
12235 now + policy.window + Duration::from_secs(1)
12236 ));
12237 assert!(
12238 state.crash_restarts.is_empty(),
12239 "reading the budget must drop the instants that left the window"
12240 );
12241 }
12242
12243 fn module_with_recovery_snapshot(
12244 state: ModuleState,
12245 enabled: bool,
12246 restart_count: u32,
12247 ) -> SupervisedModule {
12248 let registry = Arc::new(Registry::default());
12249 let supervisor =
12250 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
12251 let runtime = supervisor.runtime_config();
12252 let spec = ModuleSpec {
12253 module_id: "recovery-snapshot".to_string(),
12254 program: fake_aft_stub_path(),
12255 args: Vec::new(),
12256 env: Vec::new(),
12257 reserved: false,
12258 reserved_prefixes: Vec::new(),
12259 protocol: ModuleProtocol::Subc,
12260 overlap: Default::default(),
12261 };
12262 let mut snapshot = SupervisorSnapshot::new(state, enabled);
12263 seed_crash_restarts(&mut snapshot, restart_count);
12264 let (commands, _rx) = mpsc::channel(4);
12267 SupervisedModule {
12268 inner: Arc::new(SupervisedModuleInner {
12269 module_id: spec.module_id.clone(),
12270 registry,
12271 snapshot: Arc::new(Mutex::new(snapshot)),
12272 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12273 spec,
12274 health: runtime.health,
12275 })),
12276 stderr_ring: runtime.stderr_ring,
12277 terminal_ring: runtime.terminal_ring,
12278 commands,
12279 monitor: Mutex::new(None),
12280 restart_policy: runtime.restart_policy,
12281 effective_drain_timeout: runtime.effective_drain_timeout,
12282 provenance_probe: supervisor.provenance_probe.clone(),
12283 }),
12284 }
12285 }
12286
12287 #[cfg(target_os = "linux")]
12288 #[tokio::test]
12289 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12290 let supervisor =
12291 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12292 .with_cgroup_placement(None);
12293 let result = supervisor.spawn(ModuleSpec {
12294 module_id: "no-cgroup-placement".to_string(),
12295 program: fake_aft_stub_path(),
12296 args: Vec::new(),
12297 env: Vec::new(),
12298 reserved: false,
12299 reserved_prefixes: Vec::new(),
12300 protocol: ModuleProtocol::Subc,
12301 overlap: Default::default(),
12302 });
12303
12304 assert!(
12305 result.is_ok(),
12306 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12307 );
12308 }
12309
12310 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12311 async fn undecided_snapshot_uses_shared_restart_predicate() {
12312 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12313 .will_recover_after_connection_loss()
12314 .unwrap());
12315 assert!(
12316 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12317 .will_recover_after_connection_loss()
12318 .unwrap()
12319 );
12320 }
12321
12322 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12323 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12324 assert!(
12325 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12326 .will_recover_after_connection_loss()
12327 .unwrap()
12328 );
12329 }
12330
12331 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12332 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12333 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12334 .will_recover_after_connection_loss()
12335 .unwrap());
12336 assert!(
12337 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12338 .will_recover_after_connection_loss()
12339 .unwrap()
12340 );
12341 }
12342
12343 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12344 async fn warming_snapshot_is_limited_to_startup_phases() {
12345 for state in [
12346 ModuleState::Starting,
12347 ModuleState::Running,
12348 ModuleState::Restarting,
12349 ] {
12350 assert!(
12351 module_with_recovery_snapshot(state, true, 0)
12352 .is_warming()
12353 .unwrap(),
12354 "{state:?} should be warming"
12355 );
12356 }
12357 for state in [
12358 ModuleState::Unresponsive,
12359 ModuleState::Draining,
12360 ModuleState::Stopped,
12361 ModuleState::Failed,
12362 ModuleState::Disabled,
12363 ] {
12364 assert!(
12365 !module_with_recovery_snapshot(state, true, 0)
12366 .is_warming()
12367 .unwrap(),
12368 "{state:?} should not be warming"
12369 );
12370 }
12371 }
12372
12373 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12374 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12375 let registry = Arc::new(Registry::default());
12376 let supervisor =
12377 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
12378 let module = supervisor
12379 .spawn(ModuleSpec {
12380 module_id: "terminal-history".to_string(),
12381 program: fake_aft_stub_path(),
12382 args: Vec::new(),
12383 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12384 reserved: false,
12385 reserved_prefixes: Vec::new(),
12386 protocol: ModuleProtocol::Subc,
12387 overlap: Default::default(),
12388 })
12389 .unwrap();
12390
12391 let deadline = Instant::now() + Duration::from_secs(5);
12392 loop {
12393 let history = module.terminal_history();
12394 if history.entries.len() == 2 {
12395 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12396 assert_eq!(history.dropped, 0);
12397 assert_eq!(
12398 history
12399 .entries
12400 .iter()
12401 .map(|entry| entry.exit_code)
12402 .collect::<Vec<_>>(),
12403 vec![Some(23), Some(23)]
12404 );
12405 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
12406 return;
12407 }
12408 assert!(
12409 Instant::now() < deadline,
12410 "module did not retain two terminal exits: {history:?}"
12411 );
12412 sleep(Duration::from_millis(10)).await;
12413 }
12414 }
12415
12416 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12420 async fn disable_during_crash_backoff_cancels_pending_respawn() {
12421 let backoff = Duration::from_secs(2);
12422 let supervisor = Supervisor::new_for_test(
12423 Arc::new(Registry::default()),
12424 RestartPolicy::new(10, backoff),
12425 );
12426 let module = supervisor
12427 .spawn(ModuleSpec {
12428 module_id: "disable-during-backoff".to_string(),
12429 program: fake_aft_stub_path(),
12430 args: Vec::new(),
12431 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12432 reserved: false,
12433 reserved_prefixes: Vec::new(),
12434 protocol: ModuleProtocol::Subc,
12435 overlap: Default::default(),
12436 })
12437 .unwrap();
12438
12439 let deadline = Instant::now() + Duration::from_secs(5);
12441 loop {
12442 if module.status().unwrap().state == ModuleState::Restarting {
12443 break;
12444 }
12445 assert!(
12446 Instant::now() < deadline,
12447 "module never entered the crash backoff"
12448 );
12449 sleep(Duration::from_millis(10)).await;
12450 }
12451
12452 let started = Instant::now();
12453 module.set_enabled(false).await.unwrap();
12454 let waited = started.elapsed();
12455
12456 assert!(
12457 waited < backoff / 2,
12458 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12459 );
12460 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12461
12462 sleep(backoff + Duration::from_millis(500)).await;
12464 let status = module.status().unwrap();
12465 assert_eq!(status.state, ModuleState::Disabled);
12466 assert_eq!(
12467 status.spawn_generation, 1,
12468 "module respawned after the operator disabled it"
12469 );
12470 }
12471
12472 #[cfg(unix)]
12475 fn protocol_none_sigterm_exits_clean_spec(
12476 module_id: &str,
12477 dir: &std::path::Path,
12478 ) -> (ModuleSpec, PathBuf, PathBuf) {
12479 let ready = dir.join("ready");
12480 let marker = dir.join("sigterm");
12481 let spec = ModuleSpec {
12482 module_id: module_id.to_string(),
12483 program: fake_aft_stub_path(),
12484 args: Vec::new(),
12485 env: vec![
12486 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12487 (
12488 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12489 marker.display().to_string(),
12490 ),
12491 (
12492 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12493 ready.display().to_string(),
12494 ),
12495 ],
12496 reserved: false,
12497 reserved_prefixes: Vec::new(),
12498 protocol: ModuleProtocol::None,
12499 overlap: Default::default(),
12500 };
12501 (spec, ready, marker)
12502 }
12503
12504 #[cfg(unix)]
12508 async fn wait_for_file(path: &std::path::Path) {
12509 let deadline = Instant::now() + Duration::from_secs(10);
12510 while !path.exists() {
12511 assert!(
12512 Instant::now() < deadline,
12513 "{} never appeared",
12514 path.display()
12515 );
12516 sleep(Duration::from_millis(10)).await;
12517 }
12518 }
12519
12520 #[cfg(unix)]
12524 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12525 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12526 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12527 let (spec, ready, marker) =
12528 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12529 let supervisor = Supervisor::new_for_test(
12530 Arc::new(Registry::default()),
12531 RestartPolicy::new(3, Duration::ZERO),
12532 );
12533 let module = supervisor.spawn(spec).unwrap();
12534 wait_for_file(&ready).await;
12535 let deadline = Instant::now() + Duration::from_secs(10);
12539 let first_pid = loop {
12540 let status = module.status().unwrap();
12541 if status.state == ModuleState::Running {
12542 if let Some(pid) = status.pid {
12543 break pid;
12544 }
12545 }
12546 assert!(
12547 Instant::now() < deadline,
12548 "a running module must report its pid after exec confirmation: {status:?}"
12549 );
12550 sleep(Duration::from_millis(10)).await;
12551 };
12552
12553 rustix::process::kill_process(
12554 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12555 rustix::process::Signal::TERM,
12556 )
12557 .unwrap();
12558
12559 let deadline = Instant::now() + Duration::from_secs(10);
12560 let respawned = loop {
12561 let status = module.status().unwrap();
12562 if status.state == ModuleState::Running
12563 && status.pid.is_some_and(|pid| pid != first_pid)
12564 {
12565 break status;
12566 }
12567 assert!(
12568 Instant::now() < deadline,
12569 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12570 );
12571 sleep(Duration::from_millis(10)).await;
12572 };
12573 assert_eq!(respawned.spawn_generation, 2);
12574 assert!(
12575 marker.exists(),
12576 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12577 );
12578
12579 let history = module.terminal_history();
12580 assert_eq!(history.entries.len(), 1, "{history:?}");
12581 let entry = &history.entries[0];
12582 assert_eq!(entry.exit_code, Some(0));
12583 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12584 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12585
12586 module.stop().await.unwrap();
12587 }
12588
12589 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12593 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12594 let supervisor = Supervisor::new_for_test(
12595 Arc::new(Registry::default()),
12596 RestartPolicy::new(1, Duration::ZERO),
12597 );
12598 let module = supervisor
12599 .spawn(ModuleSpec {
12600 module_id: "none-clean-exit-budget".to_string(),
12601 program: fake_aft_stub_path(),
12602 args: Vec::new(),
12603 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12604 reserved: false,
12605 reserved_prefixes: Vec::new(),
12606 protocol: ModuleProtocol::None,
12607 overlap: Default::default(),
12608 })
12609 .unwrap();
12610
12611 let deadline = Instant::now() + Duration::from_secs(10);
12614 loop {
12615 let status = module.status().unwrap();
12616 if status.state == ModuleState::Failed {
12617 break;
12618 }
12619 assert!(
12620 Instant::now() < deadline,
12621 "module never exhausted its budget: {status:?} {:?}",
12622 module.terminal_history()
12623 );
12624 sleep(Duration::from_millis(10)).await;
12625 }
12626 let history = module.terminal_history();
12627 assert_eq!(
12628 history
12629 .entries
12630 .iter()
12631 .map(|entry| (entry.exit_code, entry.disposition.clone()))
12632 .collect::<Vec<_>>(),
12633 vec![
12634 (Some(0), TerminalDisposition::Restarting),
12635 (Some(0), TerminalDisposition::Failed),
12636 ]
12637 );
12638 let detail = history.entries[1]
12639 .disposition_detail
12640 .as_deref()
12641 .expect("a budget failure names the budget");
12642 assert!(detail.contains("max_restarts=1"), "{detail}");
12643 assert_eq!(module.status().unwrap().spawn_generation, 2);
12644 }
12645
12646 #[test]
12647 fn restart_budget_failure_is_published_after_its_terminal_record() {
12648 use std::io::{Read, Write};
12649 let supervisor = Supervisor::new_for_test(
12650 Arc::new(Registry::default()),
12651 RestartPolicy::new(0, Duration::ZERO),
12652 );
12653 let runtime = supervisor.runtime_config();
12654 let spec = ModuleSpec {
12655 protocol: ModuleProtocol::None,
12656 ..windowed_crash_spec("budget-publication-order")
12657 };
12658 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12659 ModuleState::Running,
12660 true,
12661 )));
12662 lock_snapshot(&snapshot).unwrap().process_alive = true;
12663 let reaped_snapshot = snapshot.clone();
12664 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12665 let history = runtime.terminal_ring.clone();
12666 let reap = std::thread::spawn(move || {
12667 tokio::runtime::Builder::new_current_thread()
12668 .enable_all()
12669 .build()
12670 .unwrap()
12671 .block_on(on_child_exit(
12672 &spec,
12673 runtime.restart_policy,
12674 &supervisor.registry,
12675 &reaped_snapshot,
12676 &runtime.terminal_ring,
12677 &runtime.spawn_events,
12678 &runtime.child_roster,
12679 ExitReport {
12680 kind: ExitKind::Clean,
12681 code: Some(0),
12682 signal: None,
12683 at_ms: 1,
12684 },
12685 true,
12686 ))
12687 });
12688 let mut reached = [0];
12689 barrier.read_exact(&mut reached).unwrap();
12690 assert_eq!(&reached, b"R");
12691 let (before_state, before_alive) = {
12692 let state = lock_snapshot(&snapshot).unwrap();
12693 (state.state, state.process_alive)
12694 };
12695 let before_history = history.lock().unwrap().snapshot();
12696 barrier.write_all(b"X").unwrap();
12697 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12698 assert!(before_history.entries.is_empty());
12699 assert_ne!(
12700 before_state,
12701 ModuleState::Failed,
12702 "Failed was visible before its terminal record could be written"
12703 );
12704 assert!(before_alive);
12705 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12706 let history = history.lock().unwrap().snapshot();
12707 assert_eq!(history.entries.len(), 1);
12708 assert_eq!(history.entries[0].exit_code, Some(0));
12709 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12710 }
12711
12712 #[test]
12713 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12714 use std::io::{Read, Write};
12715
12716 let supervisor = Supervisor::new_for_test(
12717 Arc::new(Registry::default()),
12718 RestartPolicy::new(3, Duration::ZERO),
12719 );
12720 let mut runtime = supervisor.runtime_config();
12721 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12722 let snapshot = module.inner.snapshot.clone();
12723 lock_snapshot(&snapshot).unwrap().process_alive = true;
12724 runtime.terminal_ring = module.inner.terminal_ring.clone();
12725 let spec = windowed_crash_spec(module.module_id());
12726 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12727 let reaped_snapshot = snapshot.clone();
12728 let reap = std::thread::spawn(move || {
12729 tokio::runtime::Builder::new_current_thread()
12730 .enable_all()
12731 .build()
12732 .unwrap()
12733 .block_on(on_child_exit(
12734 &spec,
12735 runtime.restart_policy,
12736 &supervisor.registry,
12737 &reaped_snapshot,
12738 &runtime.terminal_ring,
12739 &runtime.spawn_events,
12740 &runtime.child_roster,
12741 ExitReport {
12742 kind: ExitKind::Crash,
12743 code: Some(2),
12744 signal: None,
12745 at_ms: 1,
12746 },
12747 true,
12748 ))
12749 });
12750
12751 let mut reached = [0];
12755 barrier.read_exact(&mut reached).unwrap();
12756 assert_eq!(&reached, b"R");
12757 let before_status = module.status().unwrap();
12758 let before_history = module.terminal_history();
12759 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12760 barrier.write_all(b"X").unwrap();
12761 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12762
12763 assert_eq!(before_status.state, ModuleState::Running);
12764 assert!(before_status.process_alive);
12765 assert_eq!(before_status.restart_count, 3);
12766 assert!(before_history.entries.is_empty());
12767 assert!(
12768 before_recovers,
12769 "connection cleanup would report a terminal close for a scheduled crash restart"
12770 );
12771 let restarting = module.status().unwrap();
12772 assert_eq!(restarting.state, ModuleState::Restarting);
12773 assert_eq!(restarting.restart_count, 3);
12774 assert!(module.will_recover_after_connection_loss().unwrap());
12775 let history = module.terminal_history();
12776 assert_eq!(history.entries.len(), 1);
12777 assert_eq!(
12778 history.entries[0].disposition,
12779 TerminalDisposition::Restarting
12780 );
12781
12782 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12785 state.state = ModuleState::Running;
12786 state.respawn_pending = false;
12787 })
12788 .unwrap();
12789 assert!(!module.will_recover_after_connection_loss().unwrap());
12790 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12791 state.enabled = false;
12792 state.respawn_pending = true;
12793 })
12794 .unwrap();
12795 assert!(!module.will_recover_after_connection_loss().unwrap());
12796 }
12797
12798 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12799 use std::net::{TcpListener, TcpStream};
12800 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12801 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12802 let (barrier, _) = listener.accept().unwrap();
12803 for stream in [&writer, &barrier] {
12804 stream
12805 .set_read_timeout(Some(Duration::from_secs(10)))
12806 .unwrap();
12807 stream
12808 .set_write_timeout(Some(Duration::from_secs(10)))
12809 .unwrap();
12810 }
12811 *events.1.lock().unwrap() = Some(writer);
12812 barrier
12813 }
12814
12815 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12816 use std::io::{Read, Write};
12817 let barrier = events.1.lock().unwrap().take();
12818 if let Some(mut barrier) = barrier {
12819 barrier.write_all(b"R").unwrap();
12820 let mut release = [0];
12821 barrier.read_exact(&mut release).unwrap();
12822 assert_eq!(&release, b"X");
12823 }
12824 }
12825
12826 #[test]
12827 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12828 use std::io::{Read, Write};
12829
12830 let supervisor = Supervisor::new_for_test(
12831 Arc::new(Registry::default()),
12832 RestartPolicy::new(0, Duration::ZERO),
12833 );
12834 let mut runtime = supervisor.runtime_config();
12835 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12836 let snapshot = module.inner.snapshot.clone();
12837 runtime.terminal_ring = module.inner.terminal_ring.clone();
12838 let spec = ModuleSpec {
12839 module_id: module.module_id().into(),
12840 program: fake_aft_stub_path(),
12841 protocol: ModuleProtocol::None,
12842 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12843 ..windowed_crash_spec(module.module_id())
12844 };
12845 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12846 let reap = std::thread::spawn(move || {
12847 tokio::runtime::Builder::new_current_thread()
12848 .enable_all()
12849 .build()
12850 .unwrap()
12851 .block_on(async {
12852 let mut child =
12853 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12854 health_restart_child(
12855 &spec,
12856 &runtime,
12857 &supervisor.registry,
12858 &supervisor.process_liveness,
12859 &snapshot,
12860 &mut child,
12861 SupervisorHealthStatus::Failing,
12862 Some("health probe did not answer"),
12863 1,
12864 )
12865 .await
12866 .unwrap();
12867 assert!(child.is_none());
12868 });
12869 });
12870 let mut reached = [0];
12875 barrier.read_exact(&mut reached).unwrap();
12876 assert_eq!(&reached, b"R");
12877 let before_status = module.status().unwrap();
12878 let before_history = module.terminal_history();
12879 barrier.write_all(b"X").unwrap();
12880 reap.join().unwrap();
12881 assert!(before_history.entries.is_empty());
12882 assert_ne!(
12883 before_status.state,
12884 ModuleState::Failed,
12885 "Failed was visible before its terminal record could be written"
12886 );
12887 assert!(
12888 before_status.process_alive,
12889 "process death was visible before its terminal record could be written"
12890 );
12891 let failed = module.status().unwrap();
12892 assert_eq!(failed.state, ModuleState::Failed);
12893 assert!(!failed.process_alive);
12894 let history = module.terminal_history();
12895 assert_eq!(history.entries.len(), 1);
12896 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12897 assert_eq!(
12898 history.entries[0].disposition_detail.as_deref(),
12899 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12900 );
12901 }
12902
12903 #[test]
12904 fn observed_exits_publish_process_death_after_terminal_history() {
12905 use std::io::{Read, Write};
12906 for (kind, enabled, shutdown, wait_error, expected_state) in [
12907 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12908 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12909 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12910 (
12911 ExitKind::DeliberateSeverance,
12912 true,
12913 false,
12914 false,
12915 ModuleState::Restarting,
12916 ),
12917 #[cfg(unix)]
12918 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12919 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12920 ] {
12921 let supervisor = Supervisor::new_for_test(
12922 Arc::new(Registry::default()),
12923 RestartPolicy::new(1, Duration::ZERO),
12924 );
12925 let runtime = supervisor.runtime_config();
12926 #[cfg(unix)]
12927 if shutdown {
12928 runtime.child_roster.close();
12929 }
12930 #[cfg(not(unix))]
12931 let _ = shutdown;
12932 let spec = windowed_crash_spec("exit-publication-order");
12933 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12934 ModuleState::Running,
12935 enabled,
12936 )));
12937 lock_snapshot(&snapshot).unwrap().process_alive = true;
12938 let reaped_snapshot = snapshot.clone();
12939 let history = runtime.terminal_ring.clone();
12940 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12941 let reap = std::thread::spawn(move || {
12942 if wait_error {
12943 super::fail_after_wait_error(
12944 &spec.module_id,
12945 &reaped_snapshot,
12946 &runtime.terminal_ring,
12947 &runtime.spawn_events,
12948 "wait failed".into(),
12949 &runtime.failure_context(true),
12950 );
12951 } else {
12952 tokio::runtime::Builder::new_current_thread()
12953 .enable_all()
12954 .build()
12955 .unwrap()
12956 .block_on(on_child_exit(
12957 &spec,
12958 runtime.restart_policy,
12959 &supervisor.registry,
12960 &reaped_snapshot,
12961 &runtime.terminal_ring,
12962 &runtime.spawn_events,
12963 &runtime.child_roster,
12964 ExitReport {
12965 kind,
12966 code: Some(0),
12967 signal: None,
12968 at_ms: 1,
12969 },
12970 true,
12971 ));
12972 }
12973 });
12974 let mut reached = [0];
12975 barrier.read_exact(&mut reached).unwrap();
12976 assert_eq!(&reached, b"R");
12977 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12978 let before_history = history.lock().unwrap().snapshot();
12979 barrier.write_all(b"X").unwrap();
12980 reap.join().unwrap();
12981 assert!(
12982 before_alive,
12983 "{kind:?}: death was published before recording"
12984 );
12985 assert!(before_history.entries.is_empty());
12986 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12987 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12988 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12989 }
12990 }
12991
12992 #[tokio::test]
12993 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12994 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12995 let path = dir.join("terminals.jsonl");
12996 std::fs::create_dir(&path).unwrap();
12997 let supervisor = Supervisor::new_for_test(
12998 Arc::new(Registry::default()),
12999 RestartPolicy::new(0, Duration::ZERO),
13000 )
13001 .with_terminal_journal(path, "health-budget-journal-failure".into());
13002 let runtime = supervisor.runtime_config();
13003 let spec = ModuleSpec {
13004 program: fake_aft_stub_path(),
13005 protocol: ModuleProtocol::None,
13006 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13007 ..windowed_crash_spec("health-budget-journal-failure")
13008 };
13009 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13010 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13011 health_restart_child(
13012 &spec,
13013 &runtime,
13014 &supervisor.registry,
13015 &supervisor.process_liveness,
13016 &snapshot,
13017 &mut child,
13018 SupervisorHealthStatus::Failing,
13019 None,
13020 1,
13021 )
13022 .await
13023 .unwrap();
13024 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13025 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13026 let history = runtime
13027 .terminal_ring
13028 .lock()
13029 .unwrap()
13030 .durable_history(&spec.module_id);
13031 assert!(history.journal_write_failures > 0);
13032 assert_eq!(history.entries.len(), 1);
13033 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13034 }
13035
13036 #[test]
13037 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13038 let supervisor = Supervisor::new_for_test(
13039 Arc::new(Registry::default()),
13040 RestartPolicy::new(0, Duration::ZERO),
13041 );
13042 let runtime = supervisor.runtime_config();
13043 let spec = ModuleSpec {
13044 program: fake_aft_stub_path(),
13045 protocol: ModuleProtocol::None,
13046 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13047 ..windowed_crash_spec("health-budget-recording-panic")
13048 };
13049 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13050 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13051 tokio::runtime::Builder::new_current_thread()
13052 .enable_all()
13053 .build()
13054 .unwrap()
13055 .block_on(async {
13056 let mut child =
13057 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13058 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13059 health_restart_child(
13060 &spec,
13061 &runtime,
13062 &supervisor.registry,
13063 &supervisor.process_liveness,
13064 &snapshot,
13065 &mut child,
13066 SupervisorHealthStatus::Failing,
13067 None,
13068 1,
13069 )
13070 .await
13071 .unwrap();
13072 });
13073 }));
13074 let panic = result.expect_err("recording must still unwind");
13075 assert_eq!(
13076 panic.downcast_ref::<String>().map(String::as_str),
13077 Some("spawn event sequence exhausted")
13078 );
13079 let state = lock_snapshot(&snapshot).unwrap();
13080 assert_eq!(state.state, ModuleState::Failed);
13081 assert!(!state.process_alive);
13082 assert!(state.last_exit.is_some());
13083 }
13084
13085 #[tokio::test]
13086 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13087 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13088 let path = dir.join("terminals.jsonl");
13089 std::fs::create_dir(&path).unwrap();
13090 let supervisor = Supervisor::new_for_test(
13091 Arc::new(Registry::default()),
13092 RestartPolicy::new(0, Duration::ZERO),
13093 )
13094 .with_terminal_journal(path, "budget-journal-failure".into());
13095 let runtime = supervisor.runtime_config();
13096 let spec = windowed_crash_spec("budget-journal-failure");
13097 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13098 assert!(matches!(
13099 on_child_exit(
13100 &spec,
13101 runtime.restart_policy,
13102 &supervisor.registry,
13103 &snapshot,
13104 &runtime.terminal_ring,
13105 &runtime.spawn_events,
13106 &runtime.child_roster,
13107 crash_exit_report(1),
13108 true,
13109 )
13110 .await,
13111 NextAction::Stop { .. }
13112 ));
13113 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13114 let history = runtime
13115 .terminal_ring
13116 .lock()
13117 .unwrap()
13118 .durable_history(&spec.module_id);
13119 assert!(history.journal_write_failures > 0);
13120 assert_eq!(history.entries.len(), 1);
13121 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13122 }
13123
13124 #[test]
13125 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13126 let supervisor = Supervisor::new_for_test(
13127 Arc::new(Registry::default()),
13128 RestartPolicy::new(0, Duration::ZERO),
13129 );
13130 let runtime = supervisor.runtime_config();
13131 let spec = windowed_crash_spec("budget-recording-panic");
13132 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13133 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13134 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13137 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13138 tokio::runtime::Builder::new_current_thread()
13139 .enable_all()
13140 .build()
13141 .unwrap()
13142 .block_on(on_child_exit(
13143 &spec,
13144 runtime.restart_policy,
13145 &supervisor.registry,
13146 &snapshot,
13147 &runtime.terminal_ring,
13148 &runtime.spawn_events,
13149 &runtime.child_roster,
13150 crash_exit_report(1),
13151 true,
13152 ))
13153 }));
13154 let panic = result.err().expect("recording must still unwind");
13155 assert_eq!(
13156 panic.downcast_ref::<String>().map(String::as_str),
13157 Some("spawn event sequence exhausted")
13158 );
13159 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13160 }
13161
13162 #[cfg(unix)]
13165 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13166 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13167 for disable in [false, true] {
13168 let label = if disable {
13169 "none-requested-disable"
13170 } else {
13171 "none-requested-stop"
13172 };
13173 let dir = cortexkit_test_support::ScratchDir::new(label);
13174 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13175 let supervisor = Supervisor::new_for_test(
13176 Arc::new(Registry::default()),
13177 RestartPolicy::new(3, Duration::ZERO),
13178 );
13179 let module = supervisor.spawn(spec).unwrap();
13180 wait_for_file(&ready).await;
13181
13182 if disable {
13183 module.set_enabled(false).await.unwrap();
13184 } else {
13185 module.stop().await.unwrap();
13186 }
13187 assert!(
13188 marker.exists(),
13189 "{label}: the child must have left through its SIGTERM handler with exit 0"
13190 );
13191
13192 sleep(Duration::from_millis(500)).await;
13195 let status = module.status().unwrap();
13196 let expected = if disable {
13197 ModuleState::Disabled
13198 } else {
13199 ModuleState::Stopped
13200 };
13201 assert_eq!(status.state, expected, "{label}");
13202 assert_eq!(
13203 status.spawn_generation, 1,
13204 "{label}: respawned after a requested stop"
13205 );
13206 let history = module.terminal_history();
13207 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13208 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13209 assert_ne!(
13210 history.entries[0].disposition,
13211 TerminalDisposition::Restarting,
13212 "{label}"
13213 );
13214 }
13215 }
13216
13217 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13220 async fn subc_wire_clean_exit_is_still_a_stop() {
13221 let supervisor = Supervisor::new_for_test(
13222 Arc::new(Registry::default()),
13223 RestartPolicy::new(3, Duration::ZERO),
13224 );
13225 let module = supervisor
13226 .spawn(ModuleSpec {
13227 module_id: "wire-clean-exit".to_string(),
13228 program: fake_aft_stub_path(),
13229 args: Vec::new(),
13230 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13231 reserved: false,
13232 reserved_prefixes: Vec::new(),
13233 protocol: ModuleProtocol::Subc,
13234 overlap: Default::default(),
13235 })
13236 .unwrap();
13237
13238 let deadline = Instant::now() + Duration::from_secs(10);
13239 while module.terminal_history().entries.is_empty() {
13240 assert!(Instant::now() < deadline, "module never exited");
13241 sleep(Duration::from_millis(10)).await;
13242 }
13243 sleep(Duration::from_millis(500)).await;
13245 let status = module.status().unwrap();
13246 assert_eq!(status.state, ModuleState::Stopped);
13247 assert_eq!(status.spawn_generation, 1);
13248 let history = module.terminal_history();
13249 assert_eq!(history.entries.len(), 1, "{history:?}");
13250 assert_eq!(history.entries[0].exit_code, Some(0));
13251 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13252 }
13253
13254 #[cfg(unix)]
13255 #[tokio::test]
13256 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13257 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13258 let record = dir.join("live-children.json");
13259 let supervisor = Supervisor::new_for_test(
13260 Arc::new(Registry::default()),
13261 RestartPolicy::new(0, Duration::ZERO),
13262 );
13263 let mut runtime = supervisor.runtime_config();
13264 runtime.child_roster.record_to(record.clone());
13265 let gate = Arc::new(super::ReloadExitRecordGate::default());
13266 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13267 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13268 let spec = ModuleSpec {
13269 module_id: "reload-exit-roster".into(),
13270 program: fake_aft_stub_path(),
13271 args: Vec::new(),
13272 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13273 reserved: false,
13274 reserved_prefixes: Vec::new(),
13275 protocol: ModuleProtocol::Subc,
13276 overlap: Default::default(),
13277 };
13278 let mut child = None;
13279 let reload = super::finish_reload_child(
13280 &spec,
13281 &runtime,
13282 &supervisor.registry,
13283 &supervisor.process_liveness,
13284 &snapshot,
13285 &mut child,
13286 );
13287 tokio::pin!(reload);
13288 tokio::select! {
13289 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13290 _ = gate.reached.notified() => {}
13291 }
13292 assert!(runtime
13293 .terminal_ring
13294 .lock()
13295 .unwrap()
13296 .snapshot()
13297 .entries
13298 .is_empty());
13299 assert_eq!(
13300 crate::live_children::read_record(&record).unwrap().len(),
13301 1,
13302 "shutdown must still wait for the reaped child until its terminal record exists"
13303 );
13304 runtime.child_roster.close();
13305 gate.resume.notify_one();
13306 assert!(reload.await.is_err());
13307 assert!(crate::live_children::read_record(&record)
13308 .unwrap()
13309 .is_empty());
13310 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13311 assert_eq!(history.entries.len(), 1);
13312 assert_eq!(
13313 history.entries[0].disposition,
13314 TerminalDisposition::DaemonShutdown
13315 );
13316 }
13317
13318 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13322 async fn every_restart_increment_path_advances_lifetime_count() {
13323 let supervisor = Supervisor::new_for_test(
13324 Arc::new(Registry::default()),
13325 RestartPolicy::new(1, Duration::ZERO),
13326 );
13327 let runtime = supervisor.runtime_config();
13328 let spec = ModuleSpec {
13329 module_id: "lifetime-increment-path".to_string(),
13330 program: PathBuf::from("/unused/lifetime-increment-path"),
13331 args: Vec::new(),
13332 env: Vec::new(),
13333 reserved: false,
13334 reserved_prefixes: Vec::new(),
13335 protocol: ModuleProtocol::Subc,
13336 overlap: Default::default(),
13337 };
13338
13339 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13340 assert!(matches!(
13341 on_child_exit(
13342 &spec,
13343 runtime.restart_policy,
13344 &supervisor.registry,
13345 &crash_snapshot,
13346 &runtime.terminal_ring,
13347 &runtime.spawn_events,
13348 &runtime.child_roster,
13349 ExitReport {
13350 kind: ExitKind::Crash,
13351 code: Some(1),
13352 signal: None,
13353 at_ms: 1,
13354 },
13355 true,
13356 )
13357 .await,
13358 NextAction::Restart { schedule: _ }
13359 ));
13360 let (crash_restarts, crash_lifetime) = {
13361 let state = lock_snapshot(&crash_snapshot).unwrap();
13362 (state.crash_restarts.len(), state.lifetime_restarts)
13363 };
13364 assert_eq!(crash_restarts, 1);
13365 assert_eq!(crash_lifetime, 1);
13366
13367 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13368 let mut health_child = None;
13369 assert!(matches!(
13370 health_restart_child(
13371 &spec,
13372 &runtime,
13373 &supervisor.registry,
13374 &supervisor.process_liveness,
13375 &health_snapshot,
13376 &mut health_child,
13377 SupervisorHealthStatus::Failing,
13378 None,
13379 2,
13380 )
13381 .await,
13382 Ok(())
13383 ));
13384 assert!(health_child.is_none());
13385 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13386 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13387 let (health_restarts, health_lifetime) = {
13388 let state = lock_snapshot(&health_snapshot).unwrap();
13389 (state.crash_restarts.len(), state.lifetime_restarts)
13390 };
13391 assert_eq!(health_restarts, 1);
13392 assert_eq!(health_lifetime, 1);
13393
13394 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13395 let mut reload_child = None;
13396 assert!(matches!(
13397 handle_reload_spawn_failure(
13398 &spec,
13399 &runtime,
13400 &supervisor.process_liveness,
13401 &reload_snapshot,
13402 &mut reload_child,
13403 "forced reload spawn failure".to_string(),
13404 )
13405 .await,
13406 Err(SuperviseError::ReloadFailed { .. })
13407 ));
13408 let (reload_restarts, reload_lifetime) = {
13409 let state = lock_snapshot(&reload_snapshot).unwrap();
13410 (state.crash_restarts.len(), state.lifetime_restarts)
13411 };
13412 assert_eq!(reload_restarts, 1);
13413 assert_eq!(reload_lifetime, 1);
13414 }
13415
13416 #[tokio::test]
13417 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
13418 let supervisor = Supervisor::new_for_test(
13419 Arc::new(Registry::default()),
13420 RestartPolicy::new(3, Duration::ZERO),
13421 );
13422 let runtime = supervisor.runtime_config();
13423 let spec = ModuleSpec {
13424 module_id: "deliberately-severed".to_string(),
13425 program: PathBuf::from("/unused/deliberately-severed"),
13426 args: Vec::new(),
13427 env: Vec::new(),
13428 reserved: false,
13429 reserved_prefixes: Vec::new(),
13430 protocol: ModuleProtocol::Subc,
13431 overlap: Default::default(),
13432 };
13433 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13434 let process = ProcessIdentity {
13435 pid: 41,
13436 start_time: 101,
13437 };
13438 record_deliberate_severance(&snapshot, process).unwrap();
13439 let exit_report = apply_deliberate_severance_marker(
13440 &snapshot,
13441 Some(process),
13442 ExitReport {
13443 kind: ExitKind::Crash,
13444 code: Some(1),
13445 signal: None,
13446 at_ms: 1,
13447 },
13448 );
13449 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
13450
13451 assert!(matches!(
13452 on_child_exit(
13453 &spec,
13454 runtime.restart_policy,
13455 &supervisor.registry,
13456 &snapshot,
13457 &runtime.terminal_ring,
13458 &runtime.spawn_events,
13459 &runtime.child_roster,
13460 exit_report,
13461 true,
13462 )
13463 .await,
13464 NextAction::Restart { schedule: _ }
13465 ));
13466 let state = lock_snapshot(&snapshot).unwrap();
13467 assert_eq!(state.lifetime_restarts, 1);
13468 assert_eq!(state.crash_restarts.len(), 0);
13469 }
13470
13471 #[tokio::test]
13472 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13473 let supervisor = Supervisor::new_for_test(
13474 Arc::new(Registry::default()),
13475 RestartPolicy::new(3, Duration::ZERO),
13476 );
13477 let runtime = supervisor.runtime_config();
13478 let spec = ModuleSpec {
13479 module_id: "genuine-crash".to_string(),
13480 program: PathBuf::from("/unused/genuine-crash"),
13481 args: Vec::new(),
13482 env: Vec::new(),
13483 reserved: false,
13484 reserved_prefixes: Vec::new(),
13485 protocol: ModuleProtocol::Subc,
13486 overlap: Default::default(),
13487 };
13488 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13489
13490 assert!(matches!(
13491 on_child_exit(
13492 &spec,
13493 runtime.restart_policy,
13494 &supervisor.registry,
13495 &snapshot,
13496 &runtime.terminal_ring,
13497 &runtime.spawn_events,
13498 &runtime.child_roster,
13499 ExitReport {
13500 kind: ExitKind::Crash,
13501 code: Some(1),
13502 signal: None,
13503 at_ms: 1,
13504 },
13505 true,
13506 )
13507 .await,
13508 NextAction::Restart { schedule: _ }
13509 ));
13510 let state = lock_snapshot(&snapshot).unwrap();
13511 assert_eq!(state.lifetime_restarts, 1);
13512 assert_eq!(state.crash_restarts.len(), 1);
13513 }
13514
13515 fn crash_exit_report(at_ms: u64) -> ExitReport {
13516 ExitReport {
13517 kind: ExitKind::Crash,
13518 code: Some(1),
13519 signal: None,
13520 at_ms,
13521 }
13522 }
13523
13524 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13525 ModuleSpec {
13526 module_id: module_id.to_string(),
13527 program: PathBuf::from("/unused").join(module_id),
13528 args: Vec::new(),
13529 env: Vec::new(),
13530 reserved: false,
13531 reserved_prefixes: Vec::new(),
13532 protocol: ModuleProtocol::Subc,
13533 overlap: Default::default(),
13534 }
13535 }
13536
13537 #[tokio::test]
13543 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13544 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13545 let supervisor = Supervisor::new_for_test(
13546 Arc::new(Registry::default()),
13547 RestartPolicy::new(2, Duration::ZERO),
13548 );
13549 let runtime = supervisor.runtime_config();
13550 let spec = windowed_crash_spec("crash-loop-in-window");
13551 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13552
13553 for attempt in 1..=2 {
13554 assert!(
13555 matches!(
13556 on_child_exit(
13557 &spec,
13558 runtime.restart_policy,
13559 &supervisor.registry,
13560 &snapshot,
13561 &runtime.terminal_ring,
13562 &runtime.spawn_events,
13563 &runtime.child_roster,
13564 crash_exit_report(attempt),
13565 true,
13566 )
13567 .await,
13568 NextAction::Restart { schedule: _ }
13569 ),
13570 "crash {attempt} is inside the budget and must respawn"
13571 );
13572 }
13573
13574 assert!(matches!(
13575 on_child_exit(
13576 &spec,
13577 runtime.restart_policy,
13578 &supervisor.registry,
13579 &snapshot,
13580 &runtime.terminal_ring,
13581 &runtime.spawn_events,
13582 &runtime.child_roster,
13583 crash_exit_report(3),
13584 true,
13585 )
13586 .await,
13587 NextAction::Stop { .. }
13588 ));
13589
13590 {
13591 let state = lock_snapshot(&snapshot).unwrap();
13592 assert_eq!(state.state, ModuleState::Failed);
13593 assert_eq!(state.crash_restarts.len(), 2);
13594 assert_eq!(state.lifetime_restarts, 2);
13595 }
13596
13597 let history = runtime
13598 .terminal_ring
13599 .lock()
13600 .expect("terminal ring is not poisoned")
13601 .snapshot();
13602 let last = history
13603 .entries
13604 .last()
13605 .expect("the refused crash is retained");
13606 assert_eq!(last.disposition, TerminalDisposition::Failed);
13607 assert_eq!(
13608 last.disposition_detail.as_deref(),
13609 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13610 );
13611
13612 let captured = crate::router::test_log::captured_logs(&logs);
13613 assert!(
13614 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13615 "the stop must be logged with its window: {captured}"
13616 );
13617 }
13618
13619 #[tokio::test]
13627 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13628 let supervisor = Supervisor::new_for_test(
13629 Arc::new(Registry::default()),
13630 RestartPolicy::new(2, Duration::ZERO),
13631 );
13632 let runtime = supervisor.runtime_config();
13633 let spec = windowed_crash_spec("crash-across-windows");
13634 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13635
13636 for attempt in 1..=2 {
13637 assert!(matches!(
13638 on_child_exit(
13639 &spec,
13640 runtime.restart_policy,
13641 &supervisor.registry,
13642 &snapshot,
13643 &runtime.terminal_ring,
13644 &runtime.spawn_events,
13645 &runtime.child_roster,
13646 crash_exit_report(attempt),
13647 true,
13648 )
13649 .await,
13650 NextAction::Restart { schedule: _ }
13651 ));
13652 }
13653
13654 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13657 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13658 })
13659 .unwrap();
13660
13661 assert!(
13662 matches!(
13663 on_child_exit(
13664 &spec,
13665 runtime.restart_policy,
13666 &supervisor.registry,
13667 &snapshot,
13668 &runtime.terminal_ring,
13669 &runtime.spawn_events,
13670 &runtime.child_roster,
13671 crash_exit_report(3),
13672 true,
13673 )
13674 .await,
13675 NextAction::Restart { schedule: _ }
13676 ),
13677 "a crash older than the window must not hold a budget slot"
13678 );
13679
13680 let state = lock_snapshot(&snapshot).unwrap();
13681 assert_eq!(state.state, ModuleState::Restarting);
13682 assert_eq!(
13683 state.crash_restarts.len(),
13684 2,
13685 "the aged instant is dropped and the new one takes its place"
13686 );
13687 assert_eq!(
13688 state.lifetime_restarts, 3,
13689 "the ledger counts every restart, including the ones the window forgot"
13690 );
13691 }
13692
13693 #[tokio::test]
13698 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13699 let supervisor = Supervisor::new_for_test(
13700 Arc::new(Registry::default()),
13701 RestartPolicy::new(2, Duration::ZERO),
13702 );
13703 let runtime = supervisor.runtime_config();
13704 let spec = windowed_crash_spec("operator-cleared-budget");
13705 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13706
13707 for attempt in 1..=2 {
13708 assert!(matches!(
13709 on_child_exit(
13710 &spec,
13711 runtime.restart_policy,
13712 &supervisor.registry,
13713 &snapshot,
13714 &runtime.terminal_ring,
13715 &runtime.spawn_events,
13716 &runtime.child_roster,
13717 crash_exit_report(attempt),
13718 true,
13719 )
13720 .await,
13721 NextAction::Restart { schedule: _ }
13722 ));
13723 }
13724
13725 reset_restart_count(&snapshot, &spec.module_id).unwrap();
13726 {
13727 let state = lock_snapshot(&snapshot).unwrap();
13728 assert!(
13729 state.crash_restarts.is_empty(),
13730 "an operator restart returns the full budget"
13731 );
13732 assert_eq!(
13733 state.lifetime_restarts, 2,
13734 "clearing the budget must not unmake the crashes"
13735 );
13736 }
13737
13738 assert!(
13739 matches!(
13740 on_child_exit(
13741 &spec,
13742 runtime.restart_policy,
13743 &supervisor.registry,
13744 &snapshot,
13745 &runtime.terminal_ring,
13746 &runtime.spawn_events,
13747 &runtime.child_roster,
13748 crash_exit_report(3),
13749 true,
13750 )
13751 .await,
13752 NextAction::Restart { schedule: _ }
13753 ),
13754 "the cleared budget must be spendable again"
13755 );
13756 let state = lock_snapshot(&snapshot).unwrap();
13757 assert_eq!(state.crash_restarts.len(), 1);
13758 assert_eq!(state.lifetime_restarts, 3);
13759 }
13760
13761 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13762 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13763 let severed = ProcessIdentity {
13764 pid: 41,
13765 start_time: 101,
13766 };
13767 let successor = ProcessIdentity {
13768 pid: 41,
13769 start_time: 202,
13770 };
13771 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13772 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13773 state.pid = Some(successor.pid);
13774 state.process_start_time = Some(successor.start_time);
13775 })
13776 .unwrap();
13777 assert!(!module.record_deliberate_severance(severed).unwrap());
13778
13779 let exit_report = apply_deliberate_severance_marker(
13780 &module.inner.snapshot,
13781 Some(successor),
13782 ExitReport {
13783 kind: ExitKind::Crash,
13784 code: Some(1),
13785 signal: None,
13786 at_ms: 1,
13787 },
13788 );
13789
13790 assert_eq!(exit_report.kind, ExitKind::Crash);
13791 }
13792
13793 #[tokio::test]
13794 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13795 let registry = Registry::default();
13796 let supervisor = Supervisor::new_for_test(
13797 Arc::new(Registry::default()),
13798 RestartPolicy::new(3, Duration::ZERO),
13799 );
13800 let runtime = supervisor.runtime_config();
13801 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13802 let spec = ModuleSpec {
13803 module_id: "drain-deliberate-severance".to_string(),
13804 program: fake_aft_stub_path(),
13805 args: Vec::new(),
13806 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13807 reserved: false,
13808 reserved_prefixes: Vec::new(),
13809 protocol: ModuleProtocol::Subc,
13810 overlap: Default::default(),
13811 };
13812 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13813 let process = ProcessIdentity {
13814 pid: 41,
13815 start_time: 101,
13816 };
13817 child.process_identity = Some(process);
13818 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13819 state.pid = Some(process.pid);
13820 state.process_start_time = Some(process.start_time);
13821 })
13822 .unwrap();
13823 record_deliberate_severance(&snapshot, process).unwrap();
13824
13825 drain_child_to_state(
13826 &spec.module_id,
13827 spec.protocol,
13828 StopNotice::SentOverConnection,
13831 ®istry,
13832 None,
13833 &snapshot,
13834 &runtime.terminal_ring,
13835 &runtime.spawn_events,
13836 child,
13837 Duration::from_secs(1),
13838 ModuleState::Stopped,
13839 Some(false),
13840 &super::FailureContext::operator(),
13841 )
13842 .await
13843 .unwrap();
13844
13845 let state = lock_snapshot(&snapshot).unwrap();
13846 assert_eq!(
13847 state.last_exit.as_ref().map(|exit| exit.kind),
13848 Some(ExitKind::DeliberateSeverance)
13849 );
13850 assert_eq!(state.lifetime_restarts, 1);
13851 assert_eq!(state.crash_restarts.len(), 0);
13852 drop(state);
13853 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13854 assert_eq!(
13855 history.entries[0].exit_kind,
13856 subc_control::TerminalExitKind::DeliberateSeverance
13857 );
13858 }
13859
13860 #[tokio::test]
13861 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13862 let registry = Registry::default();
13863 let supervisor = Supervisor::new_for_test(
13864 Arc::new(Registry::default()),
13865 RestartPolicy::new(3, Duration::ZERO),
13866 );
13867 let runtime = supervisor.runtime_config();
13868 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13869 let spec = ModuleSpec {
13870 module_id: "ordinary-drain".to_string(),
13871 program: fake_aft_stub_path(),
13872 args: Vec::new(),
13873 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13874 reserved: false,
13875 reserved_prefixes: Vec::new(),
13876 protocol: ModuleProtocol::Subc,
13877 overlap: Default::default(),
13878 };
13879 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13880
13881 drain_child_to_state(
13882 &spec.module_id,
13883 spec.protocol,
13884 StopNotice::SentOverConnection,
13887 ®istry,
13888 None,
13889 &snapshot,
13890 &runtime.terminal_ring,
13891 &runtime.spawn_events,
13892 child,
13893 Duration::from_secs(1),
13894 ModuleState::Stopped,
13895 Some(false),
13896 &super::FailureContext::operator(),
13897 )
13898 .await
13899 .unwrap();
13900
13901 let state = lock_snapshot(&snapshot).unwrap();
13902 assert_eq!(
13903 state.last_exit.as_ref().map(|exit| exit.kind),
13904 Some(ExitKind::Crash)
13905 );
13906 assert_eq!(state.lifetime_restarts, 0);
13907 assert_eq!(state.crash_restarts.len(), 0);
13908 }
13909
13910 #[test]
13911 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13912 assert!(!include_str!("server.rs")
13918 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13919 }
13920
13921 #[test]
13928 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13929 assert!(drained_after_quiescence_wait(&Ok(true)));
13930 assert!(!drained_after_quiescence_wait(&Ok(false)));
13931 assert!(!drained_after_quiescence_wait(&Err(
13932 SuperviseError::StatePoisoned { module_id: None }
13933 )));
13934 }
13935
13936 #[test]
13945 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13946 let ring = Arc::new(Mutex::new(TerminalRing::new(
13947 TerminalRingConfig::default(),
13948 0,
13949 )));
13950 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13951
13952 let snapshot = ring.lock().unwrap().snapshot();
13953 assert_eq!(snapshot.entries.len(), 1);
13954 let entry = &snapshot.entries[0];
13955 assert_eq!(entry.exit_code, None);
13956 assert_eq!(entry.exit_signal, None);
13957 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13958 }
13959
13960 #[test]
13961 fn wait_error_exit_path_preserves_spawn_event_density() {
13962 let feed = super::SpawnEventFeed::default();
13963 feed.configure_incarnation("wait-error-density".to_string());
13964 feed.emit_spawned("wait-error", 41, 1);
13965 let ring = Arc::new(Mutex::new(TerminalRing::new(
13966 TerminalRingConfig::default(),
13967 0,
13968 )));
13969
13970 record_wait_error_terminal("wait-error", &ring, &feed);
13971 feed.emit_spawned("after-wait-error", 42, 2);
13972
13973 let state = feed.0.lock().unwrap();
13974 let sequences = state
13975 .events
13976 .iter()
13977 .map(|event| event.cursor.seq)
13978 .collect::<Vec<_>>();
13979 assert_eq!(sequences, vec![1, 2, 3]);
13980 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13981 assert_eq!(state.events[1].exit_code, None);
13982 assert_eq!(state.events[1].exit_signal, None);
13983 }
13984
13985 #[test]
13989 fn wait_error_exit_report_is_classified_as_a_crash() {
13990 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13991 }
13992}
13993
13994#[cfg(all(test, unix))]
13995mod restart_ordering_tests {
13996 use super::*;
13997 use std::io::Write;
13998 use tokio::io::AsyncReadExt;
13999
14000 #[test]
14001 #[ignore = "child process fixture"]
14002 fn serving_child() {
14003 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14004 return;
14005 };
14006 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14007 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14008 for stream in listener.incoming() {
14009 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14010 }
14011 }
14012
14013 async fn restart_before_publication(requests: usize) {
14014 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14015 let address_file = home.path().join("address");
14016 let spec = ModuleSpec {
14017 module_id: "restart-serving-child".into(),
14018 program: std::env::current_exe().unwrap(),
14019 args: vec![
14020 "--exact".into(),
14021 "supervise::restart_ordering_tests::serving_child".into(),
14022 "--ignored".into(),
14023 "--nocapture".into(),
14024 ],
14025 env: vec![(
14026 "RESTART_TEST_ADDRESS_FILE".into(),
14027 address_file.display().to_string(),
14028 )],
14029 reserved: false,
14030 reserved_prefixes: vec![],
14031 protocol: ModuleProtocol::None,
14032 overlap: Default::default(),
14033 };
14034 let mut supervisor =
14035 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14036 let (spawned_tx, spawned_rx) = oneshot::channel();
14037 let spawned_tx = Mutex::new(Some(spawned_tx));
14038 let (release_tx, release_rx) = std::sync::mpsc::channel();
14039 let release_rx = Mutex::new(release_rx);
14040 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14043 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14044 release_rx
14045 .lock()
14046 .unwrap()
14047 .recv_timeout(Duration::from_secs(20))
14048 .unwrap();
14049 })));
14050 let registered = supervisor
14051 .register_configured_with_health(
14052 spec.clone(),
14053 true,
14054 HealthConfig::default(),
14055 None,
14056 RestartPolicy::new(3, Duration::ZERO),
14057 DEFAULT_REGISTER_TIMEOUT,
14058 )
14059 .unwrap();
14060 let module = registered.module.clone();
14061 let startup = tokio::task::spawn_blocking(move || {
14062 supervisor.start_registered_configured(registered, None, true)
14063 });
14064 spawned_rx.await.unwrap();
14065 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14066 let address = loop {
14067 if let Ok(address) = std::fs::read_to_string(&address_file) {
14068 if !address.is_empty() {
14069 break address;
14070 }
14071 }
14072 tokio::time::sleep(Duration::from_millis(1)).await;
14073 };
14074 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14075 let mut reply = String::new();
14076 stream.read_to_string(&mut reply).await.unwrap();
14077 reply.trim().parse::<u32>().unwrap()
14078 })
14079 .await
14080 .expect("child must serve before generation publication");
14081 assert_eq!(
14082 lock_snapshot(&module.inner.snapshot)
14083 .unwrap()
14084 .spawn_generation,
14085 0
14086 );
14087 let mut replies = Vec::new();
14088 for _ in 0..requests {
14089 let mut restart = Box::pin(module.restart(Some(0)));
14090 std::future::poll_fn(|cx| {
14093 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14094 std::task::Poll::Ready(())
14095 })
14096 .await;
14097 replies.push(restart);
14098 }
14099 release_tx.send(()).unwrap();
14100 startup.await.unwrap().unwrap();
14101 for reply in replies {
14102 reply.await.unwrap();
14103 }
14104 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14107 loop {
14108 if lock_snapshot(&module.inner.snapshot)
14109 .unwrap()
14110 .spawn_generation
14111 > 1
14112 {
14113 break;
14114 }
14115 tokio::time::sleep(Duration::from_millis(1)).await;
14116 }
14117 })
14118 .await;
14119 module.update_spec_for_test(spec).await.unwrap();
14123 let (generation, pid) = {
14124 let state = lock_snapshot(&module.inner.snapshot).unwrap();
14125 (state.spawn_generation, state.pid)
14126 };
14127 module.stop().await.unwrap();
14128 replacement.expect("restart must publish a replacement for the already-serving child");
14129 assert_ne!(
14130 pid,
14131 Some(old_pid),
14132 "a serving child predating the restart must be replaced"
14133 );
14134 assert_eq!(generation, 2, "queued restarts must share one replacement");
14135 }
14136
14137 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14138 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14139 restart_before_publication(1).await;
14140 }
14141
14142 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14143 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14144 restart_before_publication(2).await;
14145 }
14146
14147 #[tokio::test]
14148 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14149 let supervisor =
14150 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14151 let registered = supervisor
14152 .register_configured_with_health(
14153 ModuleSpec {
14154 module_id: "queued-before-replacement".into(),
14155 program: "/bin/sleep".into(),
14160 args: vec!["600".into()],
14161 env: vec![],
14162 reserved: false,
14163 reserved_prefixes: vec![],
14164 protocol: ModuleProtocol::None,
14165 overlap: Default::default(),
14166 },
14167 true,
14168 HealthConfig::default(),
14169 None,
14170 RestartPolicy::default(),
14171 DEFAULT_REGISTER_TIMEOUT,
14172 )
14173 .unwrap();
14174 let RegisteredConfiguredModule {
14175 mut spec,
14176 mut runtime,
14177 module,
14178 mut commands,
14179 } = registered;
14180 let snapshot = &module.inner.snapshot;
14181 let mut restart = Box::pin(module.restart(Some(0)));
14182 std::future::poll_fn(|cx| {
14183 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14184 std::task::Poll::Ready(())
14185 })
14186 .await;
14187 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14190 let replacement_pid = child.as_ref().unwrap().id();
14191 let command = commands.recv().await.unwrap();
14192 assert!(
14193 handle_supervisor_command(
14194 command,
14195 &mut spec,
14196 &mut runtime,
14197 &supervisor.registry,
14198 &supervisor.process_liveness,
14199 snapshot,
14200 &mut child,
14201 &mut commands,
14202 &mut VecDeque::new(),
14203 )
14204 .await
14205 );
14206 restart.await.unwrap();
14207 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14208 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14209 if let Some(mut child) = child {
14210 child.start_kill().unwrap();
14211 child.wait().await.unwrap();
14212 }
14213 assert_eq!(
14214 kept_pid, replacement_pid,
14215 "a newer replacement must survive"
14216 );
14217 assert!(
14218 !pending,
14219 "a satisfied restart must not schedule another spawn"
14220 );
14221 }
14222}
14223
14224#[cfg(all(test, unix))]
14225mod health_event_tests {
14226 use super::*;
14227 use cortexkit_test_support::ScratchDir;
14228 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14229
14230 struct Actor {
14231 module: SupervisedModule,
14232 registry: Arc<Registry>,
14233 forwarding: Arc<ForwardingTable>,
14234 spec: ModuleSpec,
14235 health: HealthConfig,
14236 rx: mpsc::Receiver<crate::router::OutboundFrame>,
14237 roster: ChildRoster,
14238 _home: ScratchDir,
14239 }
14240
14241 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14242 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14246 let tick = Instant::now();
14247 loop {
14248 if let Some(value) = observe() {
14249 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14250 return value;
14251 }
14252 assert!(
14253 std::time::Instant::now() < deadline,
14254 "timed out waiting for {reason}"
14255 );
14256 tokio::task::yield_now().await;
14257 }
14258 }
14259
14260 impl Actor {
14261 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14262 Self::start_with_policy(
14263 protocol,
14264 cadence,
14265 DEFAULT_REGISTER_TIMEOUT,
14266 RestartPolicy::new(3, Duration::ZERO),
14267 )
14268 .await
14269 }
14270
14271 async fn start_with_policy(
14272 protocol: ModuleProtocol,
14273 cadence: Duration,
14274 register_timeout: Duration,
14275 policy: RestartPolicy,
14276 ) -> Self {
14277 let home = ScratchDir::new("health-events");
14278 let registry = Arc::new(Registry::default());
14279 let forwarding = Arc::new(ForwardingTable::default());
14280 let health = HealthConfig {
14281 cadence,
14282 deadline: Duration::from_secs(3600),
14283 ..HealthConfig::default()
14284 };
14285 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14286 .with_forwarding(forwarding.clone())
14287 .with_health_config(health.clone());
14288 let spec = ModuleSpec {
14289 module_id: "event-health-child".into(),
14290 program: PathBuf::from("/bin/sleep"),
14298 args: vec!["600".into()],
14301 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14302 .into_iter()
14303 .map(|key| (key.into(), home.path().display().to_string()))
14304 .collect(),
14305 reserved: false,
14306 reserved_prefixes: vec![],
14307 protocol,
14308 overlap: Default::default(),
14309 };
14310 let module = supervisor
14311 .supervise_configured_requested(
14312 spec.clone(),
14313 true,
14314 health.clone(),
14315 Some(0),
14316 policy,
14317 register_timeout,
14318 None,
14319 )
14320 .unwrap();
14321 let (_, rx) = mpsc::channel(8);
14322 let actor = Self {
14323 module,
14324 registry,
14325 forwarding,
14326 spec,
14327 health,
14328 rx,
14329 roster: supervisor.child_roster.clone(),
14330 _home: home,
14331 };
14332 actor
14333 .wait_for_select(|checkpoint| checkpoint.generation > 0)
14334 .await;
14335 actor
14336 }
14337
14338 async fn expire_registration_wait(&self, remaining: Duration) {
14339 tokio::time::advance(remaining).await;
14340 wait_for("registration timeout TERM grace", || {
14341 lock_snapshot(&self.module.inner.snapshot)
14342 .unwrap()
14343 .actor_select
14344 .is_none()
14345 .then_some(())
14346 })
14347 .await;
14348 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14349 }
14350
14351 fn turns(&self) -> u64 {
14352 lock_snapshot(&self.module.inner.snapshot)
14353 .unwrap()
14354 .actor_turns
14355 }
14356
14357 async fn wait_for_select(
14358 &self,
14359 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14360 ) -> ActorSelectCheckpoint {
14361 wait_for("actor select checkpoint", || {
14362 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14363 state.actor_select.filter(|checkpoint| {
14364 checkpoint.generation == state.spawn_generation
14365 && state.state == ModuleState::Running
14366 && state.process_alive
14367 && state.reported_pid().is_some()
14368 && expected(checkpoint)
14369 })
14370 })
14371 .await
14372 }
14373
14374 fn hello(&mut self, connection: u64, health: bool) {
14375 let manifest =
14376 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14377 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14378 .build();
14379 let connection = ConnectionId::new(connection);
14380 let (tx, rx) = mpsc::channel(8);
14381 self.rx = rx;
14382 self.forwarding
14383 .register_module_connection(
14384 connection,
14385 self.spec.module_id.clone(),
14386 subc_protocol::PROTOCOL_VERSION,
14387 Concurrency::ModuleManaged,
14388 FrameSink::new(tx),
14389 )
14390 .unwrap();
14391 self.registry
14392 .register_with_control_ops(
14393 manifest,
14394 subc_protocol::PROTOCOL_VERSION,
14395 connection,
14396 if health {
14397 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
14398 } else {
14399 vec![]
14400 },
14401 )
14402 .unwrap();
14403 }
14404
14405 async fn probe(&mut self) -> crate::router::OutboundFrame {
14406 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
14407 Ok(frame) => Some(frame),
14408 Err(mpsc::error::TryRecvError::Empty) => None,
14409 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
14410 })
14411 .await;
14412 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14413 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14414 frame
14415 }
14416
14417 fn no_probe(&mut self) {
14418 assert!(matches!(
14419 self.rx.try_recv(),
14420 Err(mpsc::error::TryRecvError::Empty)
14421 ));
14422 }
14423
14424 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
14425 self.forwarding
14426 .complete_module_control_rpc(
14427 ConnectionId::new(connection),
14428 frame.header.corr,
14429 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14430 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14431 status: HealthStatus::Ok,
14432 detail: None,
14433 metrics: None,
14434 }),
14435 )
14436 .unwrap();
14437 }
14438 }
14439
14440 #[tokio::test(start_paused = true)]
14441 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
14442 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14443 let initial = actor.module.status().unwrap();
14444 tokio::time::advance(Duration::from_secs(119)).await;
14445 tokio::task::yield_now().await;
14446 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
14447 assert!(actor.module.terminal_history().entries.is_empty());
14448 actor.expire_registration_wait(Duration::from_secs(1)).await;
14449 actor
14450 .wait_for_select(|at| at.generation > initial.spawn_generation)
14451 .await;
14452 let status = actor.module.status().unwrap();
14453 assert_ne!(status.pid, initial.pid);
14454 assert!(!cortexkit_test_support::process_alive(
14455 initial.pid.unwrap() as i32
14456 ));
14457 assert_eq!(status.restart_count, 1);
14458 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
14459 let entries = actor.module.terminal_history().entries;
14460 assert_eq!(entries.len(), 1);
14461 assert_eq!(
14462 entries[0].disposition_detail.as_deref(),
14463 Some("registration_timeout")
14464 );
14465 actor.module.stop().await.unwrap();
14466 }
14467
14468 #[tokio::test(start_paused = true)]
14469 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
14470 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14471 let pid = actor.module.status().unwrap().pid;
14472 tokio::time::advance(Duration::from_secs(119)).await;
14473 actor.hello(1, false);
14474 let registered = actor
14475 .wait_for_select(|at| at.registered_connection.is_some())
14476 .await;
14477 assert_eq!(
14478 registered.wake_after, None,
14479 "HELLO must cancel the deadline"
14480 );
14481 tokio::time::advance(Duration::from_secs(300)).await;
14482 tokio::task::yield_now().await;
14483 assert_eq!(actor.module.status().unwrap().pid, pid);
14484 assert!(actor.module.terminal_history().entries.is_empty());
14485 actor
14487 .registry
14488 .deregister_connection(ConnectionId::new(1))
14489 .unwrap();
14490 actor
14491 .wait_for_select(|at| at.registered_connection.is_none())
14492 .await;
14493 tokio::time::advance(Duration::from_secs(300)).await;
14494 tokio::task::yield_now().await;
14495 assert_eq!(actor.module.status().unwrap().pid, pid);
14496 assert!(actor.module.terminal_history().entries.is_empty());
14497 actor.module.stop().await.unwrap();
14498
14499 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14500 let pid = transient.module.status().unwrap().pid;
14501 tokio::time::advance(Duration::from_secs(119)).await;
14502 let turns = transient.turns();
14503 transient.hello(1, false);
14506 transient
14507 .registry
14508 .deregister_connection(ConnectionId::new(1))
14509 .unwrap();
14510 let parked = transient.wait_for_select(|at| at.turn > turns).await;
14511 assert_eq!(
14512 parked.wake_after, None,
14513 "transient HELLO must cancel the deadline"
14514 );
14515 tokio::time::advance(Duration::from_secs(300)).await;
14516 tokio::task::yield_now().await;
14517 assert_eq!(transient.module.status().unwrap().pid, pid);
14518 assert!(transient.module.terminal_history().entries.is_empty());
14519 transient.module.stop().await.unwrap();
14520 }
14521
14522 #[tokio::test(start_paused = true)]
14523 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
14524 let actor = Actor::start_with_policy(
14525 ModuleProtocol::Subc,
14526 Duration::from_secs(30),
14527 Duration::from_secs(10),
14528 RestartPolicy::new(1, Duration::ZERO),
14529 )
14530 .await;
14531 let generation = actor.module.status().unwrap().spawn_generation;
14532 actor
14533 .expire_registration_wait(Duration::from_secs(10))
14534 .await;
14535 actor.wait_for_select(|at| at.generation > generation).await;
14536 actor
14537 .expire_registration_wait(Duration::from_secs(10))
14538 .await;
14539 let retry = wait_for("failed retry schedule", || {
14540 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
14541 (state.state == ModuleState::Failed)
14542 .then_some(state.failed_retry.pending)
14543 .flatten()
14544 })
14545 .await;
14546 assert_eq!(retry.delay, Duration::from_secs(600));
14547 assert_eq!(retry.attempt, 1);
14548 let entries = actor.module.terminal_history().entries;
14549 assert_eq!(entries.len(), 2);
14550 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
14551 assert!(entries[1]
14552 .disposition_detail
14553 .as_deref()
14554 .unwrap()
14555 .contains("registration_timeout"));
14556 tokio::time::advance(Duration::from_secs(599)).await;
14557 tokio::task::yield_now().await;
14558 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
14559 tokio::time::advance(Duration::from_secs(1)).await;
14560 actor
14561 .wait_for_select(|at| at.generation > generation + 1)
14562 .await;
14563 actor
14565 .expire_registration_wait(Duration::from_secs(10))
14566 .await;
14567 wait_for("slow retry registration timeout", || {
14568 (actor.module.terminal_history().entries.len() == 3).then_some(())
14569 })
14570 .await;
14571 actor.module.stop().await.unwrap();
14572 }
14573
14574 #[tokio::test(start_paused = true)]
14575 async fn registration_timeout_honours_ten_second_override() {
14576 let actor = Actor::start_with_policy(
14577 ModuleProtocol::Subc,
14578 Duration::from_secs(30),
14579 Duration::from_secs(10),
14580 RestartPolicy::new(3, Duration::ZERO),
14581 )
14582 .await;
14583 let generation = actor.module.status().unwrap().spawn_generation;
14584 tokio::time::advance(Duration::from_secs(9)).await;
14585 tokio::task::yield_now().await;
14586 assert!(actor.module.terminal_history().entries.is_empty());
14587 actor.expire_registration_wait(Duration::from_secs(1)).await;
14588 actor.wait_for_select(|at| at.generation > generation).await;
14589 assert_eq!(actor.module.terminal_history().entries.len(), 1);
14590 actor.module.stop().await.unwrap();
14591 }
14592
14593 #[tokio::test]
14594 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
14595 use std::os::unix::process::{CommandExt, ExitStatusExt};
14596
14597 struct Helper(std::process::Child);
14598 impl Drop for Helper {
14599 fn drop(&mut self) {
14600 if matches!(self.0.try_wait(), Ok(None)) {
14603 let _ = self.0.kill();
14604 let _ = self.0.wait();
14605 }
14606 }
14607 }
14608
14609 let home = ScratchDir::new("registration-timeout-helper");
14610 let supervisor = Supervisor::new_for_test(
14611 Arc::new(Registry::default()),
14612 RestartPolicy::new(0, Duration::ZERO),
14613 );
14614 let spec = ModuleSpec {
14615 module_id: "registration-timeout-group".into(),
14616 program: "/bin/sleep".into(),
14617 args: vec!["120".into()],
14618 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14619 .into_iter()
14620 .map(|key| (key.into(), home.path().display().to_string()))
14621 .collect(),
14622 reserved: false,
14623 reserved_prefixes: vec![],
14624 protocol: ModuleProtocol::Subc,
14625 overlap: Default::default(),
14626 };
14627 let module = supervisor
14628 .supervise_configured_requested(
14629 spec,
14630 true,
14631 HealthConfig::default(),
14632 None,
14633 RestartPolicy::new(0, Duration::ZERO),
14634 Duration::from_secs(10),
14635 None,
14636 )
14637 .unwrap();
14638 let leader = timeout(Duration::from_secs(5), async {
14639 loop {
14640 if let Some(pid) = module.status().unwrap().pid {
14641 break pid;
14642 }
14643 sleep(Duration::from_millis(5)).await;
14644 }
14645 })
14646 .await
14647 .expect("supervised group leader must be reported before the deadline");
14648 assert!(leader > 1);
14649 let ready = home.join("helper.ready");
14650 let mut helper = Helper(
14653 std::process::Command::new("/bin/sh")
14654 .args([
14655 "-c",
14656 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
14657 "helper",
14658 ])
14659 .arg(&ready)
14660 .process_group(i32::try_from(leader).unwrap())
14661 .stdout(std::process::Stdio::null())
14662 .stderr(std::process::Stdio::null())
14663 .spawn()
14664 .unwrap(),
14665 );
14666 timeout(Duration::from_secs(5), async {
14667 while !ready.exists() {
14668 assert!(helper.0.try_wait().unwrap().is_none());
14669 sleep(Duration::from_millis(5)).await;
14670 }
14671 })
14672 .await
14673 .expect("helper must install TERM ignore before the registration deadline");
14674 let helper_pid =
14675 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
14676 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
14677 sleep(Duration::from_millis(20)).await;
14678 assert!(
14679 helper.0.try_wait().unwrap().is_none(),
14680 "helper must ignore TERM"
14681 );
14682
14683 timeout(Duration::from_secs(20), async {
14684 while module.state().unwrap() != ModuleState::Failed {
14685 sleep(Duration::from_millis(5)).await;
14686 }
14687 })
14688 .await
14689 .expect("registration timeout must stop and reap the group leader");
14690 let helper_exit = timeout(Duration::from_secs(3), async {
14691 loop {
14692 if let Some(status) = helper.0.try_wait().unwrap() {
14693 break status;
14694 }
14695 sleep(Duration::from_millis(5)).await;
14696 }
14697 })
14698 .await
14699 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
14700 assert_eq!(helper_exit.signal(), Some(9));
14701 assert!(!cortexkit_test_support::process_alive(
14702 i32::try_from(leader).unwrap()
14703 ));
14704 assert_eq!(
14705 module.status().unwrap().last_exit.unwrap().kind,
14706 ExitKind::Crash
14707 );
14708 assert!(module.terminal_history().entries[0]
14709 .disposition_detail
14710 .as_deref()
14711 .unwrap()
14712 .contains("registration_timeout"));
14713 module.stop().await.unwrap();
14714 }
14715
14716 #[tokio::test(start_paused = true)]
14717 async fn registration_timeout_does_not_count_an_earlier_exit() {
14718 let actor = Actor::start_with_policy(
14719 ModuleProtocol::Subc,
14720 Duration::from_secs(30),
14721 DEFAULT_REGISTER_TIMEOUT,
14722 RestartPolicy::new(0, Duration::ZERO),
14723 )
14724 .await;
14725 let pid = actor.module.status().unwrap().pid.unwrap();
14726 rustix::process::kill_process(
14727 rustix::process::Pid::from_raw(pid as i32).unwrap(),
14728 rustix::process::Signal::TERM,
14729 )
14730 .unwrap();
14731 wait_for("early exit", || {
14732 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
14733 })
14734 .await;
14735 tokio::time::advance(Duration::from_secs(121)).await;
14736 tokio::task::yield_now().await;
14737 let entries = actor.module.terminal_history().entries;
14738 assert_eq!(entries.len(), 1);
14739 assert!(!entries[0]
14740 .disposition_detail
14741 .as_deref()
14742 .unwrap()
14743 .contains("registration_timeout"));
14744 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
14745 actor.module.stop().await.unwrap();
14746 }
14747
14748 #[tokio::test(start_paused = true)]
14749 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
14750 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14751 let pid = actor.module.status().unwrap().pid;
14752 tokio::time::advance(Duration::from_secs(119)).await;
14753 actor.roster.close();
14754 let turns = actor.turns();
14755 tokio::time::advance(Duration::from_secs(300)).await;
14756 let parked = actor.wait_for_select(|at| at.turn > turns).await;
14757 assert_eq!(parked.wake_after, None);
14758 assert_eq!(actor.module.status().unwrap().pid, pid);
14759 assert!(actor.module.terminal_history().entries.is_empty());
14760 actor.module.stop().await.unwrap();
14761 let status = actor.module.status().unwrap();
14762 assert_eq!(status.restart_count, 0);
14763 assert_eq!(
14764 actor.module.terminal_history().entries[0].disposition,
14765 TerminalDisposition::DaemonShutdown
14766 );
14767 }
14768
14769 #[tokio::test(start_paused = true)]
14770 async fn no_health_children_park_for_sixty_seconds() {
14771 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14772 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14773 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14774 unadvertised.hello(1, false);
14775 for _ in 0..6000 {
14780 tokio::time::advance(Duration::from_millis(10)).await;
14781 tokio::task::yield_now().await;
14782 }
14783 for actor in [&none, &unregistered, &unadvertised] {
14784 assert!(
14785 actor.turns() <= 5,
14786 "no-health actor woke {} times",
14787 actor.turns()
14788 );
14789 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
14790 }
14791 }
14792
14793 #[tokio::test(start_paused = true)]
14794 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
14795 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
14799 let tick = Instant::now();
14800 actor.hello(1, true);
14801 actor.probe().await;
14802 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
14803 }
14804
14805 #[tokio::test(start_paused = true)]
14806 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
14807 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14808 let hello_tick = Instant::now();
14809 actor.hello(1, true);
14810 let armed = actor
14811 .wait_for_select(|checkpoint| {
14812 checkpoint.registered_connection == Some(ConnectionId::new(1))
14813 && checkpoint.next_probe_at.is_some()
14814 })
14815 .await;
14816 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14817 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
14818 let first_deadline = hello_tick + delay;
14819 assert_eq!(armed.next_probe_at, Some(first_deadline));
14820 tokio::time::advance(Duration::from_secs(29)).await;
14821 let turns = actor.turns();
14822 actor
14823 .registry
14824 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
14825 .unwrap();
14826 let updated = actor
14827 .wait_for_select(|checkpoint| checkpoint.turn > turns)
14828 .await;
14829 assert_eq!(
14830 updated.next_probe_at,
14831 Some(first_deadline),
14832 "catalog update must not reset cadence"
14833 );
14834 actor.no_probe();
14835 tokio::time::advance(first_deadline - Instant::now()).await;
14836 let frame = actor.probe().await;
14837 actor.answer(1, frame);
14838 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
14839 let next_deadline = Instant::now() + next;
14840 let rearmed = actor
14841 .wait_for_select(|checkpoint| {
14842 checkpoint
14843 .next_probe_at
14844 .is_some_and(|deadline| deadline > first_deadline)
14845 })
14846 .await;
14847 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14848 tokio::time::advance(next - Duration::from_nanos(1)).await;
14849 actor.no_probe();
14850 tokio::time::advance(Duration::from_nanos(1)).await;
14851 actor.probe().await;
14852 }
14853
14854 #[tokio::test(start_paused = true)]
14855 async fn disconnect_disarms_and_reconnect_rearms_health() {
14856 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14857 actor.hello(1, true);
14858 actor
14859 .wait_for_select(|checkpoint| {
14860 checkpoint.registered_connection == Some(ConnectionId::new(1))
14861 && checkpoint.next_probe_at.is_some()
14862 })
14863 .await;
14864 let turns = actor.turns();
14865 let tick = Instant::now();
14866 actor
14867 .registry
14868 .deregister_connection(ConnectionId::new(1))
14869 .unwrap();
14870 let disarmed = actor
14871 .wait_for_select(|checkpoint| {
14872 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14873 })
14874 .await;
14875 assert_eq!(disarmed.next_probe_at, None);
14876 assert_eq!(disarmed.wake_after, None);
14877 assert_eq!(
14878 Instant::now(),
14879 tick,
14880 "disconnect must disarm in the notification tick"
14881 );
14882 tokio::time::advance(Duration::from_secs(60)).await;
14883 actor.no_probe();
14884 let reconnect_tick = Instant::now();
14885 actor.hello(2, true);
14886 let rearmed = actor
14887 .wait_for_select(|checkpoint| {
14888 checkpoint.registered_connection == Some(ConnectionId::new(2))
14889 && checkpoint.next_probe_at.is_some()
14890 })
14891 .await;
14892 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14893 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14894 tokio::time::advance(delay).await;
14895 actor.probe().await;
14896 }
14897
14898 #[tokio::test(start_paused = true)]
14899 async fn reregistration_without_health_disarms_the_old_schedule() {
14900 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14901 actor.hello(1, true);
14902 actor
14903 .wait_for_select(|checkpoint| {
14904 checkpoint.registered_connection == Some(ConnectionId::new(1))
14905 && checkpoint.next_probe_at.is_some()
14906 })
14907 .await;
14908 actor
14909 .registry
14910 .deregister_connection(ConnectionId::new(1))
14911 .unwrap();
14912 actor.hello(2, false);
14913 let disarmed = actor
14914 .wait_for_select(|checkpoint| {
14915 checkpoint.registered_connection == Some(ConnectionId::new(2))
14916 })
14917 .await;
14918 assert_eq!(disarmed.next_probe_at, None);
14919 assert_eq!(disarmed.wake_after, None);
14920 tokio::time::advance(Duration::from_secs(60)).await;
14921 actor.no_probe();
14922 }
14923
14924 #[tokio::test(start_paused = true)]
14925 async fn restart_waits_for_its_own_hello_before_rearming_health() {
14926 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14927 actor.hello(1, true);
14928 let armed = actor
14929 .wait_for_select(|checkpoint| {
14930 checkpoint.registered_connection == Some(ConnectionId::new(1))
14931 && checkpoint.next_probe_at.is_some()
14932 })
14933 .await;
14934 actor
14938 .registry
14939 .deregister_connection(ConnectionId::new(1))
14940 .unwrap();
14941 actor.module.restart(Some(0)).await.unwrap();
14942 let parked = actor
14943 .wait_for_select(|checkpoint| {
14944 checkpoint.generation > armed.generation
14945 && checkpoint.registered_connection.is_none()
14946 && checkpoint.next_probe_at.is_none()
14947 })
14948 .await;
14949 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
14950 actor.rx.close();
14951 while actor.rx.try_recv().is_ok() {}
14954 let turns = actor.turns();
14955 tokio::time::advance(Duration::from_secs(60)).await;
14956 let after = actor.turns();
14957 eprintln!(
14958 "restart park turns: before={turns} after={after} delta={}",
14959 after - turns
14960 );
14961 assert_eq!(after, turns, "replacement must park before HELLO");
14962 let hello_tick = Instant::now();
14963 actor.hello(2, true);
14964 let rearmed = actor
14965 .wait_for_select(|checkpoint| {
14966 checkpoint.registered_connection == Some(ConnectionId::new(2))
14967 && checkpoint.next_probe_at.is_some()
14968 })
14969 .await;
14970 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14971 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14972 tokio::time::advance(delay).await;
14973 actor.probe().await;
14974 }
14975
14976 #[tokio::test(start_paused = true)]
14977 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14978 use tokio::{
14979 io::{AsyncReadExt, AsyncWriteExt},
14980 net::TcpListener,
14981 };
14982 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14983 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14984 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14985 let (started_tx, mut started_rx) = mpsc::channel(8);
14986 let peer = tokio::spawn(async move {
14987 loop {
14988 let (mut socket, _) = listener.accept().await.unwrap();
14989 let mut request = [0; 1024];
14990 assert!(socket.read(&mut request).await.unwrap() > 0);
14991 started_tx.send(Instant::now()).await.unwrap();
14992 socket
14993 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14994 .await
14995 .unwrap();
14996 }
14997 });
14998 actor
14999 .module
15000 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15001 .await
15002 .unwrap();
15003 let first_tick = Instant::now();
15004 let first = actor
15005 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15006 .await;
15007 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15008 assert!(started_rx.try_recv().is_err());
15009
15010 tokio::time::advance(
15013 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15014 )
15015 .await;
15016 actor.module.restart(Some(0)).await.unwrap();
15017 let spawn_tick = Instant::now();
15018 let replacement = actor
15019 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15020 .await;
15021 let deadline = replacement.next_probe_at.unwrap();
15022 assert!(
15023 deadline >= spawn_tick + actor.health.cadence,
15024 "a replacement must not inherit its predecessor's probe deadline"
15025 );
15026 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15027 assert!(
15028 started_rx.try_recv().is_err(),
15029 "no HTTP probe during startup grace"
15030 );
15031 tokio::time::advance(deadline - Instant::now()).await;
15032 let landed = wait_for("first replacement HTTP probe", || {
15033 started_rx.try_recv().ok()
15034 })
15035 .await;
15036 assert!(landed >= spawn_tick + actor.health.cadence);
15037 wait_for("healthy replacement", || {
15038 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15039 .then_some(())
15040 })
15041 .await;
15042 peer.abort();
15043 }
15044
15045 #[tokio::test(start_paused = true)]
15046 async fn rescan_adds_and_removes_http_health_without_polling() {
15047 use tokio::{
15048 io::{AsyncReadExt, AsyncWriteExt},
15049 net::TcpListener,
15050 };
15051 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15052 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15053 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15054 actor.health.cadence = Duration::from_millis(1);
15056 let (started_tx, mut started_rx) = mpsc::channel(8);
15057 let peer = tokio::spawn(async move {
15058 loop {
15059 let (mut socket, _) = listener.accept().await.unwrap();
15060 let mut request = [0; 1024];
15061 assert!(socket.read(&mut request).await.unwrap() > 0);
15062 started_tx.send(()).await.unwrap();
15063 socket
15064 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15065 .await
15066 .unwrap();
15067 }
15068 });
15069 let tick = Instant::now();
15070 let turns = actor.turns();
15071 actor
15072 .module
15073 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15074 .await
15075 .unwrap();
15076 let armed = actor
15077 .wait_for_select(|checkpoint| {
15078 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15079 })
15080 .await;
15081 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15082 assert_eq!(Instant::now(), tick);
15083 tokio::time::advance(Duration::from_millis(1)).await;
15084 let mut started = false;
15085 wait_for("successful HTTP probe", || {
15086 started |= started_rx.try_recv().is_ok();
15087 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15088 .then_some(())
15089 })
15090 .await;
15091 assert!(started, "added HTTP check must start at its first deadline");
15092 assert_eq!(
15093 actor.module.status().unwrap().health.status,
15094 SupervisorHealthStatus::Ok
15095 );
15096 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15097 actor.health.http = None;
15098 let turns = actor.turns();
15099 actor
15100 .module
15101 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15102 .await
15103 .unwrap();
15104 let parked = actor
15105 .wait_for_select(|checkpoint| {
15106 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15107 })
15108 .await;
15109 assert_eq!(parked.wake_after, None);
15110 let turns = actor.turns();
15111 tokio::time::advance(Duration::from_secs(60)).await;
15112 assert_eq!(
15113 actor.turns(),
15114 turns,
15115 "removed HTTP check must park the actor"
15116 );
15117 assert!(
15118 started_rx.try_recv().is_err(),
15119 "removed HTTP check must stay disarmed"
15120 );
15121 assert_eq!(
15122 actor.module.status().unwrap().health,
15123 ModuleHealthStatus::default()
15124 );
15125 peer.abort();
15126 }
15127}
15128
15129#[cfg(test)]
15130mod drain_busy_reprobe_tests {
15131 use super::*;
15135 use std::collections::BTreeMap;
15136 use std::sync::Mutex as StdMutex;
15137 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15138
15139 const MODULE_ID: &str = "drain-reprobe-module";
15140 const CONNECTION: u64 = 7;
15141 const GAUGE: &str = "runs_in_flight";
15142 const CADENCE: Duration = Duration::from_secs(30);
15145 const CEILING: Duration = Duration::from_secs(30);
15146
15147 #[derive(Clone, Copy)]
15149 enum Answer {
15150 BusyUntil { until: Instant, latency: Duration },
15153 OmitsGauge,
15155 Silent,
15157 }
15158
15159 struct Drain {
15160 forwarding: Arc<ForwardingTable>,
15161 runtime: SupervisorRuntimeConfig,
15162 endpoint: crate::ModuleEndpointId,
15163 probes: Arc<StdMutex<Vec<Instant>>>,
15165 responder: tokio::task::JoinHandle<()>,
15166 }
15167
15168 impl Drain {
15169 fn start(answer: Answer) -> Self {
15170 let registry = Arc::new(Registry::default());
15171 let forwarding = Arc::new(ForwardingTable::default());
15172 let health = HealthConfig {
15173 cadence: CADENCE,
15174 ..HealthConfig::default()
15175 };
15176 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15177 .with_forwarding(forwarding.clone())
15178 .with_health_config(health)
15179 .runtime_config();
15180 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15181 let endpoint = forwarding
15182 .register_module_connection(
15183 ConnectionId::new(CONNECTION),
15184 MODULE_ID.to_string(),
15185 subc_protocol::PROTOCOL_VERSION,
15186 Concurrency::ModuleManaged,
15187 FrameSink::new(tx),
15188 )
15189 .unwrap();
15190 let probes = Arc::new(StdMutex::new(Vec::new()));
15191 let responder = tokio::spawn({
15192 let forwarding = forwarding.clone();
15193 let probes = probes.clone();
15194 async move {
15195 while let Some(frame) = rx.recv().await {
15196 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15197 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15198 probes.lock().unwrap().push(Instant::now());
15199 let metrics = match answer {
15200 Answer::BusyUntil { until, latency } => {
15201 sleep(latency).await;
15202 let busy = u64::from(Instant::now() < until);
15203 serde_json::json!({ GAUGE: busy })
15204 }
15205 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15206 Answer::Silent => continue,
15207 };
15208 let _ = forwarding.complete_module_control_rpc(
15211 ConnectionId::new(CONNECTION),
15212 frame.header.corr,
15213 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15214 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15215 status: HealthStatus::Ok,
15216 detail: None,
15217 metrics: Some(metrics),
15218 }),
15219 );
15220 }
15221 }
15222 });
15223 Self {
15224 forwarding,
15225 runtime,
15226 endpoint,
15227 probes,
15228 responder,
15229 }
15230 }
15231
15232 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15235 let (drained, elapsed, _) = self.run_traced(ceiling).await;
15236 (drained, elapsed)
15237 }
15238
15239 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15241 let started = Instant::now();
15242 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15243 let drained = wait_for_forwarding_quiescence(
15244 &self.forwarding,
15245 MODULE_ID,
15246 &self.runtime,
15247 self.endpoint,
15248 started + ceiling,
15249 &[GAUGE.to_string()],
15250 DrainScope::Active,
15251 &mut trace,
15252 )
15253 .await
15254 .unwrap();
15255 (drained, started.elapsed(), trace)
15256 }
15257
15258 fn probes(&self) -> Vec<Instant> {
15259 self.probes.lock().unwrap().clone()
15260 }
15261 }
15262
15263 impl Drop for Drain {
15264 fn drop(&mut self) {
15265 self.responder.abort();
15266 }
15267 }
15268
15269 #[tokio::test(start_paused = true)]
15270 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15271 let drain = Drain::start(Answer::BusyUntil {
15272 until: Instant::now() + Duration::from_secs(1),
15273 latency: Duration::ZERO,
15274 });
15275 let (drained, elapsed) = drain.run(CEILING).await;
15276 assert!(
15277 drained,
15278 "gauges that read zero during the drain must let it finish drained, \
15279 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15280 );
15281 assert!(
15282 elapsed < Duration::from_secs(3),
15283 "drain took {elapsed:?} after the gauges went quiet at 1s"
15284 );
15285 let probes = drain.probes();
15286 assert!(
15287 probes.len() >= 2,
15288 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15289 );
15290 }
15291
15292 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15297 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15298 let drain = Drain::start(Answer::BusyUntil {
15299 until: Instant::now() + Duration::from_millis(600),
15300 latency: Duration::ZERO,
15301 });
15302 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15303 assert!(drained, "the gauge fell at 600ms: {trace}");
15304 assert!(
15305 elapsed < Duration::from_secs(3),
15306 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15307 );
15308 let probes = drain.probes();
15309 assert!(
15310 probes.len() >= 2 && trace.probes as usize == probes.len(),
15311 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15312 );
15313 }
15314
15315 #[tokio::test(start_paused = true)]
15316 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15317 let started = Instant::now();
15318 let drain = Drain::start(Answer::BusyUntil {
15319 until: started + Duration::from_secs(3600),
15320 latency: Duration::ZERO,
15321 });
15322 let (drained, elapsed) = drain.run(CEILING).await;
15323 assert!(!drained, "a module busy throughout must not drain");
15324 assert!(
15325 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15326 "a busy module is held until the ceiling and no longer: {elapsed:?}"
15327 );
15328 let probes = drain.probes();
15331 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15332 assert!(
15333 probes.len() as u128 <= max_probes,
15334 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15335 probes.len()
15336 );
15337 for pair in probes.windows(2) {
15338 assert!(
15339 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15340 "consecutive probes {:?} apart",
15341 pair[1] - pair[0]
15342 );
15343 }
15344 }
15345
15346 #[tokio::test(start_paused = true)]
15347 async fn a_slow_probe_is_never_overlapped_by_the_next() {
15348 let latency = Duration::from_secs(1);
15351 let drain = Drain::start(Answer::BusyUntil {
15352 until: Instant::now() + Duration::from_secs(3600),
15353 latency,
15354 });
15355 let (drained, _) = drain.run(Duration::from_secs(5)).await;
15356 assert!(!drained);
15357 let probes = drain.probes();
15358 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15359 for pair in probes.windows(2) {
15360 assert!(
15361 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15362 "a probe started {:?} after the previous one, before it could finish",
15363 pair[1] - pair[0]
15364 );
15365 }
15366 }
15367
15368 #[tokio::test(start_paused = true)]
15369 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15370 let drain = Drain::start(Answer::Silent);
15371 let ceiling = Duration::from_secs(2);
15372 let (drained, elapsed) = drain.run(ceiling).await;
15373 assert!(!drained, "an unanswered probe counts as busy");
15374 assert!(
15375 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15376 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15377 );
15378 assert_eq!(
15379 drain.probes().len(),
15380 1,
15381 "the unanswered probe holds the only slot until the ceiling"
15382 );
15383 }
15384
15385 #[derive(Clone, Default)]
15387 struct LogCapture {
15388 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
15389 }
15390
15391 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
15392 fn on_event(
15393 &self,
15394 event: &tracing::Event<'_>,
15395 _context: tracing_subscriber::layer::Context<'_, S>,
15396 ) {
15397 struct Fields(BTreeMap<String, String>);
15398 impl tracing::field::Visit for Fields {
15399 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
15400 self.0
15401 .insert(field.name().to_string(), format!("{value:?}"));
15402 }
15403 }
15404 let mut fields = Fields(BTreeMap::new());
15405 event.record(&mut fields);
15406 self.events.lock().unwrap().push(fields.0);
15407 }
15408 }
15409
15410 impl LogCapture {
15411 fn install(&self) -> tracing::subscriber::DefaultGuard {
15412 use tracing_subscriber::layer::SubscriberExt;
15413 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
15414 }
15415
15416 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
15417 self.events
15418 .lock()
15419 .unwrap()
15420 .iter()
15421 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
15422 .cloned()
15423 .collect()
15424 }
15425 }
15426
15427 #[tokio::test(start_paused = true)]
15432 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
15433 let capture = LogCapture::default();
15434 let _guard = capture.install();
15435 let drain = Drain::start(Answer::OmitsGauge);
15436 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
15437 assert!(!drained, "a gauge never reported cannot read quiet");
15438 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
15439
15440 let missing = capture.with_message("lacks a declared busy gauge");
15441 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
15442 assert_eq!(
15443 missing[0].get("missing_gauges").map(String::as_str),
15444 Some(GAUGE)
15445 );
15446
15447 warn_drain_timed_out(
15448 MODULE_ID,
15449 Duration::from_secs(2),
15450 RouteCloseReason::Restart,
15451 &Default::default(),
15452 &trace,
15453 );
15454 let timeouts = capture.with_message("route drain timed out");
15455 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
15456 let gauges = &timeouts[0]["busy_gauges"];
15457 assert!(
15458 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
15459 "the timeout line must name the gauge and why it read busy: {gauges}"
15460 );
15461 assert!(
15462 gauges.ends_with("; no value ever read"),
15463 "no value was ever read for the gauge: {gauges}"
15464 );
15465 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
15466 }
15467
15468 #[tokio::test(start_paused = true)]
15472 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
15473 let busy = Drain::start(Answer::BusyUntil {
15474 until: Instant::now() + Duration::from_secs(3600),
15475 latency: Duration::ZERO,
15476 });
15477 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
15478 let rendered = trace.to_string();
15479 assert!(
15480 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
15481 && rendered.contains("; last value non_zero(1) @"),
15482 "{rendered}"
15483 );
15484
15485 let silent = Drain::start(Answer::Silent);
15486 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
15487 assert_eq!(
15490 trace.to_string(),
15491 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
15492 );
15493 assert_eq!(trace.probes, 1);
15494 }
15495
15496 #[test]
15498 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
15499 let gauges = [GAUGE.to_string()];
15500 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
15501 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
15502 assert_eq!(
15503 busy_gauge_observation(&verdicts),
15504 BusyGaugeObservation::Busy
15505 );
15506 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
15507 assert_eq!(
15508 busy_gauge_observation(&zero),
15509 BusyGaugeObservation::Quiescent
15510 );
15511 let none = gauge_verdicts(None, &gauges);
15512 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
15513 }
15514}
15515
15516#[cfg(test)]
15517mod health_evidence_tests {
15518 use super::{HealthProbeError, HealthProbeEvidence};
15519 use std::collections::HashSet;
15520
15521 #[test]
15529 fn only_a_dead_lane_is_proof_of_death() {
15530 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
15531 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
15535 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
15536 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
15537 }
15538
15539 #[test]
15545 fn every_evidence_class_has_a_distinct_label() {
15546 let labels = [
15547 HealthProbeError::lane_dead("").label(),
15548 HealthProbeError::no_answer("").label(),
15549 HealthProbeError::bad_answer("").label(),
15550 HealthProbeError::misconfigured("").label(),
15551 ];
15552 let unique: HashSet<_> = labels.iter().collect();
15553 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
15554 }
15555
15556 #[test]
15562 fn classification_preserves_the_original_message() {
15563 let err = HealthProbeError::no_answer("module did not answer within 5s");
15564 assert_eq!(err.to_string(), "module did not answer within 5s");
15565 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15566 }
15567}
15568
15569#[cfg(test)]
15570mod health_tombstone_tests {
15571 use std::{path::PathBuf, sync::Arc, time::Duration};
15572
15573 use subc_protocol::{
15574 manifest::Concurrency,
15575 session::{HealthStatus, ModuleControlResponse},
15576 };
15577 use tokio::sync::mpsc;
15578
15579 use super::{
15580 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
15581 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
15582 };
15583 use crate::{
15584 control::ControlHandler,
15585 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
15586 registry::{ConnectionId, Registry},
15587 router::FrameSink,
15588 };
15589
15590 struct ProbeHarness {
15591 spec: ModuleSpec,
15592 runtime: SupervisorRuntimeConfig,
15593 forwarding: Arc<ForwardingTable>,
15594 module_connection: ConnectionId,
15595 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15596 handler: ControlHandler,
15597 module: super::SupervisedModule,
15598 }
15599
15600 fn probe_harness() -> ProbeHarness {
15601 let registry = Arc::new(Registry::default());
15602 let forwarding = Arc::new(ForwardingTable::default());
15603 let supervisor_handle = super::SupervisorHandle::new();
15604 let health = HealthConfig {
15605 http: None,
15606 cadence: Duration::from_secs(30),
15607 deadline: Duration::from_secs(5),
15608 failure_threshold: 3,
15609 on_degraded: HealthAction::Report,
15610 on_failing: HealthAction::Report,
15611 critical: false,
15612 };
15613 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15614 .with_forwarding(Arc::clone(&forwarding))
15615 .with_handle(supervisor_handle.clone())
15616 .with_health_config(health);
15617 let spec = ModuleSpec {
15618 module_id: "late-health-module".to_string(),
15619 program: PathBuf::from("disabled-module"),
15620 args: Vec::new(),
15621 env: Vec::new(),
15622 reserved: false,
15623 reserved_prefixes: Vec::new(),
15624 protocol: ModuleProtocol::Subc,
15625 overlap: Default::default(),
15626 };
15627 let module = supervisor
15628 .supervise_configured(spec.clone(), false)
15629 .unwrap();
15630 let runtime = supervisor.runtime_config();
15631 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
15632 .with_supervisor(supervisor_handle);
15633 let module_connection = ConnectionId::new(700);
15634 let (module_tx, module_rx) = mpsc::channel(8);
15635 forwarding
15636 .register_module_connection(
15637 module_connection,
15638 spec.module_id.clone(),
15639 subc_protocol::PROTOCOL_VERSION,
15640 Concurrency::ModuleManaged,
15641 FrameSink::new(module_tx),
15642 )
15643 .unwrap();
15644
15645 ProbeHarness {
15646 spec,
15647 runtime,
15648 forwarding,
15649 module_connection,
15650 module_rx,
15651 handler,
15652 module,
15653 }
15654 }
15655
15656 async fn finish_after(
15657 harness: &mut ProbeHarness,
15658 stall: Duration,
15659 ) -> ModuleControlRpcCompletion {
15660 assert!(stall > harness.runtime.health.deadline);
15661 let deadline = harness.runtime.health.deadline;
15662 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15663 let answer = async {
15664 let frame = harness.module_rx.recv().await.expect("health.check frame");
15665 tokio::time::advance(deadline).await;
15666 tokio::task::yield_now().await;
15667 tokio::time::advance(stall - deadline).await;
15668 harness
15669 .forwarding
15670 .complete_module_control_rpc(
15671 harness.module_connection,
15672 frame.header.corr,
15673 Some("health.check"),
15674 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15675 status: HealthStatus::Ok,
15676 detail: None,
15677 metrics: None,
15678 }),
15679 )
15680 .unwrap()
15681 };
15682 let (probe_result, completion) = tokio::join!(probe, answer);
15683 let err = probe_result.expect_err("probe must miss its deadline");
15684 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15685 completion
15686 }
15687
15688 async fn time_out_without_answer(harness: &mut ProbeHarness) {
15689 let deadline = harness.runtime.health.deadline;
15690 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15691 let exhaust_deadline = async {
15692 let _frame = harness.module_rx.recv().await.expect("health.check frame");
15693 tokio::time::advance(deadline).await;
15694 tokio::task::yield_now().await;
15695 };
15696 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
15697 let err = probe_result.expect_err("probe must miss its deadline");
15698 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15699 }
15700
15701 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
15702 let registry = Arc::clone(&harness.module.inner.registry);
15703 let snapshot = Arc::clone(&harness.module.inner.snapshot);
15704 let process_liveness = super::SupervisorProcessLiveness::default();
15705 let mut child = None;
15706 let cycle = super::run_health_probe_cycle(
15707 &harness.spec,
15708 &harness.runtime,
15709 ®istry,
15710 &process_liveness,
15711 &snapshot,
15712 &mut child,
15713 );
15714 let peer = async {
15715 let frame = harness.module_rx.recv().await.expect("health.check frame");
15716 if answer {
15717 harness
15718 .forwarding
15719 .complete_module_control_rpc(
15720 harness.module_connection,
15721 frame.header.corr,
15722 Some("health.check"),
15723 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15724 status: HealthStatus::Ok,
15725 detail: None,
15726 metrics: Some(serde_json::json!({"ready": true})),
15727 }),
15728 )
15729 .unwrap();
15730 } else {
15731 tokio::time::advance(harness.runtime.health.deadline).await;
15732 tokio::task::yield_now().await;
15733 }
15734 };
15735 tokio::join!(cycle, peer);
15736 }
15737
15738 #[tokio::test(start_paused = true)]
15739 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
15740 let mut harness = probe_harness();
15741 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
15745 monitor.abort();
15746 let _ = monitor.await;
15747 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
15748 state.enabled = true;
15749 state.state = super::ModuleState::Running;
15750 state.process_alive = true;
15751 })
15752 .unwrap();
15753
15754 run_probe_cycle(&mut harness, true).await;
15755 assert_eq!(
15756 harness.module.status().unwrap().health.status,
15757 super::SupervisorHealthStatus::Ok
15758 );
15759
15760 for failures in 1..harness.runtime.health.failure_threshold {
15761 run_probe_cycle(&mut harness, false).await;
15762 let status = harness.module.status().unwrap();
15763 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15764 assert_eq!(status.health.consecutive_failures, failures);
15765 assert!(status.health.last_probe_ms.is_some());
15766 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
15767 assert!(status.health.metrics.is_none());
15768 assert_eq!(status.state, super::ModuleState::Running);
15769 assert!(status.process_alive);
15770 assert_eq!(status.restart_count, 0);
15771 assert_eq!(status.lifetime_restarts, 0);
15772 assert!(status.health.last_action.is_none());
15773 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15774 }
15775
15776 run_probe_cycle(&mut harness, true).await;
15777 let recovered = harness.module.status().unwrap();
15778 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
15779 assert_eq!(recovered.health.consecutive_failures, 0);
15780 assert!(recovered.health.detail.is_none());
15781 assert_eq!(
15782 recovered.health.metrics,
15783 Some(serde_json::json!({"ready": true}))
15784 );
15785 assert_eq!(recovered.lifetime_restarts, 0);
15786
15787 for failures in 1..=harness.runtime.health.failure_threshold {
15788 run_probe_cycle(&mut harness, false).await;
15789 let status = harness.module.status().unwrap();
15790 assert_eq!(status.health.consecutive_failures, failures);
15791 if failures < harness.runtime.health.failure_threshold {
15792 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15793 assert_eq!(status.state, super::ModuleState::Running);
15794 assert_eq!(status.lifetime_restarts, 0);
15795 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15796 } else {
15797 assert_eq!(
15798 status.health.status,
15799 super::SupervisorHealthStatus::Unresponsive
15800 );
15801 assert_eq!(status.state, super::ModuleState::Restarting);
15802 assert_eq!(status.restart_count, 1);
15803 assert_eq!(status.lifetime_restarts, 1);
15804 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
15805 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
15806 }
15807 }
15808 }
15809
15810 #[tokio::test(start_paused = true)]
15811 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
15812 let mut harness = probe_harness();
15813
15814 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
15815 let first_latency = match &first {
15816 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15817 other => panic!("late answer was not retained: {other:?}"),
15818 };
15819 assert!(harness.handler.observe_module_control_completion(first));
15820
15821 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
15822 let second_latency = match &second {
15823 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15824 other => panic!("late answer was not retained: {other:?}"),
15825 };
15826 assert!(harness.handler.observe_module_control_completion(second));
15827
15828 assert_eq!(first_latency, Duration::from_secs(8));
15829 assert_eq!(
15830 second_latency - first_latency,
15831 Duration::from_secs(3),
15832 "latency must grow linearly with the additional stall"
15833 );
15834 let health = harness.module.status().unwrap().health;
15835 assert_eq!(health.late_answer_count, 2);
15836 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
15837 }
15838
15839 #[tokio::test(start_paused = true)]
15847 async fn late_answer_clears_the_consecutive_failure_streak() {
15848 let mut harness = probe_harness();
15849
15850 time_out_without_answer(&mut harness).await;
15852 harness
15853 .module
15854 .record_health_probe_failure_for_test("[no-answer] test miss")
15855 .unwrap();
15856 assert_eq!(
15857 harness.module.status().unwrap().health.consecutive_failures,
15858 1,
15859 "precondition: the miss must be on the streak before the late answer"
15860 );
15861
15862 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15864 assert!(matches!(
15865 late,
15866 ModuleControlRpcCompletion::LateHealthAnswer { .. }
15867 ));
15868 assert!(harness.handler.observe_module_control_completion(late));
15869
15870 let health = harness.module.status().unwrap().health;
15871 assert_eq!(
15872 health.consecutive_failures, 0,
15873 "a late answer is an answer: the streak must reset"
15874 );
15875 assert_eq!(health.late_answer_count, 1);
15876 }
15877
15878 #[tokio::test(start_paused = true)]
15879 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15880 let mut harness = probe_harness();
15881
15882 for _ in 0..20 {
15883 time_out_without_answer(&mut harness).await;
15884 assert_eq!(
15885 harness.forwarding.health_probe_tombstone_count().unwrap(),
15886 1
15887 );
15888 }
15889 }
15890}
15891
15892#[cfg(test)]
15893mod child_env_tests {
15894 #[cfg(windows)]
15895 use super::{
15896 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15897 };
15898 #[cfg(windows)]
15899 use std::sync::{Arc, OnceLock};
15900 #[cfg(windows)]
15901 #[test]
15902 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15903 let spec = spec(Vec::new());
15904 let mut command = Command::new("unused");
15905 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15906 .unwrap()
15907 .unwrap();
15908 let name = command
15909 .as_std()
15910 .get_envs()
15911 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15912 .and_then(|(_, value)| value)
15913 .unwrap();
15914 assert_eq!(name, OsStr::new(handoff.name()));
15915 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15916 assert!(command.as_std().get_envs().any(|(key, value)| key
15917 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15918 && value == Some(OsStr::new("env"))));
15919 assert!(command
15920 .as_std()
15921 .get_envs()
15922 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15923 }
15924
15925 #[cfg(windows)]
15926 #[test]
15927 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15928 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15929 snapshot.process_alive = true;
15930 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15931 assert_eq!(
15932 observed_launch_nonce_source(&snapshot).as_deref(),
15933 Some("env")
15934 );
15935 let consumed = Arc::new(OnceLock::new());
15936 snapshot.pipe_consumed = Some(consumed.clone());
15937 assert_eq!(
15938 observed_launch_nonce_source(&snapshot).as_deref(),
15939 Some("env")
15940 );
15941 consumed.set(()).unwrap();
15942 assert_eq!(
15943 observed_launch_nonce_source(&snapshot).as_deref(),
15944 Some("pipe")
15945 );
15946 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15947 assert_eq!(
15948 observed_launch_nonce_source(&snapshot).as_deref(),
15949 Some("env")
15950 );
15951 clear_current_process_facts(&mut snapshot);
15952 assert_eq!(observed_launch_nonce_source(&snapshot), None);
15953 }
15954 use super::{
15955 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15956 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15957 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15958 };
15959 use std::{ffi::OsStr, path::PathBuf};
15960 use tokio::process::Command;
15961
15962 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15963 ModuleSpec {
15964 module_id: "env-plan".to_string(),
15965 program: PathBuf::from("/nonexistent"),
15966 args: Vec::new(),
15967 env,
15968 reserved: false,
15969 reserved_prefixes: Vec::new(),
15970 protocol: ModuleProtocol::Subc,
15971 overlap: Default::default(),
15972 }
15973 }
15974
15975 #[test]
15976 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15977 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15978 let digest = LaunchSpecDigest::from_spec(&base);
15979 let mut reordered = base.clone();
15980 reordered.env.reverse();
15981 reordered.module_id = "different-id".into();
15982 reordered.reserved = true;
15983 reordered.overlap = Default::default();
15984 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15985 let mut delimiter_collision = base.clone();
15986 delimiter_collision.env[0] = ("A=B".into(), "C".into());
15987 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15988 let mut changed = base.clone();
15989 changed.protocol = ModuleProtocol::None;
15990 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15991 changed = base.clone();
15992 changed.args.push("--flag".into());
15993 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15994 changed = base.clone();
15995 changed.program = PathBuf::from("/other");
15996 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15997 changed = base.clone();
15998 changed.env[1].1 = "other".into();
15999 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16000 assert!(!format!("{digest:?}").contains("B=C"));
16001 assert!(!format!("{digest:?}").contains("[u8"));
16002 }
16003
16004 #[cfg(unix)]
16005 #[test]
16006 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16007 use std::os::unix::ffi::OsStringExt;
16008 let mut left = spec(Vec::new());
16009 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16010 let mut right = left.clone();
16011 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16012 assert_eq!(
16013 left.program.to_string_lossy(),
16014 right.program.to_string_lossy()
16015 );
16016 assert_ne!(
16017 LaunchSpecDigest::from_spec(&left),
16018 LaunchSpecDigest::from_spec(&right)
16019 );
16020 }
16021
16022 #[test]
16036 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16037 let mut command = Command::new("/nonexistent");
16038 apply_child_env(&mut command, &spec(Vec::new()));
16039 let removed = command
16040 .as_std()
16041 .get_envs()
16042 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16043 assert!(
16044 removed,
16045 "ambient CK_LOG must be explicitly removed for an unconfigured module"
16046 );
16047
16048 let mut configured = Command::new("/nonexistent");
16049 apply_child_env(
16050 &mut configured,
16051 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16052 );
16053 let effective = configured
16054 .as_std()
16055 .get_envs()
16056 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16057 .last()
16058 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16059 assert_eq!(
16060 effective,
16061 Some(Some("debug".to_string())),
16062 "a module's configured CK_LOG must survive the ambient removal"
16063 );
16064 }
16065
16066 #[test]
16075 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16076 let connection_file = std::path::Path::new("/run/subc-connection.json");
16077 let handle = SupervisorHandle::new();
16078
16079 let mut none_spec = spec(Vec::new());
16080 none_spec.protocol = ModuleProtocol::None;
16081 let mut none = Command::new("/nonexistent");
16082 let none_handoff =
16083 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16084 .expect("protocol-none spawn args apply");
16085 assert!(
16086 none_handoff.is_none(),
16087 "protocol:none spawn must not receive a nonce descriptor"
16088 );
16089 #[cfg(windows)]
16090 assert!(!none.as_std().get_envs().any(|(key, value)| key
16091 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16092 && value.is_some()));
16093 assert!(
16094 !none.as_std().get_envs().any(|(key, value)| key
16095 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16096 && value.is_some()),
16097 "protocol:none spawn must not name a nonce descriptor"
16098 );
16099 let none_args: Vec<String> = none
16100 .as_std()
16101 .get_args()
16102 .map(|a| a.to_string_lossy().into_owned())
16103 .collect();
16104 assert!(
16105 !none_args.iter().any(|a| a == SUBC_ARG),
16106 "protocol:none argv must not carry --subc; got {none_args:?}"
16107 );
16108 let none_has_nonce = none
16109 .as_std()
16110 .get_envs()
16111 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16112 assert!(
16113 !none_has_nonce,
16114 "protocol:none spawn must not receive a launch nonce"
16115 );
16116 let none_has_module_id = none
16117 .as_std()
16118 .get_envs()
16119 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16120 assert!(
16121 none_has_module_id,
16122 "SUBC_MODULE_ID is inert and stays on every path"
16123 );
16124 assert!(
16125 handle.spawn_nonce(&none_spec.module_id).is_none(),
16126 "no nonce record for a process that will never present one"
16127 );
16128
16129 let wire_spec = spec(Vec::new());
16131 let mut wire = Command::new("/nonexistent");
16132 let wire_handoff =
16133 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16134 .expect("subc-wire spawn args apply");
16135 let wire_fd_env = wire
16136 .as_std()
16137 .get_envs()
16138 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16139 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16140 #[cfg(unix)]
16141 assert_eq!(
16142 wire_fd_env,
16143 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16144 "a subc-wire spawn names the pipe it will receive at descriptor 3"
16145 );
16146 #[cfg(windows)]
16147 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16148 #[cfg(not(any(unix, windows)))]
16149 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16150 let wire_args: Vec<String> = wire
16151 .as_std()
16152 .get_args()
16153 .map(|a| a.to_string_lossy().into_owned())
16154 .collect();
16155 assert_eq!(
16156 wire_args,
16157 vec![
16158 SUBC_ARG.to_string(),
16159 connection_file.to_string_lossy().into_owned()
16160 ],
16161 "a subc-wire spawn still carries --subc <path>"
16162 );
16163 assert_eq!(
16164 wire.as_std()
16165 .get_envs()
16166 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16167 !cfg!(unix),
16168 "only Windows supplies the environment nonce"
16169 );
16170 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16171 }
16172
16173 #[test]
16183 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16184 let role = |command: &Command| {
16185 command
16186 .as_std()
16187 .get_envs()
16188 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16189 .last()
16190 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16191 };
16192 let forged = spec(vec![(
16193 SUBC_SPAWN_ROLE_ENV.to_string(),
16194 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16195 )]);
16196
16197 let mut plain = Command::new("/nonexistent");
16198 apply_child_env(&mut plain, &forged);
16199 apply_spawn_role(&mut plain, SpawnRole::Plain);
16200 assert_eq!(
16201 role(&plain),
16202 Some(None),
16203 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16204 );
16205
16206 let mut candidate = Command::new("/nonexistent");
16207 apply_child_env(&mut candidate, &spec(Vec::new()));
16208 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16209 assert_eq!(
16210 role(&candidate),
16211 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16212 );
16213 }
16214
16215 #[test]
16221 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16222 let mut command = Command::new("/nonexistent");
16223 apply_child_env(
16224 &mut command,
16225 &spec(vec![
16226 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16227 ("KEPT".to_string(), "yes".to_string()),
16228 ]),
16229 );
16230 let keys: Vec<String> = command
16231 .as_std()
16232 .get_envs()
16233 .filter(|(_, value)| value.is_some())
16234 .map(|(key, _)| key.to_string_lossy().into_owned())
16235 .collect();
16236 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16237 assert!(
16238 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16239 "daemon-private capture key leaked to the child: {keys:?}"
16240 );
16241 }
16242}
16243
16244#[cfg(test)]
16245mod jitter_tests {
16246 use super::jittered_health_delay;
16247 use std::{collections::HashSet, time::Duration};
16248
16249 const FLEET: [&str; 14] = [
16258 "aft",
16259 "alfonso-core",
16260 "magic-context",
16261 "broca",
16262 "thalamus",
16263 "quota",
16264 "engram",
16265 "plexus",
16266 "cerebellum",
16267 "astrocyte",
16268 "synapse",
16269 "subc-mcp",
16270 "cortexkit-credentials",
16271 "subc-federation",
16272 ];
16273
16274 #[test]
16282 fn probe_delays_disperse_across_the_fleet() {
16283 let cadence = Duration::from_secs(30);
16284 let delays: HashSet<Duration> = FLEET
16285 .iter()
16286 .map(|id| jittered_health_delay(id, 0, cadence))
16287 .collect();
16288 assert_eq!(
16289 delays.len(),
16290 FLEET.len(),
16291 "every supervised module must land on its own probe offset"
16292 );
16293 }
16294
16295 #[test]
16301 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16302 let cadence = Duration::from_secs(30);
16303 let span = cadence / 10;
16304 for id in FLEET {
16305 for probe_index in 0..8 {
16306 let delay = jittered_health_delay(id, probe_index, cadence);
16307 assert!(
16308 delay >= cadence,
16309 "{id}#{probe_index}: jitter must not shorten the cadence"
16310 );
16311 assert!(
16312 delay < cadence + span,
16313 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16314 );
16315 }
16316 }
16317 }
16318
16319 #[test]
16325 fn a_module_offset_is_stable_across_restarts() {
16326 let cadence = Duration::from_secs(30);
16327 for id in FLEET {
16328 assert_eq!(
16329 jittered_health_delay(id, 0, cadence),
16330 jittered_health_delay(id, 0, cadence),
16331 "{id}: the same module and probe index must produce the same offset"
16332 );
16333 }
16334 }
16335
16336 #[test]
16338 fn zero_cadence_yields_zero_delay() {
16339 assert_eq!(
16340 jittered_health_delay("aft", 0, Duration::ZERO),
16341 Duration::ZERO
16342 );
16343 }
16344}
16345
16346#[cfg(all(test, target_os = "linux"))]
16347mod cgroup_placement_tests {
16348 use super::{
16349 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16350 SuperviseError, SupervisedChild,
16351 };
16352 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16353 use cortexkit_test_support::ScratchDir;
16354 use std::{
16355 fs, io,
16356 path::{Path, PathBuf},
16357 sync::{Arc, Mutex},
16358 };
16359 use tokio::process::Command;
16360
16361 #[tokio::test]
16362 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16363 use super::*;
16364 let dir = ScratchDir::new("unique-spawn-cgroups");
16365 let root = PathBuf::from(format!(
16366 "/sys/fs/cgroup/subc-unique-test-{}-{}",
16367 std::process::id(),
16368 unix_ms_now()
16369 ));
16370 if let Err(error) = fs::create_dir(&root) {
16371 assert!(
16372 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16373 "required cgroup test cannot execute: {error}"
16374 );
16375 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16376 return;
16377 }
16378 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16379 let group_count = || {
16380 fs::read_dir(root.join("subc-modules"))
16381 .unwrap()
16382 .map(|entry| entry.unwrap().file_type().unwrap())
16383 .filter(|kind| kind.is_dir())
16384 .count()
16385 };
16386 let supervisor =
16387 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16388 .with_cgroup_placement(Some(placement.clone()));
16389 let runtime = supervisor.runtime_config();
16390 let mut spec = ModuleSpec {
16391 module_id: "unique-spawn".into(),
16392 program: PathBuf::from("/bin/sleep"),
16393 args: vec!["60".into()],
16394 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16395 .into_iter()
16396 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
16397 .collect(),
16398 reserved: false,
16399 reserved_prefixes: vec![],
16400 protocol: ModuleProtocol::None,
16401 overlap: Default::default(),
16402 };
16403 let spawn = |spec: &ModuleSpec| {
16404 spawn_child(
16405 spec,
16406 None,
16407 None,
16408 &runtime.stderr_ring,
16409 None,
16410 &runtime.child_roster,
16411 Some(&placement),
16412 )
16413 .unwrap()
16414 };
16415 let mut live = spawn(&spec);
16416 for _ in 0..3 {
16417 let next = spawn(&spec);
16419 assert_ne!(live.module_id, next.module_id);
16420 live.start_kill().unwrap();
16421 live.wait().await.unwrap();
16422 live = next;
16423 assert!(
16424 live.child.try_wait().unwrap().is_none(),
16425 "retiring the old slot must not kill the replacement"
16426 );
16427 assert_eq!(
16428 group_count(),
16429 1,
16430 "only the live spawn's cgroup should remain"
16431 );
16432 }
16433 supervisor.begin_daemon_shutdown();
16434 let reap = tokio::spawn(async move {
16435 live.wait().await.unwrap();
16436 });
16437 supervisor
16438 .end_children_for_daemon_shutdown(false, std::future::pending())
16439 .await;
16440 reap.await.unwrap();
16441 assert_eq!(group_count(), 0);
16442 spec.program = PathBuf::from("/bin/true");
16444 spec.args.clear();
16445 let fresh_roster = ChildRoster::default();
16446 let mut short = spawn_child(
16447 &spec,
16448 None,
16449 None,
16450 &runtime.stderr_ring,
16451 None,
16452 &fresh_roster,
16453 Some(&placement),
16454 )
16455 .unwrap();
16456 short.wait().await.unwrap();
16457 assert_eq!(group_count(), 0);
16458 spec.module_id = "_".repeat(255);
16459 let mut long_id = spawn_child(
16460 &spec,
16461 None,
16462 None,
16463 &runtime.stderr_ring,
16464 None,
16465 &fresh_roster,
16466 Some(&placement),
16467 )
16468 .unwrap();
16469 long_id.wait().await.unwrap();
16470 assert_eq!(
16471 group_count(),
16472 0,
16473 "valid long module IDs must not exceed cgroup NAME_MAX"
16474 );
16475 fs::remove_dir(root.join("subc-modules")).unwrap();
16476 fs::remove_dir(root).unwrap();
16477 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
16478 }
16479
16480 #[test]
16481 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
16482 let path = Path::new("/definitely-missing-subc-cgroup");
16483 let mut command = Command::new("true");
16484 let error = apply_cgroup_placement(
16485 &mut command,
16486 &ModuleSpec {
16487 module_id: "broken-cgroup".to_string(),
16488 program: PathBuf::from("true"),
16489 args: Vec::new(),
16490 env: Vec::new(),
16491 reserved: false,
16492 reserved_prefixes: Vec::new(),
16493 protocol: ModuleProtocol::Subc,
16494 overlap: Default::default(),
16495 },
16496 path,
16497 )
16498 .expect_err("a parent cgroup open failure must reject the supervised spawn");
16499 let reason = error.to_string();
16500
16501 assert!(
16502 matches!(error, SuperviseError::Cgroup { .. }),
16503 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
16504 );
16505 assert!(
16506 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
16507 "parent cgroup open failure must name cgroup.procs: {reason}"
16508 );
16509 }
16510
16511 #[tokio::test]
16512 async fn reaping_a_child_removes_its_empty_module_cgroup() {
16513 let root = ScratchDir::new("supervisor-reap-cgroup");
16514 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16515 let placement = subc_cgroup::prepare_at(&root)
16516 .expect("prepare scratch cgroup root")
16517 .expect("scratch root has a cgroup.procs marker");
16518 let module_id = "reaped-module";
16519 let module = placement
16520 .module_path(module_id)
16521 .expect("create scratch module cgroup");
16522 let child = Command::new("true")
16523 .env("XDG_DATA_HOME", root.path())
16524 .env("XDG_RUNTIME_DIR", root.path())
16525 .env("XDG_CONFIG_HOME", root.path())
16526 .spawn()
16527 .expect("spawn short-lived child");
16528 let pid = child.id().expect("spawned child has pid");
16529 let mut child = SupervisedChild {
16530 child,
16531 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
16532 protocol: ModuleProtocol::Subc,
16533 module_id: module_id.to_string(),
16534 cgroup_placement: Some(placement),
16535 stdout_pump: None,
16536 stderr_pump: None,
16537 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
16538 spawned_at: tokio::time::Instant::now(),
16539 registration_deadline: None,
16540 registration_watch: None,
16541 spawned_at_ms: 0,
16542 spawned_from: PathBuf::from("true"),
16543 launch_spec_digest: LaunchSpecDigest([0; 32]),
16544 spawned_file_identity: None,
16545 process_start_time: None,
16546 process_identity: None,
16547 pid,
16548 roster_guard: None,
16549 #[cfg(target_os = "macos")]
16550 privacy_exec: None,
16551 spawn_failure: None,
16552 };
16553
16554 child.wait().await.expect("reap short-lived child");
16555
16556 assert!(
16557 !module.exists(),
16558 "reaping the supervised child must remove its empty cgroup"
16559 );
16560 }
16561
16562 #[test]
16563 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
16564 let root = ScratchDir::new("supervisor-non-empty-cgroup");
16565 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16566 let placement = subc_cgroup::prepare_at(&root)
16567 .expect("prepare scratch cgroup root")
16568 .expect("scratch root has a cgroup.procs marker");
16569 let module = placement
16570 .module_path("surviving-module")
16571 .expect("create scratch module cgroup");
16572 fs::write(module.join("surviving-process"), b"still present")
16573 .expect("make scratch cgroup non-empty");
16574 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
16575
16576 remove_module_cgroup(&placement, "surviving-module");
16577
16578 let logs = crate::router::test_log::captured_logs(&logs);
16579 assert!(
16580 module.exists(),
16581 "failed removal must leave the cgroup intact"
16582 );
16583 assert!(
16584 logs.contains("could not remove module cgroup after process exit; continuing teardown")
16585 && logs.contains("surviving-module"),
16586 "best-effort removal must report the failure without returning it: {logs}"
16587 );
16588 }
16589
16590 #[test]
16591 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
16592 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
16593 let reason = SuperviseError::Spawn {
16594 program: PathBuf::from("/bin/true"),
16595 source: io::Error::from_raw_os_error(13),
16596 cgroup_path: Some(cgroup_path.clone()),
16597 }
16598 .to_string();
16599
16600 assert!(
16601 reason.contains(&cgroup_path.display().to_string()),
16602 "a pre_exec spawn failure must name the cgroup path: {reason}"
16603 );
16604 }
16605}
16606
16607#[cfg(test)]
16608mod spawn_subscriber_lag_tests {
16609 use super::*;
16610
16611 #[tokio::test]
16616 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
16617 let feed = SpawnEventFeed::default();
16618 feed.configure_incarnation("lag-incarnation".to_string());
16619 let (tx, mut rx) = mpsc::channel(1);
16622 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
16623 .expect("subscribe");
16624 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
16625 for index in 0..emitted {
16626 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
16627 tokio::task::yield_now().await;
16630 }
16631 assert_eq!(
16632 feed.subscriber_count(),
16633 0,
16634 "the lagged subscriber must be removed"
16635 );
16636
16637 let mut data = Vec::new();
16638 let mut last = None;
16639 loop {
16640 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16641 .await
16642 .expect("the forwarder must finish once the subscriber is dropped");
16643 let Some(outbound) = next else { break };
16644 let frame = outbound.frame;
16645 if frame.header.ty == FrameType::StreamData {
16646 assert!(last.is_none(), "no data may follow the terminal frame");
16647 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
16648 data.push(event.cursor.seq);
16649 } else {
16650 assert!(last.is_none(), "exactly one terminal frame");
16651 last = Some(frame);
16652 }
16653 }
16654 assert!(!data.is_empty(), "queued frames drain before the terminal");
16655 for pair in data.windows(2) {
16656 assert_eq!(
16657 pair[1],
16658 pair[0] + 1,
16659 "queued frames arrive dense and in order"
16660 );
16661 }
16662 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
16663 assert_eq!(terminal.header.ty, FrameType::Error);
16664 assert_eq!(terminal.header.corr, 7);
16665 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
16666 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
16667 let detail = body.detail.expect("lagged error carries detail");
16668 assert_eq!(
16669 detail["first_undelivered_cursor"]["seq"],
16670 data.last().unwrap() + 1,
16671 "the named cursor is the first event the subscriber did not receive"
16672 );
16673 assert_eq!(
16674 detail["first_undelivered_cursor"]["daemon_incarnation"],
16675 "lag-incarnation"
16676 );
16677 }
16678}
16679
16680#[cfg(test)]
16681mod terminal_history_read_concurrency_tests {
16682 use super::*;
16683 use crate::terminal_journal::read_pause;
16684 use cortexkit_test_support::ScratchDir;
16685 use std::sync::mpsc as std_mpsc;
16686
16687 fn journaled_ring(
16688 journal: &Arc<crate::terminal_journal::TerminalJournal>,
16689 ) -> Arc<Mutex<TerminalRing>> {
16690 Arc::new(Mutex::new(
16691 TerminalRing::new(TerminalRingConfig::default(), 1)
16692 .with_journal(Some(Arc::clone(journal))),
16693 ))
16694 }
16695
16696 fn crash(at_ms: u64) -> ExitReport {
16697 ExitReport {
16698 kind: ExitKind::Crash,
16699 code: Some(1),
16700 signal: None,
16701 at_ms,
16702 }
16703 }
16704
16705 fn record_within(
16708 module_id: &'static str,
16709 ring: &Arc<Mutex<TerminalRing>>,
16710 at_ms: u64,
16711 bound: Duration,
16712 ) -> bool {
16713 let ring = Arc::clone(ring);
16714 let (done, done_rx) = std_mpsc::channel();
16715 std::thread::spawn(move || {
16716 record_terminal(
16717 module_id,
16718 &ring,
16719 &SpawnEventFeed::default(),
16720 &crash(at_ms),
16721 TerminalDisposition::Restarting,
16722 );
16723 let _ = done.send(());
16724 });
16725 done_rx.recv_timeout(bound).is_ok()
16726 }
16727
16728 #[test]
16733 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
16734 let dir = ScratchDir::new("terminal-history-concurrent-read");
16735 let path = dir.join("terminals.jsonl");
16736 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
16737 path.clone(),
16738 "daemon".into(),
16739 ));
16740 let reader_ring = journaled_ring(&journal);
16741 let other_ring = journaled_ring(&journal);
16742 assert!(record_within(
16743 "reader-module",
16744 &reader_ring,
16745 10,
16746 Duration::from_secs(5)
16747 ));
16748
16749 let (started, release) = read_pause::install(&path);
16750 let reading = {
16751 let ring = Arc::clone(&reader_ring);
16752 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
16753 };
16754 started
16755 .recv_timeout(Duration::from_secs(5))
16756 .expect("the history read reached its pause");
16757
16758 let bound = Duration::from_secs(1);
16759 assert!(
16760 record_within("other-module", &other_ring, 20, bound),
16761 "another module's exit waited on a history read (journal writer held)"
16762 );
16763 assert!(
16764 record_within("reader-module", &reader_ring, 30, bound),
16765 "the read module's own exit waited on its history read (ring held)"
16766 );
16767
16768 drop(release);
16769 let paused = reading.join().unwrap();
16770 assert_eq!(
16771 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16772 vec![10],
16773 "an exit recorded after the read began lands in neither half of it"
16774 );
16775 assert_eq!(paused.journal_skipped_lines, 0);
16776 assert_eq!(paused.journal_read_errors, 0);
16777
16778 let after = durable_terminal_history_of(&reader_ring, "reader-module");
16779 assert_eq!(
16780 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16781 vec![10, 30],
16782 "the next read merges ring and journal with no duplicate"
16783 );
16784 assert_eq!(after.journal_skipped_lines, 0);
16785 }
16786}
16787
16788#[cfg(test)]
16793mod stderr_settle_tests {
16794 use std::{
16795 future::Future,
16796 io,
16797 pin::Pin,
16798 sync::{Arc, Mutex},
16799 task::{Context, Poll},
16800 time::Duration,
16801 };
16802
16803 use tokio::{
16804 io::{AsyncRead, ReadBuf},
16805 sync::oneshot,
16806 time::Instant,
16807 };
16808
16809 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
16810 use crate::stderr_tail::{
16811 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
16812 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
16813 };
16814
16815 const BOUND: Duration = Duration::from_millis(250);
16816
16817 struct HeldReader {
16821 before: Option<Vec<u8>>,
16822 gate: Option<oneshot::Receiver<()>>,
16823 after: io::Cursor<Vec<u8>>,
16824 }
16825
16826 impl AsyncRead for HeldReader {
16827 fn poll_read(
16828 mut self: Pin<&mut Self>,
16829 cx: &mut Context<'_>,
16830 buf: &mut ReadBuf<'_>,
16831 ) -> Poll<io::Result<()>> {
16832 if let Some(bytes) = self.before.take() {
16833 buf.put_slice(&bytes);
16834 return Poll::Ready(Ok(()));
16835 }
16836 if let Some(gate) = self.gate.as_mut() {
16837 match Pin::new(gate).poll(cx) {
16838 Poll::Pending => return Poll::Pending,
16839 Poll::Ready(_) => self.gate = None,
16840 }
16841 }
16842 Pin::new(&mut self.after).poll_read(cx, buf)
16843 }
16844 }
16845
16846 struct DiscardSink;
16847
16848 impl OutputSink for DiscardSink {
16849 fn write_line(&mut self, _line: &[u8]) {}
16850 }
16851
16852 fn line(text: &str) -> TailEntry {
16853 TailEntry::Line {
16854 text: text.to_string(),
16855 truncated: false,
16856 at_ms: None,
16857 }
16858 }
16859
16860 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16861 ring.lock().unwrap()
16862 }
16863
16864 fn held_pump(
16868 ring: &Arc<Mutex<StderrRing>>,
16869 before: &str,
16870 after: &str,
16871 ) -> (StderrPump, oneshot::Sender<()>) {
16872 let generation = lock(ring).begin_process();
16873 let (release, gate) = oneshot::channel();
16874 let reader = HeldReader {
16875 before: Some(before.as_bytes().to_vec()),
16876 gate: Some(gate),
16877 after: io::Cursor::new(after.as_bytes().to_vec()),
16878 };
16879 let task = tokio::spawn(pump_stderr_to(
16880 reader,
16881 Arc::clone(ring),
16882 generation,
16883 DiscardSink,
16884 ));
16885 (StderrPump { task, generation }, release)
16886 }
16887
16888 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16889 for _ in 0..1000 {
16890 if done(&lock(ring)) {
16891 return;
16892 }
16893 tokio::time::sleep(Duration::from_millis(1)).await;
16894 }
16895 panic!(
16896 "ring never reached the expected state: {:?}",
16897 lock(ring).snapshot(None, None)
16898 );
16899 }
16900
16901 #[tokio::test(start_paused = true)]
16902 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16903 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16904 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16905
16906 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16907 let before_release = lock(&ring).snapshot(None, None);
16908 assert!(
16909 matches!(before_release.capture, CaptureState::Incomplete { .. }),
16910 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16911 );
16912
16913 let next = lock(&ring).begin_process();
16916 lock(&ring).push_line_from(next, "next process booting");
16917 release.send(()).unwrap();
16918 wait_until(&ring, |ring| {
16919 ring.snapshot(None, None).capture == CaptureState::Captured
16920 })
16921 .await;
16922
16923 assert_eq!(
16924 untimed(lock(&ring).snapshot(None, None).entries),
16925 vec![
16926 line("booting"),
16927 line("config error: missing storage"),
16928 TailEntry::ProcessStart,
16929 line("next process booting"),
16930 ],
16931 "the crash's last line must survive a slow reader and stay in the crashed process's section"
16932 );
16933 }
16934
16935 #[tokio::test(start_paused = true)]
16936 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16937 ) {
16938 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16939 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16942
16943 let started = Instant::now();
16944 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16945 assert_eq!(
16946 started.elapsed(),
16947 BOUND,
16948 "the restart must wait exactly the bound for a pipe that stays open, no longer"
16949 );
16950
16951 let next = lock(&ring).begin_process();
16952 lock(&ring).push_line_from(next, "next process booting");
16953 tokio::time::sleep(Duration::from_secs(60)).await;
16954
16955 let snapshot = lock(&ring).snapshot(None, None);
16956 match &snapshot.capture {
16957 CaptureState::Incomplete { reason } => assert!(
16958 reason.contains("had not reached EOF") && reason.contains("250ms"),
16959 "the reason must say what is missing and after how long: {reason}"
16960 ),
16961 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16962 }
16963 assert_eq!(
16964 untimed(snapshot.entries),
16965 vec![
16966 line("parent exiting"),
16967 TailEntry::ProcessStart,
16968 line("next process booting"),
16969 ]
16970 );
16971 }
16972
16973 #[tokio::test(start_paused = true)]
16974 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16975 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16976 let (pump, release) = held_pump(&ring, "one\n", "two\n");
16977 release.send(()).unwrap();
16978
16979 settle_stderr_pump("clean", &ring, pump, BOUND).await;
16980
16981 let snapshot = lock(&ring).snapshot(None, None);
16982 assert_eq!(snapshot.capture, CaptureState::Captured);
16983 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16984 }
16985
16986 #[tokio::test(start_paused = true)]
16987 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16988 const BURST: usize = 150;
16989 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16990 let path = capture.join("two-pipe.stderr.log");
16991 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16992 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16993 let generation = lock(&ring).begin_process();
16994 let stdout = (0..BURST)
16995 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16996 .collect::<String>();
16997 let stderr = (0..BURST)
16998 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16999 .collect::<String>();
17000 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17005 let (release, gate) = oneshot::channel();
17006 let reader = HeldReader {
17007 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17008 gate: Some(gate),
17009 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17010 };
17011 let (finished, completion) = oneshot::channel();
17012 let stdout_sink = sink.clone();
17013 let pump = tokio::spawn(async move {
17014 pump_stdout_to(reader, stdout_sink).await;
17015 finished.send(()).unwrap();
17016 });
17017 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17018
17019 let started = Instant::now();
17020 settle_stdout_pump("two-pipe", pump, BOUND).await;
17021 assert_eq!(
17022 started.elapsed(),
17023 BOUND,
17024 "a late reader must not hold up restart"
17025 );
17026 assert_eq!(
17027 std::fs::read_to_string(&path).unwrap().lines().count(),
17028 BURST + 1
17029 );
17030
17031 release
17032 .send(())
17033 .expect("stdout reader must survive the exit drain bound");
17034 tokio::time::timeout(Duration::from_secs(1), completion)
17035 .await
17036 .expect("released stdout reader must reach EOF")
17037 .expect("stdout capture must finish at EOF, not be cancelled");
17038 let contents = std::fs::read_to_string(&path).unwrap();
17039 let mut lines = contents
17040 .lines()
17041 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17042 .collect::<Vec<_>>();
17043 let mut expected = (0..BURST)
17044 .flat_map(|i| {
17045 [
17046 format!("out-{i:04}-{}", "o".repeat(64)),
17047 format!("err-{i:04}-{}", "e".repeat(64)),
17048 ]
17049 })
17050 .collect::<Vec<_>>();
17051 lines.sort_unstable();
17052 expected.sort_unstable();
17053 assert_eq!(
17054 lines, expected,
17055 "both pipes must retain every complete line exactly once"
17056 );
17057 }
17058}
17059
17060#[cfg(all(test, windows))]
17074mod job_containment_tests {
17075 use super::*;
17076 use cortexkit_test_support::ScratchDir;
17077 use std::{
17078 path::{Path, PathBuf},
17079 sync::{Arc, Mutex},
17080 time::{Duration, Instant},
17081 };
17082
17083 fn stub_path() -> PathBuf {
17089 let mut path = std::env::current_exe().expect("current_exe available in tests");
17090 path.pop();
17091 path.pop();
17092 path.push("fake-aft-stub.exe");
17093 assert!(
17094 path.exists(),
17095 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17096 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17097 path.display()
17098 );
17099 path
17100 }
17101
17102 fn read_grandchild_pid(path: &Path) -> u32 {
17104 let deadline = Instant::now() + Duration::from_secs(10);
17105 loop {
17106 if let Ok(contents) = std::fs::read_to_string(path) {
17107 if let Ok(pid) = contents.trim().parse() {
17108 return pid;
17109 }
17110 }
17111 assert!(
17112 Instant::now() < deadline,
17113 "the stub never recorded a grandchild pid at {}",
17114 path.display()
17115 );
17116 std::thread::sleep(Duration::from_millis(10));
17117 }
17118 }
17119
17120 struct Fixture {
17123 _dir: ScratchDir,
17124 module_id: String,
17125 grandchild: u32,
17126 child: Option<SupervisedChild>,
17127 registry: Arc<Registry>,
17128 snapshot: Arc<Mutex<SupervisorSnapshot>>,
17129 terminal_ring: Arc<Mutex<TerminalRing>>,
17130 spawn_events: SpawnEventFeed,
17131 }
17132
17133 fn fixture(label: &str, module_id: &str) -> Fixture {
17134 let dir = ScratchDir::new(label);
17135 let pid_file = dir.join("grandchild.pid");
17136 let supervisor = Supervisor::new_for_test(
17137 Arc::new(Registry::default()),
17138 RestartPolicy::new(3, Duration::ZERO),
17139 );
17140 let runtime = supervisor.runtime_config();
17141 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17142 let spec = ModuleSpec {
17143 module_id: module_id.to_string(),
17144 program: stub_path(),
17145 args: Vec::new(),
17149 env: vec![
17150 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17151 (
17152 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17153 pid_file.display().to_string(),
17154 ),
17155 ],
17156 reserved: false,
17157 reserved_prefixes: Vec::new(),
17158 protocol: ModuleProtocol::Subc,
17159 overlap: Default::default(),
17160 };
17161 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17162 .expect("spawn the supervised fixture");
17163 let grandchild = read_grandchild_pid(&pid_file);
17164 Fixture {
17165 _dir: dir,
17166 module_id: module_id.to_string(),
17167 grandchild,
17168 child: Some(child),
17169 registry: Arc::new(Registry::default()),
17170 snapshot,
17171 terminal_ring: Arc::clone(&runtime.terminal_ring),
17172 spawn_events: SpawnEventFeed::default(),
17173 }
17174 }
17175
17176 impl Fixture {
17177 async fn drain(&mut self) {
17179 let child = self
17180 .child
17181 .take()
17182 .expect("the fixture child is still present");
17183 drain_child_to_state(
17184 &self.module_id,
17185 ModuleProtocol::Subc,
17186 StopNotice::NotSent,
17189 &self.registry,
17190 None,
17191 &self.snapshot,
17192 &self.terminal_ring,
17193 &self.spawn_events,
17194 child,
17195 Duration::from_millis(500),
17196 ModuleState::Stopped,
17197 Some(false),
17198 &FailureContext::operator(),
17199 )
17200 .await
17201 .expect("drain the supervised fixture");
17202 }
17203 }
17204
17205 #[tokio::test]
17211 async fn teardown_reaps_the_grandchild() {
17212 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17213 let grandchild = fixture.grandchild;
17214
17215 assert!(
17216 subc_jobobject::process_exists(grandchild),
17217 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17218 );
17219
17220 fixture.drain().await;
17221
17222 assert!(
17223 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17224 "grandchild {grandchild} outlived module teardown: the tree was not contained"
17225 );
17226 }
17227
17228 #[test]
17241 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17242 let dir = ScratchDir::new("teardown-uncontained");
17243 let pid_file = dir.join("grandchild.pid");
17244 let mut child = std::process::Command::new(stub_path())
17245 .env("FAKE_AFT_NEVER_CONNECT", "1")
17246 .env(
17247 "FAKE_AFT_GRANDCHILD_PID_FILE",
17248 pid_file.display().to_string(),
17249 )
17250 .stdin(std::process::Stdio::null())
17251 .stdout(std::process::Stdio::null())
17252 .stderr(std::process::Stdio::null())
17253 .spawn()
17254 .expect("spawn the uncontained fixture");
17255 let grandchild = read_grandchild_pid(&pid_file);
17256
17257 child.kill().expect("kill the direct child");
17259 let _ = child.wait();
17260
17261 assert!(
17262 subc_jobobject::process_exists(grandchild),
17263 "grandchild {grandchild} died with the direct child, so this control no longer \
17264 distinguishes contained from uncontained teardown and the regression test is \
17265 passing vacuously"
17266 );
17267
17268 kill_tree(grandchild);
17271 }
17272
17273 #[tokio::test]
17282 async fn dropping_containment_reaps_the_grandchild() {
17283 let mut fixture = fixture("drop-containment", "tree-drop");
17284 let grandchild = fixture.grandchild;
17285
17286 assert!(subc_jobobject::process_exists(grandchild));
17287
17288 fixture.child.as_mut().expect("child present").job = None;
17290
17291 assert!(
17292 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17293 "grandchild {grandchild} survived the containment handle closing, so a daemon \
17294 crash would leave the tree behind"
17295 );
17296 }
17297
17298 fn kill_tree(pid: u32) {
17300 let _ = std::process::Command::new("taskkill.exe")
17301 .args(["/PID", &pid.to_string(), "/T", "/F"])
17302 .stdin(std::process::Stdio::null())
17303 .stdout(std::process::Stdio::null())
17304 .stderr(std::process::Stdio::null())
17305 .status();
17306 assert!(
17307 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17308 "could not clean up grandchild {pid}"
17309 );
17310 }
17311}
17312
17313#[cfg(test)]
17314mod privacy_trampoline_configuration_tests {
17315 #[tokio::test]
17316 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17317 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17318 #[cfg(target_os = "macos")]
17319 {
17320 let supervisor = super::Supervisor::new(
17321 std::sync::Arc::new(crate::Registry::default()),
17322 super::RestartPolicy::default(),
17323 );
17324 let error = supervisor.spawn(spec()).unwrap_err();
17325 assert!(
17326 error
17327 .to_string()
17328 .contains("no privacy trampoline configured"),
17329 "{error}"
17330 );
17331 }
17332 }
17333
17334 #[tokio::test]
17335 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17336 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17337 #[cfg(target_os = "macos")]
17338 {
17339 let supervisor = super::Supervisor::new(
17340 std::sync::Arc::new(crate::Registry::default()),
17341 super::RestartPolicy::default(),
17342 )
17343 .with_privacy_trampoline(std::env::current_exe().unwrap());
17344 let error = supervisor.spawn(spec()).unwrap_err();
17345 assert!(
17346 error
17347 .to_string()
17348 .contains("binary does not implement the privacy trampoline protocol"),
17349 "{error}"
17350 );
17351 }
17352 }
17353
17354 #[cfg(target_os = "macos")]
17355 fn spec() -> super::ModuleSpec {
17356 super::ModuleSpec {
17357 module_id: "privacy-configuration".into(),
17358 program: "/bin/sleep".into(),
17359 args: vec!["30".into()],
17360 env: vec![],
17361 reserved: false,
17362 reserved_prefixes: vec![],
17363 protocol: subc_control::ModuleProtocol::None,
17364 overlap: super::ModuleOverlap::Exclusive,
17365 }
17366 }
17367}
17368
17369#[cfg(test)]
17370mod privacy_exec_boundary_tests {
17371 #[cfg(target_os = "macos")]
17372 use super::*;
17373 #[cfg(target_os = "macos")]
17374 use std::{
17375 io::{Read, Write},
17376 net::{TcpListener, TcpStream},
17377 };
17378
17379 #[cfg(target_os = "macos")]
17382 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
17383 if let Some((_, path)) = spec
17384 .env
17385 .iter()
17386 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
17387 {
17388 let mut barrier = TcpStream::connect(path).unwrap();
17389 barrier
17390 .set_read_timeout(Some(Duration::from_secs(30)))
17391 .unwrap();
17392 barrier.write_all(&pid.to_ne_bytes()).unwrap();
17393 let mut release = [0];
17394 barrier.read_exact(&mut release).unwrap();
17395 assert_eq!(&release, b"X");
17396 }
17397 }
17398
17399 #[cfg(target_os = "macos")]
17400 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
17401 ModuleSpec {
17402 module_id: "privacy-boundary".into(),
17403 program: program.into(),
17404 args: args.iter().map(|arg| (*arg).into()).collect(),
17405 env: vec![],
17406 reserved: false,
17407 reserved_prefixes: vec![],
17408 protocol: ModuleProtocol::None,
17409 overlap: ModuleOverlap::Exclusive,
17410 }
17411 }
17412
17413 #[cfg(target_os = "macos")]
17414 async fn accept(listener: TcpListener) -> TcpStream {
17415 let listener = tokio::net::TcpListener::from_std({
17417 listener.set_nonblocking(true).unwrap();
17418 listener
17419 })
17420 .unwrap();
17421 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
17422 .await
17423 .unwrap()
17424 .unwrap();
17425 let stream = stream.into_std().unwrap();
17426 stream.set_nonblocking(false).unwrap();
17427 stream
17428 .set_read_timeout(Some(Duration::from_secs(30)))
17429 .unwrap();
17430 stream
17431 }
17432
17433 #[tokio::test]
17434 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17435 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
17436 #[cfg(target_os = "macos")]
17437 {
17438 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
17439 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17442 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17443 let record = root.join("live-children.json");
17444 let supervisor =
17445 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17446 .with_live_children_record(&record);
17447 let runtime = supervisor.runtime_config();
17448 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17449 let mut spec = spec("/bin/sleep", &["30"]);
17450 spec.env = vec![
17451 (
17452 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
17453 exec_listener.local_addr().unwrap().to_string(),
17454 ),
17455 (
17456 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
17457 sample_listener.local_addr().unwrap().to_string(),
17458 ),
17459 ];
17460 let spawn = tokio::task::spawn_blocking(move || {
17461 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
17462 });
17463 let mut sample = accept(sample_listener).await;
17464 let mut pid = [0; 4];
17465 sample.read_exact(&mut pid).unwrap();
17466 let pid = u32::from_ne_bytes(pid);
17467 let mut exec = accept(exec_listener).await;
17468 let mut ready = [0];
17469 exec.read_exact(&mut ready).unwrap();
17470 assert_eq!(&ready, b"R");
17471 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
17474 assert_eq!(
17475 observe_spawned_image(pid).unwrap().executable,
17476 Some(trampoline)
17477 );
17478 sample.write_all(b"X").unwrap();
17479 let mut child = spawn.await.unwrap();
17480 let early = crate::live_children::read_record(&record).unwrap();
17481 assert_eq!(early.len(), 1);
17482 assert_eq!(early[0].pid, pid);
17483 assert_eq!(
17484 early[0].executable, None,
17485 "unconfirmed trampoline image entered the roster"
17486 );
17487 assert!(child.report_ready.get().is_none());
17488 child.privacy_exec.as_mut().unwrap().deadline =
17491 tokio::time::Instant::now() + Duration::from_secs(30);
17492 exec.write_all(b"X").unwrap();
17493 child.confirm_privacy_exec().await;
17494 assert_eq!(child.spawn_failure, None);
17495 assert!(child.report_ready.get().is_some());
17496 let confirmed = crate::live_children::read_record(&record).unwrap();
17497 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
17498 assert_ne!(module, trampoline);
17499 assert_eq!(confirmed[0].executable, Some(module.into()));
17500 child.start_kill().unwrap();
17501 child.wait().await.unwrap();
17502 child.release_roster();
17503 }
17504 }
17505
17506 #[tokio::test]
17507 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17508 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
17509 #[cfg(target_os = "macos")]
17510 {
17511 let registry = Arc::new(Registry::default());
17512 let policy = RestartPolicy::new(0, Duration::ZERO);
17513 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
17514 let runtime = supervisor.runtime_config();
17515 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17516 let spec = spec("/bin/sh", &["-c", "exit 121"]);
17517 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17521 let pid = child.pid;
17522 tokio::task::spawn_blocking(move || {
17523 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
17524 })
17525 .await
17526 .unwrap()
17527 .unwrap();
17528 child.privacy_exec.as_mut().unwrap().deadline =
17529 tokio::time::Instant::now() + Duration::from_secs(30);
17530 let status = child.wait().await.unwrap();
17531 assert_eq!(status.code(), Some(121));
17532 assert!(child.privacy_exec.is_none());
17533 assert!(
17534 child.report_ready.get().is_none(),
17535 "an exited module must not publish a live pid"
17536 );
17537 let report = classify_reaped_child_exit(&snapshot, &child, &status);
17538 on_child_exit(
17539 &spec,
17540 policy,
17541 ®istry,
17542 &snapshot,
17543 &runtime.terminal_ring,
17544 &runtime.spawn_events,
17545 &runtime.child_roster,
17546 report,
17547 true,
17548 )
17549 .await;
17550 let state = lock_snapshot(&snapshot).unwrap();
17551 assert_eq!(state.state, ModuleState::Failed);
17552 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
17553 assert_eq!(state.reported_pid(), None);
17554 drop(state);
17555 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17556 assert_eq!(history.entries.len(), 1);
17557 let terminal = &history.entries[0];
17558 assert_eq!(terminal.exit_code, Some(121));
17559 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
17560 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
17561 assert_eq!(
17562 terminal.disposition_detail.as_deref(),
17563 Some(policy.budget_exhausted_detail().as_str()),
17564 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
17565 );
17566 assert_eq!(child.spawn_failure, None);
17567 child.release_roster();
17568 }
17569 }
17570}
17571
17572#[cfg(all(test, unix))]
17576mod launch_nonce_descriptor_tests {
17577 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
17578 use crate::stderr_tail::{StderrRing, StderrTailConfig};
17579 use cortexkit_test_support::ScratchDir;
17580 use std::{
17581 path::PathBuf,
17582 sync::{Arc, Mutex},
17583 time::{Duration, Instant},
17584 };
17585
17586 async fn probe(role: super::SpawnRole) {
17587 let scratch = ScratchDir::new("launch-nonce-descriptor");
17588 let fd_copy = scratch.join("from-descriptor");
17589 let env_copy = scratch.join("environment");
17590 let script = format!(
17591 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
17592 fd = fd_copy.display(), env = env_copy.display(),
17593 );
17594 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
17595 let spec = ModuleSpec {
17596 module_id: "nonce-descriptor-probe".to_string(),
17597 program: PathBuf::from("/bin/sh"),
17598 args: vec!["-c".to_string(), script],
17599 env: vec![
17600 xdg("XDG_DATA_HOME"),
17601 xdg("XDG_RUNTIME_DIR"),
17602 xdg("XDG_CONFIG_HOME"),
17603 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
17604 ],
17605 reserved: true,
17606 reserved_prefixes: Vec::new(),
17607 protocol: ModuleProtocol::Subc,
17608 overlap: Default::default(),
17609 };
17610 let handle = SupervisorHandle::new();
17611 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17612 let roster = ChildRoster::default();
17613 #[cfg(target_os = "macos")]
17614 {
17615 let path = super::test_privacy_trampoline();
17616 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
17617 }
17618 let child = super::spawn_child_in_slot(
17619 &spec,
17620 None,
17621 Some(&handle),
17622 &ring,
17623 None,
17624 &roster,
17625 #[cfg(target_os = "linux")]
17626 None,
17627 role,
17628 matches!(role, super::SpawnRole::SwapCandidate),
17629 )
17630 .expect("spawn probe");
17631 let deadline = Instant::now() + Duration::from_secs(10);
17632 while !(fd_copy.exists() && env_copy.exists()) {
17633 assert!(Instant::now() < deadline, "probe never wrote its copies");
17634 tokio::time::sleep(Duration::from_millis(20)).await;
17635 }
17636 let nonce = std::fs::read_to_string(fd_copy).unwrap();
17637 assert!(!nonce.is_empty());
17638 let environment = std::fs::read_to_string(env_copy).unwrap();
17639 assert!(environment
17640 .lines()
17641 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
17642 let copy = environment
17643 .lines()
17644 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
17645 assert_eq!(
17646 copy, None,
17647 "Unix children must never receive the environment nonce"
17648 );
17649 if matches!(role, super::SpawnRole::Plain) {
17650 assert_eq!(
17651 handle.spawn_nonce(&spec.module_id).as_deref(),
17652 Some(nonce.as_str())
17653 );
17654 }
17655 drop(child);
17656 }
17657
17658 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17659 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
17660 probe(super::SpawnRole::Plain).await;
17661 }
17662
17663 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17664 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
17665 probe(super::SpawnRole::SwapCandidate).await;
17666 }
17667}
17668
17669#[cfg(all(test, target_os = "linux"))]
17670#[path = "supervise/cgroup_reap_tests.rs"]
17671mod cgroup_reap_tests;
17672
17673#[cfg(all(test, target_os = "linux"))]
17674mod cgroup_containment_tests {
17675 use super::*;
17676 use cortexkit_test_support::ScratchDir;
17677
17678 fn running(pid: u32) -> bool {
17679 std::fs::read_to_string(format!("/proc/{pid}/stat"))
17681 .ok()
17682 .and_then(|stat| {
17683 stat.rsplit_once(") ")
17684 .map(|(_, rest)| rest.starts_with('Z'))
17685 })
17686 .is_some_and(|zombie| !zombie)
17687 }
17688
17689 #[tokio::test]
17690 async fn linux_teardown_reaps_the_grandchild() {
17691 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
17692 }
17693
17694 #[tokio::test]
17695 async fn linux_shutdown_straggler_reaps_the_grandchild() {
17696 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
17697 }
17698
17699 async fn teardown_tree(test_name: &str, shutdown: bool) {
17700 let dir = ScratchDir::new(test_name);
17701 let root = PathBuf::from(format!(
17702 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
17703 std::process::id(),
17704 unix_ms_now()
17705 ));
17706 if let Err(error) = std::fs::create_dir(&root) {
17707 assert!(
17708 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17709 "required cgroup test cannot execute: {error}"
17710 );
17711 eprintln!(
17712 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
17713 root.display()
17714 );
17715 return;
17716 }
17717 let placement = subc_cgroup::prepare_at(&root)
17718 .expect("prepare isolated kernel cgroup")
17719 .expect("isolated cgroup is delegated");
17720 let module_id = "tree-teardown";
17721 let module = placement
17722 .module_path(module_id)
17723 .expect("create isolated module cgroup");
17724 if !module.join("cgroup.kill").exists() {
17725 std::fs::remove_dir(&module).unwrap();
17726 std::fs::remove_dir(root.join("subc-modules")).unwrap();
17727 std::fs::remove_dir(&root).unwrap();
17728 assert!(
17729 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17730 "required cgroup.kill interface unavailable"
17731 );
17732 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
17733 return;
17734 }
17735 let supervisor = Supervisor::new_for_test(
17736 Arc::new(Registry::default()),
17737 RestartPolicy::new(3, Duration::ZERO),
17738 )
17739 .with_cgroup_placement(Some(placement));
17740 let mut runtime = supervisor.runtime_config();
17741 runtime.child_roster = runtime
17742 .child_roster
17743 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
17744 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17745 let pid_file = dir.join("grandchild.pid");
17746 let spec = ModuleSpec {
17747 module_id: module_id.to_string(),
17748 program: PathBuf::from("/bin/sh"),
17749 args: vec![
17750 "-c".into(),
17751 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
17752 "fixture".into(),
17753 pid_file.display().to_string(),
17754 ],
17755 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17756 .into_iter()
17757 .map(|key| (key.to_string(), dir.display().to_string()))
17758 .collect(),
17759 reserved: false,
17760 reserved_prefixes: Vec::new(),
17761 protocol: ModuleProtocol::None,
17762 overlap: Default::default(),
17763 };
17764 let child =
17765 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
17766 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17767 let grandchild: u32 = loop {
17768 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
17769 if let Ok(pid) = contents.trim().parse() {
17770 break pid;
17771 }
17772 }
17773 assert!(
17774 tokio::time::Instant::now() < deadline,
17775 "grandchild pid was not recorded"
17776 );
17777 tokio::time::sleep(Duration::from_millis(10)).await;
17778 };
17779 assert!(
17780 running(grandchild),
17781 "grandchild must be alive before teardown"
17782 );
17783 if shutdown {
17784 let mut child = child;
17785 crate::child_roster::end_children_for_daemon_shutdown(
17786 &runtime.child_roster,
17787 false,
17788 std::future::pending(),
17789 )
17790 .await;
17791 child.wait().await.expect("reap shutdown straggler");
17792 } else {
17793 drain_child_to_state(
17794 module_id,
17795 ModuleProtocol::None,
17796 StopNotice::NotSent,
17797 &Registry::default(),
17798 None,
17799 &snapshot,
17800 &runtime.terminal_ring,
17801 &SpawnEventFeed::default(),
17802 child,
17803 Duration::from_millis(100),
17804 ModuleState::Stopped,
17805 Some(false),
17806 &FailureContext::operator(),
17807 )
17808 .await
17809 .expect("real supervisor teardown");
17810 }
17811 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
17812 while running(grandchild) && tokio::time::Instant::now() < deadline {
17813 tokio::time::sleep(Duration::from_millis(10)).await;
17814 }
17815 let survived = running(grandchild);
17816 if survived {
17818 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
17819 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
17820 tokio::time::sleep(Duration::from_millis(100)).await;
17821 }
17822 if module.exists() {
17823 std::fs::remove_dir(&module).expect("remove empty module cgroup");
17824 }
17825 std::fs::remove_dir(root.join("subc-modules")).unwrap();
17826 std::fs::remove_dir(&root).unwrap();
17827 assert!(
17828 !survived,
17829 "grandchild {grandchild} outlived module teardown"
17830 );
17831 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
17832 }
17833}
17834
17835#[cfg(all(test, unix))]
17838mod record_write_stall_tests {
17839 use super::*;
17840 use std::sync::atomic::{AtomicBool, Ordering};
17841 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
17842
17843 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
17846 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
17850
17851 fn sleeping_spec() -> ModuleSpec {
17863 ModuleSpec {
17864 module_id: "record-write-stall".into(),
17865 program: "/bin/sleep".into(),
17866 args: vec!["600".into()],
17867 env: vec![],
17868 reserved: false,
17869 reserved_prefixes: vec![],
17870 protocol: ModuleProtocol::None,
17871 overlap: Default::default(),
17872 }
17873 }
17874
17875 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17876 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17877 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17878 let supervisor =
17879 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17880 .with_live_children_record(home.join("live-children.json"));
17881 let module = supervisor.spawn(sleeping_spec()).unwrap();
17882 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17887 loop {
17888 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17889 .unwrap()
17890 .reported_pid()
17891 {
17892 return pid;
17893 }
17894 tokio::time::sleep(Duration::from_millis(10)).await;
17895 }
17896 })
17897 .await
17898 .expect("the first process was admitted as the module");
17899
17900 let key = vec![0x42; 32];
17903 let daemon_id = [0x24; 16];
17904 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17905 let port = listener.local_addr().unwrap().port();
17906 let server = tokio::spawn(crate::server::serve_listener(
17907 listener,
17908 Arc::new(crate::router::Router::with_default_self_handler()),
17909 crate::server::ServerAuth::with_limits(
17910 key.clone(),
17911 daemon_id,
17912 "test",
17913 crate::server::DEFAULT_AUTH_DEADLINE,
17914 4,
17915 ),
17916 ));
17917
17918 let (held_tx, held_rx) = std::sync::mpsc::channel();
17921 let armed = AtomicBool::new(true);
17922 supervisor
17923 .child_roster
17924 .set_record_write_hook_for_test(move |live| {
17925 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17926 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17927 let _ = held_tx.send(());
17928 std::thread::sleep(HELD_RECORD_WRITE);
17929 }
17930 });
17931
17932 let module_process = std::thread::spawn(move || {
17935 held_rx
17936 .recv_timeout(Duration::from_secs(30))
17937 .expect("the restart reached the replacement's record write");
17938 let conn = ConnectionInfo {
17939 schema: SCHEMA_VERSION,
17940 wire_version: None,
17941 endpoints: vec![Endpoint {
17942 host: "127.0.0.1".to_owned(),
17943 port,
17944 }],
17945 key,
17946 daemon_id,
17947 pid: std::process::id(),
17948 daemon_ver: "test".to_owned(),
17949 };
17950 tokio::runtime::Builder::new_current_thread()
17951 .enable_all()
17952 .build()
17953 .unwrap()
17954 .block_on(async {
17955 let started = Instant::now();
17956 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17957 .await
17958 .unwrap();
17959 let result =
17960 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17961 (result, started.elapsed())
17962 })
17963 });
17964
17965 module.restart(Some(0)).await.unwrap();
17966 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17967 .await
17968 .unwrap();
17969 assert!(
17970 auth.is_ok(),
17971 "the daemon must answer a module's handshake while a restart waits on the \
17972 live-children record write; got {auth:?} after {elapsed:?}"
17973 );
17974
17975 tokio::time::timeout(Duration::from_secs(20), async {
17976 while lock_snapshot(&module.inner.snapshot)
17977 .unwrap()
17978 .reported_pid()
17979 .is_none_or(|pid| pid == old_pid)
17980 {
17981 tokio::time::sleep(Duration::from_millis(10)).await;
17982 }
17983 })
17984 .await
17985 .expect("the restart published its replacement");
17986 module.stop().await.unwrap();
17987 server.abort();
17988 }
17989}