1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62mod slow_retry;
63use slow_retry::{FailedRetry, FailureContext};
64
65#[cfg(all(test, unix))]
66mod slow_retry_tests;
67
68pub const SUBC_ARG: &str = "--subc";
74
75const DEFAULT_MAX_RESTARTS: u32 = 3;
76const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
77const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
78const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
82pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
93const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
94const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
95const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
107const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
130pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
132const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
133pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
138
139#[cfg(target_os = "macos")]
140struct PrivacyExec {
141 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
142 deadline: tokio::time::Instant,
143 expected: Option<subc_os::FileIdentity>,
144 trampoline: Option<subc_os::FileIdentity>,
145 script: bool,
146 module_id: String,
147 #[cfg(test)]
148 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
149}
150
151#[cfg(all(test, target_os = "macos"))]
152pub(crate) fn test_privacy_trampoline() -> PathBuf {
153 let path = std::env::current_exe()
156 .unwrap()
157 .parent()
158 .unwrap()
159 .parent()
160 .unwrap()
161 .join("privacy-trampoline-fixture");
162 assert!(
166 path.exists(),
167 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
168 --bins --features test-support` or `cargo test -p subc-daemon` first",
169 path.display()
170 );
171 path
172}
173
174#[cfg(target_os = "macos")]
175fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
176 use std::io::Read;
177 let mut probe = std::process::Command::new(path)
178 .args(["__disclaim-exec", "--probe"])
179 .stdin(Stdio::null())
180 .stdout(Stdio::piped())
181 .stderr(Stdio::piped())
182 .spawn()
183 .map_err(|error| {
184 format!(
185 "privacy trampoline probe failed for {}: {error}",
186 path.display()
187 )
188 })?;
189 let deadline = std::time::Instant::now() + Duration::from_secs(5);
190 let status = loop {
191 match probe.try_wait() {
192 Ok(Some(status)) => break status,
193 Ok(None) if std::time::Instant::now() < deadline => {
194 std::thread::sleep(Duration::from_millis(5))
195 }
196 result => {
197 let _ = probe.kill();
198 let _ = probe.wait();
199 return Err(format!(
200 "privacy trampoline probe failed or timed out for {}: {result:?}",
201 path.display()
202 ));
203 }
204 }
205 };
206 let mut answer = String::new();
207 if let Some(stdout) = probe.stdout.take() {
208 let _ = stdout.take(256).read_to_string(&mut answer);
209 }
210 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
211 return Ok(());
212 }
213 let mut diagnostic = String::new();
214 if let Some(stderr) = probe.stderr.take() {
215 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
216 }
217 let cause = diagnostic
218 .trim()
219 .strip_prefix("ck-subc: own privacy identity refused: ")
220 .unwrap_or("binary does not implement the privacy trampoline protocol");
221 Err(format!(
222 "{cause}: probe of {} exited {status}",
223 path.display()
224 ))
225}
226
227#[cfg(target_os = "macos")]
228fn privacy_command(
229 spec: &ModuleSpec,
230 roster: &ChildRoster,
231) -> Result<
232 (
233 Command,
234 Option<PrivacyExec>,
235 subc_os::privacy_identity::ExecAcknowledgement,
236 ),
237 SuperviseError,
238> {
239 let failure = |cause: String| {
240 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
241 SuperviseError::Spawn {
242 program: spec.program.clone(),
243 source: io::Error::other(cause),
244 cgroup_path: None,
245 }
246 };
247 let trampoline = roster.privacy_trampoline().map_err(failure)?;
248 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
253 let path = spec
254 .env
255 .iter()
256 .find(|(key, _)| key == "PATH")
257 .map(|(_, value)| std::ffi::OsString::from(value))
258 .or_else(|| std::env::var_os("PATH"))
259 .unwrap_or_else(|| "/usr/bin:/bin".into());
260 std::env::split_paths(&path)
261 .map(|dir| dir.join(&spec.program))
262 .find(|path| path.is_file())
263 .unwrap_or_else(|| spec.program.clone())
264 } else {
265 spec.program.clone()
266 };
267 let expected = subc_os::file_identity(&program);
268 let trampoline_image = subc_os::file_identity(&trampoline);
269 let script = {
270 use std::io::Read;
271 let mut prefix = [0u8; 2];
272 std::fs::File::open(&program)
273 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
274 };
275 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
276 return Err(failure(
277 "privacy identity module executable is missing or is the trampoline itself".to_string(),
278 ));
279 }
280 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
281 .map_err(|error| failure(error.to_string()))?;
282 let reader =
283 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
284 let mut command = Command::new(&trampoline);
285 command
286 .arg("__disclaim-exec")
287 .arg(ack.fd().to_string())
288 .arg(&program);
289 ack.install(command.as_std_mut());
290 Ok((
291 command,
292 Some(PrivacyExec {
293 reader,
294 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
295 expected,
296 trampoline: trampoline_image,
297 script,
298 module_id: spec.module_id.clone(),
299 #[cfg(test)]
300 confirmation_pause: spec
301 .env
302 .iter()
303 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
304 .map(|(_, address)| (address.clone(), None)),
305 }),
306 ack,
307 ))
308}
309
310struct SupervisedChild {
311 child: Child,
312 exec: crate::activity::ExecOwner,
313 #[cfg(windows)]
314 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
315 #[cfg(target_os = "macos")]
316 privacy_exec: Option<PrivacyExec>,
317 #[cfg(target_os = "macos")]
324 report_ready: Arc<OnceLock<()>>,
325 spawn_failure: Option<String>,
327 protocol: ModuleProtocol,
332 #[cfg(target_os = "linux")]
338 module_id: String,
339 #[cfg(target_os = "linux")]
340 cgroup_placement: Option<subc_cgroup::Placement>,
341 #[cfg(windows)]
363 job: Option<subc_jobobject::JobObject>,
364 stdout_pump: Option<JoinHandle<()>>,
365 stderr_pump: Option<StderrPump>,
366 stderr_ring: Arc<Mutex<StderrRing>>,
367 spawned_at: Instant,
370 spawned_at_ms: u64,
371 spawned_from: PathBuf,
372 launch_spec_digest: LaunchSpecDigest,
373 spawned_file_identity: Option<SpawnedFileIdentity>,
374 process_start_time: Option<u64>,
375 process_identity: Option<ProcessIdentity>,
376 pid: u32,
377 roster_guard: Option<crate::child_roster::RosterGuard>,
380}
381
382impl SupervisedChild {
383 fn id(&self) -> Option<u32> {
384 Some(self.pid)
385 }
386
387 fn process_identity(&self) -> Option<ProcessIdentity> {
388 self.process_identity
389 }
390
391 async fn wait(&mut self) -> io::Result<ExitStatus> {
392 #[cfg(target_os = "macos")]
393 self.confirm_privacy_exec().await;
394 let result = self.child.wait().await;
402 #[cfg(target_os = "linux")]
403 if result.is_ok() {
404 if let Some(placement) = self.cgroup_placement.as_ref() {
405 cleanup_reaped_cgroup(placement, &self.module_id).await;
406 self.cgroup_placement = None;
409 }
410 }
411 result
412 }
413
414 #[cfg(target_os = "macos")]
415 async fn confirm_privacy_exec(&mut self) {
416 let Some(pending) = &mut self.privacy_exec else {
417 return;
418 };
419 let result = tokio::time::timeout_at(pending.deadline, async {
420 let mut record = Vec::new();
421 loop {
422 let mut ready = pending.reader.readable().await?;
423 let read = ready.try_io(|reader| {
424 use std::io::Read;
425 let mut reader = reader.get_ref();
426 let mut buffer = [0u8; 256];
427 reader.read(&mut buffer).map(|count| (count, buffer))
428 });
429 match read {
430 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
431 Ok(Ok((count, buffer))) => {
432 if record.len() + count > 1024 {
433 return Err(io::Error::other(
434 "privacy exec refusal record is too long",
435 ));
436 }
437 record.extend_from_slice(&buffer[..count]);
438 }
439 Ok(Err(error)) => return Err(error),
440 Err(_) => continue,
441 }
442 }
443 })
444 .await;
445 #[cfg(test)]
446 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
447 use tokio::io::{AsyncReadExt, AsyncWriteExt};
448 if stream.is_none() {
449 *stream = Some(
450 tokio::net::TcpStream::connect(address.as_str())
451 .await
452 .unwrap(),
453 );
454 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
455 stream
456 .as_mut()
457 .unwrap()
458 .write_all(&self.pid.to_ne_bytes())
459 .await
460 .unwrap();
461 }
462 let mut reply = [0];
463 stream
464 .as_mut()
465 .unwrap()
466 .read_exact(&mut reply)
467 .await
468 .unwrap();
469 pending.confirmation_pause = None;
470 if &reply == b"F" {
471 Ok(Err(io::Error::other("injected late exec refusal")))
472 } else {
473 assert_eq!(&reply, b"X");
474 result
475 }
476 } else {
477 result
478 };
479 let pending = self.privacy_exec.as_ref().expect("pending exec");
482 let cause = match result {
483 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
484 Ok(Err(error)) => Some(format!(
485 "privacy identity exec acknowledgement failed: {error}"
486 )),
487 Ok(Ok(record)) if !record.is_empty() => Some(
488 std::str::from_utf8(&record)
489 .ok()
490 .and_then(|record| {
491 record
492 .trim()
493 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
494 })
495 .filter(|cause| !cause.is_empty())
496 .unwrap_or("invalid privacy exec refusal record")
497 .to_string(),
498 ),
499 Ok(Ok(_)) => match self.child.try_wait() {
500 Ok(Some(_status)) => None,
504 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
505 Ok(None) => {
506 let image = observe_spawned_image(self.pid);
507 if let Some(image) = image.filter(|image| {
508 image.executable.is_some()
509 && image.executable != pending.trampoline
510 && (image.executable == pending.expected || pending.script)
511 }) {
512 if let Err(error) = self
513 .roster_guard
514 .as_ref()
515 .map(|guard| guard.confirm_executable(image))
516 .transpose()
517 {
518 Some(format!(
519 "could not durably record confirmed module image: {error}"
520 ))
521 } else {
522 let _ = self.report_ready.set(());
523 info!(module_id = %pending.module_id, pid = self.pid,
524 "module spawned with own privacy identity (responsibility disclaimed)");
525 None
526 }
527 } else if image.is_none()
528 || image.is_some_and(|image| image.executable.is_none())
529 {
530 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
537 Ok(Ok(_status)) => None,
538 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
539 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
540 }
541 } else {
542 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
543 }
544 }
545 },
546 };
547 let pending = self.privacy_exec.take().expect("pending exec");
548 self.exec.0.finish(if cause.is_some() {
551 crate::activity::ExecState::Refused
552 } else {
553 crate::activity::ExecState::CommittedExec
554 });
555 if let Some(cause) = cause {
556 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
557 self.spawn_failure = Some(cause);
558 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
561 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
562 }
563 let _ = self.child.start_kill();
564 }
565 }
566
567 fn release_roster(&mut self) {
571 self.roster_guard = None;
572 }
573
574 fn start_kill(&mut self) -> io::Result<()> {
587 #[cfg(windows)]
588 if let Some(job) = &self.job {
589 if let Err(error) = job.terminate() {
590 debug!(
591 error = %error,
592 "job termination failed; the direct-child kill still owns the outcome"
593 );
594 }
595 }
596 #[cfg(target_os = "linux")]
597 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
598 self.child.start_kill()
599 }
600
601 async fn drain_stderr(&mut self, module_id: &str) {
602 if let Some(pump) = self.stdout_pump.take() {
603 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
604 }
605
606 let Some(pump) = self.stderr_pump.take() else {
607 return;
608 };
609 settle_stderr_pump(
610 module_id,
611 &self.stderr_ring,
612 pump,
613 STDERR_PUMP_DRAIN_TIMEOUT,
614 )
615 .await;
616 }
617}
618
619async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
623 match timeout(bound, &mut pump).await {
624 Ok(Ok(())) => {}
625 Ok(Err(error)) => {
626 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
627 }
628 Err(_) => {
629 drop(pump);
630 warn!(
631 module_id,
632 waited = ?bound,
633 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
634 );
635 }
636 }
637}
638
639struct StderrPump {
642 task: JoinHandle<()>,
643 generation: u64,
644}
645
646async fn settle_stderr_pump(
652 module_id: &str,
653 ring: &Arc<Mutex<StderrRing>>,
654 pump: StderrPump,
655 bound: Duration,
656) {
657 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
658 let StderrPump {
659 mut task,
660 generation,
661 } = pump;
662 lock().retire_pump(generation);
663 match timeout(bound, &mut task).await {
664 Ok(Ok(())) => {}
665 Ok(Err(err)) => {
666 let mut ring = lock();
667 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
668 ring.finish_pump(generation);
669 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
670 }
671 Err(_) => {
672 drop(task);
674 lock().mark_pump_late(
675 generation,
676 format!(
677 "stderr of the exited process had not reached EOF {bound:?} after it was \
678 retired (a descendant may still hold the pipe open); lines it still \
679 writes are kept in that process's section"
680 ),
681 );
682 warn!(
683 module_id,
684 waited = ?bound,
685 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
686 );
687 }
688 }
689}
690
691fn registration_release_events() -> &'static watch::Sender<u64> {
692 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
693 EVENTS.get_or_init(|| {
694 let (sender, _receiver) = watch::channel(0);
695 sender
696 })
697}
698
699pub(crate) fn notify_registration_release() {
700 let events = registration_release_events();
701 let next_generation = (*events.borrow()).wrapping_add(1);
702 events.send_replace(next_generation);
703}
704
705#[derive(Debug, Clone, PartialEq, Eq)]
707pub struct ModuleSpec {
708 pub module_id: String,
709 pub program: PathBuf,
710 pub args: Vec<String>,
711 pub env: Vec<(String, String)>,
712 pub reserved: bool,
717 pub reserved_prefixes: Vec<String>,
722 pub protocol: ModuleProtocol,
741 pub overlap: ModuleOverlap,
746}
747
748#[derive(Clone, PartialEq, Eq)]
751pub(crate) struct LaunchSpecDigest([u8; 32]);
752
753impl fmt::Debug for LaunchSpecDigest {
754 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
755 f.write_str("LaunchSpecDigest([redacted])")
756 }
757}
758
759impl LaunchSpecDigest {
760 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
761 fn field(hash: &mut Sha256, bytes: &[u8]) {
762 hash.update((bytes.len() as u64).to_le_bytes());
763 hash.update(bytes);
764 }
765 let mut hash = Sha256::new();
766 if let Some(program) = spec.program.to_str() {
767 field(&mut hash, b"utf8-program");
768 field(&mut hash, program.as_bytes());
769 } else {
770 #[cfg(unix)]
773 {
774 use std::os::unix::ffi::OsStrExt;
775 field(&mut hash, b"unix-program");
776 field(&mut hash, spec.program.as_os_str().as_bytes());
777 }
778 #[cfg(windows)]
779 {
780 use std::os::windows::ffi::OsStrExt;
781 field(&mut hash, b"windows-program");
782 let bytes = spec
783 .program
784 .as_os_str()
785 .encode_wide()
786 .flat_map(u16::to_le_bytes)
787 .collect::<Vec<_>>();
788 field(&mut hash, &bytes);
789 }
790 #[cfg(not(any(unix, windows)))]
791 field(&mut hash, spec.program.to_string_lossy().as_bytes());
792 }
793 hash.update((spec.args.len() as u64).to_le_bytes());
794 for arg in &spec.args {
795 field(&mut hash, arg.as_bytes());
796 }
797 let mut env = spec.env.iter().collect::<Vec<_>>();
798 env.sort_unstable();
799 hash.update((env.len() as u64).to_le_bytes());
800 for (key, value) in env {
801 field(&mut hash, key.as_bytes());
802 field(&mut hash, value.as_bytes());
803 }
804 field(
805 &mut hash,
806 match spec.protocol {
807 ModuleProtocol::Subc => b"subc",
808 ModuleProtocol::None => b"none",
809 },
810 );
811 Self(hash.finalize().into())
812 }
813}
814
815#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
822pub enum ModuleOverlap {
823 #[default]
825 Exclusive,
826 Safe,
838}
839
840impl ModuleOverlap {
841 pub fn as_str(self) -> &'static str {
842 match self {
843 Self::Exclusive => "exclusive",
844 Self::Safe => "safe",
845 }
846 }
847}
848
849pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
859pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
861pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
866
867#[derive(Debug, Clone, Copy, PartialEq, Eq)]
885pub struct RestartPolicy {
886 pub max_restarts: u32,
887 pub backoff: Duration,
890 pub max_backoff: Duration,
892 pub window: Duration,
896}
897
898impl RestartPolicy {
899 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
903 Self {
904 max_restarts,
905 backoff,
906 max_backoff: DEFAULT_MAX_BACKOFF,
907 window: DEFAULT_RESTART_WINDOW,
908 }
909 }
910
911 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
912 self.max_backoff = max_backoff;
913 self
914 }
915
916 pub fn with_window(mut self, window: Duration) -> Self {
917 self.window = window;
918 self
919 }
920
921 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
926 if self.backoff.is_zero() || self.max_backoff.is_zero() {
927 return Duration::ZERO;
928 }
929
930 let mut delay = self.backoff;
931 for _ in 0..restart_in_window {
932 if delay >= self.max_backoff {
933 return self.max_backoff;
934 }
935 delay = delay
936 .checked_mul(10)
937 .unwrap_or(self.max_backoff)
938 .min(self.max_backoff);
939 }
940 delay.min(self.max_backoff)
941 }
942
943 fn budget_exhausted_detail(&self) -> String {
948 format!(
949 "crash budget exhausted: max_restarts={} within window_secs={}",
950 self.max_restarts,
951 self.window.as_secs()
952 )
953 }
954}
955
956impl Default for RestartPolicy {
957 fn default() -> Self {
958 Self {
959 max_restarts: DEFAULT_MAX_RESTARTS,
960 backoff: DEFAULT_BACKOFF,
961 max_backoff: DEFAULT_MAX_BACKOFF,
962 window: DEFAULT_RESTART_WINDOW,
963 }
964 }
965}
966
967#[derive(Debug, Clone, Copy, PartialEq, Eq)]
968struct CrashRestartSchedule {
969 restart_in_window: u32,
970 delay: Duration,
971}
972
973fn daemon_will_restart(
980 state: &mut SupervisorSnapshot,
981 policy: &RestartPolicy,
982 now: Instant,
983) -> bool {
984 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
985}
986
987const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
988const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
989const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
990const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
991
992#[derive(Debug, Clone, Copy, PartialEq, Eq)]
993pub enum HealthAction {
994 Report,
995 Restart,
996 Alert,
997}
998
999impl fmt::Display for HealthAction {
1000 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1001 f.write_str(match self {
1002 Self::Report => "report",
1003 Self::Restart => "restart",
1004 Self::Alert => "alert",
1005 })
1006 }
1007}
1008
1009#[derive(Debug, Clone, PartialEq, Eq)]
1010pub struct HealthConfig {
1011 pub http: Option<String>,
1015 pub cadence: Duration,
1016 pub deadline: Duration,
1017 pub failure_threshold: u32,
1018 pub on_degraded: HealthAction,
1019 pub on_failing: HealthAction,
1020 pub critical: bool,
1021}
1022
1023impl Default for HealthConfig {
1024 fn default() -> Self {
1025 Self {
1026 http: None,
1027 cadence: DEFAULT_HEALTH_CADENCE,
1028 deadline: DEFAULT_HEALTH_DEADLINE,
1029 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1030 on_degraded: HealthAction::Report,
1031 on_failing: HealthAction::Report,
1032 critical: false,
1033 }
1034 }
1035}
1036
1037#[derive(Debug, Clone, PartialEq)]
1055pub struct ModuleHealthStatus {
1056 pub status: SupervisorHealthStatus,
1057 pub last_probe_ms: Option<u64>,
1058 pub detail: Option<String>,
1059 pub metrics: Option<Value>,
1060 pub consecutive_failures: u32,
1061 pub late_answer_count: u64,
1064 pub last_late_answer_latency_ms: Option<u64>,
1066 pub last_action: Option<String>,
1067 pub last_action_ms: Option<u64>,
1071}
1072
1073impl Default for ModuleHealthStatus {
1074 fn default() -> Self {
1075 Self {
1076 status: SupervisorHealthStatus::Unknown,
1077 last_probe_ms: None,
1078 detail: None,
1079 metrics: None,
1080 consecutive_failures: 0,
1081 late_answer_count: 0,
1082 last_late_answer_latency_ms: None,
1083 last_action: None,
1084 last_action_ms: None,
1085 }
1086 }
1087}
1088
1089#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1091pub enum ModuleState {
1092 Starting,
1093 Running,
1094 Unresponsive,
1095 Restarting,
1096 Draining,
1097 Stopped,
1098 Failed,
1099 Disabled,
1100}
1101
1102impl fmt::Display for ModuleState {
1103 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1104 f.write_str(match self {
1105 Self::Starting => "starting",
1106 Self::Running => "running",
1107 Self::Unresponsive => "unresponsive",
1108 Self::Restarting => "restarting",
1109 Self::Draining => "draining",
1110 Self::Stopped => "stopped",
1111 Self::Failed => "failed",
1112 Self::Disabled => "disabled",
1113 })
1114 }
1115}
1116
1117#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1119pub enum ExitKind {
1120 Clean,
1121 Crash,
1122 DeliberateSeverance,
1123}
1124
1125impl From<ExitKind> for TerminalExitKind {
1126 fn from(kind: ExitKind) -> Self {
1127 match kind {
1128 ExitKind::Clean => Self::Clean,
1129 ExitKind::Crash => Self::Crash,
1130 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1131 }
1132 }
1133}
1134
1135#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1138pub(crate) struct ProcessIdentity {
1139 pub(crate) pid: u32,
1140 pub(crate) start_time: u64,
1141}
1142
1143#[derive(Debug, Clone, PartialEq, Eq)]
1145pub struct ExitReport {
1146 pub kind: ExitKind,
1147 pub code: Option<i32>,
1148 pub signal: Option<i32>,
1149 pub at_ms: u64,
1150}
1151
1152#[derive(Debug, Clone, PartialEq)]
1155pub struct ModuleStatus {
1156 pub module_id: String,
1157 pub waiting_for_previous_pid: Option<u32>,
1159 pub waiting_for_previous_detail: Option<String>,
1161 pub launch_nonce_source: Option<String>,
1166 pub state: ModuleState,
1167 pub enabled: bool,
1168 pub process_alive: bool,
1169 pub registration_active: bool,
1170 pub protocol: ModuleProtocol,
1175 pub live: bool,
1186 pub restart_count: u32,
1190 pub lifetime_restarts: u32,
1194 pub spawn_generation: u64,
1195 pub max_restarts: u32,
1200 pub restart_window: Duration,
1204 pub drain_timeout: Duration,
1208 pub restart_backoff: Duration,
1209 pub restart_max_backoff: Duration,
1210 pub pid: Option<u32>,
1215 pub spawned_at_ms: Option<u64>,
1216 pub spawned_from: Option<PathBuf>,
1217 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1218 pub process_start_time: Option<u64>,
1219 pub last_exit: Option<ExitReport>,
1220 pub health: ModuleHealthStatus,
1221}
1222
1223#[cfg(test)]
1224#[derive(Debug, Clone, Copy, PartialEq)]
1225struct ActorSelectCheckpoint {
1226 generation: u64,
1227 turn: u64,
1228 registered_connection: Option<ConnectionId>,
1229 next_probe_at: Option<Instant>,
1230 wake_after: Option<Duration>,
1231}
1232
1233#[derive(Debug, Clone, PartialEq)]
1234struct SupervisorSnapshot {
1235 failed_retry: FailedRetry,
1236 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1237 #[cfg(windows)]
1238 pipe_consumed: Option<Arc<OnceLock<()>>>,
1239 #[cfg(test)]
1241 actor_turns: u64,
1242 #[cfg(test)]
1245 actor_select: Option<ActorSelectCheckpoint>,
1246 state: ModuleState,
1247 enabled: bool,
1248 process_alive: bool,
1249 spawned_protocol: Option<ModuleProtocol>,
1250 spawn_failure: Option<String>,
1251 waiting_for_previous_pid: Option<u32>,
1252 waiting_for_previous_detail: Option<String>,
1253 crash_restarts: VecDeque<Instant>,
1259 lifetime_restarts: u32,
1260 spawn_generation: u64,
1269 pid: Option<u32>,
1270 #[cfg(target_os = "macos")]
1271 report_ready: Option<Arc<OnceLock<()>>>,
1272 reaped_pid: Option<u32>,
1274 respawn_pending: bool,
1276 coalesced_restart_pending: bool,
1278 spawned_at_ms: Option<u64>,
1279 spawned_from: Option<PathBuf>,
1280 launch_spec_digest: Option<LaunchSpecDigest>,
1281 spawned_file_identity: Option<SpawnedFileIdentity>,
1282 process_start_time: Option<u64>,
1283 deliberate_severance: Option<ProcessIdentity>,
1284 last_exit: Option<ExitReport>,
1285 drain_disposition_detail: Option<String>,
1287 health: ModuleHealthStatus,
1288 in_alternate_slot: bool,
1293 draining_to_replace: bool,
1300 configuration_updated_since_spawn: bool,
1306}
1307
1308impl SupervisorSnapshot {
1309 fn reported_pid(&self) -> Option<u32> {
1314 #[cfg(target_os = "macos")]
1315 if self
1316 .report_ready
1317 .as_ref()
1318 .is_some_and(|ready| ready.get().is_none())
1319 {
1320 return None;
1321 }
1322 self.pid
1323 }
1324
1325 fn starting() -> Self {
1326 Self::new(ModuleState::Starting, true)
1327 }
1328
1329 fn disabled() -> Self {
1330 Self::new(ModuleState::Disabled, false)
1331 }
1332
1333 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1337 while let Some(oldest) = self.crash_restarts.front() {
1338 if now.duration_since(*oldest) > window {
1339 self.crash_restarts.pop_front();
1340 } else {
1341 break;
1342 }
1343 }
1344 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1345 }
1346
1347 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1353 self.crash_restarts.push_back(now);
1354 while self.crash_restarts.len() > policy.max_restarts as usize {
1355 self.crash_restarts.pop_front();
1356 }
1357 self.lifetime_restarts += 1;
1358 }
1359
1360 fn next_crash_restart(
1364 &mut self,
1365 policy: &RestartPolicy,
1366 now: Instant,
1367 ) -> Option<CrashRestartSchedule> {
1368 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1369 if restart_in_window >= policy.max_restarts {
1370 return None;
1371 }
1372 self.record_crash_restart(policy, now);
1373 Some(CrashRestartSchedule {
1374 restart_in_window,
1375 delay: policy.delay_for_restart(restart_in_window),
1376 })
1377 }
1378
1379 fn clear_crash_restarts(&mut self) {
1384 self.crash_restarts.clear();
1385 }
1386
1387 fn new(state: ModuleState, enabled: bool) -> Self {
1388 Self {
1389 failed_retry: FailedRetry::default(),
1390 exec_outcome: None,
1391 #[cfg(test)]
1392 actor_turns: 0,
1393 #[cfg(test)]
1394 actor_select: None,
1395 state,
1396 enabled,
1397 process_alive: false,
1398 #[cfg(windows)]
1399 pipe_consumed: None,
1400 spawned_protocol: None,
1401 spawn_failure: None,
1402 waiting_for_previous_pid: None,
1403 waiting_for_previous_detail: None,
1404 crash_restarts: VecDeque::new(),
1405 lifetime_restarts: 0,
1406 spawn_generation: 0,
1407 pid: None,
1408 #[cfg(target_os = "macos")]
1409 report_ready: None,
1410 reaped_pid: None,
1411 respawn_pending: false,
1412 coalesced_restart_pending: false,
1413 spawned_at_ms: None,
1414 spawned_from: None,
1415 launch_spec_digest: None,
1416 spawned_file_identity: None,
1417 process_start_time: None,
1418 deliberate_severance: None,
1419 last_exit: None,
1420 drain_disposition_detail: None,
1421 health: ModuleHealthStatus::default(),
1422 in_alternate_slot: false,
1423 draining_to_replace: false,
1424 configuration_updated_since_spawn: false,
1425 }
1426 }
1427}
1428
1429type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1430
1431type SpawnSubscriberKey = (ConnectionId, u64);
1432
1433#[derive(Debug)]
1434struct SpawnSubscriber {
1435 version: u8,
1436 frames: mpsc::Sender<Frame>,
1437 lagged: Option<oneshot::Sender<SpawnCursor>>,
1441}
1442
1443#[derive(Debug)]
1444struct SpawnEventState {
1445 daemon_incarnation: String,
1446 seq: u64,
1447 capacity: usize,
1448 live: HashMap<String, LiveSpawn>,
1449 generations: HashMap<String, u64>,
1450 events: VecDeque<SpawnEvent>,
1451 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1452}
1453
1454impl Default for SpawnEventState {
1455 fn default() -> Self {
1456 Self {
1457 daemon_incarnation: "unconfigured".to_string(),
1458 seq: 0,
1459 capacity: SPAWN_EVENT_RING_CAPACITY,
1460 live: HashMap::new(),
1461 generations: HashMap::new(),
1462 events: VecDeque::new(),
1463 subscribers: HashMap::new(),
1464 }
1465 }
1466}
1467
1468#[derive(Debug, Clone, Default)]
1469struct SpawnEventFeed(
1470 Arc<Mutex<SpawnEventState>>,
1471 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1472);
1473
1474#[derive(Debug, Clone, PartialEq, Eq)]
1475pub(crate) enum SpawnSubscribeRefusal {
1476 ForeignIncarnation { current: String },
1477 TooOld { oldest: SpawnCursor },
1478 Frame(String),
1479}
1480
1481impl SpawnEventFeed {
1482 fn configure_incarnation(&self, daemon_incarnation: String) {
1483 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1484 state.daemon_incarnation = daemon_incarnation;
1485 state.seq = 0;
1486 state.live.clear();
1487 state.generations.clear();
1488 state.events.clear();
1489 state.subscribers.clear();
1490 }
1491
1492 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1493 SpawnCursor {
1494 daemon_incarnation: state.daemon_incarnation.clone(),
1495 seq: state.seq,
1496 }
1497 }
1498
1499 fn snapshot(&self) -> SpawnSnapshot {
1500 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1501 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1502 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1503 SpawnSnapshot {
1504 cursor: Self::cursor(&state),
1505 ring_bound: state.capacity as u64,
1506 live,
1507 }
1508 }
1509
1510 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1511 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1512 let generation = state
1513 .generations
1514 .get(module_id)
1515 .copied()
1516 .unwrap_or(0)
1517 .checked_add(1)
1518 .expect("spawn generation exhausted");
1519 state.generations.insert(module_id.to_string(), generation);
1520 let live = LiveSpawn {
1521 module_id: module_id.to_string(),
1522 spawn_generation: generation,
1523 pid,
1524 spawned_at_ms,
1525 };
1526 state.live.insert(module_id.to_string(), live);
1527 Self::emit_locked(
1528 &mut state,
1529 SpawnEventKind::Spawned,
1530 module_id.to_string(),
1531 generation,
1532 pid,
1533 None,
1534 None,
1535 );
1536 generation
1537 }
1538
1539 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1540 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1541 let Some(live) = state.live.remove(module_id) else {
1542 warn!(
1543 module_id,
1544 "terminal record had no live spawn event identity"
1545 );
1546 return;
1547 };
1548 Self::emit_locked(
1549 &mut state,
1550 SpawnEventKind::Exited,
1551 module_id.to_string(),
1552 live.spawn_generation,
1553 live.pid,
1554 exit_code,
1555 exit_signal,
1556 );
1557 }
1558
1559 fn emit_superseded_exited(
1566 &self,
1567 module_id: &str,
1568 spawn_generation: u64,
1569 pid: u32,
1570 exit_code: Option<i32>,
1571 exit_signal: Option<i32>,
1572 ) {
1573 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1574 if state
1575 .live
1576 .get(module_id)
1577 .is_some_and(|live| live.spawn_generation == spawn_generation)
1578 {
1579 state.live.remove(module_id);
1580 }
1581 Self::emit_locked(
1582 &mut state,
1583 SpawnEventKind::Exited,
1584 module_id.to_string(),
1585 spawn_generation,
1586 pid,
1587 exit_code,
1588 exit_signal,
1589 );
1590 }
1591
1592 #[allow(clippy::too_many_arguments)]
1593 fn emit_locked(
1594 state: &mut SpawnEventState,
1595 kind: SpawnEventKind,
1596 module_id: String,
1597 spawn_generation: u64,
1598 pid: u32,
1599 exit_code: Option<i32>,
1600 exit_signal: Option<i32>,
1601 ) {
1602 state.seq = state
1603 .seq
1604 .checked_add(1)
1605 .expect("spawn event sequence exhausted");
1606 let event = SpawnEvent {
1607 cursor: Self::cursor(state),
1608 kind,
1609 module_id,
1610 spawn_generation,
1611 pid,
1612 exit_code,
1613 exit_signal,
1614 };
1615 state.events.push_back(event.clone());
1616 while state.events.len() > state.capacity {
1617 state.events.pop_front();
1618 }
1619 let body = match serde_json::to_vec(&event) {
1620 Ok(body) => body,
1621 Err(error) => {
1622 error!(%error, "failed to serialize supervisor spawn event");
1623 return;
1624 }
1625 };
1626 state.subscribers.retain(|(connection_id, corr), subscriber| {
1627 let frame = Frame::build_with_version(
1628 subscriber.version,
1629 FrameType::StreamData,
1630 control_flags(),
1631 0,
1632 0,
1633 *corr,
1634 body.clone(),
1635 );
1636 match frame {
1637 Ok(frame) => {
1638 if subscriber.frames.try_send(frame).is_ok() {
1639 true
1640 } else {
1641 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1642 if let Some(lagged) = subscriber.lagged.take() {
1643 let _ = lagged.send(event.cursor.clone());
1644 }
1645 false
1646 }
1647 }
1648 Err(error) => {
1649 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1650 false
1651 }
1652 }
1653 });
1654 }
1655
1656 fn subscribe(
1657 &self,
1658 connection_id: ConnectionId,
1659 corr: u64,
1660 version: u8,
1661 since: Option<SpawnCursor>,
1662 sink: FrameSink,
1663 ) -> Result<(), SpawnSubscribeRefusal> {
1664 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1665 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1666 {
1667 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1668 let replay = if let Some(since) = since {
1669 if since.daemon_incarnation != state.daemon_incarnation {
1670 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1671 current: state.daemon_incarnation.clone(),
1672 });
1673 }
1674 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1675 if since.seq < oldest.seq.saturating_sub(1) {
1676 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1677 }
1678 }
1679 state
1680 .events
1681 .iter()
1682 .filter(|event| event.cursor.seq > since.seq)
1683 .cloned()
1684 .collect::<Vec<_>>()
1685 } else {
1686 Vec::new()
1687 };
1688 for event in replay {
1689 let body = serde_json::to_vec(&event)
1690 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1691 let frame = Frame::build_with_version(
1692 version,
1693 FrameType::StreamData,
1694 control_flags(),
1695 0,
1696 0,
1697 corr,
1698 body,
1699 )
1700 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1701 frames
1702 .try_send(frame)
1703 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1704 }
1705 state.subscribers.insert(
1706 (connection_id, corr),
1707 SpawnSubscriber {
1708 version,
1709 frames,
1710 lagged: Some(lagged),
1711 },
1712 );
1713 }
1714 tokio::spawn(async move {
1725 while let Some(frame) = receiver.recv().await {
1726 if sink.send(frame).await.is_err() {
1727 return;
1728 }
1729 }
1730 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1731 return;
1732 };
1733 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1734 Ok(frame) => {
1735 let _ = sink.send(frame).await;
1736 }
1737 Err(error) => {
1738 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1739 }
1740 }
1741 });
1742 Ok(())
1743 }
1744
1745 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1746 let Some(subscriber) = self
1747 .0
1748 .lock()
1749 .unwrap_or_else(|p| p.into_inner())
1750 .subscribers
1751 .remove(&(connection_id, corr))
1752 else {
1753 return false;
1754 };
1755 if let Ok(frame) = Frame::build_with_version(
1756 subscriber.version,
1757 FrameType::StreamEnd,
1758 control_flags(),
1759 0,
1760 0,
1761 corr,
1762 Vec::new(),
1763 ) {
1764 tokio::spawn(async move {
1765 let _ = subscriber.frames.send(frame).await;
1766 });
1767 }
1768 true
1769 }
1770
1771 fn remove_connection(&self, connection_id: ConnectionId) {
1772 self.0
1773 .lock()
1774 .unwrap_or_else(|p| p.into_inner())
1775 .subscribers
1776 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1777 }
1778
1779 #[cfg(any(test, feature = "test-support"))]
1780 fn set_capacity(&self, capacity: usize) {
1781 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1782 }
1783
1784 #[cfg(any(test, feature = "test-support"))]
1785 fn subscriber_count(&self) -> usize {
1786 self.0
1787 .lock()
1788 .unwrap_or_else(|p| p.into_inner())
1789 .subscribers
1790 .len()
1791 }
1792}
1793
1794fn spawn_subscriber_lagged_frame(
1797 version: u8,
1798 corr: u64,
1799 first_undelivered: SpawnCursor,
1800) -> Result<Frame, String> {
1801 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1802 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1803 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1804 .to_string(),
1805 detail: Some(serde_json::json!({
1806 "first_undelivered_cursor": first_undelivered
1807 })),
1808 })
1809 .map_err(|error| error.to_string())?;
1810 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1811 .map_err(|error| error.to_string())
1812}
1813
1814pub trait ModuleProcessLiveness: Send + Sync {
1815 fn process_live(&self, module_id: &str) -> Option<bool>;
1816
1817 fn process_replacing(&self, _module_id: &str) -> bool {
1823 false
1824 }
1825}
1826
1827#[derive(Debug, Clone, Default)]
1829pub struct SupervisorProcessLiveness {
1830 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1831}
1832
1833impl SupervisorProcessLiveness {
1834 pub fn new() -> Self {
1835 Self::default()
1836 }
1837
1838 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1839 let mut snapshots = self
1840 .snapshots
1841 .lock()
1842 .unwrap_or_else(|poisoned| poisoned.into_inner());
1843 snapshots.insert(module_id, snapshot);
1844 }
1845
1846 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1847 let mut snapshots = self
1848 .snapshots
1849 .lock()
1850 .unwrap_or_else(|poisoned| poisoned.into_inner());
1851 let is_current = snapshots
1852 .get(module_id)
1853 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1854 .unwrap_or(false);
1855 if is_current {
1856 snapshots.remove(module_id);
1857 }
1858 }
1859}
1860
1861impl ModuleProcessLiveness for SupervisorProcessLiveness {
1862 fn process_live(&self, module_id: &str) -> Option<bool> {
1863 let snapshot = {
1864 let snapshots = self
1865 .snapshots
1866 .lock()
1867 .unwrap_or_else(|poisoned| poisoned.into_inner());
1868 snapshots.get(module_id).cloned()
1869 }?;
1870 let snapshot = snapshot
1871 .lock()
1872 .unwrap_or_else(|poisoned| poisoned.into_inner());
1873 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1874 }
1875
1876 fn process_replacing(&self, module_id: &str) -> bool {
1877 let Some(snapshot) = self
1878 .snapshots
1879 .lock()
1880 .unwrap_or_else(|poisoned| poisoned.into_inner())
1881 .get(module_id)
1882 .cloned()
1883 else {
1884 return false;
1885 };
1886 let snapshot = snapshot
1887 .lock()
1888 .unwrap_or_else(|poisoned| poisoned.into_inner());
1889 snapshot.enabled
1890 && match snapshot.state {
1891 ModuleState::Restarting => true,
1892 ModuleState::Draining => snapshot.draining_to_replace,
1893 ModuleState::Starting
1894 | ModuleState::Running
1895 | ModuleState::Unresponsive
1896 | ModuleState::Stopped
1897 | ModuleState::Failed
1898 | ModuleState::Disabled => false,
1899 }
1900 }
1901}
1902
1903#[cfg(test)]
1904#[derive(Debug, Default)]
1905struct ReloadExitRecordGate {
1906 reached: tokio::sync::Notify,
1907 resume: tokio::sync::Notify,
1908}
1909
1910#[derive(Debug, Clone, Copy)]
1911enum RespawnKind {
1912 Spawn,
1913 Reload,
1914}
1915
1916#[derive(Debug, Clone, Copy)]
1917struct PendingRespawn {
1918 deadline: Instant,
1919 kind: RespawnKind,
1920 daemon_initiated: bool,
1921}
1922
1923type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1924
1925#[derive(Debug, Clone)]
1926struct SupervisorRuntimeConfig {
1927 recovery_is_daemon: bool,
1929 start_requester: Option<Principal>,
1930 pending_activity_restart: bool,
1931 provenance_probe: ExecutableIdentityProbe,
1932 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1934 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1936 restart_policy: RestartPolicy,
1937 drain_timeout: Duration,
1940 effective_drain_timeout: Arc<Mutex<Duration>>,
1943 default_drain_timeout: Duration,
1946 health: HealthConfig,
1947 connection_file_path: Option<PathBuf>,
1948 capture_logs_dir: Option<PathBuf>,
1949 forwarding: Option<Arc<ForwardingTable>>,
1950 supervisor_handle: Option<SupervisorHandle>,
1953 stderr_ring: Arc<Mutex<StderrRing>>,
1960 terminal_ring: Arc<Mutex<TerminalRing>>,
1961 spawn_events: SpawnEventFeed,
1962 child_roster: ChildRoster,
1963 #[cfg(target_os = "linux")]
1964 cgroup_placement: Option<subc_cgroup::Placement>,
1965 #[cfg(test)]
1966 test_seed_stale_facts_before_enable_spawn: bool,
1967 #[cfg(test)]
1968 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1969}
1970
1971impl SupervisorRuntimeConfig {
1972 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
1973 FailureContext {
1974 daemon_initiated,
1975 shutdown: self.child_roster.shutdown_flag(),
1976 }
1977 }
1978}
1979
1980#[derive(Debug, Clone, PartialEq, Eq)]
1981struct SupervisedConfiguration {
1982 spec: ModuleSpec,
1983 health: HealthConfig,
1984}
1985
1986#[derive(Debug, Clone, Default)]
1992pub struct SupervisorHandle {
1993 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1994 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1995 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1996 configured_ids: Arc<Mutex<HashSet<String>>>,
2008 spawn_events: SpawnEventFeed,
2009 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2020 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2026 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2030 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2038 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2044 promotion_observer: PromotionObserverSlot,
2046 operation_lock: Arc<AsyncMutex<()>>,
2050}
2051
2052pub(crate) trait SwapPromotionObserver: Send + Sync {
2061 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2062}
2063
2064#[derive(Clone, Default)]
2068struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2069
2070impl fmt::Debug for PromotionObserverSlot {
2071 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2072 f.write_str("PromotionObserverSlot")
2073 }
2074}
2075
2076#[derive(Debug, Clone)]
2078struct OpenSwap {
2079 candidate_nonce: String,
2082 incumbent_nonce: Option<String>,
2087 candidate_admitted: bool,
2091}
2092
2093#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2096pub(crate) enum SwapHelloAdmission {
2097 NotSwapping,
2100 Candidate,
2102 Refused,
2105}
2106
2107#[derive(Debug, Clone, PartialEq, Eq)]
2108pub(crate) enum ReservedHelloRejection {
2109 Exact {
2110 module_id: String,
2111 },
2112 Prefix {
2113 prefix: String,
2114 owner_module_id: String,
2115 },
2116}
2117
2118impl SupervisorHandle {
2119 pub fn new() -> Self {
2120 Self::default()
2121 }
2122
2123 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2124 self.spawn_events.snapshot()
2125 }
2126
2127 pub(crate) fn subscribe_spawns(
2128 &self,
2129 connection_id: ConnectionId,
2130 corr: u64,
2131 version: u8,
2132 since: Option<SpawnCursor>,
2133 sink: FrameSink,
2134 ) -> Result<(), SpawnSubscribeRefusal> {
2135 self.spawn_events
2136 .subscribe(connection_id, corr, version, since, sink)
2137 }
2138
2139 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2140 self.spawn_events.cancel(connection_id, corr)
2141 }
2142
2143 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2144 self.spawn_events.remove_connection(connection_id);
2145 }
2146
2147 #[cfg(any(test, feature = "test-support"))]
2148 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2149 assert!(capacity > 0, "spawn event capacity must be non-zero");
2150 self.spawn_events.set_capacity(capacity);
2151 }
2152
2153 #[cfg(any(test, feature = "test-support"))]
2154 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2155 self.spawn_events.subscriber_count()
2156 }
2157
2158 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2161 self.spawn_nonces
2162 .lock()
2163 .unwrap_or_else(|poisoned| poisoned.into_inner())
2164 .insert(module_id.to_string(), nonce);
2165 }
2166
2167 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2170 self.reserved_nonces
2171 .lock()
2172 .unwrap_or_else(|poisoned| poisoned.into_inner())
2173 .insert(module_id.to_string(), Some(nonce));
2174 }
2175
2176 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2178 let mut owners = self
2179 .reserved_prefix_owners
2180 .lock()
2181 .unwrap_or_else(|poisoned| poisoned.into_inner());
2182 owners.retain(|_, owner| owner != owner_module_id);
2183 for prefix in prefixes {
2184 owners.insert(prefix.clone(), owner_module_id.to_string());
2185 }
2186 }
2187
2188 #[cfg(test)]
2190 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2191 self.spawn_nonces
2192 .lock()
2193 .unwrap_or_else(|poisoned| poisoned.into_inner())
2194 .get(module_id)
2195 .cloned()
2196 }
2197
2198 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2199 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2200 let spawn_nonce = self
2201 .spawn_nonces
2202 .lock()
2203 .unwrap_or_else(|poisoned| poisoned.into_inner())
2204 .get(&spec.module_id)
2205 .cloned();
2206 let mut reserved_nonces = self
2207 .reserved_nonces
2208 .lock()
2209 .unwrap_or_else(|poisoned| poisoned.into_inner());
2210 if spec.reserved {
2211 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2216 }
2217 drop(reserved_nonces);
2218 self.removal_tombstones
2222 .lock()
2223 .unwrap_or_else(|poisoned| poisoned.into_inner())
2224 .remove(&spec.module_id);
2225 }
2226
2227 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2232 self.reserved_hello_verification(module_id, presented)
2233 .is_ok()
2234 }
2235
2236 pub(crate) fn reserved_hello_verification(
2239 &self,
2240 module_id: &str,
2241 presented: Option<&str>,
2242 ) -> Result<bool, ReservedHelloRejection> {
2243 let nonces = self
2244 .reserved_nonces
2245 .lock()
2246 .unwrap_or_else(|poisoned| poisoned.into_inner());
2247 if let Some(expected) = nonces.get(module_id) {
2248 let authorized = match expected {
2252 Some(expected) => {
2253 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2254 }
2255 None => false,
2256 };
2257 if authorized {
2258 return Ok(true);
2259 }
2260 return Err(ReservedHelloRejection::Exact {
2261 module_id: module_id.to_string(),
2262 });
2263 }
2264 drop(nonces);
2265
2266 let matched_prefix = self
2267 .reserved_prefix_owners
2268 .lock()
2269 .unwrap_or_else(|poisoned| poisoned.into_inner())
2270 .iter()
2271 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2272 .max_by_key(|(prefix, _)| prefix.len())
2273 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2274 let Some((prefix, owner_module_id)) = matched_prefix else {
2275 return Ok(false);
2276 };
2277
2278 let authorized = presented.is_some_and(|presented| {
2279 self.spawn_nonces
2280 .lock()
2281 .unwrap_or_else(|poisoned| poisoned.into_inner())
2282 .get(&owner_module_id)
2283 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2284 || self.swap_nonce_matches(&owner_module_id, presented)
2287 });
2288 if authorized {
2289 Ok(true)
2290 } else {
2291 Err(ReservedHelloRejection::Prefix {
2292 prefix,
2293 owner_module_id,
2294 })
2295 }
2296 }
2297
2298 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2303 if presented.is_empty() {
2304 return false;
2305 }
2306 let nonces = self
2307 .spawn_nonces
2308 .lock()
2309 .unwrap_or_else(|poisoned| poisoned.into_inner());
2310 let current = nonces
2311 .get(module_id)
2312 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2313 drop(nonces);
2314 current || self.swap_nonce_matches(module_id, presented)
2319 }
2320
2321 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2323 let swaps = self
2324 .swaps
2325 .lock()
2326 .unwrap_or_else(|poisoned| poisoned.into_inner());
2327 swaps.get(module_id).is_some_and(|swap| {
2328 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2329 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2330 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2331 })
2332 })
2333 }
2334
2335 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2338 let incumbent_nonce = self
2339 .spawn_nonces
2340 .lock()
2341 .unwrap_or_else(|poisoned| poisoned.into_inner())
2342 .get(module_id)
2343 .cloned();
2344 self.swaps
2345 .lock()
2346 .unwrap_or_else(|poisoned| poisoned.into_inner())
2347 .insert(
2348 module_id.to_string(),
2349 OpenSwap {
2350 candidate_nonce,
2351 incumbent_nonce,
2352 candidate_admitted: false,
2353 },
2354 );
2355 }
2356
2357 pub(crate) fn close_swap(&self, module_id: &str) {
2360 self.swaps
2361 .lock()
2362 .unwrap_or_else(|poisoned| poisoned.into_inner())
2363 .remove(module_id);
2364 }
2365
2366 pub(crate) fn set_swap_promotion_observer(
2369 &self,
2370 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2371 ) {
2372 *self
2373 .promotion_observer
2374 .0
2375 .lock()
2376 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2377 }
2378
2379 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2382 let observer = self
2383 .promotion_observer
2384 .0
2385 .lock()
2386 .unwrap_or_else(|poisoned| poisoned.into_inner())
2387 .as_ref()
2388 .and_then(std::sync::Weak::upgrade);
2389 if let Some(observer) = observer {
2390 observer.swap_promoted(registration);
2391 }
2392 }
2393
2394 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2396 self.swaps
2397 .lock()
2398 .unwrap_or_else(|poisoned| poisoned.into_inner())
2399 .contains_key(module_id)
2400 }
2401
2402 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2407 let candidate_nonce = self
2408 .swaps
2409 .lock()
2410 .unwrap_or_else(|poisoned| poisoned.into_inner())
2411 .get(module_id)
2412 .map(|swap| swap.candidate_nonce.clone());
2413 let Some(nonce) = candidate_nonce else {
2414 return;
2415 };
2416 self.set_spawn_nonce(module_id, nonce.clone());
2417 if reserved {
2418 self.set_reserved_nonce(module_id, nonce);
2419 }
2420 }
2421
2422 pub(crate) fn swap_hello_admission(
2437 &self,
2438 module_id: &str,
2439 presented: Option<&str>,
2440 ) -> SwapHelloAdmission {
2441 let swaps = self
2442 .swaps
2443 .lock()
2444 .unwrap_or_else(|poisoned| poisoned.into_inner());
2445 let Some(swap) = swaps.get(module_id) else {
2446 return SwapHelloAdmission::NotSwapping;
2447 };
2448 let Some(presented) = presented else {
2449 return SwapHelloAdmission::Refused;
2450 };
2451 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2452 return if swap.candidate_admitted {
2453 SwapHelloAdmission::Refused
2454 } else {
2455 SwapHelloAdmission::Candidate
2456 };
2457 }
2458 if swap
2459 .incumbent_nonce
2460 .as_deref()
2461 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2462 {
2463 return SwapHelloAdmission::NotSwapping;
2464 }
2465 SwapHelloAdmission::Refused
2466 }
2467
2468 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2471 if let Some(swap) = self
2472 .swaps
2473 .lock()
2474 .unwrap_or_else(|poisoned| poisoned.into_inner())
2475 .get_mut(module_id)
2476 {
2477 swap.candidate_admitted = true;
2478 }
2479 }
2480
2481 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2483 self.spawn_nonces
2484 .lock()
2485 .unwrap_or_else(|poisoned| poisoned.into_inner())
2486 .get(module_id)
2487 .cloned()
2488 }
2489
2490 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2492 self.reserved_nonces
2493 .lock()
2494 .unwrap_or_else(|poisoned| poisoned.into_inner())
2495 .get(module_id)
2496 .cloned()
2497 .flatten()
2498 }
2499
2500 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2501 self.mark_configured(module.module_id());
2505 let mut modules = self
2506 .modules
2507 .lock()
2508 .unwrap_or_else(|poisoned| poisoned.into_inner());
2509 modules.insert(module.module_id().to_string(), module)
2510 }
2511
2512 fn mark_configured(&self, module_id: &str) {
2516 self.configured_ids
2517 .lock()
2518 .unwrap_or_else(|poisoned| poisoned.into_inner())
2519 .insert(module_id.to_string());
2520 }
2521
2522 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2526 let modules = self
2527 .modules
2528 .lock()
2529 .unwrap_or_else(|poisoned| poisoned.into_inner());
2530 if !modules.contains_key(module_id) {
2531 self.configured_ids
2532 .lock()
2533 .unwrap_or_else(|poisoned| poisoned.into_inner())
2534 .remove(module_id);
2535 }
2536 }
2537
2538 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2546 self.configured_ids
2547 .lock()
2548 .unwrap_or_else(|poisoned| poisoned.into_inner())
2549 .contains(module_id)
2550 }
2551
2552 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2553 let modules = self
2554 .modules
2555 .lock()
2556 .unwrap_or_else(|poisoned| poisoned.into_inner());
2557 modules.get(module_id).cloned()
2558 }
2559
2560 pub(crate) fn record_late_health_answer(
2561 &self,
2562 module_id: &str,
2563 latency_ms: u64,
2564 ) -> Result<bool, SuperviseError> {
2565 let Some(module) = self.get(module_id) else {
2566 return Ok(false);
2567 };
2568 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2569 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2570 state.health.last_late_answer_latency_ms = Some(latency_ms);
2571 state.health.consecutive_failures = 0;
2579 })?;
2580 Ok(true)
2581 }
2582
2583 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2589 let Some(module) = self.get(module_id) else {
2590 return Ok(false);
2591 };
2592 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2593 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2594 return Ok(false);
2595 };
2596 drop(snapshot);
2597 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2598 }
2599
2600 pub fn list(&self) -> Vec<SupervisedModule> {
2601 let modules = self
2602 .modules
2603 .lock()
2604 .unwrap_or_else(|poisoned| poisoned.into_inner());
2605 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2606 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2607 modules
2608 }
2609
2610 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2611 self.activity_images
2612 .lock()
2613 .unwrap_or_else(|p| p.into_inner())
2614 .remove(module_id);
2615 self.spawn_nonces
2616 .lock()
2617 .unwrap_or_else(|poisoned| poisoned.into_inner())
2618 .remove(module_id);
2619 self.close_swap(module_id);
2620 let mut reserved_nonces = self
2621 .reserved_nonces
2622 .lock()
2623 .unwrap_or_else(|poisoned| poisoned.into_inner());
2624 if reserved_nonces.contains_key(module_id) {
2625 reserved_nonces.insert(module_id.to_string(), None);
2628 }
2629 drop(reserved_nonces);
2630 self.reserved_prefix_owners
2631 .lock()
2632 .unwrap_or_else(|poisoned| poisoned.into_inner())
2633 .retain(|_, owner| owner != module_id);
2634 let removed = self
2635 .modules
2636 .lock()
2637 .unwrap_or_else(|poisoned| poisoned.into_inner())
2638 .remove(module_id);
2639 self.configured_ids
2640 .lock()
2641 .unwrap_or_else(|poisoned| poisoned.into_inner())
2642 .remove(module_id);
2643 removed
2644 }
2645
2646 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2649 self.removal_tombstones
2650 .lock()
2651 .unwrap_or_else(|poisoned| poisoned.into_inner())
2652 .insert(module_id.to_string(), unix_ms_now());
2653 }
2654
2655 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2657 self.removal_tombstones
2658 .lock()
2659 .unwrap_or_else(|poisoned| poisoned.into_inner())
2660 .get(module_id)
2661 .copied()
2662 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2663 }
2664
2665 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2670 if self.get(module_id).is_some() {
2671 return false;
2672 }
2673 let mut reserved_nonces = self
2674 .reserved_nonces
2675 .lock()
2676 .unwrap_or_else(|poisoned| poisoned.into_inner());
2677 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2678 return false;
2679 }
2680 reserved_nonces.remove(module_id);
2681 true
2682 }
2683
2684 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2685 Arc::clone(&self.operation_lock)
2686 }
2687}
2688
2689#[derive(Debug, Clone)]
2691pub struct Supervisor {
2692 registry: Arc<Registry>,
2693 restart_policy: RestartPolicy,
2694 drain_timeout: Duration,
2695 connection_file_path: Option<PathBuf>,
2696 capture_logs_dir: Option<PathBuf>,
2697 forwarding: Option<Arc<ForwardingTable>>,
2698 process_liveness: Arc<SupervisorProcessLiveness>,
2699 supervisor_handle: Option<SupervisorHandle>,
2700 health: HealthConfig,
2701 daemon_start_clock: crate::clock::StartClock,
2702 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2703 spawn_events: SpawnEventFeed,
2704 provenance_probe: ExecutableIdentityProbe,
2705 child_roster: ChildRoster,
2708 #[cfg(target_os = "linux")]
2709 cgroup_placement: Option<subc_cgroup::Placement>,
2710 #[cfg(test)]
2711 test_after_first_spawn: AfterFirstSpawnHook,
2712}
2713
2714pub(crate) struct RegisteredConfiguredModule {
2718 spec: ModuleSpec,
2719 runtime: SupervisorRuntimeConfig,
2720 module: SupervisedModule,
2721 commands: mpsc::Receiver<SupervisorCommand>,
2722}
2723
2724#[cfg(test)]
2730#[derive(Clone, Default)]
2731struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2732
2733#[cfg(test)]
2734type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2735
2736#[cfg(test)]
2737impl fmt::Debug for AfterFirstSpawnHook {
2738 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2739 f.write_str("AfterFirstSpawnHook")
2740 }
2741}
2742
2743#[cfg(test)]
2744impl AfterFirstSpawnHook {
2745 fn run(&self, module_id: &str) {
2746 if let Some(hook) = &self.0 {
2747 hook(module_id);
2748 }
2749 }
2750}
2751
2752impl Supervisor {
2753 #[cfg(test)]
2754 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2755 let supervisor = Self::new(registry, policy);
2756 #[cfg(target_os = "macos")]
2757 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2758 supervisor
2759 }
2760 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2765 let path = path.into();
2766 #[cfg(target_os = "macos")]
2767 {
2768 let result = probe_privacy_trampoline(&path).map(|()| path);
2769 if let Err(cause) = &result {
2770 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2771 }
2772 self.child_roster.set_privacy_trampoline(result);
2773 }
2774 #[cfg(not(target_os = "macos"))]
2775 let _ = path;
2776 self
2777 }
2778 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2781 self.child_roster.shutdown_flag()
2782 }
2783
2784 #[cfg(unix)]
2795 pub(crate) fn begin_daemon_shutdown(&self) {
2796 self.child_roster.close();
2797 if let Some(journal) = &self.terminal_journal {
2798 journal.stamp_shutdown();
2799 }
2800 }
2801
2802 #[cfg(unix)]
2806 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2807 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2808 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2809 let Some(forwarding) = &self.forwarding else {
2810 return Ok(());
2811 };
2812 let notice_started = Instant::now();
2813 let modules = forwarding
2814 .begin_daemon_drain()
2815 .map_err(SuperviseError::Forwarding)?;
2816 let deadline_ms =
2817 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2818 let mut notices = tokio::task::JoinSet::new();
2819 let mut drains = Vec::new();
2820 for (module_id, endpoint) in modules {
2821 let Some(target) = forwarding
2822 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2823 .map_err(SuperviseError::Forwarding)?
2824 else {
2825 continue;
2826 };
2827 let routes = forwarding
2828 .endpoint_routes(target.endpoint)
2829 .map_err(SuperviseError::Forwarding)?;
2830 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2836 reason: RouteCloseReason::Restart,
2837 deadline_ms,
2838 })
2839 .expect("module draining serializes");
2840 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2841 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2842 for route in routes {
2843 let client = route.goodbye_target;
2844 if let Some((_, channels)) = clients
2845 .iter_mut()
2846 .find(|(existing, _)| existing.connection_id == client.connection_id)
2847 {
2848 channels.push(client.channel);
2849 } else {
2850 let channel = client.channel;
2851 clients.push((client, vec![channel]));
2852 }
2853 }
2854 for (client, mut channels) in clients {
2855 channels.sort_unstable();
2856 channels.dedup();
2857 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2858 module_id: module_id.clone(),
2859 channels,
2860 reason: RouteCloseReason::Restart,
2861 })
2862 .expect("route closing serializes");
2863 recipients.push((client.sink, client.negotiated_ver, closing));
2864 }
2865 for (sink, version, body) in recipients {
2866 notices.spawn(async move {
2867 let frame = Frame::build_with_version(
2868 version,
2869 FrameType::Push,
2870 control_flags(),
2871 0,
2872 0,
2873 0,
2874 body,
2875 )
2876 .expect("bounded lifecycle notice frame builds");
2877 sink.send_flushed(frame).await
2878 });
2879 }
2880 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2881 drains.push((module_id, target.endpoint, gauges));
2882 }
2883 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2886 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2887 if !matches!(result, Ok(Ok(()))) {
2888 warn!(?result, "daemon shutdown notice delivery failed");
2889 }
2890 }
2891 notices.abort_all();
2892 let notice_elapsed = notice_started.elapsed();
2893 let drain_started = Instant::now();
2894 let deadline = drain_started + DRAIN_BUDGET;
2895 let mut waits = tokio::task::JoinSet::new();
2896 for (module_id, endpoint, gauges) in drains {
2897 let forwarding = Arc::clone(forwarding);
2898 let mut runtime = self.runtime_config();
2899 runtime.health.cadence = Duration::from_millis(100);
2900 waits.spawn(async move {
2901 let wait_result = wait_for_forwarding_quiescence(
2902 &forwarding,
2903 &module_id,
2904 &runtime,
2905 endpoint,
2906 deadline,
2907 &gauges,
2908 DrainScope::Endpoint(endpoint),
2909 &mut DrainGaugeTrace::new(&gauges),
2910 )
2911 .await;
2912 forwarding
2916 .record_daemon_drain_result(
2917 endpoint,
2918 drained_after_quiescence_wait(&wait_result),
2919 )
2920 .map_err(SuperviseError::Forwarding)?;
2921 wait_result
2922 });
2923 }
2924 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2925 if !matches!(result, Ok(Ok(true))) {
2926 warn!(?result, "daemon shutdown drain did not reach quiescence");
2927 }
2928 }
2929 info!(
2932 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
2933 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
2934 "daemon shutdown forwarding drain complete"
2935 );
2936 Ok(())
2937 }
2938
2939 #[cfg(unix)]
2951 pub(crate) async fn end_children_for_daemon_shutdown(
2952 &self,
2953 already_escalated: bool,
2954 escalate: impl std::future::Future<Output = ()>,
2955 ) {
2956 tokio::pin!(escalate);
2957 let mut escalated = already_escalated;
2958 if let Some(forwarding) = &self.forwarding {
2959 let reason = CloseReason::new(
2960 "daemon_shutdown",
2961 "the daemon is exiting after its shutdown notice and drain",
2962 );
2963 if escalated {
2964 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2967 } else {
2968 tokio::select! {
2969 biased;
2970 _ = escalate.as_mut() => {
2971 info!("second SIGTERM: abandoning module GOODBYE delivery");
2972 escalated = true;
2973 }
2974 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2975 }
2976 }
2977 let closed = forwarding.close_all_connections(&reason);
2978 debug!(closed, "closed established connections for daemon shutdown");
2979 }
2980 let escalated_here = escalated && !already_escalated;
2984 let remaining_escalate = async move {
2985 if escalated_here {
2986 std::future::pending::<()>().await;
2987 } else {
2988 escalate.await;
2989 }
2990 };
2991 crate::child_roster::end_children_for_daemon_shutdown(
2992 &self.child_roster,
2993 escalated,
2994 remaining_escalate,
2995 )
2996 .await;
2997 }
2998
2999 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3000 Self {
3001 registry,
3002 restart_policy,
3003 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3004 connection_file_path: None,
3005 capture_logs_dir: None,
3006 forwarding: None,
3007 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3008 supervisor_handle: None,
3009 health: HealthConfig::default(),
3010 daemon_start_clock: crate::clock::StartClock::capture(),
3011 terminal_journal: None,
3012 spawn_events: SpawnEventFeed::default(),
3013 provenance_probe: ExecutableIdentityProbe::default(),
3014 child_roster: ChildRoster::default(),
3015 #[cfg(target_os = "linux")]
3016 cgroup_placement: None,
3017 #[cfg(test)]
3018 test_after_first_spawn: AfterFirstSpawnHook::default(),
3019 }
3020 }
3021
3022 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3023 self.drain_timeout = drain_timeout;
3024 self
3025 }
3026
3027 pub(crate) fn default_drain_timeout(&self) -> Duration {
3029 self.drain_timeout
3030 }
3031
3032 pub fn with_process_liveness(
3033 mut self,
3034 process_liveness: Arc<SupervisorProcessLiveness>,
3035 ) -> Self {
3036 self.process_liveness = process_liveness;
3037 self
3038 }
3039
3040 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3041 self.connection_file_path = Some(connection_file_path.into());
3042 self
3043 }
3044
3045 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3047 self.capture_logs_dir = Some(logs_dir.into());
3048 self
3049 }
3050
3051 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3054 self.spawn_events.configure_incarnation(daemon_incarnation);
3058 self
3059 }
3060
3061 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3064 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3065 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3066 path,
3067 daemon_incarnation,
3068 )));
3069 this
3070 }
3071
3072 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3073 self.forwarding = Some(forwarding);
3074 self
3075 }
3076
3077 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3078 self.spawn_events = supervisor_handle.spawn_events.clone();
3079 self.supervisor_handle = Some(supervisor_handle);
3080 self
3081 }
3082
3083 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3084 self.health = health;
3085 self
3086 }
3087
3088 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3092 self.child_roster.record_to(path.into());
3093 self
3094 }
3095
3096 #[cfg(target_os = "linux")]
3097 pub fn with_cgroup_placement(
3098 mut self,
3099 cgroup_placement: Option<subc_cgroup::Placement>,
3100 ) -> Self {
3101 self.cgroup_placement = cgroup_placement;
3102 self
3103 }
3104
3105 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3111 validate_spec(&spec)?;
3112 self.establish_identity(&spec);
3113
3114 let runtime = self.runtime_config();
3115 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3116 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3117 let spawned = spawn_child(
3118 &spec,
3119 runtime.connection_file_path.as_deref(),
3120 self.supervisor_handle.as_ref(),
3121 &runtime.stderr_ring,
3122 runtime.capture_logs_dir.as_deref(),
3123 &runtime.child_roster,
3124 #[cfg(target_os = "linux")]
3125 runtime.cgroup_placement.as_ref(),
3126 );
3127 #[cfg(test)]
3128 self.test_after_first_spawn.run(&spec.module_id);
3129 let child = match spawned {
3130 Ok(child) => child,
3131 Err(err) => {
3132 self.abandon_unrostered(&spec.module_id);
3135 return Err(err);
3136 }
3137 };
3138 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3139 if let Some(spawn) = activity_spawn {
3140 spawn.complete(&snapshot);
3141 }
3142
3143 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3144 }
3145
3146 fn establish_identity(&self, spec: &ModuleSpec) {
3159 if let Some(supervisor_handle) = &self.supervisor_handle {
3160 supervisor_handle.apply_identity_configuration(spec);
3161 supervisor_handle.mark_configured(&spec.module_id);
3162 }
3163 }
3164
3165 fn abandon_unrostered(&self, module_id: &str) {
3168 if let Some(supervisor_handle) = &self.supervisor_handle {
3169 supervisor_handle.unmark_configured_unless_rostered(module_id);
3170 }
3171 }
3172
3173 fn mark_first_process_running(
3176 &self,
3177 spec: &ModuleSpec,
3178 runtime: &SupervisorRuntimeConfig,
3179 snapshot: &SharedSnapshot,
3180 child: &SupervisedChild,
3181 ) -> Result<(), SuperviseError> {
3182 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3183 self.abandon_unrostered(&spec.module_id);
3184 return Err(err);
3185 }
3186 self.process_liveness
3187 .track(spec.module_id.clone(), Arc::clone(snapshot));
3188 Ok(())
3189 }
3190
3191 pub fn supervise_configured(
3197 &self,
3198 spec: ModuleSpec,
3199 enabled: bool,
3200 ) -> Result<SupervisedModule, SuperviseError> {
3201 self.supervise_configured_requested(
3202 spec,
3203 enabled,
3204 self.health.clone(),
3205 None,
3206 self.restart_policy,
3207 None,
3208 )
3209 }
3210
3211 pub fn supervise_configured_with_health(
3217 &self,
3218 spec: ModuleSpec,
3219 enabled: bool,
3220 health: HealthConfig,
3221 drain_timeout_ms: Option<u64>,
3222 restart_policy: RestartPolicy,
3223 ) -> Result<SupervisedModule, SuperviseError> {
3224 self.supervise_configured_requested(
3225 spec,
3226 enabled,
3227 health,
3228 drain_timeout_ms,
3229 restart_policy,
3230 None,
3231 )
3232 }
3233
3234 pub(crate) fn supervise_configured_requested(
3235 &self,
3236 spec: ModuleSpec,
3237 enabled: bool,
3238 health: HealthConfig,
3239 drain_timeout_ms: Option<u64>,
3240 restart_policy: RestartPolicy,
3241 requester: Option<Principal>,
3242 ) -> Result<SupervisedModule, SuperviseError> {
3243 let registered = self.register_configured_with_health(
3244 spec,
3245 enabled,
3246 health,
3247 drain_timeout_ms,
3248 restart_policy,
3249 )?;
3250 let daemon_initiated = requester.is_none();
3251 self.start_registered_configured(registered, requester, daemon_initiated)
3252 }
3253
3254 pub(crate) fn register_configured_with_health(
3259 &self,
3260 spec: ModuleSpec,
3261 enabled: bool,
3262 health: HealthConfig,
3263 drain_timeout_ms: Option<u64>,
3264 restart_policy: RestartPolicy,
3265 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3266 validate_spec(&spec)?;
3267 self.establish_identity(&spec);
3268
3269 let mut runtime = self.runtime_config();
3270 runtime.health = health;
3271 runtime.restart_policy = restart_policy;
3272 if let Some(ms) = drain_timeout_ms {
3273 runtime.drain_timeout = Duration::from_millis(ms);
3274 *runtime
3275 .effective_drain_timeout
3276 .lock()
3277 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3278 }
3279 let snapshot = if enabled {
3280 SupervisorSnapshot::starting()
3281 } else {
3282 SupervisorSnapshot::disabled()
3283 };
3284 let snapshot = {
3285 let mut snapshot = snapshot;
3286 if enabled {
3287 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3288 {
3289 snapshot.waiting_for_previous_pid = Some(pid);
3290 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3291 }
3292 }
3293 snapshot
3294 };
3295 let (module, commands) =
3296 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3297 Ok(RegisteredConfiguredModule {
3298 spec,
3299 runtime,
3300 module,
3301 commands,
3302 })
3303 }
3304
3305 pub(crate) fn start_registered_configured(
3308 &self,
3309 registered: RegisteredConfiguredModule,
3310 requester: Option<Principal>,
3311 daemon_initiated: bool,
3312 ) -> Result<SupervisedModule, SuperviseError> {
3313 let RegisteredConfiguredModule {
3314 spec,
3315 mut runtime,
3316 module,
3317 commands,
3318 } = registered;
3319 runtime.start_requester = requester;
3320 runtime.recovery_is_daemon = daemon_initiated;
3321 let snapshot = &module.inner.snapshot;
3322 let should_spawn = {
3323 let state = lock_snapshot(snapshot)?;
3324 state.enabled && state.waiting_for_previous_pid.is_none()
3325 };
3326 let child = if should_spawn {
3327 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3328 let spawned = spawn_child(
3329 &spec,
3330 runtime.connection_file_path.as_deref(),
3331 self.supervisor_handle.as_ref(),
3332 &runtime.stderr_ring,
3333 runtime.capture_logs_dir.as_deref(),
3334 &runtime.child_roster,
3335 #[cfg(target_os = "linux")]
3336 runtime.cgroup_placement.as_ref(),
3337 );
3338 #[cfg(test)]
3339 self.test_after_first_spawn.run(&spec.module_id);
3340 match spawned {
3341 Ok(child) => {
3342 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3343 if let Some(spawn) = activity_spawn {
3344 spawn.complete(snapshot);
3345 }
3346 Some(child)
3347 }
3348 Err(err) => {
3349 if runtime.health.critical {
3350 error!(
3351 module_id = %spec.module_id,
3352 program = %spec.program.display(),
3353 error = %err,
3354 "critical configured module failed to spawn; marking failed and alerting"
3355 );
3356 } else {
3357 error!(
3358 module_id = %spec.module_id,
3359 program = %spec.program.display(),
3360 error = %err,
3361 "configured module failed to spawn; marking failed and continuing"
3362 );
3363 }
3364 fail_snapshot(
3365 snapshot,
3366 Some(&spec.module_id),
3367 None,
3368 &runtime.failure_context(daemon_initiated),
3369 );
3370 None
3371 }
3372 }
3373 } else {
3374 None
3375 };
3376 self.start_monitor(spec, runtime, &module, child, commands);
3377 Ok(module)
3378 }
3379
3380 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3381 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3382 SupervisorRuntimeConfig {
3383 recovery_is_daemon: true,
3384 start_requester: None,
3385 pending_activity_restart: false,
3386 provenance_probe: self.provenance_probe.clone(),
3387 scheduled_respawn: Arc::default(),
3388 deferred_reload_reply: Arc::default(),
3389 restart_policy: self.restart_policy,
3390 drain_timeout: self.drain_timeout,
3391 child_roster: self
3394 .child_roster
3395 .for_module(Arc::clone(&effective_drain_timeout)),
3396 effective_drain_timeout,
3397 default_drain_timeout: self.drain_timeout,
3398 health: self.health.clone(),
3399 connection_file_path: self.connection_file_path.clone(),
3400 capture_logs_dir: self.capture_logs_dir.clone(),
3401 forwarding: self.forwarding.clone(),
3402 supervisor_handle: self.supervisor_handle.clone(),
3403 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3404 terminal_ring: Arc::new(Mutex::new(
3405 TerminalRing::new(
3406 TerminalRingConfig::default(),
3407 self.daemon_start_clock.started_at_ms(),
3408 )
3409 .with_start_clock(self.daemon_start_clock)
3410 .with_journal(self.terminal_journal.clone())
3411 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3412 )),
3413 spawn_events: self.spawn_events.clone(),
3414 #[cfg(target_os = "linux")]
3415 cgroup_placement: self.cgroup_placement.clone(),
3416 #[cfg(test)]
3417 test_seed_stale_facts_before_enable_spawn: false,
3418 #[cfg(test)]
3419 test_reload_exit_record_gate: None,
3420 }
3421 }
3422
3423 fn supervised_module(
3424 &self,
3425 spec: ModuleSpec,
3426 runtime: SupervisorRuntimeConfig,
3427 snapshot: SharedSnapshot,
3428 child: Option<SupervisedChild>,
3429 ) -> SupervisedModule {
3430 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3431 self.start_monitor(spec, runtime, &module, child, commands);
3432 module
3433 }
3434
3435 fn register_module(
3436 &self,
3437 spec: &ModuleSpec,
3438 runtime: &SupervisorRuntimeConfig,
3439 snapshot: SharedSnapshot,
3440 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3441 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3442 spec: spec.clone(),
3443 health: runtime.health.clone(),
3444 }));
3445 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3446 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3447 let restart_policy = runtime.restart_policy;
3451 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3452 let (tx, rx) = mpsc::channel(4);
3453 let module_id = spec.module_id.clone();
3454 let module = SupervisedModule {
3455 inner: Arc::new(SupervisedModuleInner {
3456 module_id: module_id.clone(),
3457 registry: Arc::clone(&self.registry),
3458 snapshot,
3459 configuration,
3460 stderr_ring,
3461 terminal_ring,
3462 commands: tx,
3463 monitor: Mutex::new(None),
3464 restart_policy,
3465 effective_drain_timeout,
3466 provenance_probe: self.provenance_probe.clone(),
3467 }),
3468 };
3469 if let Some(supervisor_handle) = &self.supervisor_handle {
3472 supervisor_handle.insert(module.clone());
3473 }
3474 (module, rx)
3475 }
3476
3477 fn start_monitor(
3478 &self,
3479 spec: ModuleSpec,
3480 runtime: SupervisorRuntimeConfig,
3481 module: &SupervisedModule,
3482 child: Option<SupervisedChild>,
3483 commands: mpsc::Receiver<SupervisorCommand>,
3484 ) {
3485 let monitor = tokio::spawn(supervise_loop(
3486 spec,
3487 runtime,
3488 Arc::clone(&self.registry),
3489 Arc::clone(&self.process_liveness),
3490 Arc::clone(&module.inner.snapshot),
3491 child,
3492 commands,
3493 ));
3494 *module
3495 .inner
3496 .monitor
3497 .lock()
3498 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3499 }
3500}
3501
3502impl Default for Supervisor {
3503 fn default() -> Self {
3504 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3505 }
3506}
3507
3508#[derive(Clone)]
3510pub struct SupervisedModule {
3511 inner: Arc<SupervisedModuleInner>,
3512}
3513
3514struct SupervisedModuleInner {
3515 module_id: String,
3516 registry: Arc<Registry>,
3517 snapshot: SharedSnapshot,
3518 configuration: Arc<Mutex<SupervisedConfiguration>>,
3519 stderr_ring: Arc<Mutex<StderrRing>>,
3520 terminal_ring: Arc<Mutex<TerminalRing>>,
3521 commands: mpsc::Sender<SupervisorCommand>,
3522 monitor: Mutex<Option<JoinHandle<()>>>,
3523 restart_policy: RestartPolicy,
3527 effective_drain_timeout: Arc<Mutex<Duration>>,
3528 provenance_probe: ExecutableIdentityProbe,
3529}
3530
3531impl fmt::Debug for SupervisedModule {
3532 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3533 f.debug_struct("SupervisedModule")
3534 .field("module_id", &self.inner.module_id)
3535 .field("status", &self.status())
3536 .finish_non_exhaustive()
3537 }
3538}
3539
3540impl SupervisedModule {
3541 pub fn module_id(&self) -> &str {
3542 &self.inner.module_id
3543 }
3544
3545 #[cfg(test)]
3549 pub(crate) fn record_health_probe_failure_for_test(
3550 &self,
3551 detail: &str,
3552 ) -> Result<(), SuperviseError> {
3553 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3554 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3555 state.health.detail = Some(detail.to_string());
3556 })
3557 }
3558
3559 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3560 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3561 }
3562
3563 pub(crate) fn registration_end_reason(
3564 &self,
3565 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3566 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3567 Ok(match snapshot.state {
3568 ModuleState::Draining if snapshot.draining_to_replace => {
3569 Some(RegistrationEndReason::SupervisorRestart)
3570 }
3571 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3572 ModuleState::Restarting if snapshot.draining_to_replace => {
3573 Some(RegistrationEndReason::SupervisorRestart)
3574 }
3575 _ => None,
3576 })
3577 }
3578
3579 pub fn stderr_tail(
3586 &self,
3587 max_lines: Option<usize>,
3588 max_bytes: Option<usize>,
3589 ) -> StderrTailSnapshot {
3590 self.inner
3591 .stderr_ring
3592 .lock()
3593 .unwrap_or_else(|poisoned| poisoned.into_inner())
3594 .snapshot(max_lines, max_bytes)
3595 }
3596
3597 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3602 self.inner
3603 .terminal_ring
3604 .lock()
3605 .unwrap_or_else(|poisoned| poisoned.into_inner())
3606 .snapshot()
3607 }
3608
3609 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3614 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3615 }
3616
3617 pub(crate) async fn read_durable_terminal_history(
3622 &self,
3623 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3624 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3625 let module_id = self.inner.module_id.clone();
3626 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3627 .await
3628 }
3629
3630 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3631 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3632 .map(|(status, _)| status)
3633 }
3634
3635 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3636 let state = lock_snapshot(&self.inner.snapshot)?;
3637 Ok(state
3638 .failed_retry
3639 .pending
3640 .filter(|_| state.state == ModuleState::Failed && state.enabled)
3641 .map(|retry| {
3642 unix_ms_now().saturating_add(
3643 u64::try_from(
3644 retry
3645 .deadline
3646 .saturating_duration_since(Instant::now())
3647 .as_millis(),
3648 )
3649 .unwrap_or(u64::MAX),
3650 )
3651 }))
3652 }
3653
3654 pub(crate) fn record_deliberate_severance(
3655 &self,
3656 identity: ProcessIdentity,
3657 ) -> Result<bool, SuperviseError> {
3658 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3659 if snapshot.pid != Some(identity.pid)
3660 || snapshot.process_start_time != Some(identity.start_time)
3661 {
3662 return Ok(false);
3663 }
3664 snapshot.deliberate_severance = Some(identity);
3665 Ok(true)
3666 }
3667
3668 pub(crate) fn status_for_control(
3673 &self,
3674 caller: &'static str,
3675 ) -> Result<ModuleStatus, SuperviseError> {
3676 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3677 .map(|(status, _)| status)
3678 }
3679
3680 fn status_with_snapshot_lock(
3681 &self,
3682 snapshot: &SharedSnapshot,
3683 caller: Option<&'static str>,
3684 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3685 let mut guard = match caller {
3686 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3687 None => lock_snapshot(snapshot)?,
3688 };
3689 let restart_count =
3692 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3693 let snapshot = guard.clone();
3694 drop(guard);
3695 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3696 SuperviseError::StatePoisoned {
3697 module_id: Some(self.inner.module_id.clone()),
3698 }
3699 })?;
3700 let registration_active = self
3701 .inner
3702 .registry
3703 .get_module(&self.inner.module_id)
3704 .map_err(SuperviseError::Registry)?
3705 .is_some();
3706 let protocol = snapshot
3707 .spawned_protocol
3708 .unwrap_or(self.declared_protocol()?);
3709 let running_process =
3710 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3711 let live = match protocol {
3717 ModuleProtocol::Subc => running_process && registration_active,
3718 ModuleProtocol::None => running_process,
3719 };
3720
3721 Ok((
3722 ModuleStatus {
3723 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3724 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3725 module_id: self.inner.module_id.clone(),
3726 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3727 state: snapshot.state,
3728 enabled: snapshot.enabled,
3729 process_alive: snapshot.process_alive,
3730 registration_active,
3731 protocol,
3732 live,
3733 restart_count,
3734 lifetime_restarts: snapshot.lifetime_restarts,
3735 spawn_generation: snapshot.spawn_generation,
3736 max_restarts: self.inner.restart_policy.max_restarts,
3737 restart_window: self.inner.restart_policy.window,
3738 drain_timeout,
3739 restart_backoff: self.inner.restart_policy.backoff,
3740 restart_max_backoff: self.inner.restart_policy.max_backoff,
3741 pid: snapshot.reported_pid(),
3742 spawned_at_ms: snapshot.spawned_at_ms,
3743 spawned_from: snapshot.spawned_from,
3744 launch_spec_digest: snapshot.launch_spec_digest,
3745 process_start_time: snapshot.process_start_time,
3746 last_exit: snapshot.last_exit,
3747 health: snapshot.health,
3748 },
3749 snapshot.spawned_file_identity.clone(),
3750 ))
3751 }
3752
3753 #[cfg(test)]
3754 pub(crate) fn hold_snapshot_for_test(
3755 &self,
3756 acquired: std::sync::mpsc::Sender<()>,
3757 hold: Duration,
3758 ) -> std::thread::JoinHandle<()> {
3759 let snapshot = Arc::clone(&self.inner.snapshot);
3760 std::thread::spawn(move || {
3761 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3762 acquired
3763 .send(())
3764 .expect("test receiver waits for snapshot lock");
3765 std::thread::sleep(hold);
3766 })
3767 }
3768
3769 pub(crate) async fn status_and_running_image_agreement(
3775 &self,
3776 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3777 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3778 let image = self
3779 .inner
3780 .provenance_probe
3781 .observe(
3782 status.pid,
3783 status.spawned_from.as_deref(),
3784 identity,
3785 status.process_start_time,
3786 )
3787 .await;
3788 Ok((status, image))
3789 }
3790
3791 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3792 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3793 Ok(snapshot) => snapshot.clone(),
3794 Err(_) => {
3795 return subc_control::RunningImageAgreement::Unavailable {
3796 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3797 };
3798 }
3799 };
3800 self.inner
3801 .provenance_probe
3802 .observe(
3803 snapshot.reported_pid(),
3804 snapshot.spawned_from.as_deref(),
3805 snapshot.spawned_file_identity.clone(),
3806 snapshot.process_start_time,
3807 )
3808 .await
3809 }
3810
3811 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3814 #[cfg(windows)]
3815 {
3816 let image = lock_snapshot(&self.inner.snapshot)
3817 .ok()
3818 .and_then(|snapshot| {
3819 snapshot
3820 .spawned_file_identity
3821 .as_ref()
3822 .and_then(|identity| identity.windows_image.clone())
3823 });
3824 match image {
3825 Some(image) => {
3826 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3827 }
3828 None => subc_control::ChildResourceUsage::Unavailable {
3829 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3830 },
3831 }
3832 }
3833 #[cfg(not(windows))]
3834 {
3835 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3836 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3837 Err(_) => {
3838 return subc_control::ChildResourceUsage::Unavailable {
3839 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3840 }
3841 }
3842 };
3843 crate::child_resources::read(pid, start_time)
3844 }
3845 }
3846
3847 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3848 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3849 Ok(match snapshot.state {
3850 ModuleState::Restarting => true,
3851 ModuleState::Failed | ModuleState::Disabled => false,
3852 _ if snapshot.enabled && snapshot.respawn_pending => true,
3853 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3854 })
3855 }
3856
3857 #[cfg(test)]
3858 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3859 self.is_warming_with_snapshot_lock(None)
3860 }
3861
3862 pub(crate) fn is_warming_for_control(
3863 &self,
3864 caller: &'static str,
3865 ) -> Result<bool, SuperviseError> {
3866 self.is_warming_with_snapshot_lock(Some(caller))
3867 }
3868
3869 fn is_warming_with_snapshot_lock(
3870 &self,
3871 caller: Option<&'static str>,
3872 ) -> Result<bool, SuperviseError> {
3873 let snapshot = match caller {
3874 Some(caller) => {
3875 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3876 }
3877 None => lock_snapshot(&self.inner.snapshot)?,
3878 }
3879 .clone();
3880 Ok(matches!(
3881 snapshot.state,
3882 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3883 ))
3884 }
3885
3886 pub async fn drain(&self) -> Result<(), SuperviseError> {
3888 self.stop().await
3889 }
3890
3891 pub(crate) async fn retire_requested(
3892 &self,
3893 requester: Option<Principal>,
3894 ) -> Result<(), SuperviseError> {
3895 match self.state()? {
3896 ModuleState::Stopped => return Ok(()),
3897 ModuleState::Failed
3898 if lock_snapshot(&self.inner.snapshot)?
3899 .failed_retry
3900 .pending
3901 .is_none() =>
3902 {
3903 return Ok(())
3904 }
3905 ModuleState::Failed
3906 | ModuleState::Starting
3907 | ModuleState::Running
3908 | ModuleState::Unresponsive
3909 | ModuleState::Restarting
3910 | ModuleState::Draining
3911 | ModuleState::Disabled => {}
3912 }
3913
3914 let (reply_tx, reply_rx) = oneshot::channel();
3915 self.inner
3916 .commands
3917 .send(SupervisorCommand::Retire {
3918 requester,
3919 reply: reply_tx,
3920 })
3921 .await
3922 .map_err(|_| SuperviseError::CommandClosed {
3923 module_id: self.inner.module_id.clone(),
3924 })?;
3925 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3926 module_id: self.inner.module_id.clone(),
3927 })?
3928 }
3929
3930 pub async fn stop(&self) -> Result<(), SuperviseError> {
3931 self.stop_requested(None).await
3932 }
3933
3934 pub(crate) async fn stop_requested(
3935 &self,
3936 requester: Option<Principal>,
3937 ) -> Result<(), SuperviseError> {
3938 match self.state()? {
3939 ModuleState::Stopped => return Ok(()),
3940 ModuleState::Failed
3941 if lock_snapshot(&self.inner.snapshot)?
3942 .failed_retry
3943 .pending
3944 .is_none() =>
3945 {
3946 return Ok(())
3947 }
3948 ModuleState::Failed
3949 | ModuleState::Starting
3950 | ModuleState::Running
3951 | ModuleState::Unresponsive
3952 | ModuleState::Restarting
3953 | ModuleState::Draining
3954 | ModuleState::Disabled => {}
3955 }
3956
3957 let (reply_tx, reply_rx) = oneshot::channel();
3958 self.inner
3959 .commands
3960 .send(SupervisorCommand::Drain {
3961 requester,
3962 reply: reply_tx,
3963 })
3964 .await
3965 .map_err(|_| SuperviseError::CommandClosed {
3966 module_id: self.inner.module_id.clone(),
3967 })?;
3968 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3969 module_id: self.inner.module_id.clone(),
3970 })?
3971 }
3972
3973 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3974 self.restart_requested(drain_timeout_ms, None).await
3975 }
3976
3977 pub(crate) async fn restart_requested(
3978 &self,
3979 drain_timeout_ms: Option<u64>,
3980 requester: Option<Principal>,
3981 ) -> Result<(), SuperviseError> {
3982 let queued_at = Instant::now();
3983 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3984 let (reply_tx, reply_rx) = oneshot::channel();
3985 self.inner
3986 .commands
3987 .send(SupervisorCommand::Restart {
3988 requester,
3989 drain_timeout_ms,
3990 received_at_generation,
3991 queued_at,
3992 reply: reply_tx,
3993 })
3994 .await
3995 .map_err(|_| SuperviseError::CommandClosed {
3996 module_id: self.inner.module_id.clone(),
3997 })?;
3998 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3999 module_id: self.inner.module_id.clone(),
4000 })?
4001 }
4002
4003 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4008 self.swap_requested(ready_timeout, None).await
4009 }
4010
4011 pub(crate) async fn swap_requested(
4012 &self,
4013 ready_timeout: Option<Duration>,
4014 requester: Option<Principal>,
4015 ) -> Result<(), SuperviseError> {
4016 let (reply_tx, reply_rx) = oneshot::channel();
4017 self.inner
4018 .commands
4019 .send(SupervisorCommand::Swap {
4020 requester,
4021 ready_timeout,
4022 reply: reply_tx,
4023 })
4024 .await
4025 .map_err(|_| SuperviseError::CommandClosed {
4026 module_id: self.inner.module_id.clone(),
4027 })?;
4028 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4029 module_id: self.inner.module_id.clone(),
4030 })?
4031 }
4032
4033 pub async fn reload(&self) -> Result<(), SuperviseError> {
4034 self.reload_requested(None).await
4035 }
4036
4037 pub(crate) async fn reload_requested(
4038 &self,
4039 requester: Option<Principal>,
4040 ) -> Result<(), SuperviseError> {
4041 let (reply_tx, reply_rx) = oneshot::channel();
4042 self.inner
4043 .commands
4044 .send(SupervisorCommand::Reload {
4045 requester,
4046 reply: reply_tx,
4047 })
4048 .await
4049 .map_err(|_| SuperviseError::CommandClosed {
4050 module_id: self.inner.module_id.clone(),
4051 })?;
4052 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4053 module_id: self.inner.module_id.clone(),
4054 })?
4055 }
4056
4057 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4058 self.set_enabled_requested(enabled, None).await
4059 }
4060
4061 pub(crate) async fn set_enabled_requested(
4062 &self,
4063 enabled: bool,
4064 requester: Option<Principal>,
4065 ) -> Result<bool, SuperviseError> {
4066 let (reply_tx, reply_rx) = oneshot::channel();
4067 self.inner
4068 .commands
4069 .send(SupervisorCommand::SetEnabled {
4070 requester,
4071 enabled,
4072 reply: reply_tx,
4073 })
4074 .await
4075 .map_err(|_| SuperviseError::CommandClosed {
4076 module_id: self.inner.module_id.clone(),
4077 })?;
4078 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4079 module_id: self.inner.module_id.clone(),
4080 })?
4081 }
4082
4083 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4087 let configured = self
4088 .inner
4089 .configuration
4090 .lock()
4091 .map_err(|_| SuperviseError::StatePoisoned {
4092 module_id: Some(self.inner.module_id.clone()),
4093 })?
4094 .spec
4095 .protocol;
4096 let state = lock_snapshot(&self.inner.snapshot)?;
4097 Ok(state.spawned_protocol.unwrap_or(configured))
4098 }
4099
4100 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4101 let configuration =
4102 self.inner
4103 .configuration
4104 .lock()
4105 .map_err(|_| SuperviseError::StatePoisoned {
4106 module_id: Some(self.inner.module_id.clone()),
4107 })?;
4108 Ok((configuration.spec.clone(), configuration.health.clone()))
4109 }
4110
4111 #[cfg(any(test, feature = "test-support"))]
4115 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4116 let (_, health) = self.configuration()?;
4117 let drain_timeout_ms = u64::try_from(
4118 self.inner
4119 .effective_drain_timeout
4120 .lock()
4121 .unwrap_or_else(|poisoned| poisoned.into_inner())
4122 .as_millis(),
4123 )
4124 .ok();
4125 self.update_configuration(spec, health, drain_timeout_ms)
4126 .await
4127 }
4128
4129 pub(crate) async fn update_configuration(
4130 &self,
4131 spec: ModuleSpec,
4132 health: HealthConfig,
4133 drain_timeout_ms: Option<u64>,
4134 ) -> Result<(), SuperviseError> {
4135 if spec.module_id != self.inner.module_id {
4136 return Err(SuperviseError::InvalidSpec {
4137 reason: "a supervised module's module_id cannot be changed".to_string(),
4138 });
4139 }
4140 validate_spec(&spec)?;
4141 let (reply_tx, reply_rx) = oneshot::channel();
4142 self.inner
4143 .commands
4144 .send(SupervisorCommand::UpdateConfiguration {
4145 spec: spec.clone(),
4146 health: health.clone(),
4147 drain_timeout_ms,
4148 reply: reply_tx,
4149 })
4150 .await
4151 .map_err(|_| SuperviseError::CommandClosed {
4152 module_id: self.inner.module_id.clone(),
4153 })?;
4154 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4155 module_id: self.inner.module_id.clone(),
4156 })?;
4157 let mut configuration =
4158 self.inner
4159 .configuration
4160 .lock()
4161 .map_err(|_| SuperviseError::StatePoisoned {
4162 module_id: Some(self.inner.module_id.clone()),
4163 })?;
4164 configuration.spec = spec;
4165 configuration.health = health;
4166 Ok(())
4167 }
4168}
4169
4170impl Drop for SupervisedModuleInner {
4171 fn drop(&mut self) {
4172 let Ok(mut monitor) = self.monitor.lock() else {
4173 return;
4174 };
4175 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4176 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4177 state.state = ModuleState::Stopped;
4178 clear_current_process_facts(state);
4179 });
4180 monitor.abort();
4181 }
4182 let _ = monitor.take();
4183 }
4184}
4185
4186#[derive(Debug)]
4187enum SupervisorCommand {
4188 Drain {
4189 requester: Option<Principal>,
4190 reply: oneshot::Sender<Result<(), SuperviseError>>,
4191 },
4192 Retire {
4193 requester: Option<Principal>,
4194 reply: oneshot::Sender<Result<(), SuperviseError>>,
4195 },
4196 Restart {
4197 requester: Option<Principal>,
4198 drain_timeout_ms: Option<u64>,
4203 received_at_generation: u64,
4207 queued_at: Instant,
4210 reply: oneshot::Sender<Result<(), SuperviseError>>,
4211 },
4212 Reload {
4213 requester: Option<Principal>,
4214 reply: oneshot::Sender<Result<(), SuperviseError>>,
4215 },
4216 SetEnabled {
4217 requester: Option<Principal>,
4218 enabled: bool,
4219 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4220 },
4221 UpdateConfiguration {
4222 spec: ModuleSpec,
4223 health: HealthConfig,
4224 drain_timeout_ms: Option<u64>,
4227 reply: oneshot::Sender<()>,
4228 },
4229 Swap {
4230 requester: Option<Principal>,
4231 ready_timeout: Option<Duration>,
4234 reply: oneshot::Sender<Result<(), SuperviseError>>,
4236 },
4237}
4238
4239#[derive(Debug)]
4240pub enum SuperviseError {
4241 InvalidSpec {
4242 reason: String,
4243 },
4244 Spawn {
4245 program: PathBuf,
4246 source: io::Error,
4247 cgroup_path: Option<PathBuf>,
4248 },
4249 Cgroup {
4250 module_id: String,
4251 source: io::Error,
4252 },
4253 LaunchNonce {
4256 reason: String,
4257 },
4258 Wait {
4259 module_id: String,
4260 source: io::Error,
4261 },
4262 Kill {
4263 module_id: String,
4264 source: io::Error,
4265 },
4266 Forwarding(ForwardingError),
4267 Registry(RegistryError),
4268 ReloadUnavailable {
4269 module_id: String,
4270 reason: String,
4271 },
4272 Disabled {
4277 module_id: String,
4278 },
4279 ReloadFailed {
4280 module_id: String,
4281 reason: String,
4282 },
4283 RegistrationStillActive {
4284 module_id: String,
4285 waited: Duration,
4286 },
4287 StatePoisoned {
4288 module_id: Option<String>,
4289 },
4290 CommandClosed {
4291 module_id: String,
4292 },
4293 SwapInProgress {
4297 module_id: String,
4298 },
4299 SwapRefused {
4301 module_id: String,
4302 reason: SwapRefusal,
4303 },
4304 SwapFailed {
4308 module_id: String,
4309 arm: SwapFailureArm,
4310 detail: String,
4311 candidate_exit: Option<ExitReport>,
4314 },
4315}
4316
4317#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4319pub enum SwapRefusal {
4320 OverlapExclusive,
4322 NotRegistered,
4325 ProtocolNone,
4328 NotConfigured,
4331 AlreadySwapping,
4333}
4334
4335impl SwapRefusal {
4336 pub fn as_str(self) -> &'static str {
4337 match self {
4338 Self::OverlapExclusive => "overlap_exclusive",
4339 Self::NotRegistered => "not_registered",
4340 Self::ProtocolNone => "protocol_none",
4341 Self::NotConfigured => "not_configured",
4342 Self::AlreadySwapping => "already_swapping",
4343 }
4344 }
4345}
4346
4347#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4350pub enum SwapFailureArm {
4351 SpawnFailed,
4353 NeverRegistered,
4355 NeverReady,
4357 CandidateExited,
4359 CandidateUnhealthy,
4361 Interrupted,
4365 CutoverLost,
4370}
4371
4372impl SwapFailureArm {
4373 pub fn as_str(self) -> &'static str {
4374 match self {
4375 Self::SpawnFailed => "spawn_failed",
4376 Self::NeverRegistered => "never_registered",
4377 Self::NeverReady => "never_ready",
4378 Self::CandidateExited => "candidate_exited",
4379 Self::CandidateUnhealthy => "candidate_unhealthy",
4380 Self::Interrupted => "interrupted",
4381 Self::CutoverLost => "cutover_lost",
4382 }
4383 }
4384}
4385
4386impl fmt::Display for SuperviseError {
4387 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4388 match self {
4389 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4390 Self::Spawn {
4391 program,
4392 source,
4393 cgroup_path: Some(cgroup_path),
4394 } => write!(
4395 f,
4396 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4397 cgroup_path.display(),
4398 program.display()
4399 ),
4400 Self::Spawn {
4401 program,
4402 source,
4403 cgroup_path: None,
4404 } => write!(
4405 f,
4406 "failed to spawn module '{}': {source}",
4407 program.display()
4408 ),
4409 Self::Cgroup { module_id, source } => {
4410 write!(
4411 f,
4412 "failed to prepare cgroup for module '{module_id}': {source}"
4413 )
4414 }
4415 Self::LaunchNonce { reason } => {
4416 write!(
4417 f,
4418 "failed to generate reserved-module launch nonce: {reason}"
4419 )
4420 }
4421 Self::Wait { module_id, source } => {
4422 write!(f, "failed to wait for module '{module_id}': {source}")
4423 }
4424 Self::Kill { module_id, source } => {
4425 write!(f, "failed to kill module '{module_id}': {source}")
4426 }
4427 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4428 Self::Registry(err) => write!(f, "registry error: {err}"),
4429 Self::ReloadUnavailable { module_id, reason } => {
4430 write!(f, "reload unavailable for module '{module_id}': {reason}")
4431 }
4432 Self::Disabled { module_id } => {
4433 write!(
4434 f,
4435 "module '{module_id}' is disabled; enable it before restart or reload"
4436 )
4437 }
4438 Self::ReloadFailed { module_id, reason } => {
4439 write!(f, "reload failed for module '{module_id}': {reason}")
4440 }
4441 Self::RegistrationStillActive { module_id, waited } => write!(
4442 f,
4443 "module '{module_id}' registration remained active after waiting {waited:?}"
4444 ),
4445 Self::StatePoisoned { module_id } => match module_id {
4446 Some(module_id) => {
4447 write!(f, "supervisor state for module '{module_id}' was poisoned")
4448 }
4449 None => write!(f, "supervisor state was poisoned"),
4450 },
4451 Self::CommandClosed { module_id } => {
4452 write!(
4453 f,
4454 "supervisor command channel for module '{module_id}' is closed"
4455 )
4456 }
4457 Self::SwapInProgress { module_id } => write!(
4458 f,
4459 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4460 ),
4461 Self::SwapRefused { module_id, reason } => match reason {
4462 SwapRefusal::OverlapExclusive => write!(
4463 f,
4464 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4465 ),
4466 SwapRefusal::NotRegistered => write!(
4467 f,
4468 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4469 ),
4470 SwapRefusal::ProtocolNone => write!(
4471 f,
4472 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4473 ),
4474 SwapRefusal::NotConfigured => write!(
4475 f,
4476 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4477 ),
4478 SwapRefusal::AlreadySwapping => {
4479 write!(f, "module '{module_id}' is already being swapped")
4480 }
4481 },
4482 Self::SwapFailed {
4483 module_id,
4484 arm,
4485 detail,
4486 ..
4487 } => write!(
4488 f,
4489 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4490 arm.as_str()
4491 ),
4492 }
4493 }
4494}
4495
4496impl Error for SuperviseError {
4497 fn source(&self) -> Option<&(dyn Error + 'static)> {
4498 match self {
4499 Self::Spawn { source, .. }
4500 | Self::Cgroup { source, .. }
4501 | Self::Wait { source, .. }
4502 | Self::Kill { source, .. } => Some(source),
4503 Self::Forwarding(err) => Some(err),
4504 Self::Registry(err) => Some(err),
4505 Self::LaunchNonce { .. }
4506 | Self::InvalidSpec { .. }
4507 | Self::ReloadUnavailable { .. }
4508 | Self::Disabled { .. }
4509 | Self::ReloadFailed { .. }
4510 | Self::RegistrationStillActive { .. }
4511 | Self::StatePoisoned { .. }
4512 | Self::CommandClosed { .. }
4513 | Self::SwapInProgress { .. }
4514 | Self::SwapRefused { .. }
4515 | Self::SwapFailed { .. } => None,
4516 }
4517 }
4518}
4519
4520pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4521 if spec.module_id.trim().is_empty() {
4522 return Err(SuperviseError::InvalidSpec {
4523 reason: "module_id must not be empty".to_string(),
4524 });
4525 }
4526
4527 Ok(())
4528}
4529
4530#[derive(Debug, Default)]
4531struct HealthProbeRuntime {
4532 spawn_generation: Option<u64>,
4533 configured_health: Option<HealthConfig>,
4534 registered_connection: Option<crate::ConnectionId>,
4535 advertised: bool,
4536 next_probe_at: Option<Instant>,
4537 probe_index: u64,
4538}
4539
4540fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4541 lock_snapshot(snapshot)
4542 .ok()
4543 .and_then(|state| state.spawned_protocol)
4544 .unwrap_or(spec.protocol)
4545}
4546
4547impl HealthProbeRuntime {
4548 fn refresh_registration(
4549 &mut self,
4550 spec: &ModuleSpec,
4551 runtime: &SupervisorRuntimeConfig,
4552 registry: &Registry,
4553 snapshot: &SharedSnapshot,
4554 ) {
4555 let generation = lock_snapshot(snapshot)
4556 .ok()
4557 .map(|state| state.spawn_generation);
4558 if self.spawn_generation != generation {
4559 *self = Self {
4563 spawn_generation: generation,
4564 ..Self::default()
4565 };
4566 }
4567 if self.configured_health.as_ref() != Some(&runtime.health) {
4568 self.configured_health = Some(runtime.health.clone());
4569 self.next_probe_at = None;
4570 self.registered_connection = None;
4571 self.probe_index = 0;
4572 }
4573 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4577 self.registered_connection = None;
4578 self.advertised = runtime.health.http.is_some();
4579 if !self.advertised {
4580 self.next_probe_at = None;
4581 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4582 let unknown = ModuleHealthStatus::default();
4583 if state.health != unknown {
4584 state.health = unknown;
4585 }
4586 });
4587 } else if self.next_probe_at.is_none() {
4588 self.next_probe_at = Some(
4589 Instant::now()
4590 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4591 );
4592 }
4593 return;
4594 }
4595
4596 let registration = match registry.get_module(&spec.module_id) {
4597 Ok(registration) => registration,
4598 Err(err) => {
4599 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4600 self.advertised = false;
4601 self.next_probe_at = None;
4602 return;
4603 }
4604 };
4605
4606 let Some(registration) = registration else {
4607 self.registered_connection = None;
4608 self.advertised = false;
4609 self.next_probe_at = None;
4610 return;
4611 };
4612
4613 let advertised = registration
4614 .control_ops
4615 .iter()
4616 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4617 if !advertised {
4618 self.registered_connection = Some(registration.connection_id);
4619 self.advertised = false;
4620 self.next_probe_at = None;
4621 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4622 if state.health.status != SupervisorHealthStatus::Unknown
4623 || state.health.consecutive_failures != 0
4624 || state.health.last_probe_ms.is_some()
4625 || state.health.detail.is_some()
4626 || state.health.metrics.is_some()
4627 {
4628 state.health.status = SupervisorHealthStatus::Unknown;
4629 state.health.consecutive_failures = 0;
4630 state.health.last_probe_ms = None;
4631 state.health.detail = None;
4632 state.health.metrics = None;
4633 }
4634 });
4635 return;
4636 }
4637
4638 let reregistered = self.registered_connection != Some(registration.connection_id);
4639 self.registered_connection = Some(registration.connection_id);
4640 self.advertised = true;
4641 if reregistered || self.next_probe_at.is_none() {
4642 self.probe_index = 0;
4643 self.next_probe_at = Some(
4644 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4645 );
4646 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4647 state.health.status = SupervisorHealthStatus::Unknown;
4648 state.health.consecutive_failures = 0;
4649 state.health.detail = None;
4650 state.health.metrics = None;
4651 });
4652 }
4653 }
4654
4655 fn wake_after(&self) -> Option<Duration> {
4656 if !self.advertised {
4657 return None;
4658 }
4659 self.next_probe_at
4660 .map(|next| next.saturating_duration_since(Instant::now()))
4661 }
4662
4663 fn due(&self) -> bool {
4664 self.advertised
4665 && self
4666 .next_probe_at
4667 .is_some_and(|next| Instant::now() >= next)
4668 }
4669
4670 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4671 self.probe_index = self.probe_index.wrapping_add(1);
4672 self.next_probe_at = Some(
4673 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4674 );
4675 }
4676}
4677
4678#[derive(Debug)]
4713enum HealthProbeEvidence {
4714 LaneDead,
4716 NoAnswer,
4718 BadAnswer,
4720 Misconfigured,
4722}
4723
4724#[derive(Debug)]
4725struct HealthProbeError {
4726 evidence: HealthProbeEvidence,
4727 message: String,
4728}
4729
4730impl HealthProbeError {
4731 fn lane_dead(message: impl Into<String>) -> Self {
4732 Self::with(HealthProbeEvidence::LaneDead, message)
4733 }
4734
4735 fn no_answer(message: impl Into<String>) -> Self {
4736 Self::with(HealthProbeEvidence::NoAnswer, message)
4737 }
4738
4739 fn bad_answer(message: impl Into<String>) -> Self {
4740 Self::with(HealthProbeEvidence::BadAnswer, message)
4741 }
4742
4743 fn misconfigured(message: impl Into<String>) -> Self {
4744 Self::with(HealthProbeEvidence::Misconfigured, message)
4745 }
4746
4747 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4748 Self {
4749 evidence,
4750 message: message.into(),
4751 }
4752 }
4753
4754 #[allow(dead_code)]
4768 fn is_proof_of_death(&self) -> bool {
4769 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4770 }
4771
4772 fn label(&self) -> &'static str {
4780 match self.evidence {
4781 HealthProbeEvidence::LaneDead => "lane-dead",
4782 HealthProbeEvidence::NoAnswer => "no-answer",
4783 HealthProbeEvidence::BadAnswer => "bad-answer",
4784 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4785 }
4786 }
4787}
4788
4789impl fmt::Display for HealthProbeError {
4790 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4791 f.write_str(&self.message)
4792 }
4793}
4794
4795async fn run_health_probe_cycle(
4796 spec: &ModuleSpec,
4797 runtime: &SupervisorRuntimeConfig,
4798 registry: &Registry,
4799 process_liveness: &SupervisorProcessLiveness,
4800 snapshot: &SharedSnapshot,
4801 child: &mut Option<SupervisedChild>,
4802) {
4803 let now_ms = unix_ms_now();
4804 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4805 .then_some(runtime.health.http.as_deref())
4806 .flatten();
4807 let result = match http {
4808 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4809 None => probe_module_health(&spec.module_id, runtime, None).await,
4810 };
4811 match result {
4812 Ok(report) => {
4813 handle_health_report(
4814 spec,
4815 runtime,
4816 registry,
4817 process_liveness,
4818 snapshot,
4819 child,
4820 report,
4821 now_ms,
4822 )
4823 .await;
4824 }
4825 Err(err) => {
4826 if http.is_some() {
4827 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4828 state.health.status = SupervisorHealthStatus::Failing;
4829 });
4830 }
4831 handle_health_probe_failure(
4832 spec,
4833 runtime,
4834 registry,
4835 process_liveness,
4836 snapshot,
4837 child,
4838 err,
4839 now_ms,
4840 )
4841 .await;
4842 }
4843 }
4844}
4845
4846pub(crate) struct HttpProbeTarget<'a> {
4847 address: std::net::SocketAddr,
4848 localhost: bool,
4849 authority: &'a str,
4850 path: String,
4851}
4852
4853pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4856 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4857 return Err("must not contain whitespace, controls, or a fragment".into());
4858 }
4859 let rest = url
4860 .strip_prefix("http://")
4861 .ok_or("must use plain http://")?;
4862 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4863 let (authority, suffix) = rest.split_at(split);
4864 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4865 ("::1", rest)
4866 } else {
4867 let split = authority.find(':').unwrap_or(authority.len());
4868 authority.split_at(split)
4869 };
4870 let ip: std::net::IpAddr = match host {
4871 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4872 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4873 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4874 };
4875 let port = if port.is_empty() {
4876 80
4877 } else {
4878 port.strip_prefix(':')
4879 .and_then(|p| p.parse::<u16>().ok())
4880 .filter(|p| *p > 0)
4881 .ok_or("must have a valid nonzero TCP port")?
4882 };
4883 let path = if suffix.is_empty() {
4884 "/".into()
4885 } else if suffix.starts_with('?') {
4886 format!("/{suffix}")
4887 } else {
4888 suffix.into()
4889 };
4890 Ok(HttpProbeTarget {
4891 address: std::net::SocketAddr::new(ip, port),
4892 localhost: host == "localhost",
4893 authority,
4894 path,
4895 })
4896}
4897
4898async fn probe_http_health(
4899 url: &str,
4900 deadline: Duration,
4901) -> Result<HealthReport, HealthProbeError> {
4902 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4903 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4904 let mut response_status = String::new();
4907 let mut body = Vec::new();
4908 let probe = async {
4909 let connection = match tokio::net::TcpStream::connect(target.address).await {
4912 Err(_) if target.localhost => {
4913 tokio::net::TcpStream::connect((
4914 std::net::Ipv6Addr::LOCALHOST,
4915 target.address.port(),
4916 ))
4917 .await
4918 }
4919 result => result,
4920 };
4921 let mut stream = connection.map_err(|error| {
4922 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4923 })?;
4924 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4925 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4926 let mut reader = BufReader::new(stream);
4927 let mut budget = 16 * 1024;
4928 let status = http_line(&mut reader, &mut budget).await?;
4929 let mut words = status.split_ascii_whitespace();
4930 let version = words.next();
4931 let code = words
4932 .next()
4933 .filter(|word| word.len() == 3)
4934 .and_then(|word| word.parse::<u16>().ok());
4935 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4936 || !code.is_some_and(|code| (100..600).contains(&code))
4937 {
4938 return Err(HealthProbeError::bad_answer(format!(
4939 "invalid HTTP status: {status}"
4940 )));
4941 }
4942 let code = code.expect("validated status code");
4943 response_status = status.clone();
4944 let mut length = None;
4945 let mut chunked = false;
4946 loop {
4947 let line = http_line(&mut reader, &mut budget).await?;
4948 if line.is_empty() {
4949 break;
4950 }
4951 if let Some((name, value)) = line.split_once(':') {
4952 if name.eq_ignore_ascii_case("content-length") {
4953 length = Some(value.trim().parse::<u64>().map_err(|_| {
4954 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4955 })?);
4956 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4957 chunked = value.trim().eq_ignore_ascii_case("chunked");
4958 }
4959 }
4960 }
4961 if chunked {
4962 while body.len() < 200 {
4963 let line = http_line(&mut reader, &mut budget).await?;
4964 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4965 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4966 if size == 0 {
4967 break;
4968 }
4969 let count = size.min((200 - body.len()) as u64) as usize;
4970 let start = body.len();
4971 (&mut reader)
4972 .take(count as u64)
4973 .read_to_end(&mut body)
4974 .await
4975 .map_err(|error| {
4976 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4977 })?;
4978 if body.len() - start != count {
4979 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4980 }
4981 if size > count as u64 || body.len() == 200 {
4982 break;
4983 }
4984 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4985 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4986 }
4987 }
4988 } else {
4989 reader
4990 .take(length.unwrap_or(200).min(200))
4991 .read_to_end(&mut body)
4992 .await
4993 .map_err(|error| {
4994 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4995 })?;
4996 }
4997 if (200..300).contains(&code) {
4998 Ok(HealthReport::ok())
4999 } else {
5000 Err(HealthProbeError::bad_answer(
5001 "HTTP health endpoint returned non-2xx",
5002 ))
5003 }
5004 };
5005 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5006 Err(HealthProbeError::no_answer(format!(
5007 "HTTP probe timed out after {deadline:?}"
5008 )))
5009 });
5010 if let Err(error) = &mut result {
5011 if !response_status.is_empty() {
5012 error.message = format!(
5013 "{}; {response_status}: {}",
5014 error.message,
5015 String::from_utf8_lossy(&body)
5016 );
5017 }
5018 }
5019 result
5020}
5021
5022async fn http_line(
5023 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5024 remaining: &mut usize,
5025) -> Result<String, HealthProbeError> {
5026 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5027 let mut line = Vec::new();
5028 (&mut *reader)
5029 .take(*remaining as u64)
5030 .read_until(b'\n', &mut line)
5031 .await
5032 .map_err(|error| {
5033 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5034 })?;
5035 *remaining -= line.len();
5036 if !line.ends_with(b"\r\n") {
5037 return Err(HealthProbeError::bad_answer(
5038 "HTTP headers are incomplete or exceed 16 KiB",
5039 ));
5040 }
5041 line.truncate(line.len() - 2);
5042 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5043}
5044
5045async fn probe_module_health(
5046 module_id: &str,
5047 runtime: &SupervisorRuntimeConfig,
5048 drain_deadline: Option<Instant>,
5049) -> Result<HealthReport, HealthProbeError> {
5050 let Some(forwarding) = runtime.forwarding.as_ref() else {
5051 return Err(HealthProbeError::misconfigured(
5052 "supervisor was not configured with a forwarding table",
5053 ));
5054 };
5055 let probe_started_at = Instant::now();
5056 let mut deadline = probe_started_at + runtime.health.deadline;
5057 if let Some(drain_deadline) = drain_deadline {
5058 deadline = deadline.min(drain_deadline);
5059 }
5060 let pending = if drain_deadline.is_some() {
5061 forwarding.begin_drain_health_probe_rpc_for(
5062 module_id,
5063 MODULE_CONTROL_OP_HEALTH_CHECK,
5064 probe_started_at,
5065 deadline,
5066 )
5067 } else {
5068 forwarding.begin_health_probe_rpc_for(
5069 module_id,
5070 MODULE_CONTROL_OP_HEALTH_CHECK,
5071 probe_started_at,
5072 deadline,
5073 )
5074 }
5075 .map_err(|err| {
5076 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5079 })?;
5080 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5081}
5082
5083async fn probe_endpoint_health(
5090 endpoint: crate::ModuleEndpointId,
5091 runtime: &SupervisorRuntimeConfig,
5092 deadline_cap: Option<Instant>,
5093) -> Result<HealthReport, HealthProbeError> {
5094 let Some(forwarding) = runtime.forwarding.as_ref() else {
5095 return Err(HealthProbeError::misconfigured(
5096 "supervisor was not configured with a forwarding table",
5097 ));
5098 };
5099 let probe_started_at = Instant::now();
5100 let mut deadline = probe_started_at + runtime.health.deadline;
5101 if let Some(cap) = deadline_cap {
5102 deadline = deadline.min(cap);
5103 }
5104 let pending = forwarding
5105 .begin_endpoint_health_probe_rpc_for(
5106 endpoint,
5107 MODULE_CONTROL_OP_HEALTH_CHECK,
5108 probe_started_at,
5109 deadline,
5110 )
5111 .map_err(|err| {
5112 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5113 })?;
5114 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5115}
5116
5117async fn await_health_probe(
5119 forwarding: &ForwardingTable,
5120 pending: PendingModuleControlRpc,
5121 deadline: Instant,
5122 probe_budget: Duration,
5123) -> Result<HealthReport, HealthProbeError> {
5124 let PendingModuleControlRpc {
5125 endpoint,
5126 module_sink,
5127 negotiated_ver,
5128 corr,
5129 receiver,
5130 } = pending;
5131 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5132 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5133 })?;
5134 let frame = Frame::build_with_version(
5135 negotiated_ver,
5136 FrameType::Request,
5137 control_flags(),
5138 0,
5139 0,
5140 corr,
5141 body,
5142 )
5143 .map_err(|err| {
5144 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5145 })?;
5146
5147 match timeout_at(deadline, module_sink.send(frame)).await {
5153 Ok(Ok(())) => {}
5154 Ok(Err(err)) => {
5155 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5156 return Err(HealthProbeError::lane_dead(format!(
5159 "failed to send health.check: {err}"
5160 )));
5161 }
5162 Err(_elapsed) => {
5163 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5164 return Err(HealthProbeError::no_answer(
5168 "health.check send timed out before enqueue (module egress full)",
5169 ));
5170 }
5171 }
5172
5173 match timeout_at(deadline, receiver).await {
5174 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5178 response.health_report().ok_or_else(|| {
5179 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5180 })
5181 }
5182 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5183 format!("health.check rejected: {}", body.message),
5184 )),
5185 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5186 Err(HealthProbeError::lane_dead(message))
5187 }
5188 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5189 Err(HealthProbeError::bad_answer(message))
5190 }
5191 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5192 Err(HealthProbeError::bad_answer(format!(
5193 "expected module-control op '{expected}', got '{actual}'"
5194 )))
5195 }
5196 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5200 "module answered health.check after its daemon deadline",
5201 )),
5202 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5203 "health.check waiter was canceled before the module responded",
5204 )),
5205 Err(_) => {
5206 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5207 Err(HealthProbeError::no_answer(format!(
5208 "module did not answer health.check within {probe_budget:?}"
5209 )))
5210 }
5211 }
5212}
5213
5214#[allow(clippy::too_many_arguments)]
5215async fn handle_health_report(
5216 spec: &ModuleSpec,
5217 runtime: &SupervisorRuntimeConfig,
5218 registry: &Registry,
5219 process_liveness: &SupervisorProcessLiveness,
5220 snapshot: &SharedSnapshot,
5221 child: &mut Option<SupervisedChild>,
5222 report: HealthReport,
5223 now_ms: u64,
5224) {
5225 let status = supervisor_health_status(report.status);
5226 let detail = report.detail.clone();
5227 let metrics = truncate_health_metrics(report.metrics);
5228 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5229 state.health.status = status;
5230 state.health.last_probe_ms = Some(now_ms);
5231 state.health.detail = detail.clone();
5232 state.health.metrics = metrics.clone();
5233 state.health.consecutive_failures = 0;
5234 state.failed_retry.observe_health(
5235 report.status == HealthStatus::Ok,
5236 runtime.restart_policy.window,
5237 );
5238 });
5239
5240 let action = match report.status {
5241 HealthStatus::Ok => return,
5242 HealthStatus::Degraded => runtime.health.on_degraded,
5243 HealthStatus::Failing => runtime.health.on_failing,
5244 };
5245 apply_l3_health_action(
5246 spec,
5247 runtime,
5248 registry,
5249 process_liveness,
5250 snapshot,
5251 child,
5252 status,
5253 detail.as_deref(),
5254 action,
5255 now_ms,
5256 )
5257 .await;
5258}
5259
5260#[allow(clippy::too_many_arguments)]
5261async fn handle_health_probe_failure(
5262 spec: &ModuleSpec,
5263 runtime: &SupervisorRuntimeConfig,
5264 registry: &Registry,
5265 process_liveness: &SupervisorProcessLiveness,
5266 snapshot: &SharedSnapshot,
5267 child: &mut Option<SupervisedChild>,
5268 err: HealthProbeError,
5269 now_ms: u64,
5270) {
5271 let threshold = runtime.health.failure_threshold.max(1);
5272 let mut failures = 0;
5273 let detail = format!("[{}] {err}", err.label());
5278 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5279 state
5280 .failed_retry
5281 .observe_health(false, runtime.restart_policy.window);
5282 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5285 state.health.status = SupervisorHealthStatus::Unknown;
5286 }
5287 state.health.last_probe_ms = Some(now_ms);
5288 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5289 state.health.detail = Some(detail.clone());
5290 state.health.metrics = None;
5291 failures = state.health.consecutive_failures;
5292 });
5293
5294 if failures < threshold {
5295 warn!(
5296 module_id = %spec.module_id,
5297 consecutive_failures = failures,
5298 threshold,
5299 evidence = err.label(),
5300 detail = %detail,
5301 "health.check probe failed"
5302 );
5303 return;
5304 }
5305
5306 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5307 state.state = ModuleState::Unresponsive;
5308 state.health.status = SupervisorHealthStatus::Unresponsive;
5309 });
5310 if runtime.health.critical {
5314 error!(
5315 module_id = %spec.module_id,
5316 status = "unresponsive",
5317 evidence = err.label(),
5318 detail = %detail,
5319 "critical module health alert"
5320 );
5321 } else {
5322 warn!(
5323 module_id = %spec.module_id,
5324 status = "unresponsive",
5325 evidence = err.label(),
5326 detail = %detail,
5327 "module health threshold breached"
5328 );
5329 }
5330 if let Err(err) = health_restart_child(
5331 spec,
5332 runtime,
5333 registry,
5334 process_liveness,
5335 snapshot,
5336 child,
5337 SupervisorHealthStatus::Unresponsive,
5338 Some(&detail),
5339 now_ms,
5340 )
5341 .await
5342 {
5343 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5344 }
5345}
5346
5347#[allow(clippy::too_many_arguments)]
5348async fn apply_l3_health_action(
5349 spec: &ModuleSpec,
5350 runtime: &SupervisorRuntimeConfig,
5351 registry: &Registry,
5352 process_liveness: &SupervisorProcessLiveness,
5353 snapshot: &SharedSnapshot,
5354 child: &mut Option<SupervisedChild>,
5355 status: SupervisorHealthStatus,
5356 detail: Option<&str>,
5357 action: HealthAction,
5358 now_ms: u64,
5359) {
5360 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5361 match action {
5362 HealthAction::Report => {
5363 info!(
5364 module_id = %spec.module_id,
5365 status = ?status,
5366 detail,
5367 "module reported non-ok health"
5368 );
5369 }
5370 HealthAction::Alert => {
5371 error!(
5372 module_id = %spec.module_id,
5373 status = ?status,
5374 detail,
5375 "module health alert"
5376 );
5377 }
5378 HealthAction::Restart => {
5379 if let Err(err) = health_restart_child(
5380 spec,
5381 runtime,
5382 registry,
5383 process_liveness,
5384 snapshot,
5385 child,
5386 status,
5387 detail,
5388 now_ms,
5389 )
5390 .await
5391 {
5392 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5393 }
5394 }
5395 }
5396}
5397
5398#[allow(clippy::too_many_arguments)]
5399async fn health_restart_child(
5400 spec: &ModuleSpec,
5401 runtime: &SupervisorRuntimeConfig,
5402 registry: &Registry,
5403 process_liveness: &SupervisorProcessLiveness,
5404 snapshot: &SharedSnapshot,
5405 child: &mut Option<SupervisedChild>,
5406 status: SupervisorHealthStatus,
5407 detail: Option<&str>,
5408 now_ms: u64,
5409) -> Result<(), SuperviseError> {
5410 let (enabled, schedule) = {
5411 let mut state = lock_snapshot(snapshot)?;
5412 let enabled = state.enabled;
5413 let schedule = if enabled {
5414 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5415 } else {
5416 None
5417 };
5418 (enabled, schedule)
5419 };
5420
5421 if !enabled {
5422 return Err(SuperviseError::Disabled {
5423 module_id: spec.module_id.clone(),
5424 });
5425 }
5426
5427 if schedule.is_none() {
5428 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5429 error!(
5430 module_id = %spec.module_id,
5431 status = ?status,
5432 detail,
5433 max_restarts = runtime.restart_policy.max_restarts,
5434 window_secs = runtime.restart_policy.window.as_secs(),
5435 reason = %runtime.restart_policy.budget_exhausted_detail(),
5436 "health restart budget exhausted; marking module failed"
5437 );
5438 let stop_notice = begin_forwarding_drain_if_configured(
5439 spec,
5440 runtime,
5441 registry,
5442 snapshot,
5443 Some(true),
5444 RouteCloseReason::Disable,
5445 )
5446 .await?;
5447 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5448 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5449 })?;
5450 let result = drain_optional_child(
5451 &spec.module_id,
5452 spec.protocol,
5453 stop_notice,
5454 registry,
5455 runtime.forwarding.as_deref(),
5456 snapshot,
5457 &runtime.terminal_ring,
5458 &runtime.spawn_events,
5459 child,
5460 runtime.drain_timeout,
5461 ModuleState::Failed,
5462 Some(true),
5463 &runtime.failure_context(true),
5464 )
5465 .await;
5466 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5467 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5468 state.failed_retry.schedule(&spec.module_id);
5469 }
5470 })?;
5471 result?;
5472 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5473 return Ok(());
5474 }
5475
5476 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5477 let mut restart_count = 0;
5478 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5479 restart_count = state.crash_restarts.len();
5480 state.state = ModuleState::Unresponsive;
5481 state.health.status = status;
5482 state.health.last_action = Some(HealthAction::Restart.to_string());
5483 state.health.last_action_ms = Some(now_ms);
5484 })?;
5485 warn!(
5486 module_id = %spec.module_id,
5487 status = ?status,
5488 detail,
5489 restart_count,
5490 restart_in_window = schedule.restart_in_window,
5491 delay_ms = schedule.delay.as_millis() as u64,
5492 "health-triggered module restart"
5493 );
5494
5495 let stop_notice = begin_forwarding_drain_if_configured(
5496 spec,
5497 runtime,
5498 registry,
5499 snapshot,
5500 Some(true),
5501 RouteCloseReason::Restart,
5502 )
5503 .await?;
5504 drain_optional_child(
5505 &spec.module_id,
5506 spec.protocol,
5507 stop_notice,
5508 registry,
5509 runtime.forwarding.as_deref(),
5510 snapshot,
5511 &runtime.terminal_ring,
5512 &runtime.spawn_events,
5513 child,
5514 runtime.drain_timeout,
5515 ModuleState::Restarting,
5516 Some(true),
5517 &runtime.failure_context(true),
5518 )
5519 .await?;
5520 schedule_respawn(
5521 runtime,
5522 snapshot,
5523 &spec.module_id,
5524 schedule.delay,
5525 RespawnKind::Spawn,
5526 true,
5527 )
5528}
5529
5530fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5531 if let Some(reply) = runtime
5532 .deferred_reload_reply
5533 .lock()
5534 .unwrap_or_else(|p| p.into_inner())
5535 .take()
5536 {
5537 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5538 module_id: module_id.to_string(),
5539 reason: reason.to_string(),
5540 }));
5541 }
5542}
5543
5544fn schedule_respawn(
5545 runtime: &SupervisorRuntimeConfig,
5546 snapshot: &SharedSnapshot,
5547 module_id: &str,
5548 delay: Duration,
5549 kind: RespawnKind,
5550 daemon_initiated: bool,
5551) -> Result<(), SuperviseError> {
5552 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5553 update_snapshot(snapshot, Some(module_id), |state| {
5554 state.respawn_pending = true
5555 })?;
5556 *runtime
5557 .scheduled_respawn
5558 .lock()
5559 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5560 deadline: Instant::now() + delay,
5561 kind,
5562 daemon_initiated,
5563 });
5564 Ok(())
5565}
5566
5567fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5568 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5569 state.health.last_action = Some(action);
5570 state.health.last_action_ms = Some(now_ms);
5571 });
5572}
5573
5574fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5575 match status {
5576 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5577 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5578 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5579 }
5580}
5581
5582fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5594 let metrics = metrics?;
5595 match serde_json::to_vec(&metrics) {
5596 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5597 "truncated": true,
5598 "original_bytes": encoded.len(),
5599 })),
5600 Ok(_) | Err(_) => Some(metrics),
5601 }
5602}
5603
5604fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5610 if cadence.is_zero() {
5611 return Duration::ZERO;
5612 }
5613 let cadence_ms = cadence.as_millis() as u64;
5614 if cadence_ms == 0 {
5630 return cadence;
5631 }
5632 let jitter_span = (cadence_ms / 10).max(1);
5647 let hash = module_id.as_bytes().iter().fold(
5648 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5649 |acc, byte| {
5650 acc.wrapping_mul(1099511628211)
5651 .wrapping_add(u64::from(*byte))
5652 },
5653 );
5654 cadence + Duration::from_millis(hash % jitter_span)
5655}
5656
5657#[cfg(test)]
5658mod tests {
5659 use super::*;
5660
5661 #[test]
5662 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5663 let handle = SupervisorHandle::new();
5664 let module_id = "readded-tombstone";
5665 handle.record_rescan_removal(module_id);
5666 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5667
5668 handle.apply_identity_configuration(&ModuleSpec {
5669 module_id: module_id.to_string(),
5670 program: PathBuf::from("/test/module"),
5671 args: Vec::new(),
5672 env: Vec::new(),
5673 reserved: false,
5674 reserved_prefixes: Vec::new(),
5675 protocol: ModuleProtocol::Subc,
5676 overlap: Default::default(),
5677 });
5678
5679 assert!(
5680 handle.removal_tombstone_age_ms(module_id).is_none(),
5681 "a re-added module must not retain a stale removal tombstone"
5682 );
5683 }
5684
5685 #[derive(Debug, PartialEq, Eq)]
5688 struct OwnerInSpawnWindow {
5689 module_id: String,
5690 configured: bool,
5691 on_roster: bool,
5692 admission_refusal: Option<&'static str>,
5693 }
5694
5695 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5706 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5707 use crate::scopes::ScopeTable;
5708 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5709
5710 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5711 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5712 module_id: module_id.to_string(),
5713 program,
5714 args: Vec::new(),
5715 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5716 .into_iter()
5717 .map(|key| (key.to_string(), dir.path().display().to_string()))
5718 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5719 .collect(),
5720 reserved: false,
5721 reserved_prefixes: Vec::new(),
5722 protocol: ModuleProtocol::Subc,
5723 overlap: Default::default(),
5724 };
5725 let live = super::terminal_history_tests::fake_aft_stub_path();
5726 let missing = dir.path().join("definitely-missing-module");
5727
5728 let handle = SupervisorHandle::new();
5729 let mut supervisor =
5730 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5731 .with_handle(handle.clone());
5732 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5733 let hook_handle = handle.clone();
5734 let hook_observed = Arc::clone(&observed);
5735 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5736 let configured = hook_handle.is_configured(module_id);
5739 let selector = ScopeSelector::new(
5740 Principal::Reserved {
5741 module_id: module_id.to_string(),
5742 },
5743 "s".to_string(),
5744 Some(1),
5745 );
5746 let carrier = Principal::Reserved {
5747 module_id: "carrier".to_string(),
5748 };
5749 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5750 .admit(&carrier, module_id, &selector, configured)
5751 {
5752 Ok(_) => None,
5753 Err(refusal) => Some(refusal.code),
5754 };
5755 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5756 module_id: module_id.to_string(),
5757 configured,
5758 on_roster: hook_handle.get(module_id).is_some(),
5759 admission_refusal,
5760 });
5761 })));
5762
5763 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5764 let configured = supervisor
5765 .supervise_configured(stub("configured", live.clone()), true)
5766 .unwrap();
5767 let with_health = supervisor
5768 .supervise_configured_with_health(
5769 stub("with-health", live.clone()),
5770 true,
5771 HealthConfig::default(),
5772 None,
5773 RestartPolicy::default(),
5774 )
5775 .unwrap();
5776 let failed = supervisor
5779 .supervise_configured_with_health(
5780 stub("failed-spawn", missing.clone()),
5781 true,
5782 HealthConfig::default(),
5783 None,
5784 RestartPolicy::default(),
5785 )
5786 .unwrap();
5787 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5790
5791 let expected = [
5792 "plain",
5793 "configured",
5794 "with-health",
5795 "failed-spawn",
5796 "spawn-error",
5797 ]
5798 .into_iter()
5799 .map(|module_id| OwnerInSpawnWindow {
5800 module_id: module_id.to_string(),
5801 configured: true,
5802 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5803 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5804 })
5805 .collect::<Vec<_>>();
5806 assert_eq!(*observed.lock().unwrap(), expected);
5807
5808 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5809 assert!(
5810 handle.get(module_id).is_some(),
5811 "{module_id} is on the roster"
5812 );
5813 assert!(
5814 handle.is_configured(module_id),
5815 "{module_id} stays configured"
5816 );
5817 }
5818 assert!(handle.get("spawn-error").is_none());
5819 assert!(
5820 !handle.is_configured("spawn-error"),
5821 "a plain spawn that failed must not leave its module marked configured"
5822 );
5823
5824 handle.retire("failed-spawn");
5826 assert!(!handle.is_configured("failed-spawn"));
5827
5828 for module in [plain, configured, with_health] {
5829 module.stop().await.unwrap();
5830 }
5831 drop(failed);
5832 }
5833
5834 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5841 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5842 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5843 let spec = ModuleSpec {
5844 module_id: "spawned-not-admitted".to_string(),
5845 program: super::terminal_history_tests::fake_aft_stub_path(),
5846 args: Vec::new(),
5847 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5848 .into_iter()
5849 .map(|key| (key.to_string(), dir.path().display().to_string()))
5850 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5851 .collect(),
5852 reserved: false,
5853 reserved_prefixes: Vec::new(),
5854 protocol: ModuleProtocol::Subc,
5855 overlap: Default::default(),
5856 };
5857 let handle = SupervisorHandle::new();
5858 let mut supervisor =
5859 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5860 .with_handle(handle.clone());
5861 let observed = Arc::new(Mutex::new(None));
5862 let hook_handle = handle.clone();
5863 let hook_observed = Arc::clone(&observed);
5864 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5865 let state = hook_handle
5866 .get(module_id)
5867 .expect("a configured module is on the roster before it is spawned")
5868 .state()
5869 .unwrap();
5870 *hook_observed.lock().unwrap() = Some(state);
5871 })));
5872
5873 let module = supervisor.supervise_configured(spec, true).unwrap();
5874 assert_eq!(
5875 *observed.lock().unwrap(),
5876 Some(ModuleState::Starting),
5877 "between the spawn and its admission the supervisor still reports starting"
5878 );
5879 assert_eq!(module.state().unwrap(), ModuleState::Running);
5880 module.stop().await.unwrap();
5881 }
5882
5883 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5884 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5885 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5886 snapshot.process_alive = true;
5887 snapshot.pid = Some(41);
5888 snapshot.spawned_at_ms = Some(42);
5889 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5890 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5891 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5892 device: 43,
5893 inode: 44,
5894 #[cfg(windows)]
5895 windows_image: None,
5896 });
5897 })
5898 .unwrap();
5899 snapshot
5900 }
5901
5902 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5903 let snapshot = lock_snapshot(snapshot).unwrap();
5904 assert!(!snapshot.process_alive);
5905 assert_eq!(snapshot.pid, None);
5906 assert_eq!(snapshot.spawned_at_ms, None);
5907 assert_eq!(snapshot.spawned_from, None);
5908 assert_eq!(snapshot.launch_spec_digest, None);
5909 assert_eq!(snapshot.spawned_file_identity, None);
5910 }
5911
5912 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5913 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5914 let supervisor =
5915 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5916 let mut runtime = supervisor.runtime_config();
5917 runtime.test_seed_stale_facts_before_enable_spawn = true;
5918 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5919 let mut child = None;
5920 let spec = ModuleSpec {
5921 module_id: "failed-enable-clears-facts".to_string(),
5922 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5923 args: Vec::new(),
5924 env: Vec::new(),
5925 reserved: false,
5926 reserved_prefixes: Vec::new(),
5927 protocol: ModuleProtocol::Subc,
5928 overlap: Default::default(),
5929 };
5930
5931 let result = set_child_enabled(
5932 &spec,
5933 &runtime,
5934 &supervisor.registry,
5935 &supervisor.process_liveness,
5936 &snapshot,
5937 &mut child,
5938 true,
5939 &runtime.failure_context(false),
5940 )
5941 .await;
5942
5943 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5944 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5945 assert_snapshot_process_facts_cleared(&snapshot);
5946 }
5947
5948 #[tokio::test]
5949 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5950 let supervisor =
5951 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5952 let runtime = supervisor.runtime_config();
5953 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5954 ModuleState::Restarting,
5955 true,
5956 )));
5957 let spec = ModuleSpec {
5958 module_id: "start-stranded-restarting".to_string(),
5959 program: super::terminal_history_tests::fake_aft_stub_path(),
5960 args: Vec::new(),
5961 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5962 reserved: false,
5963 reserved_prefixes: Vec::new(),
5964 protocol: ModuleProtocol::None,
5965 overlap: Default::default(),
5966 };
5967 let mut child = None;
5968 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5969 assert!(!super::set_child_enabled(
5970 &spec,
5971 &runtime,
5972 &Registry::default(),
5973 &supervisor.process_liveness,
5974 &snapshot,
5975 &mut child,
5976 true,
5977 &runtime.failure_context(false),
5978 )
5979 .await
5980 .unwrap());
5981 assert!(child.is_none());
5982 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5983 assert!(super::set_child_enabled(
5984 &spec,
5985 &runtime,
5986 &Registry::default(),
5987 &supervisor.process_liveness,
5988 &snapshot,
5989 &mut child,
5990 true,
5991 &runtime.failure_context(false),
5992 )
5993 .await
5994 .unwrap());
5995 assert_eq!(
5996 lock_snapshot(&snapshot).unwrap().state,
5997 ModuleState::Running
5998 );
5999 let mut child = child.unwrap();
6000 child.start_kill().unwrap();
6001 child.wait().await.unwrap();
6002 }
6003
6004 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6005 async fn failed_reload_spawn_clears_current_process_facts() {
6006 let supervisor =
6007 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6008 let mut runtime = supervisor.runtime_config();
6009 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6010 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6011 let mut child = None;
6012 let spec = ModuleSpec {
6013 module_id: "failed-reload-clears-facts".to_string(),
6014 program: PathBuf::from("/unused/failed-reload-module"),
6015 args: Vec::new(),
6016 env: Vec::new(),
6017 reserved: false,
6018 reserved_prefixes: Vec::new(),
6019 protocol: ModuleProtocol::Subc,
6020 overlap: Default::default(),
6021 };
6022
6023 let result = handle_reload_spawn_failure(
6024 &spec,
6025 &runtime,
6026 &supervisor.process_liveness,
6027 &snapshot,
6028 &mut child,
6029 "forced reload spawn failure".to_string(),
6030 )
6031 .await;
6032
6033 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6034 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6035 assert_snapshot_process_facts_cleared(&snapshot);
6036 }
6037
6038 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6039 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6040 let supervisor =
6041 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6042 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6043 let module = supervisor.supervised_module(
6044 ModuleSpec {
6045 module_id: "drop-clears-facts".to_string(),
6046 program: PathBuf::from("/unused/drop-module"),
6047 args: Vec::new(),
6048 env: Vec::new(),
6049 reserved: false,
6050 reserved_prefixes: Vec::new(),
6051 protocol: ModuleProtocol::Subc,
6052 overlap: Default::default(),
6053 },
6054 supervisor.runtime_config(),
6055 Arc::clone(&snapshot),
6056 None,
6057 );
6058 assert!(!module
6059 .inner
6060 .monitor
6061 .lock()
6062 .unwrap()
6063 .as_ref()
6064 .unwrap()
6065 .is_finished());
6066
6067 drop(module);
6068
6069 assert_eq!(
6070 lock_snapshot(&snapshot).unwrap().state,
6071 ModuleState::Stopped
6072 );
6073 assert_snapshot_process_facts_cleared(&snapshot);
6074 }
6075
6076 #[cfg(unix)]
6077 #[tokio::test]
6078 async fn rescan_preserves_running_protocol_until_respawn() {
6079 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6080 let initial = ModuleSpec {
6081 module_id: "rescan-protocol".into(),
6082 program: PathBuf::from("/bin/sleep"),
6083 args: vec!["60".into()],
6084 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6085 .into_iter()
6086 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6087 .collect(),
6088 reserved: false,
6089 reserved_prefixes: vec![],
6090 protocol: ModuleProtocol::None,
6091 overlap: Default::default(),
6092 };
6093 let supervisor =
6094 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6095 let module = supervisor.spawn(initial.clone()).unwrap();
6096 assert!(module.status().unwrap().live);
6097 let mut next = initial;
6098 next.protocol = ModuleProtocol::Subc;
6099 module
6100 .update_configuration(next.clone(), HealthConfig::default(), None)
6101 .await
6102 .unwrap();
6103 assert!(
6104 module.status().unwrap().live,
6105 "rescan must not require HELLO from the old non-wire process"
6106 );
6107 let runtime = supervisor.runtime_config();
6108 let action = on_child_exit(
6109 &next,
6110 RestartPolicy::default(),
6111 &supervisor.registry,
6112 &module.inner.snapshot,
6113 &runtime.terminal_ring,
6114 &runtime.spawn_events,
6115 &runtime.child_roster,
6116 ExitReport {
6117 kind: ExitKind::Clean,
6118 code: Some(0),
6119 signal: None,
6120 at_ms: unix_ms_now(),
6121 },
6122 true,
6123 )
6124 .await;
6125 assert!(
6126 matches!(action, NextAction::Restart { .. }),
6127 "the old non-wire process's clean exit must restart"
6128 );
6129 module.drain().await.unwrap();
6130 }
6131
6132 #[cfg(unix)]
6133 #[tokio::test]
6134 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6135 use std::os::unix::fs::PermissionsExt;
6136 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6137 let script = dir.join("module.sh");
6138 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6139 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6140 let record_path = dir.join("live-children.json");
6141 let supervisor =
6142 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6143 .with_live_children_record(&record_path);
6144 for (program, args) in [
6145 (PathBuf::from("sleep"), vec!["60".into()]),
6146 (script, vec![]),
6147 ] {
6148 let spec = ModuleSpec {
6149 module_id: "image-identity".into(),
6150 program: program.clone(),
6151 args,
6152 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6153 .into_iter()
6154 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6155 .collect(),
6156 reserved: false,
6157 reserved_prefixes: vec![],
6158 protocol: ModuleProtocol::None,
6159 overlap: Default::default(),
6160 };
6161 let module = supervisor.spawn(spec).unwrap();
6162 #[cfg(target_os = "macos")]
6163 {
6164 let deadline = Instant::now() + Duration::from_secs(5);
6167 while crate::live_children::read_record(&record_path)
6168 .unwrap()
6169 .iter()
6170 .all(|entry| entry.executable.is_none())
6171 {
6172 assert!(Instant::now() < deadline, "module image was not confirmed");
6173 tokio::time::sleep(Duration::from_millis(5)).await;
6174 }
6175 }
6176 let entry = crate::live_children::read_record(&record_path)
6177 .unwrap()
6178 .pop()
6179 .unwrap();
6180 let observed = subc_os::Process::open(entry.pid)
6181 .unwrap()
6182 .unwrap()
6183 .observe()
6184 .unwrap();
6185 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6186 module.drain().await.unwrap();
6187 assert_eq!(
6188 verdict,
6189 crate::live_children::IdentityVerdict::Matches,
6190 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6191 );
6192 }
6193 }
6194
6195 #[cfg(unix)]
6196 fn http_fixture(
6197 dir: &std::path::Path,
6198 url: &str,
6199 threshold: u32,
6200 ) -> crate::daemon_config::ConfiguredModule {
6201 let path = dir.join("subc.jsonc");
6202 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6203 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6204 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6205 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6206 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6207 }}}).to_string()).unwrap();
6208 crate::daemon_config::load(&path)
6209 .unwrap()
6210 .unwrap()
6211 .modules
6212 .pop()
6213 .unwrap()
6214 }
6215
6216 #[cfg(unix)]
6217 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6218 timeout(Duration::from_secs(5), async {
6219 loop {
6220 if module.status().unwrap().health.status == status {
6221 break;
6222 }
6223 sleep(Duration::from_millis(5)).await;
6224 }
6225 })
6226 .await
6227 .unwrap_or_else(|_| {
6228 panic!(
6229 "expected {status:?}, got {:?}",
6230 module.status().unwrap().health
6231 )
6232 });
6233 }
6234
6235 #[cfg(unix)]
6236 #[tokio::test]
6237 async fn http_health_status_flips_ok_failing_ok() {
6238 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6239 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6240 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6241 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6242 let serving_status = status.clone();
6243 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6244 let server = tokio::spawn(async move {
6245 loop {
6246 let (mut stream, _) = listener.accept().await.unwrap();
6247 let mut request = [0u8; 2048];
6248 let count = stream.read(&mut request).await.unwrap();
6249 assert!(count > 0, "a probe must send an HTTP request");
6250 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6251 let body = if code == 200 {
6252 "ready"
6253 } else {
6254 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6255 };
6256 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6257 let _ = stream.write_all(response.as_bytes()).await;
6258 }
6259 });
6260 let configured = http_fixture(&dir, &url, 1000);
6261 let module =
6262 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6263 .supervise_configured_with_health(
6264 configured.module_spec(),
6265 true,
6266 configured.health,
6267 configured.drain_timeout_ms,
6268 configured.restart,
6269 )
6270 .unwrap();
6271 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6272 status.store(503, std::sync::atomic::Ordering::SeqCst);
6273 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6274 assert!(module
6275 .status()
6276 .unwrap()
6277 .health
6278 .detail
6279 .unwrap()
6280 .contains("scratch failure"));
6281 status.store(200, std::sync::atomic::Ordering::SeqCst);
6282 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6283 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6284 let before = module.status().unwrap();
6285 let (spec, mut health) = module.configuration().unwrap();
6286 health.http = None;
6287 module
6288 .update_configuration(spec.clone(), health.clone(), Some(10))
6289 .await
6290 .unwrap();
6291 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6292 health.http = Some(url);
6293 module
6294 .update_configuration(spec, health, Some(10))
6295 .await
6296 .unwrap();
6297 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6298 assert_eq!(
6299 module.status().unwrap().pid,
6300 before.pid,
6301 "changing a probe must apply live, not restart its process"
6302 );
6303 let (spec, mut health) = module.configuration().unwrap();
6304 health.failure_threshold = 2;
6305 module
6306 .update_configuration(spec, health, Some(10))
6307 .await
6308 .unwrap();
6309 status.store(503, std::sync::atomic::Ordering::SeqCst);
6310 timeout(Duration::from_secs(5), async {
6311 while module.status().unwrap().spawn_generation == before.spawn_generation {
6312 sleep(Duration::from_millis(5)).await;
6313 }
6314 })
6315 .await
6316 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6317 module.drain().await.unwrap();
6318 server.abort();
6319 }
6320
6321 #[cfg(unix)]
6322 #[tokio::test]
6323 async fn http_health_replacement_recovers_after_one_restart() {
6324 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6325 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6326 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6327 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6328 let mut configured = http_fixture(&dir, &url, 2);
6329 configured.restart.max_restarts = 2;
6330 let module =
6331 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6332 .supervise_configured_with_health(
6333 configured.module_spec(),
6334 true,
6335 configured.health,
6336 configured.drain_timeout_ms,
6337 configured.restart,
6338 )
6339 .unwrap();
6340 let first_generation = module.status().unwrap().spawn_generation;
6341 let observed = module.clone();
6342 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6343 let server = tokio::spawn(async move {
6344 let mut generation = first_generation;
6345 let mut probes = 0;
6346 loop {
6347 let (mut stream, _) = listener.accept().await.unwrap();
6348 let mut request = [0; 2048];
6349 assert!(stream.read(&mut request).await.unwrap() > 0);
6350 let current = observed.status().unwrap().spawn_generation;
6351 if current != generation {
6352 generation = current;
6353 probes = 0;
6354 }
6355 probes += 1;
6356 let healthy = generation != first_generation && probes > 1;
6359 let code = if healthy { 200 } else { 503 };
6360 let response = format!(
6361 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6362 );
6363 let _ = stream.write_all(response.as_bytes()).await;
6364 if healthy {
6365 healthy_tx.send(()).await.unwrap();
6366 }
6367 }
6368 });
6369 timeout(Duration::from_secs(5), async {
6370 let mut healthy_probes = 0;
6371 loop {
6372 if module.state().unwrap() == ModuleState::Failed {
6373 break;
6374 }
6375 while healthy_rx.try_recv().is_ok() {
6376 healthy_probes += 1;
6377 }
6378 if healthy_probes >= 3
6379 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6380 {
6381 break;
6382 }
6383 sleep(Duration::from_millis(5)).await;
6384 }
6385 })
6386 .await
6387 .expect("the replacement must settle healthy without exhausting its restart budget");
6388 let status = module.status().unwrap();
6389 module.drain().await.unwrap();
6390 server.abort();
6391 assert_eq!(status.state, ModuleState::Running);
6392 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6393 assert_eq!(status.health.consecutive_failures, 0);
6394 assert_eq!(status.lifetime_restarts, 1);
6395 assert_eq!(status.restart_count, 1);
6396 }
6397
6398 #[cfg(unix)]
6399 #[tokio::test]
6400 async fn http_health_sustained_failure_exhausts_restart_budget() {
6401 use std::collections::BTreeMap;
6402 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6403 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6404 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6405 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6406 let mut configured = http_fixture(&dir, &url, 2);
6407 configured.restart.max_restarts = 2;
6408 let module =
6409 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6410 .supervise_configured_with_health(
6411 configured.module_spec(),
6412 true,
6413 configured.health,
6414 configured.drain_timeout_ms,
6415 configured.restart,
6416 )
6417 .unwrap();
6418 let observed = module.clone();
6419 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6420 let served = probes.clone();
6421 let server = tokio::spawn(async move {
6422 loop {
6423 let (mut stream, _) = listener.accept().await.unwrap();
6424 let mut request = [0; 2048];
6425 assert!(stream.read(&mut request).await.unwrap() > 0);
6426 *served
6427 .lock()
6428 .unwrap()
6429 .entry(observed.status().unwrap().spawn_generation)
6430 .or_default() += 1;
6431 let _ = stream
6432 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6433 .await;
6434 }
6435 });
6436 timeout(Duration::from_secs(5), async {
6437 while module.state().unwrap() != ModuleState::Failed {
6438 sleep(Duration::from_millis(5)).await;
6439 }
6440 })
6441 .await
6442 .expect("sustained failures must still exhaust the health restart budget");
6443 let status = module.status().unwrap();
6444 server.abort();
6445 assert_eq!(status.restart_count, 2);
6446 assert_eq!(status.lifetime_restarts, 2);
6447 assert!(!status.process_alive);
6448 assert_eq!(
6449 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6450 vec![2, 2, 2]
6451 );
6452 }
6453
6454 #[cfg(unix)]
6455 #[tokio::test]
6456 async fn spawn_clears_the_previous_process_health() {
6457 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6458 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6459 configured.health.cadence = Duration::from_secs(3600);
6460 let module =
6461 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6462 .supervise_configured_with_health(
6463 configured.module_spec(),
6464 false,
6465 configured.health,
6466 configured.drain_timeout_ms,
6467 configured.restart,
6468 )
6469 .unwrap();
6470 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6471 state.health.status = SupervisorHealthStatus::Unresponsive;
6472 state.health.consecutive_failures = 3;
6473 state.health.detail = Some("old process refused connections".into());
6474 state.health.metrics = Some(serde_json::json!({"old": true}));
6475 state.health.last_probe_ms = Some(123);
6476 state.health.last_action = Some("restart".into());
6477 state.health.last_action_ms = Some(124);
6478 })
6479 .unwrap();
6480 module.set_enabled(true).await.unwrap();
6481 let health = module.status().unwrap().health;
6482 module.drain().await.unwrap();
6483 assert_eq!(health.consecutive_failures, 0);
6484 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6485 assert_eq!(health.detail, None);
6486 assert_eq!(health.metrics, None);
6487 assert_eq!(health.last_probe_ms, None);
6488 assert_eq!(health.last_action.as_deref(), Some("restart"));
6489 assert_eq!(health.last_action_ms, Some(124));
6490 }
6491
6492 #[cfg(unix)]
6493 #[tokio::test]
6494 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6495 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6496 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6497 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6498 drop(unused);
6499 let configured = http_fixture(&dir, &url, 2);
6500 let module =
6501 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6502 .supervise_configured_with_health(
6503 configured.module_spec(),
6504 true,
6505 configured.health,
6506 configured.drain_timeout_ms,
6507 configured.restart,
6508 )
6509 .unwrap();
6510 let before = module.status().unwrap().spawn_generation;
6511 timeout(Duration::from_secs(5), async {
6512 loop {
6513 let status = module.status().unwrap();
6514 if status.spawn_generation > before {
6515 assert!(status.lifetime_restarts > 0);
6516 break;
6517 }
6518 sleep(Duration::from_millis(5)).await;
6519 }
6520 })
6521 .await
6522 .expect("sustained HTTP refusal must trigger the health restart policy");
6523 module.drain().await.unwrap();
6524 }
6525
6526 #[cfg(unix)]
6527 #[tokio::test]
6528 async fn http_health_timeout_honours_deadline() {
6529 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6530 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6531 let server = tokio::spawn(async move {
6532 let _held = listener.accept().await.unwrap();
6533 std::future::pending::<()>().await;
6534 });
6535 let error = timeout(
6536 Duration::from_secs(1),
6537 probe_http_health(&url, Duration::from_millis(10)),
6538 )
6539 .await
6540 .expect("the probe must enforce its own deadline")
6541 .unwrap_err();
6542 server.abort();
6543 assert!(error.to_string().contains("timed out"));
6544 }
6545
6546 #[cfg(unix)]
6547 #[tokio::test]
6548 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6549 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6550 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6551 let url = format!(
6552 "http://localhost:{}/healthz",
6553 listener.local_addr().unwrap().port()
6554 );
6555 let server = tokio::spawn(async move {
6556 let (mut stream, _) = listener.accept().await.unwrap();
6557 let mut request = [0u8; 2048];
6558 assert!(stream.read(&mut request).await.unwrap() > 0);
6559 stream
6560 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6561 .await
6562 .unwrap();
6563 });
6564 assert_eq!(
6569 probe_http_health(&url, Duration::from_secs(10))
6570 .await
6571 .unwrap()
6572 .status,
6573 HealthStatus::Ok
6574 );
6575 server.await.unwrap();
6576 }
6577
6578 #[cfg(unix)]
6579 #[tokio::test]
6580 async fn http_health_timeout_keeps_partial_status_and_body() {
6581 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6582 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6583 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6584 let server = tokio::spawn(async move {
6585 let (mut stream, _) = listener.accept().await.unwrap();
6586 let mut request = [0u8; 2048];
6587 assert!(stream.read(&mut request).await.unwrap() > 0);
6588 stream
6589 .write_all(
6590 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6591 )
6592 .await
6593 .unwrap();
6594 std::future::pending::<()>().await;
6595 });
6596 let error = probe_http_health(&url, Duration::from_secs(1))
6597 .await
6598 .unwrap_err()
6599 .to_string();
6600 server.abort();
6601 assert!(
6602 error.contains("timed out")
6603 && error.contains("503 Unavailable")
6604 && error.contains("partial diagnostic"),
6605 "{error}"
6606 );
6607 }
6608
6609 #[cfg(unix)]
6610 #[tokio::test]
6611 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6612 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6613 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6614 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6615 let server = tokio::spawn(async move {
6616 let (mut stream, _) = listener.accept().await.unwrap();
6617 let mut request = [0u8; 2048];
6618 assert!(stream.read(&mut request).await.unwrap() > 0);
6619 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6620 let response = format!(
6621 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6622 body.len()
6623 );
6624 stream.write_all(response.as_bytes()).await.unwrap();
6625 });
6626 let error = probe_http_health(&url, Duration::from_secs(1))
6627 .await
6628 .unwrap_err()
6629 .to_string();
6630 server.await.unwrap();
6631 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6632 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6633 assert!(!error.contains("not-in-diagnostic"));
6634 }
6635
6636 #[cfg(unix)]
6637 #[tokio::test]
6638 async fn http_health_real_nats_server_monitoring() {
6639 if std::process::Command::new("nats-server")
6640 .arg("--version")
6641 .env("XDG_DATA_HOME", std::env::temp_dir())
6642 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6643 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6644 .output()
6645 .is_err()
6646 {
6647 eprintln!(
6648 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6649 );
6650 return;
6651 }
6652 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6653 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6654 let port = monitor.local_addr().unwrap().port();
6655 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6656 let client_port = client.local_addr().unwrap().port();
6657 let config = dir.join("server.conf");
6658 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6659 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6660 configured.program = PathBuf::from("nats-server");
6661 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6662 drop(monitor);
6663 drop(client);
6664 let module =
6665 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6666 .supervise_configured_with_health(
6667 configured.module_spec(),
6668 true,
6669 configured.health,
6670 configured.drain_timeout_ms,
6671 configured.restart,
6672 )
6673 .unwrap();
6674 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6675 module.drain().await.unwrap();
6676 }
6677
6678 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6679 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6680 let supervisor =
6681 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6682 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6683 let initial = ModuleSpec {
6684 module_id: "rescan-preserves-spawn-facts".to_string(),
6685 program: PathBuf::from("/spawned/module"),
6686 args: Vec::new(),
6687 env: Vec::new(),
6688 reserved: false,
6689 reserved_prefixes: Vec::new(),
6690 protocol: ModuleProtocol::Subc,
6691 overlap: Default::default(),
6692 };
6693 let module = supervisor.supervised_module(
6694 initial.clone(),
6695 supervisor.runtime_config(),
6696 snapshot,
6697 None,
6698 );
6699 let before = module.status().unwrap();
6700 let mut replacement = initial;
6701 replacement.program = PathBuf::from("/rescanned/replacement-module");
6702
6703 module
6704 .update_configuration(replacement, HealthConfig::default(), None)
6705 .await
6706 .unwrap();
6707
6708 let after = module.status().unwrap();
6709 assert_eq!(after.pid, before.pid);
6710 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6711 assert_eq!(after.spawned_from, before.spawned_from);
6712 drop(module);
6713 }
6714}
6715
6716fn unix_ms_now() -> u64 {
6717 SystemTime::now()
6718 .duration_since(UNIX_EPOCH)
6719 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6720 .unwrap_or(0)
6721}
6722
6723async fn supervise_loop(
6724 mut spec: ModuleSpec,
6725 mut runtime: SupervisorRuntimeConfig,
6726 registry: Arc<Registry>,
6727 process_liveness: Arc<SupervisorProcessLiveness>,
6728 snapshot: SharedSnapshot,
6729 mut child: Option<SupervisedChild>,
6730 mut commands: mpsc::Receiver<SupervisorCommand>,
6731) {
6732 let mut health_probe = HealthProbeRuntime::default();
6733 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6737 Ok(changes) => Some(changes),
6738 Err(err) => {
6739 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6742 None
6743 }
6744 };
6745 let mut pending_respawn: Option<PendingRespawn> = None;
6749 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6752 let mut previous_instance_recheck = tokio::time::interval_at(
6753 Instant::now() + Duration::from_secs(5),
6754 Duration::from_secs(5),
6755 );
6756 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6757 loop {
6758 #[cfg(test)]
6759 {
6760 lock_snapshot(&snapshot).unwrap().actor_select = None;
6761 }
6762 #[cfg(target_os = "macos")]
6763 if let Some(active) = child.as_mut() {
6764 active.confirm_privacy_exec().await;
6765 }
6766 if let Some(scheduled) = runtime
6767 .scheduled_respawn
6768 .lock()
6769 .unwrap_or_else(|p| p.into_inner())
6770 .take()
6771 {
6772 runtime.recovery_is_daemon = scheduled.daemon_initiated;
6773 pending_respawn = Some(scheduled);
6774 }
6775 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6776 pending_respawn = None;
6777 cancel_deferred_reload(
6778 &runtime,
6779 &spec.module_id,
6780 "respawn cancelled by a supervisor command",
6781 );
6782 }
6783 if child.is_none()
6784 && pending_respawn.is_none()
6785 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6786 {
6787 cancel_deferred_reload(
6788 &runtime,
6789 &spec.module_id,
6790 "respawn cancelled before a replacement was spawned",
6791 );
6792 let mut stranded = false;
6793 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6794 state.respawn_pending = false;
6795 state.coalesced_restart_pending = false;
6796 if matches!(
6797 state.state,
6798 ModuleState::Restarting
6799 | ModuleState::Starting
6800 | ModuleState::Draining
6801 | ModuleState::Unresponsive
6802 ) {
6803 error!(module_id = %spec.module_id, state = ?state.state,
6804 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6805 stranded = true;
6806 }
6807 });
6808 if stranded {
6809 fail_snapshot(
6810 &snapshot,
6811 Some(&spec.module_id),
6812 None,
6813 &runtime.failure_context(runtime.recovery_is_daemon),
6814 );
6815 }
6816 }
6817 if let Some(command) = requeued.pop_front() {
6818 if !handle_supervisor_command(
6819 command,
6820 &mut spec,
6821 &mut runtime,
6822 ®istry,
6823 &process_liveness,
6824 &snapshot,
6825 &mut child,
6826 &mut commands,
6827 &mut requeued,
6828 )
6829 .await
6830 {
6831 return;
6832 }
6833 if child.is_some() || !respawn_still_pending(&snapshot) {
6834 pending_respawn = None;
6835 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6836 state.respawn_pending = false
6837 });
6838 }
6839 continue;
6840 }
6841 if child.is_some() {
6842 runtime.recovery_is_daemon = true;
6843 #[cfg(test)]
6844 {
6845 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6846 }
6847 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6848 let no_checks = runtime.health.http.is_none()
6849 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
6850 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
6851 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
6852 state
6853 .failed_retry
6854 .uptime_reset_at(no_checks, runtime.restart_policy.window)
6855 });
6856 let wake_after = match (health_probe.wake_after(), uptime_reset) {
6857 (probe, Some(reset)) => Some(
6858 probe
6859 .unwrap_or(Duration::MAX)
6860 .min(reset.saturating_duration_since(Instant::now())),
6861 ),
6862 (probe, None) => probe,
6863 };
6864 let probe_sleep = async move {
6865 match wake_after {
6866 Some(delay) => sleep(delay).await,
6867 None => std::future::pending().await,
6868 }
6869 };
6870 tokio::pin!(probe_sleep);
6871 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6876 #[cfg(test)]
6877 {
6878 let registration_pending = wire_child
6879 && registration_changes
6880 .as_ref()
6881 .is_some_and(|changes| changes.has_changed().unwrap());
6882 let mut state = lock_snapshot(&snapshot).unwrap();
6883 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6884 generation: state.spawn_generation,
6885 turn: state.actor_turns,
6886 registered_connection: health_probe.registered_connection,
6887 next_probe_at: health_probe.next_probe_at,
6888 wake_after,
6889 });
6890 }
6891 let active_child = child.as_mut().expect("child checked above");
6892 tokio::select! {
6893 wait_result = active_child.wait() => {
6894 if let Ok(mut state) = lock_snapshot(&snapshot) {
6895 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
6896 }
6897 #[cfg(test)]
6898 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6899 let exit_report = match wait_result {
6908 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6909 Err(err) => {
6910 active_child.drain_stderr(&spec.module_id).await;
6911 fail_after_wait_error(
6912 &spec.module_id,
6913 &snapshot,
6914 &runtime.terminal_ring,
6915 &runtime.spawn_events,
6916 format!("failed to wait for supervised module: {err}"),
6917 &runtime.failure_context(true),
6918 );
6919 untrack_if_registration_released(
6920 &process_liveness,
6921 ®istry,
6922 &spec.module_id,
6923 &snapshot,
6924 );
6925 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6926 child = None;
6927 continue;
6928 }
6929 };
6930 active_child.drain_stderr(&spec.module_id).await;
6931
6932 let next = on_child_exit(
6933 &spec,
6934 runtime.restart_policy,
6935 ®istry,
6936 &snapshot,
6937 &runtime.terminal_ring,
6938 &runtime.spawn_events,
6939 &runtime.child_roster,
6940 exit_report,
6941 true,
6942 ).await;
6943 runtime.start_requester = None;
6944 runtime.pending_activity_restart = false;
6945 active_child.release_roster();
6948 match next {
6949 NextAction::Stop { registration_released } => {
6950 if registration_released {
6951 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6952 }
6953 child = None;
6954 }
6955 NextAction::Restart { schedule } => {
6956 let delay = schedule.map_or(
6957 runtime.restart_policy.delay_for_restart(0),
6958 |schedule| schedule.delay,
6959 );
6960 if let Some(schedule) = schedule {
6961 log_crash_respawn(&spec.module_id, schedule);
6962 }
6963 child = None;
6971 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
6972 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6973 }
6974 }
6975 }
6976 command = commands.recv() => {
6977 #[cfg(test)]
6978 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6979 let Some(command) = command else {
6980 return;
6981 };
6982 if !handle_supervisor_command(
6983 command,
6984 &mut spec,
6985 &mut runtime,
6986 ®istry,
6987 &process_liveness,
6988 &snapshot,
6989 &mut child,
6990 &mut commands,
6991 &mut requeued,
6992 ).await {
6993 return;
6994 }
6995 }
6996 _ = async {
6997 match registration_changes.as_mut() {
6998 Some(changes) => { let _ = changes.changed().await; }
6999 None => std::future::pending().await,
7000 }
7001 }, if wire_child => {
7002 #[cfg(test)]
7003 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7004 }
7005 _ = &mut probe_sleep => {
7006 #[cfg(test)]
7007 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7008 if health_probe.due() {
7009 run_health_probe_cycle(
7010 &spec,
7011 &runtime,
7012 ®istry,
7013 &process_liveness,
7014 &snapshot,
7015 &mut child,
7016 ).await;
7017 if child.is_none() {
7018 runtime.start_requester = None;
7019 runtime.pending_activity_restart = false;
7020 }
7021 if child.is_some() {
7022 health_probe.schedule_next(&spec, runtime.health.cadence);
7023 }
7024 }
7025 }
7026 }
7027 } else if let Some(pending) = pending_respawn {
7028 tokio::select! {
7029 _ = sleep_until(pending.deadline) => {
7030 pending_respawn = None;
7031 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7032 if !respawn_still_pending(&snapshot) {
7036 continue;
7037 }
7038 if runtime.child_roster.is_closed() {
7043 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7044 state.state = ModuleState::Stopped;
7045 });
7046 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7047 continue;
7048 }
7049 if let Err(err) = release_dead_registration(
7050 ®istry,
7051 runtime.forwarding.as_deref(),
7052 &snapshot,
7053 &spec.module_id,
7054 &runtime.failure_context(runtime.recovery_is_daemon),
7055 ).await {
7056 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7057 continue;
7058 }
7059
7060 if matches!(pending.kind, RespawnKind::Reload) {
7061 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7062 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
7063 if result.is_ok() && runtime.pending_activity_restart {
7064 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7065 }
7066 runtime.pending_activity_restart = false;
7067 if let Some(reply) = reply { let _ = reply.send(result); }
7068 continue;
7069 }
7070 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7071 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7072 Ok(next_child) => {
7073 child = Some(next_child);
7074 if runtime.pending_activity_restart {
7075 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7076 runtime.pending_activity_restart = false;
7077 }
7078 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7079 }
7080 Err(err) => {
7081 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7082 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7083 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7084 }
7085 }
7086 }
7087 command = commands.recv() => {
7088 let Some(command) = command else {
7089 return;
7090 };
7091 if !handle_supervisor_command(
7092 command,
7093 &mut spec,
7094 &mut runtime,
7095 ®istry,
7096 &process_liveness,
7097 &snapshot,
7098 &mut child,
7099 &mut commands,
7100 &mut requeued,
7101 ).await {
7102 return;
7103 }
7104 if child.is_some() || !respawn_still_pending(&snapshot) {
7109 pending_respawn = None;
7110 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7111 }
7112 }
7113 }
7114 } else {
7115 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7116 (state.enabled && state.state == ModuleState::Failed)
7117 .then_some(state.failed_retry.pending)
7118 .flatten()
7119 });
7120 let command = if lock_snapshot(&snapshot)
7121 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7122 {
7123 tokio::select! {
7124 command = commands.recv() => command,
7125 _ = previous_instance_recheck.tick() => {
7126 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7127 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7128 if enabled && !runtime.child_roster.is_closed() {
7129 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7130 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7131 Ok(next_child) => child = Some(next_child),
7132 Err(error) => {
7133 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7134 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7135 }
7136 }
7137 } else {
7138 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7139 state.waiting_for_previous_pid = None;
7140 state.waiting_for_previous_detail = None;
7141 });
7142 }
7143 }
7144 continue;
7145 }
7146 }
7147 } else {
7148 tokio::select! {
7149 biased;
7150 command = commands.recv() => command,
7151 _ = async {
7152 match slow_retry {
7153 Some(retry) => sleep_until(retry.deadline).await,
7154 None => std::future::pending().await,
7155 }
7156 } => {
7157 let retry = slow_retry.expect("only a scheduled slow retry can wake");
7158 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7159 state.failed_retry.pending = None;
7160 });
7161 if runtime.child_roster.is_closed() {
7162 continue;
7163 }
7164 info!(module_id = %spec.module_id, attempt = retry.attempt,
7165 delay_secs = retry.delay.as_secs(), "starting failed module retry");
7166 if let Err(error) = set_child_enabled(
7167 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
7168 &runtime.failure_context(true),
7169 ).await {
7170 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7171 }
7172 continue;
7173 }
7174 }
7175 };
7176 let Some(command) = command else {
7177 return;
7178 };
7179 if !handle_supervisor_command(
7180 command,
7181 &mut spec,
7182 &mut runtime,
7183 ®istry,
7184 &process_liveness,
7185 &snapshot,
7186 &mut child,
7187 &mut commands,
7188 &mut requeued,
7189 )
7190 .await
7191 {
7192 return;
7193 }
7194 }
7195 }
7196}
7197
7198fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7199 info!(
7200 module_id,
7201 restart_in_window = schedule.restart_in_window,
7202 delay_ms = schedule.delay.as_millis() as u64,
7203 "respawning after crash"
7204 );
7205}
7206
7207fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7213 matches!(
7214 lock_snapshot(snapshot),
7215 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7216 )
7217}
7218
7219enum NextAction {
7220 Stop {
7221 registration_released: bool,
7222 },
7223 Restart {
7224 schedule: Option<CrashRestartSchedule>,
7225 },
7226}
7227
7228#[allow(clippy::too_many_arguments)]
7229async fn handle_supervisor_command(
7230 command: SupervisorCommand,
7231 spec: &mut ModuleSpec,
7232 runtime: &mut SupervisorRuntimeConfig,
7233 registry: &Arc<Registry>,
7234 process_liveness: &SupervisorProcessLiveness,
7235 snapshot: &SharedSnapshot,
7236 child: &mut Option<SupervisedChild>,
7237 commands: &mut mpsc::Receiver<SupervisorCommand>,
7238 requeued: &mut VecDeque<SupervisorCommand>,
7239) -> bool {
7240 if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7241 runtime.recovery_is_daemon = false;
7242 }
7243 if matches!(
7244 command,
7245 SupervisorCommand::Drain { .. }
7246 | SupervisorCommand::Retire { .. }
7247 | SupervisorCommand::SetEnabled { .. }
7248 | SupervisorCommand::Restart { .. }
7249 ) {
7250 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7251 state.failed_retry.cancel()
7252 });
7253 }
7254 let command =
7255 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7256 match command {
7257 SupervisorCommand::Restart { reply, .. }
7258 | SupervisorCommand::Reload { reply, .. }
7259 | SupervisorCommand::Swap { reply, .. } => {
7260 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7261 return true;
7262 }
7263 SupervisorCommand::SetEnabled {
7264 enabled: true,
7265 reply,
7266 ..
7267 } => {
7268 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7269 return true;
7270 }
7271 other => other,
7272 }
7273 } else {
7274 command
7275 };
7276 match command {
7277 SupervisorCommand::Drain { requester, reply } => {
7278 let result = drain_optional_child(
7281 &spec.module_id,
7282 spec.protocol,
7283 StopNotice::NotSent,
7284 registry,
7285 runtime.forwarding.as_deref(),
7286 snapshot,
7287 &runtime.terminal_ring,
7288 &runtime.spawn_events,
7289 child,
7290 runtime.drain_timeout,
7291 ModuleState::Stopped,
7292 None,
7293 &FailureContext::operator(),
7294 )
7295 .await;
7296 let registration_released = result.is_ok();
7297 if registration_released {
7298 runtime.record_activity_transition(
7299 "subc.module_stopped",
7300 &spec.module_id,
7301 requester,
7302 );
7303 }
7304 let _ = reply.send(result);
7305 if registration_released {
7306 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7307 }
7308 false
7309 }
7310 SupervisorCommand::Retire { requester, reply } => {
7311 let result = async {
7312 let stop_notice = begin_forwarding_drain_if_configured(
7313 spec,
7314 runtime,
7315 registry,
7316 snapshot,
7317 None,
7318 RouteCloseReason::Disable,
7319 )
7320 .await?;
7321 drain_optional_child(
7322 &spec.module_id,
7323 spec.protocol,
7324 stop_notice,
7325 registry,
7326 runtime.forwarding.as_deref(),
7327 snapshot,
7328 &runtime.terminal_ring,
7329 &runtime.spawn_events,
7330 child,
7331 runtime.drain_timeout,
7332 ModuleState::Stopped,
7333 None,
7334 &FailureContext::operator(),
7335 )
7336 .await
7337 }
7338 .await;
7339 let registration_released = result.is_ok();
7340 if registration_released {
7341 runtime.record_activity_transition(
7342 "subc.module_stopped",
7343 &spec.module_id,
7344 requester,
7345 );
7346 }
7347 let _ = reply.send(result);
7348 if registration_released {
7349 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7350 }
7351 false
7352 }
7353 SupervisorCommand::Restart {
7354 requester,
7355 drain_timeout_ms,
7356 received_at_generation,
7357 queued_at,
7358 reply,
7359 } => {
7360 info!(
7364 module_id = %spec.module_id,
7365 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7366 "restart command dequeued"
7367 );
7368 let validation = match lock_snapshot(snapshot) {
7380 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7381 module_id: spec.module_id.clone(),
7382 }),
7383 Ok(_) => Ok(()),
7384 Err(err) => Err(err),
7385 };
7386 let initiated = validation.is_ok();
7387 let _ = reply.send(validation);
7388 let satisfied_by_generation = if initiated
7402 && child
7403 .as_ref()
7404 .is_some_and(|child| child.spawned_at > queued_at)
7405 {
7406 lock_snapshot(snapshot).ok().and_then(|state| {
7407 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7408 })
7409 } else {
7410 None
7411 };
7412 let satisfied_by_pending = initiated
7413 && child.is_none()
7414 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7415 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7419 if pending {
7420 state.coalesced_restart_pending = true;
7421 }
7422 pending
7423 });
7424 if satisfied_by_pending {
7425 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7426 } else if let Some(generation) = satisfied_by_generation {
7427 info!(
7428 module_id = %spec.module_id,
7429 received_at_generation,
7430 "restart already satisfied by generation {generation}; not restarting again"
7431 );
7432 } else if initiated {
7433 let drain_timeout = drain_timeout_ms
7436 .map(Duration::from_millis)
7437 .unwrap_or(runtime.drain_timeout);
7438 if let Err(err) = restart_child(
7439 spec,
7440 runtime,
7441 registry,
7442 process_liveness,
7443 snapshot,
7444 child,
7445 drain_timeout,
7446 )
7447 .await
7448 {
7449 warn!(
7450 module_id = %spec.module_id,
7451 error = %err,
7452 "operator restart failed after initiation ack; module state carries the outcome"
7453 );
7454 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7455 state.state = ModuleState::Failed;
7456 clear_current_process_facts(state);
7457 });
7458 } else {
7459 runtime.start_requester = requester;
7460 runtime.pending_activity_restart = true;
7461 }
7462 }
7463 true
7464 }
7465 SupervisorCommand::Reload { requester, reply } => {
7466 let result =
7467 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7468 if result.is_ok() {
7469 runtime.start_requester = requester;
7470 runtime.pending_activity_restart = true;
7471 }
7472 if result.is_ok()
7473 && runtime
7474 .scheduled_respawn
7475 .lock()
7476 .unwrap_or_else(|p| p.into_inner())
7477 .as_ref()
7478 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7479 {
7480 *runtime
7481 .deferred_reload_reply
7482 .lock()
7483 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7484 } else {
7485 let _ = reply.send(result);
7486 }
7487 true
7488 }
7489 SupervisorCommand::SetEnabled {
7490 enabled,
7491 requester,
7492 reply,
7493 } => {
7494 let previous_requester = runtime.start_requester.clone();
7495 if enabled {
7496 runtime.start_requester = requester.clone();
7497 }
7498 let result = set_child_enabled(
7499 spec,
7500 runtime,
7501 registry,
7502 process_liveness,
7503 snapshot,
7504 child,
7505 enabled,
7506 &runtime.failure_context(false),
7507 )
7508 .await;
7509 if matches!(result, Ok(true)) {
7510 runtime.pending_activity_restart = false;
7511 if enabled {
7512 runtime.start_requester = requester;
7513 } else {
7514 runtime.start_requester = None;
7515 runtime.record_activity_transition(
7516 "subc.module_disabled",
7517 &spec.module_id,
7518 requester,
7519 );
7520 }
7521 } else {
7522 runtime.start_requester = previous_requester;
7523 }
7524 let _ = reply.send(result);
7525 true
7526 }
7527 SupervisorCommand::UpdateConfiguration {
7528 spec: next_spec,
7529 health,
7530 drain_timeout_ms,
7531 reply,
7532 } => {
7533 if let Some(handle) = &runtime.supervisor_handle {
7534 handle.apply_identity_configuration(&next_spec);
7535 }
7536 *spec = next_spec;
7537 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7538 state.configuration_updated_since_spawn = true;
7539 });
7540 let health_changed = runtime.health != health;
7541 runtime.health = health;
7542 if health_changed {
7545 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7546 state.health = ModuleHealthStatus::default();
7547 });
7548 }
7549 runtime.drain_timeout = drain_timeout_ms
7550 .map(Duration::from_millis)
7551 .unwrap_or(runtime.default_drain_timeout);
7552 *runtime
7553 .effective_drain_timeout
7554 .lock()
7555 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7556 let _ = reply.send(());
7557 true
7558 }
7559 SupervisorCommand::Swap {
7560 requester,
7561 ready_timeout,
7562 reply,
7563 } => {
7564 let mut swap_runtime = runtime.clone();
7565 swap_runtime.start_requester = requester;
7566 let before = lock_snapshot(snapshot)
7567 .ok()
7568 .map(|state| state.spawn_generation);
7569 let end = swap::run_swap(
7570 spec,
7571 &swap_runtime,
7572 registry,
7573 process_liveness,
7574 snapshot,
7575 child,
7576 commands,
7577 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7578 reply,
7579 )
7580 .await;
7581 if before
7582 != lock_snapshot(snapshot)
7583 .ok()
7584 .map(|state| state.spawn_generation)
7585 {
7586 runtime.start_requester = swap_runtime.start_requester;
7587 }
7588 requeued.extend(end.requeue);
7589 true
7590 }
7591 }
7592}
7593
7594async fn restart_child(
7595 spec: &ModuleSpec,
7596 runtime: &SupervisorRuntimeConfig,
7597 registry: &Registry,
7598 process_liveness: &SupervisorProcessLiveness,
7599 snapshot: &SharedSnapshot,
7600 child: &mut Option<SupervisedChild>,
7601 drain_timeout: Duration,
7602) -> Result<(), SuperviseError> {
7603 if !lock_snapshot(snapshot)?.enabled {
7605 return Err(SuperviseError::Disabled {
7606 module_id: spec.module_id.clone(),
7607 });
7608 }
7609 let stop_notice = begin_forwarding_drain_with_timeout(
7610 spec,
7611 runtime,
7612 registry,
7613 snapshot,
7614 None,
7615 RouteCloseReason::Restart,
7616 drain_timeout,
7617 )
7618 .await?;
7619
7620 if child.is_some() {
7621 drain_optional_child(
7622 &spec.module_id,
7623 spec.protocol,
7624 stop_notice,
7625 registry,
7626 runtime.forwarding.as_deref(),
7627 snapshot,
7628 &runtime.terminal_ring,
7629 &runtime.spawn_events,
7630 child,
7631 drain_timeout,
7632 ModuleState::Restarting,
7633 Some(true),
7634 &FailureContext::operator(),
7635 )
7636 .await?;
7637 } else {
7638 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7639 state.enabled = true;
7640 state.state = ModuleState::Restarting;
7641 clear_current_process_facts(state);
7642 })?;
7643 release_dead_registration(
7644 registry,
7645 runtime.forwarding.as_deref(),
7646 snapshot,
7647 &spec.module_id,
7648 &runtime.failure_context(false),
7649 )
7650 .await?;
7651 }
7652
7653 reset_restart_count(snapshot, &spec.module_id)?;
7654 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7655 schedule_respawn(
7656 runtime,
7657 snapshot,
7658 &spec.module_id,
7659 runtime.restart_policy.backoff,
7660 RespawnKind::Spawn,
7661 false,
7662 )
7663}
7664
7665async fn reload_child(
7666 spec: &ModuleSpec,
7667 runtime: &SupervisorRuntimeConfig,
7668 registry: &Registry,
7669 process_liveness: &SupervisorProcessLiveness,
7670 snapshot: &SharedSnapshot,
7671 child: &mut Option<SupervisedChild>,
7672) -> Result<(), SuperviseError> {
7673 if !lock_snapshot(snapshot)?.enabled {
7675 return Err(SuperviseError::Disabled {
7676 module_id: spec.module_id.clone(),
7677 });
7678 }
7679 let stop_notice = begin_forwarding_drain(
7680 spec,
7681 runtime,
7682 registry,
7683 snapshot,
7684 Some(true),
7685 RouteCloseReason::Reload,
7686 )
7687 .await?;
7688
7689 if child.is_some() {
7690 drain_optional_child(
7691 &spec.module_id,
7692 spec.protocol,
7693 stop_notice,
7694 registry,
7695 runtime.forwarding.as_deref(),
7696 snapshot,
7697 &runtime.terminal_ring,
7698 &runtime.spawn_events,
7699 child,
7700 runtime.drain_timeout,
7701 ModuleState::Restarting,
7702 Some(true),
7703 &FailureContext::operator(),
7704 )
7705 .await?;
7706 } else {
7707 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7708 state.enabled = true;
7709 state.state = ModuleState::Restarting;
7710 clear_current_process_facts(state);
7711 })?;
7712 release_dead_registration(
7713 registry,
7714 runtime.forwarding.as_deref(),
7715 snapshot,
7716 &spec.module_id,
7717 &runtime.failure_context(false),
7718 )
7719 .await?;
7720 }
7721
7722 reset_restart_count(snapshot, &spec.module_id)?;
7723 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7724 schedule_respawn(
7725 runtime,
7726 snapshot,
7727 &spec.module_id,
7728 runtime.restart_policy.backoff,
7729 RespawnKind::Reload,
7730 false,
7731 )
7732}
7733
7734async fn finish_reload_child(
7735 spec: &ModuleSpec,
7736 runtime: &SupervisorRuntimeConfig,
7737 registry: &Registry,
7738 process_liveness: &SupervisorProcessLiveness,
7739 snapshot: &SharedSnapshot,
7740 child: &mut Option<SupervisedChild>,
7741) -> Result<(), SuperviseError> {
7742 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7743 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7744 Ok(next_child) => next_child,
7745 Err(err) => {
7746 return handle_reload_spawn_failure(
7747 spec,
7748 runtime,
7749 process_liveness,
7750 snapshot,
7751 child,
7752 format!("new child failed to spawn: {err}"),
7753 )
7754 .await;
7755 }
7756 };
7757 *child = Some(next_child);
7758
7759 let wait_outcome = {
7760 let active_child = child.as_mut().expect("new reload child was just stored");
7761 wait_for_registration_after_reload(
7762 registry,
7763 &spec.module_id,
7764 snapshot,
7765 active_child,
7766 REGISTRY_RELEASE_TIMEOUT,
7767 )
7768 .await?
7769 };
7770
7771 match wait_outcome {
7772 RegistrationWaitOutcome::Registered => {
7773 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7774 Ok(())
7775 }
7776 RegistrationWaitOutcome::Exited(exit_report) => {
7777 if let Some(active_child) = child.as_mut() {
7778 active_child.drain_stderr(&spec.module_id).await;
7779 }
7780 let mut exited_child = child.take().expect("exited reload child is still stored");
7783 #[cfg(test)]
7784 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7785 gate.reached.notify_one();
7786 gate.resume.notified().await;
7787 }
7788 let result = handle_reload_child_registration_failure(
7789 spec,
7790 runtime,
7791 registry,
7792 process_liveness,
7793 snapshot,
7794 child,
7795 ReloadRegistrationFailure {
7796 exit_report: registration_failure_exit_report(exit_report),
7797 reason: exited_child
7798 .spawn_failure
7799 .clone()
7800 .unwrap_or_else(|| "new child exited before registering".to_string()),
7801 },
7802 )
7803 .await;
7804 exited_child.release_roster();
7805 result
7806 }
7807 RegistrationWaitOutcome::TimedOut => {
7808 let mut timed_out_child = child
7809 .take()
7810 .expect("timed-out reload child is still running");
7811 timed_out_child
7812 .start_kill()
7813 .map_err(|source| SuperviseError::Kill {
7814 module_id: spec.module_id.clone(),
7815 source,
7816 })?;
7817 let status = timed_out_child
7818 .wait()
7819 .await
7820 .map_err(|source| SuperviseError::Wait {
7821 module_id: spec.module_id.clone(),
7822 source,
7823 })?;
7824 timed_out_child.drain_stderr(&spec.module_id).await;
7825 handle_reload_child_registration_failure(
7826 spec,
7827 runtime,
7828 registry,
7829 process_liveness,
7830 snapshot,
7831 child,
7832 ReloadRegistrationFailure {
7833 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7834 snapshot,
7835 &timed_out_child,
7836 &status,
7837 )),
7838 reason: format!(
7839 "new child did not register within {:?}",
7840 REGISTRY_RELEASE_TIMEOUT
7841 ),
7842 },
7843 )
7844 .await
7845 }
7846 }
7847}
7848
7849#[allow(clippy::too_many_arguments)]
7850async fn set_child_enabled(
7851 spec: &ModuleSpec,
7852 runtime: &SupervisorRuntimeConfig,
7853 registry: &Registry,
7854 process_liveness: &SupervisorProcessLiveness,
7855 snapshot: &SharedSnapshot,
7856 child: &mut Option<SupervisedChild>,
7857 enabled: bool,
7858 failure: &FailureContext,
7859) -> Result<bool, SuperviseError> {
7860 let (current_enabled, current_state, respawn_pending) = {
7861 let state = lock_snapshot(snapshot)?;
7862 (state.enabled, state.state, state.respawn_pending)
7863 };
7864 let revive_terminal = enabled
7872 && current_enabled
7873 && child.is_none()
7874 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7875 || (current_state == ModuleState::Restarting && !respawn_pending));
7876 if current_enabled == enabled && !revive_terminal {
7877 return Ok(false);
7878 }
7879
7880 if enabled {
7881 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7882 state.enabled = true;
7883 state.state = ModuleState::Starting;
7884 clear_current_process_facts(state);
7885 })?;
7886 #[cfg(test)]
7887 if runtime.test_seed_stale_facts_before_enable_spawn {
7888 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7889 state.process_alive = true;
7890 state.pid = Some(41);
7891 state.spawned_at_ms = Some(42);
7892 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7893 state.spawned_file_identity = Some(SpawnedFileIdentity {
7894 device: 43,
7895 inode: 44,
7896 #[cfg(windows)]
7897 windows_image: None,
7898 });
7899 })?;
7900 }
7901 release_dead_registration(
7902 registry,
7903 runtime.forwarding.as_deref(),
7904 snapshot,
7905 &spec.module_id,
7906 failure,
7907 )
7908 .await?;
7909 reset_restart_count(snapshot, &spec.module_id)?;
7910 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7911 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7912 Ok(next_child) => next_child,
7913 Err(err) => {
7914 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7915 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7916 return Err(err);
7917 }
7918 };
7919 *child = Some(next_child);
7920 debug!(module_id = %spec.module_id, "supervised module enabled");
7921 Ok(true)
7922 } else {
7923 let stop_notice = begin_forwarding_drain_if_configured(
7924 spec,
7925 runtime,
7926 registry,
7927 snapshot,
7928 Some(false),
7929 RouteCloseReason::Disable,
7930 )
7931 .await?;
7932 drain_optional_child(
7933 &spec.module_id,
7934 spec.protocol,
7935 stop_notice,
7936 registry,
7937 runtime.forwarding.as_deref(),
7938 snapshot,
7939 &runtime.terminal_ring,
7940 &runtime.spawn_events,
7941 child,
7942 runtime.drain_timeout,
7943 ModuleState::Disabled,
7944 Some(false),
7945 failure,
7946 )
7947 .await?;
7948 debug!(module_id = %spec.module_id, "supervised module disabled");
7949 Ok(true)
7950 }
7951}
7952
7953#[allow(clippy::too_many_arguments)]
7954async fn on_child_exit(
7955 spec: &ModuleSpec,
7956 policy: RestartPolicy,
7957 registry: &Registry,
7958 snapshot: &SharedSnapshot,
7959 terminal_ring: &Arc<Mutex<TerminalRing>>,
7960 spawn_events: &SpawnEventFeed,
7961 roster: &ChildRoster,
7962 exit_report: ExitReport,
7963 daemon_initiated: bool,
7964) -> NextAction {
7965 if roster.is_closed() {
7971 return on_child_exit_during_daemon_shutdown(
7972 spec,
7973 registry,
7974 snapshot,
7975 terminal_ring,
7976 spawn_events,
7977 exit_report,
7978 )
7979 .await;
7980 }
7981 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7996 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7997 match exit_report.kind {
7998 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7999 info!(
8000 module_id = %spec.module_id,
8001 exit_code = ?exit_report.code,
8002 exit_signal = ?exit_report.signal,
8003 "supervised module exited cleanly"
8004 );
8005 if let Err(err) = publish_after_terminal_record(
8006 &spec.module_id,
8007 || {
8008 record_terminal(
8009 &spec.module_id,
8010 terminal_ring,
8011 spawn_events,
8012 &exit_report,
8013 TerminalDisposition::Stopped,
8014 )
8015 },
8016 || {
8017 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8018 state.state = ModuleState::Stopped;
8019 clear_current_process_facts(state);
8020 state.last_exit = Some(exit_report.clone());
8021 })
8022 },
8023 ) {
8024 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8025 }
8026 let registration_released = match wait_for_registration_release(
8027 registry,
8028 &spec.module_id,
8029 REGISTRY_RELEASE_TIMEOUT,
8030 )
8031 .await
8032 {
8033 Ok(()) => true,
8034 Err(err) => {
8035 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8036 false
8037 }
8038 };
8039 NextAction::Stop {
8040 registration_released,
8041 }
8042 }
8043 ExitKind::Clean | ExitKind::Crash => {
8044 if unrequested_clean_exit_of_protocol_none {
8045 warn!(
8046 module_id = %spec.module_id,
8047 exit_code = ?exit_report.code,
8048 exit_signal = ?exit_report.signal,
8049 "protocol-none module exited cleanly without a stop request; handling it as a crash"
8050 );
8051 } else {
8052 warn!(
8053 module_id = %spec.module_id,
8054 exit_code = ?exit_report.code,
8055 exit_signal = ?exit_report.signal,
8056 "supervised module exited abnormally (crash)"
8057 );
8058 }
8059 let mut restart_schedule = None;
8060 let mut disposition = TerminalDisposition::Disabled;
8061 let mut final_state = ModuleState::Disabled;
8062 let mut disposition_detail = lock_snapshot(snapshot)
8066 .ok()
8067 .and_then(|mut state| state.spawn_failure.take());
8068 let now = Instant::now();
8069 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8070 if state.enabled {
8071 if let Some(schedule) = state.next_crash_restart(&policy, now) {
8072 state.respawn_pending = true;
8078 final_state = ModuleState::Restarting;
8079 restart_schedule = Some(schedule);
8080 disposition = TerminalDisposition::Restarting;
8081 } else {
8082 final_state = ModuleState::Failed;
8083 disposition = TerminalDisposition::Failed;
8084 let budget = policy.budget_exhausted_detail();
8085 disposition_detail =
8086 Some(disposition_detail.take().map_or_else(
8087 || budget.clone(),
8088 |cause| format!("{cause}; {budget}"),
8089 ));
8090 }
8091 } else {
8092 disposition = TerminalDisposition::Disabled;
8093 }
8094 }) {
8095 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8096 return NextAction::Stop {
8097 registration_released: false,
8098 };
8099 }
8100 if disposition == TerminalDisposition::Failed {
8101 error!(
8106 module_id = %spec.module_id,
8107 max_restarts = policy.max_restarts,
8108 window_secs = policy.window.as_secs(),
8109 "module stopped: {}",
8110 policy.budget_exhausted_detail()
8111 );
8112 }
8113 if let Err(err) = publish_after_terminal_record(
8114 &spec.module_id,
8115 || {
8116 record_terminal_with_detail(
8117 &spec.module_id,
8118 terminal_ring,
8119 spawn_events,
8120 &exit_report,
8121 disposition,
8122 disposition_detail,
8123 )
8124 },
8125 || {
8126 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8127 state.state = final_state;
8128 clear_current_process_facts(state);
8129 if final_state == ModuleState::Failed
8130 && daemon_initiated
8131 && !roster.is_closed()
8132 {
8133 state.failed_retry.schedule(&spec.module_id);
8134 }
8135 state.last_exit = Some(exit_report.clone());
8136 })
8137 },
8138 ) {
8139 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8140 }
8141
8142 if let Some(schedule) = restart_schedule {
8143 NextAction::Restart {
8144 schedule: Some(schedule),
8145 }
8146 } else {
8147 let registration_released = match wait_for_registration_release(
8148 registry,
8149 &spec.module_id,
8150 REGISTRY_RELEASE_TIMEOUT,
8151 )
8152 .await
8153 {
8154 Ok(()) => true,
8155 Err(err) => {
8156 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8157 false
8158 }
8159 };
8160 NextAction::Stop {
8161 registration_released,
8162 }
8163 }
8164 }
8165 ExitKind::DeliberateSeverance => {
8166 warn!(
8167 module_id = %spec.module_id,
8168 exit_code = ?exit_report.code,
8169 exit_signal = ?exit_report.signal,
8170 "supervised module exited after deliberate connection severance"
8171 );
8172 let should_restart = match lock_snapshot(snapshot) {
8173 Ok(state) => state.enabled,
8174 Err(err) => {
8175 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8176 return NextAction::Stop {
8177 registration_released: false,
8178 };
8179 }
8180 };
8181 let final_state = if should_restart {
8182 ModuleState::Restarting
8183 } else {
8184 ModuleState::Disabled
8185 };
8186 if let Err(err) = publish_after_terminal_record(
8187 &spec.module_id,
8188 || {
8189 record_terminal(
8190 &spec.module_id,
8191 terminal_ring,
8192 spawn_events,
8193 &exit_report,
8194 terminal_disposition(final_state),
8195 )
8196 },
8197 || {
8198 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8199 state.state = final_state;
8200 clear_current_process_facts(state);
8201 state.last_exit = Some(exit_report.clone());
8202 state.lifetime_restarts += 1;
8203 })
8204 },
8205 ) {
8206 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8207 return NextAction::Stop {
8208 registration_released: false,
8209 };
8210 }
8211
8212 if should_restart {
8213 NextAction::Restart { schedule: None }
8214 } else {
8215 let registration_released = match wait_for_registration_release(
8216 registry,
8217 &spec.module_id,
8218 REGISTRY_RELEASE_TIMEOUT,
8219 )
8220 .await
8221 {
8222 Ok(()) => true,
8223 Err(err) => {
8224 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8225 false
8226 }
8227 };
8228 NextAction::Stop {
8229 registration_released,
8230 }
8231 }
8232 }
8233 }
8234}
8235
8236async fn on_child_exit_during_daemon_shutdown(
8237 spec: &ModuleSpec,
8238 registry: &Registry,
8239 snapshot: &SharedSnapshot,
8240 terminal_ring: &Arc<Mutex<TerminalRing>>,
8241 spawn_events: &SpawnEventFeed,
8242 exit_report: ExitReport,
8243) -> NextAction {
8244 info!(
8245 module_id = %spec.module_id,
8246 exit_code = ?exit_report.code,
8247 exit_signal = ?exit_report.signal,
8248 exit_kind = ?exit_report.kind,
8249 "supervised module exited during daemon shutdown; not restarting it"
8250 );
8251 if let Err(err) = publish_after_terminal_record(
8252 &spec.module_id,
8253 || {
8254 record_terminal(
8255 &spec.module_id,
8256 terminal_ring,
8257 spawn_events,
8258 &exit_report,
8259 TerminalDisposition::DaemonShutdown,
8260 )
8261 },
8262 || {
8263 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8264 state.state = ModuleState::Stopped;
8265 clear_current_process_facts(state);
8266 state.last_exit = Some(exit_report.clone());
8267 })
8268 },
8269 ) {
8270 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8271 }
8272 let registration_released =
8273 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8274 .await
8275 .is_ok();
8276 NextAction::Stop {
8277 registration_released,
8278 }
8279}
8280
8281fn publish_after_terminal_record<T>(
8286 module_id: &str,
8287 record: impl FnOnce(),
8288 publish: impl FnOnce() -> T,
8289) -> T {
8290 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8291 let published = publish();
8292 if let Err(panic) = recorded {
8293 error!(
8294 module_id,
8295 "terminal recording panicked; published final state before resuming unwind"
8296 );
8297 std::panic::resume_unwind(panic);
8298 }
8299 published
8300}
8301
8302fn fail_after_wait_error(
8303 module_id: &str,
8304 snapshot: &SharedSnapshot,
8305 terminal_ring: &Arc<Mutex<TerminalRing>>,
8306 spawn_events: &SpawnEventFeed,
8307 detail: String,
8308 failure: &FailureContext,
8309) {
8310 let report = wait_error_exit_report();
8311 publish_after_terminal_record(
8312 module_id,
8313 || {
8314 record_terminal_with_detail(
8315 module_id,
8316 terminal_ring,
8317 spawn_events,
8318 &report,
8319 TerminalDisposition::Failed,
8320 Some(detail),
8321 )
8322 },
8323 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8324 );
8325}
8326
8327#[cfg(test)]
8328fn record_wait_error_terminal(
8329 module_id: &str,
8330 terminal_ring: &Arc<Mutex<TerminalRing>>,
8331 spawn_events: &SpawnEventFeed,
8332) {
8333 record_terminal(
8334 module_id,
8335 terminal_ring,
8336 spawn_events,
8337 &wait_error_exit_report(),
8338 TerminalDisposition::Failed,
8339 );
8340}
8341
8342fn record_terminal(
8343 module_id: &str,
8344 terminal_ring: &Arc<Mutex<TerminalRing>>,
8345 spawn_events: &SpawnEventFeed,
8346 exit_report: &ExitReport,
8347 disposition: TerminalDisposition,
8348) {
8349 record_terminal_with_detail(
8350 module_id,
8351 terminal_ring,
8352 spawn_events,
8353 exit_report,
8354 disposition,
8355 None,
8356 );
8357}
8358
8359fn durable_terminal_history_of(
8363 terminal_ring: &Mutex<TerminalRing>,
8364 module_id: &str,
8365) -> subc_control::TerminalHistory {
8366 let read = terminal_ring
8367 .lock()
8368 .unwrap_or_else(|p| p.into_inner())
8369 .capture_durable_history();
8370 read.read(module_id)
8371}
8372
8373fn record_terminal_with_detail(
8374 module_id: &str,
8375 terminal_ring: &Arc<Mutex<TerminalRing>>,
8376 spawn_events: &SpawnEventFeed,
8377 exit_report: &ExitReport,
8378 disposition: TerminalDisposition,
8379 disposition_detail: Option<String>,
8380) {
8381 #[cfg(test)]
8382 terminal_history_tests::before_terminal_record(spawn_events);
8383 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8384 let record = TerminalRecord {
8385 exit_code: exit_report.code,
8386 exit_signal: exit_report.signal,
8387 at_ms: exit_report.at_ms,
8388 disposition,
8389 exit_kind: exit_report.kind.into(),
8390 disposition_detail,
8391 };
8392 terminal_ring
8393 .lock()
8394 .unwrap_or_else(|poisoned| poisoned.into_inner())
8395 .record_exit(module_id, record);
8396}
8397
8398fn untrack_if_registration_released(
8399 process_liveness: &SupervisorProcessLiveness,
8400 registry: &Registry,
8401 module_id: &str,
8402 snapshot: &SharedSnapshot,
8403) {
8404 match registry.get_module(module_id) {
8405 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8406 Ok(Some(_)) => {}
8407 Err(err) => {
8408 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8409 }
8410 }
8411}
8412
8413#[cfg(test)]
8427fn apply_wire_spawn_args(
8428 command: &mut Command,
8429 spec: &ModuleSpec,
8430 connection_file_path: Option<&std::path::Path>,
8431 handle: Option<&SupervisorHandle>,
8432) -> Result<Option<NonceHandoff>, SuperviseError> {
8433 apply_wire_spawn_args_for_role(
8434 command,
8435 spec,
8436 connection_file_path,
8437 handle,
8438 SpawnRole::Plain,
8439 )
8440}
8441
8442#[cfg(unix)]
8447type NonceHandoff = subc_os::LaunchNonceHandoff;
8448#[cfg(windows)]
8449type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8450#[cfg(not(any(unix, windows)))]
8451type NonceHandoff = std::convert::Infallible;
8452
8453fn apply_wire_spawn_args_for_role(
8474 command: &mut Command,
8475 spec: &ModuleSpec,
8476 connection_file_path: Option<&std::path::Path>,
8477 handle: Option<&SupervisorHandle>,
8478 role: SpawnRole,
8479) -> Result<Option<NonceHandoff>, SuperviseError> {
8480 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8481 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8486 #[cfg(windows)]
8487 {
8488 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8489 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8490 }
8491 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8493 if spec.protocol == ModuleProtocol::None {
8494 return Ok(None);
8495 }
8496 if let Some(connection_file_path) = connection_file_path {
8497 command.arg(SUBC_ARG).arg(connection_file_path);
8498 }
8499
8500 let nonce = generate_launch_nonce()?;
8504 if let Some(handle) = handle {
8505 match role {
8506 SpawnRole::Plain => {
8507 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8508 if spec.reserved {
8509 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8510 }
8511 }
8512 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8513 }
8514 }
8515 #[cfg(unix)]
8516 let handoff = {
8517 let handoff =
8518 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8519 program: spec.program.clone(),
8520 source,
8521 cgroup_path: None,
8522 })?;
8523 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8524 Some(handoff)
8525 };
8526 #[cfg(windows)]
8527 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8528 #[cfg(not(any(unix, windows)))]
8529 let handoff = None;
8530 #[cfg(not(unix))]
8534 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8535 Ok(handoff)
8536}
8537
8538fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8539 command.env_remove(CK_LOG_ENV);
8540 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8547 for (key, value) in &spec.env {
8548 if matches!(
8552 key.as_str(),
8553 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8554 ) || key == SUBC_SPAWN_ROLE_ENV
8555 {
8556 continue;
8557 }
8558 command.env(key, value);
8559 }
8560}
8561
8562#[cfg(windows)]
8563fn prepare_windows_nonce_pipe(
8564 command: &mut Command,
8565 spec: &ModuleSpec,
8566 nonce: &str,
8567) -> Option<NonceHandoff> {
8568 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8569 Ok(handoff) => {
8570 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8571 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8576 Some(handoff)
8577 }
8578 Err(error) => {
8579 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8580 None
8581 }
8582 }
8583}
8584
8585#[cfg(windows)]
8586fn start_windows_nonce_pipe(
8587 handoff: Option<NonceHandoff>,
8588 pid: u32,
8589 spec: &ModuleSpec,
8590) -> Option<subc_os::LaunchNoncePipeDelivery> {
8591 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8595 Ok(delivery) => Some(delivery),
8596 Err(error) => {
8597 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8598 None
8599 }
8600 })
8601}
8602
8603fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8604 #[cfg(windows)]
8605 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8606 return Some(
8607 if snapshot
8608 .pipe_consumed
8609 .as_ref()
8610 .is_some_and(|consumed| consumed.get().is_some())
8611 {
8612 "pipe"
8613 } else {
8614 "env"
8615 }
8616 .to_owned(),
8617 );
8618 }
8619 let _ = snapshot;
8620 None
8621}
8622
8623#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8626enum SpawnRole {
8627 Plain,
8628 SwapCandidate,
8629}
8630
8631fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8634 if role == SpawnRole::SwapCandidate {
8635 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8636 }
8637}
8638
8639fn previous_instance_error(
8640 spec: &ModuleSpec,
8641 pid: u32,
8642 reason: crate::live_children::RetainedReason,
8643) -> SuperviseError {
8644 SuperviseError::Spawn {
8645 program: spec.program.clone(),
8646 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8647 cgroup_path: None,
8648 }
8649}
8650
8651fn spawn_child(
8652 spec: &ModuleSpec,
8653 connection_file_path: Option<&std::path::Path>,
8654 handle: Option<&SupervisorHandle>,
8655 ring: &Arc<Mutex<StderrRing>>,
8656 capture_logs_dir: Option<&std::path::Path>,
8657 roster: &ChildRoster,
8658 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8659) -> Result<SupervisedChild, SuperviseError> {
8660 spawn_child_in_slot(
8661 spec,
8662 connection_file_path,
8663 handle,
8664 ring,
8665 capture_logs_dir,
8666 roster,
8667 #[cfg(target_os = "linux")]
8668 cgroup_placement,
8669 SpawnRole::Plain,
8670 false,
8671 )
8672}
8673
8674#[allow(clippy::too_many_arguments)]
8687fn spawn_child_in_slot(
8688 spec: &ModuleSpec,
8689 connection_file_path: Option<&std::path::Path>,
8690 handle: Option<&SupervisorHandle>,
8691 ring: &Arc<Mutex<StderrRing>>,
8692 capture_logs_dir: Option<&std::path::Path>,
8693 roster: &ChildRoster,
8694 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8695 role: SpawnRole,
8696 alternate_slot: bool,
8697) -> Result<SupervisedChild, SuperviseError> {
8698 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8699 return Err(previous_instance_error(spec, pid, reason));
8700 }
8701 if roster.is_closed() {
8702 return Err(SuperviseError::Spawn {
8703 program: spec.program.clone(),
8704 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8705 cgroup_path: None,
8706 });
8707 }
8708 #[cfg(target_os = "linux")]
8709 let cgroup_name = {
8710 if cgroup_placement.is_none() {
8715 swap::cgroup_name(&spec.module_id, alternate_slot)
8716 } else {
8717 let nonce = generate_launch_nonce()?;
8718 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8719 let mut end = spec.module_id.len().min(64);
8722 while !spec.module_id.is_char_boundary(end) {
8723 end -= 1;
8724 }
8725 format!(
8726 "{}_{suffix}",
8727 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8728 )
8729 }
8730 };
8731 #[cfg(not(target_os = "linux"))]
8732 let _ = alternate_slot;
8733 #[cfg(target_os = "macos")]
8734 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8735 #[cfg(windows)]
8736 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8737 program: spec.program.clone(),
8738 source,
8739 cgroup_path: None,
8740 })?;
8741 #[cfg(windows)]
8742 let mut command = Command::new(image_capture.path());
8743 #[cfg(not(any(target_os = "macos", windows)))]
8744 let mut command = Command::new(&spec.program);
8745 command.args(&spec.args);
8746 apply_child_env(&mut command, spec);
8776 apply_spawn_role(&mut command, role);
8777 let nonce_handoff =
8778 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8779
8780 #[cfg(target_os = "linux")]
8781 let cgroup_path = cgroup_placement
8782 .map(|placement| placement.module_path(&cgroup_name))
8783 .transpose()
8784 .map_err(|source| SuperviseError::Cgroup {
8785 module_id: spec.module_id.clone(),
8786 source,
8787 })?;
8788 #[cfg(not(target_os = "linux"))]
8789 let cgroup_path: Option<PathBuf> = None;
8790 #[cfg(target_os = "linux")]
8791 if let Some(path) = &cgroup_path {
8792 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8793 if let Some(placement) = cgroup_placement {
8794 remove_module_cgroup(placement, &cgroup_name);
8795 }
8796 return Err(error);
8797 }
8798 }
8799
8800 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8801 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8802 match ChildOutputSink::open(&path, capture_retention(spec)) {
8803 Ok(sink) => sink,
8804 Err(error) => {
8805 warn!(
8806 module_id = %spec.module_id,
8807 path = %path.display(),
8808 error = %error,
8809 "could not open child output capture file; forwarding to stderr"
8810 );
8811 ChildOutputSink::Stderr
8812 }
8813 }
8814 } else {
8815 ChildOutputSink::Stderr
8816 };
8817
8818 command.stdout(Stdio::piped());
8819 command.stderr(Stdio::piped());
8820 command.kill_on_drop(true);
8821 #[cfg(unix)]
8838 command.process_group(0);
8839 command.stdin(Stdio::null());
8840 #[cfg(unix)]
8844 if let Some(handoff) = nonce_handoff {
8845 handoff.install_last(command.as_std_mut());
8846 }
8847 #[cfg(not(any(unix, windows)))]
8848 let _ = nonce_handoff;
8849
8850 #[cfg(windows)]
8855 {
8856 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8857 program: spec.program.clone(),
8858 source,
8859 cgroup_path: None,
8860 })?;
8861 subc_jobobject::suspend_on_create_async(&mut command);
8862 }
8863 let spawned_at = Instant::now();
8867 let mut child = match command.spawn() {
8868 Ok(child) => child,
8869 Err(source) => {
8870 #[cfg(target_os = "linux")]
8871 if let Some(placement) = cgroup_placement {
8872 remove_module_cgroup(placement, &cgroup_name);
8873 }
8874 return Err(SuperviseError::Spawn {
8875 program: spec.program.clone(),
8876 source,
8877 cgroup_path,
8878 });
8879 }
8880 };
8881 #[cfg(test)]
8882 spawn_record_tests::after_create(&child, spec);
8883 #[cfg(target_os = "macos")]
8888 drop(exec_ack);
8889
8890 #[cfg(windows)]
8895 let windows_image = match image_capture.bind_tokio(&child) {
8896 Ok(image) => Arc::new(image),
8897 Err(source) => {
8898 kill_and_reap_rejected_child(&mut child);
8899 return Err(SuperviseError::Spawn {
8900 program: spec.program.clone(),
8901 source,
8902 cgroup_path: None,
8903 });
8904 }
8905 };
8906 #[cfg(windows)]
8909 let job = contain_spawned_child(&mut child, spec)?;
8910 let spawned_at_ms = unix_ms_now();
8911 #[cfg(not(windows))]
8912 let spawned_from = spec.program.clone();
8913 #[cfg(windows)]
8914 let spawned_from = windows_image.path().to_path_buf();
8915 #[cfg(not(windows))]
8916 let spawned_file_identity = spawned_file_identity(&spawned_from);
8917 #[cfg(windows)]
8918 let spawned_file_identity = Some(SpawnedFileIdentity {
8919 device: 0,
8920 inode: 0,
8921 windows_image: Some(Arc::clone(&windows_image)),
8922 });
8923 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8924 program: spec.program.clone(),
8925 source: io::Error::other("spawned child exposed no live pid"),
8926 cgroup_path: cgroup_path.clone(),
8927 })?;
8928 #[cfg(windows)]
8929 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8930 #[cfg(not(windows))]
8931 let process_start_time = crate::provenance::process_start_time(pid);
8932 #[cfg(windows)]
8933 let process_start_time = Some(windows_image.start_time());
8934 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8935 #[cfg(all(test, target_os = "macos"))]
8936 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8937 #[cfg(not(windows))]
8942 let recorded_image = observe_spawned_image(pid);
8943 #[cfg(target_os = "macos")]
8947 let recorded_image = if privacy_exec.is_some() {
8948 None
8949 } else {
8950 recorded_image
8951 };
8952 #[cfg(target_os = "linux")]
8953 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8954 #[cfg(not(target_os = "linux"))]
8955 let recorded_cgroup_name = None;
8956 let roster_guard = roster.admit(
8957 spec.module_id.clone(),
8958 pid,
8959 spec.protocol,
8960 process_start_time,
8961 crate::child_roster::RecordedIdentity {
8962 #[cfg(not(windows))]
8963 start_time: recorded_image.map(|image| image.start_time),
8964 #[cfg(windows)]
8965 start_time: process_start_time,
8966 #[cfg(not(windows))]
8967 executable: recorded_image
8968 .and_then(|image| image.executable)
8969 .map(crate::live_children::ExecutableIdentity::from),
8970 #[cfg(windows)]
8971 executable: Some(crate::live_children::ExecutableIdentity::from(
8972 windows_image.identity(),
8973 )),
8974 cgroup_name: recorded_cgroup_name,
8975 #[cfg(target_os = "linux")]
8976 cgroup_placement: cgroup_placement.cloned(),
8977 },
8978 );
8979 let roster_guard = match roster_guard {
8980 Ok(guard) => guard,
8981 Err(failure) => {
8982 return failure.with_cleanup(|source| {
8983 #[cfg(windows)]
8984 if let Some(job) = &job {
8985 let _ = job.terminate();
8986 }
8987 #[cfg(target_os = "linux")]
8988 kill_module_cgroup(cgroup_placement, &cgroup_name);
8989 kill_and_reap_rejected_child(&mut child);
8990 #[cfg(target_os = "linux")]
8991 if let Some(placement) = cgroup_placement {
8992 remove_module_cgroup(placement, &cgroup_name);
8993 }
8994 Err(SuperviseError::Spawn {
8995 program: spec.program.clone(),
8996 source: io::Error::other(format!(
8997 "could not durably record module child: {source}"
8998 )),
8999 cgroup_path,
9000 })
9001 });
9002 }
9003 };
9004 if roster.is_closed() {
9009 #[cfg(target_os = "linux")]
9011 kill_module_cgroup(cgroup_placement, &cgroup_name);
9012 if let Err(error) = child.start_kill() {
9013 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9014 }
9015 #[cfg(target_os = "linux")]
9016 if let Some(placement) = cgroup_placement {
9017 while matches!(child.try_wait(), Ok(None)) {
9021 std::thread::yield_now();
9022 }
9023 if matches!(
9024 subc_cgroup::kill_module(Some(placement), &cgroup_name),
9025 subc_cgroup::KillOutcome::Killed
9026 ) {
9027 if let Ok(path) = placement.module_path(&cgroup_name) {
9028 while std::fs::read_to_string(path.join("cgroup.events"))
9029 .ok()
9030 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9031 {
9032 std::thread::yield_now();
9033 }
9034 }
9035 }
9036 remove_module_cgroup(placement, &cgroup_name);
9037 }
9038 drop(roster_guard);
9039 return Err(SuperviseError::Spawn {
9040 program: spec.program.clone(),
9041 source: io::Error::other(
9042 "the daemon began shutting down while this process was starting; ended it",
9043 ),
9044 cgroup_path,
9045 });
9046 }
9047
9048 #[cfg(windows)]
9049 if let Err(error) = resume_suspended_child(pid, spec) {
9050 kill_and_reap_rejected_child(&mut child);
9051 return Err(error);
9052 }
9053
9054 let stdout_pump = match child.stdout.take() {
9055 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9056 None => {
9057 warn!(
9058 module_id = %spec.module_id,
9059 "spawned child exposed no stdout pipe; file capture will be incomplete"
9060 );
9061 None
9062 }
9063 };
9064 let stderr_pump = match child.stderr.take() {
9065 Some(stderr) => {
9066 let generation = ring
9067 .lock()
9068 .unwrap_or_else(|poisoned| poisoned.into_inner())
9069 .begin_process();
9070 Some(StderrPump {
9071 task: tokio::spawn(pump_stderr_to(
9072 stderr,
9073 Arc::clone(ring),
9074 generation,
9075 output_sink,
9076 )),
9077 generation,
9078 })
9079 }
9080 None => {
9081 ring.lock()
9085 .unwrap_or_else(|poisoned| poisoned.into_inner())
9086 .mark_not_captured("stderr pipe was not available on spawn");
9087 warn!(
9088 module_id = %spec.module_id,
9089 "spawned child exposed no stderr pipe; tail will be unavailable"
9090 );
9091 None
9092 }
9093 };
9094
9095 let exec = Arc::new(crate::activity::ExecOutcome::new());
9096 #[cfg(target_os = "macos")]
9097 let pending_exec = privacy_exec.is_some();
9098 #[cfg(not(target_os = "macos"))]
9099 let pending_exec = false;
9100 if !pending_exec {
9101 exec.finish(crate::activity::ExecState::CommittedExec);
9102 }
9103 Ok(SupervisedChild {
9104 child,
9105 exec: crate::activity::ExecOwner(exec),
9106 #[cfg(windows)]
9107 nonce_delivery,
9108 protocol: spec.protocol,
9109 #[cfg(target_os = "linux")]
9110 module_id: cgroup_name,
9111 #[cfg(target_os = "linux")]
9112 cgroup_placement: cgroup_placement.cloned(),
9113 #[cfg(windows)]
9114 job,
9115 stdout_pump,
9116 stderr_pump,
9117 stderr_ring: Arc::clone(ring),
9118 spawned_at,
9119 spawned_at_ms,
9120 spawned_from,
9121 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9122 spawned_file_identity,
9123 process_start_time,
9124 process_identity,
9125 pid,
9126 roster_guard: Some(roster_guard),
9127 #[cfg(target_os = "macos")]
9128 privacy_exec,
9129 #[cfg(target_os = "macos")]
9130 report_ready: Arc::new(OnceLock::new()),
9131 spawn_failure: None,
9132 })
9133}
9134
9135#[cfg(test)]
9136mod spawn_record_tests {
9137 use super::*;
9138
9139 thread_local! {
9140 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9141 }
9142
9143 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9144 LAST_PID.set(child.id());
9145 #[cfg(windows)]
9146 if let Some((_, address)) = _spec
9147 .env
9148 .iter()
9149 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9150 {
9151 use std::io::{Read, Write};
9152 let mut stream = std::net::TcpStream::connect(address).unwrap();
9153 stream
9154 .write_all(&child.id().unwrap().to_ne_bytes())
9155 .unwrap();
9156 let mut release = [0];
9157 stream.read_exact(&mut release).unwrap();
9158 }
9159 }
9160
9161 #[cfg(windows)]
9162 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9163 if spec
9164 .env
9165 .iter()
9166 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9167 {
9168 Err(io::Error::other("injected module job assignment failure"))
9169 } else {
9170 Ok(())
9171 }
9172 }
9173
9174 fn spec() -> ModuleSpec {
9175 ModuleSpec {
9176 module_id: "spawn-record-worker".into(),
9177 #[cfg(unix)]
9178 program: "/bin/sleep".into(),
9179 #[cfg(windows)]
9180 program: std::env::current_exe().unwrap(),
9181 #[cfg(unix)]
9182 args: vec!["60".into()],
9183 #[cfg(windows)]
9184 args: vec![
9185 "--exact".into(),
9186 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9187 "--nocapture".into(),
9188 ],
9189 env: vec![],
9190 reserved: false,
9191 reserved_prefixes: vec![],
9192 protocol: ModuleProtocol::None,
9193 overlap: Default::default(),
9194 }
9195 }
9196
9197 #[tokio::test]
9198 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9199 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9200 let supervisor =
9201 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9202 .with_live_children_record(home.join("live-children.json"));
9203 crate::live_children::fail_next_file_sync();
9204 let result = supervisor.spawn(spec());
9205 let failed = result.is_err();
9206 if let Ok(module) = result {
9207 module.stop().await.unwrap();
9208 }
9209 assert!(failed, "record failure must fail spawn");
9210 let pid = LAST_PID.get().expect("the real OS child was created");
9211 assert!(
9212 subc_os::Process::open(pid).unwrap().is_none(),
9213 "child {pid} survived rejected admission or was not reaped"
9214 );
9215 }
9216
9217 #[cfg(target_os = "macos")]
9218 #[test]
9219 #[ignore = "manual on-device latency measurement"]
9220 fn durable_record_write_latency_50() {
9221 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9222 let path = home.join("live-children.json");
9223 let children = vec![crate::live_children::LiveChild {
9224 module_id: "latency-worker".into(),
9225 pid: std::process::id(),
9226 protocol: ModuleProtocol::None,
9227 start_time: Some(1),
9228 executable: None,
9229 cgroup_name: None,
9230 }];
9231 let mut samples = Vec::new();
9232 for _ in 0..50 {
9233 let start = std::time::Instant::now();
9234 crate::live_children::write_record(&path, &children).unwrap();
9235 samples.push(start.elapsed());
9236 }
9237 samples.sort();
9238 eprintln!(
9239 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9240 (samples[24] + samples[25]) / 2,
9241 samples[47]
9242 );
9243 }
9244
9245 #[cfg(windows)]
9246 #[test]
9247 fn windows_store_touch_fixture() {
9248 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9249 std::fs::write(path, b"module ran").unwrap();
9250 }
9251 }
9252
9253 #[cfg(windows)]
9254 #[tokio::test]
9255 async fn windows_daemon_fixture() {
9256 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9257 return;
9258 };
9259 let mut spec = spec();
9260 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9261 let supervisor =
9262 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9263 let _module = supervisor.spawn(spec).unwrap();
9264 }
9265
9266 #[cfg(windows)]
9267 #[test]
9268 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9269 use std::io::Read;
9270 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9271 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9274 command
9275 .args([
9276 "--exact",
9277 "supervise::spawn_record_tests::windows_daemon_fixture",
9278 "--nocapture",
9279 ])
9280 .env(
9281 "SUBC_TEST_DAEMON_BARRIER",
9282 listener.local_addr().unwrap().to_string(),
9283 );
9284 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9285 listener.set_nonblocking(true).unwrap();
9286 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9287 let mut stream = loop {
9288 if let Ok((stream, _)) = listener.accept() {
9289 break stream;
9290 }
9291 assert!(
9292 daemon.child.try_wait().unwrap().is_none(),
9293 "daemon failed before rendezvous"
9294 );
9295 assert!(
9296 std::time::Instant::now() < deadline,
9297 "daemon did not reach CreateProcess failpoint"
9298 );
9299 std::thread::yield_now();
9300 };
9301 stream
9302 .set_read_timeout(Some(Duration::from_secs(30)))
9303 .unwrap();
9304 let mut pid = [0; 4];
9305 stream.read_exact(&mut pid).unwrap();
9306 let pid = u32::from_ne_bytes(pid);
9307 assert!(subc_jobobject::process_exists(pid));
9308 daemon.child.kill().unwrap();
9311 daemon.child.wait().unwrap();
9312 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9313 if !exited {
9314 daemon.job.terminate().unwrap();
9315 }
9316 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9317 }
9318
9319 #[cfg(windows)]
9320 #[tokio::test]
9321 async fn windows_containment_failure_never_resumes() {
9322 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9323 let sentinel = home.join("store-touched");
9324 let mut spec = spec();
9325 spec.env
9326 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9327 spec.env.push((
9328 "SUBC_TEST_STORE_TOUCH".into(),
9329 sentinel.display().to_string(),
9330 ));
9331 let supervisor =
9332 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9333 let result = supervisor.spawn(spec);
9334 if let Ok(module) = result {
9337 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9338 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9339 tokio::task::yield_now().await;
9340 }
9341 })
9342 .await;
9343 let _ = module.stop().await;
9344 panic!(
9345 "assignment failure resumed a module; store touched={}",
9346 sentinel.exists()
9347 );
9348 }
9349 assert!(
9350 !sentinel.exists(),
9351 "refused module executed store-touch sentinel"
9352 );
9353 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9354 }
9355
9356 #[cfg(windows)]
9357 #[tokio::test]
9358 async fn windows_nested_daemon_fixture() {
9359 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9360 return;
9361 };
9362 let mut spec = spec();
9363 spec.env.push((
9364 "SUBC_TEST_STORE_TOUCH".into(),
9365 sentinel.to_str().unwrap().into(),
9366 ));
9367 let supervisor =
9368 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9369 let module = supervisor
9370 .spawn(spec)
9371 .expect("per-module job must nest below daemon and host jobs");
9372 tokio::time::timeout(Duration::from_secs(10), async {
9373 while !PathBuf::from(&sentinel).exists() {
9374 tokio::task::yield_now().await;
9375 }
9376 })
9377 .await
9378 .unwrap();
9379 module.stop().await.unwrap();
9380 }
9381
9382 #[cfg(windows)]
9383 #[test]
9384 fn windows_module_job_nests_under_host_job() {
9385 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9386 let sentinel = home.join("nested-store-touched");
9387 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9388 command
9389 .args([
9390 "--exact",
9391 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9392 "--nocapture",
9393 ])
9394 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9395 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9396 assert!(daemon.child.wait().unwrap().success());
9397 assert!(sentinel.exists(), "nested protocol:none module never ran");
9398 }
9399}
9400
9401#[cfg(target_os = "linux")]
9402pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9403 use subc_cgroup::KillOutcome;
9404 match subc_cgroup::kill_module(placement, module_id) {
9405 KillOutcome::Killed => {}
9406 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9407 debug!(
9408 module_id,
9409 "cgroup tree kill unavailable; using direct-child kill"
9410 );
9411 }
9412 KillOutcome::IoError { path, error } => {
9413 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9414 }
9415 }
9416}
9417
9418#[cfg(windows)]
9426fn contain_spawned_child(
9427 child: &mut Child,
9428 spec: &ModuleSpec,
9429) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9430 let assignment = || {
9431 let job = subc_jobobject::JobObject::new()?;
9432 #[cfg(test)]
9433 spawn_record_tests::assignment_failpoint(spec)?;
9434 job.assign(child)?;
9435 Ok::<_, io::Error>(job)
9436 };
9437 match assignment() {
9438 Ok(job) => Ok(Some(job)),
9439 Err(source) => {
9440 kill_and_reap_rejected_child(child);
9441 Err(SuperviseError::Spawn {
9442 program: spec.program.clone(),
9443 source: io::Error::other(format!(
9444 "Windows module job containment failed: {source}"
9445 )),
9446 cgroup_path: None,
9447 })
9448 }
9449 }
9450}
9451
9452fn kill_and_reap_rejected_child(child: &mut Child) {
9455 #[cfg(unix)]
9456 if let Some(pid) = child
9457 .id()
9458 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9459 {
9460 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9461 }
9462 let _ = child.start_kill();
9463 while matches!(child.try_wait(), Ok(None)) {
9464 std::thread::yield_now();
9465 }
9466}
9467
9468#[cfg(windows)]
9473fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9474 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9475 let _ = std::process::Command::new("taskkill.exe")
9479 .args(["/PID", &pid.to_string(), "/T", "/F"])
9480 .stdin(Stdio::null())
9481 .stdout(Stdio::null())
9482 .stderr(Stdio::null())
9483 .status();
9484 return Err(SuperviseError::Spawn {
9485 program: spec.program.clone(),
9486 source,
9487 cgroup_path: None,
9488 });
9489 }
9490 Ok(())
9491}
9492
9493#[cfg(target_os = "linux")]
9494fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9495 match placement.remove_module(module_id) {
9496 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9497 Err(error) => warn!(
9498 module_id,
9499 error = %error,
9500 "could not remove module cgroup after process exit; continuing teardown"
9501 ),
9502 }
9503}
9504
9505#[cfg(target_os = "linux")]
9506async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9507 if matches!(
9511 subc_cgroup::kill_module(Some(placement), module_id),
9512 subc_cgroup::KillOutcome::Killed
9513 ) {
9514 if let Ok(path) = placement.module_path(module_id) {
9515 while std::fs::read_to_string(path.join("cgroup.events"))
9516 .ok()
9517 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9518 {
9519 sleep(Duration::from_millis(1)).await;
9520 }
9521 }
9522 }
9523 remove_module_cgroup(placement, module_id);
9524}
9525
9526#[cfg(target_os = "linux")]
9527fn apply_cgroup_placement(
9528 command: &mut Command,
9529 spec: &ModuleSpec,
9530 path: &std::path::Path,
9531) -> Result<(), SuperviseError> {
9532 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9533 module_id: spec.module_id.clone(),
9534 source,
9535 })
9536}
9537
9538fn capture_retention(spec: &ModuleSpec) -> Retention {
9539 let defaults = Retention::default();
9540 let value = |name: &str| {
9541 spec.env
9542 .iter()
9543 .rev()
9544 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9545 };
9546 Retention {
9547 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9548 .and_then(|value| value.parse().ok())
9549 .unwrap_or(defaults.max_file_mb),
9550 keep: value(CAPTURE_KEEP_ENV)
9551 .and_then(|value| value.parse().ok())
9552 .unwrap_or(defaults.keep),
9553 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9554 .and_then(|value| value.parse().ok())
9555 .unwrap_or(defaults.max_age_days),
9556 }
9557}
9558
9559fn generate_launch_nonce() -> Result<String, SuperviseError> {
9562 let mut bytes = [0u8; 32];
9563 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9564 reason: source.to_string(),
9565 })?;
9566 let mut hex = String::with_capacity(64);
9567 for b in bytes {
9568 use std::fmt::Write;
9569 let _ = write!(hex, "{b:02x}");
9570 }
9571 Ok(hex)
9572}
9573
9574fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9577 if a.len() != b.len() {
9578 return false;
9579 }
9580 let mut diff = 0u8;
9581 for (x, y) in a.iter().zip(b.iter()) {
9582 diff |= x ^ y;
9583 }
9584 diff == 0
9585}
9586
9587#[cfg(not(windows))]
9591fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9592 let process = subc_os::Process::open(pid).ok().flatten()?;
9593 #[cfg(target_os = "linux")]
9594 return sample_after_linux_exec(
9595 || {
9596 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9597 let committed = linux_exec_committed(&stat);
9598 #[cfg(test)]
9599 spawn_image_tests::report_exec_state(committed);
9600 committed
9601 },
9602 || process.observe(),
9603 );
9604 #[cfg(not(target_os = "linux"))]
9605 process.observe()
9606}
9607
9608#[cfg(target_os = "linux")]
9612fn linux_exec_committed(stat: &str) -> Option<bool> {
9613 const PF_FORKNOEXEC: u64 = 0x40;
9614 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9615 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9616}
9617
9618#[cfg(target_os = "linux")]
9619fn sample_after_linux_exec(
9620 mut exec_committed: impl FnMut() -> Option<bool>,
9621 observe: impl FnOnce() -> Option<subc_os::Observation>,
9622) -> Option<subc_os::Observation> {
9623 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9624 while !exec_committed()? {
9625 if std::time::Instant::now() >= deadline {
9629 return None;
9630 }
9631 std::thread::sleep(Duration::from_micros(50));
9632 }
9633 observe()
9634}
9635
9636#[cfg(all(test, target_os = "linux"))]
9637mod spawn_image_tests {
9638 use super::*;
9639 use std::cell::{Cell, RefCell};
9640
9641 enum ExecEvent {
9642 Uncommitted,
9643 Image(Option<subc_os::Observation>),
9644 }
9645
9646 thread_local! {
9647 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9648 }
9649
9650 pub(super) fn report_exec_state(committed: Option<bool>) {
9651 if committed == Some(false) {
9652 EXEC_EVENTS.with(|events| {
9653 if let Some(events) = events.borrow_mut().take() {
9654 let _ = events.send(ExecEvent::Uncommitted);
9655 }
9656 });
9657 }
9658 }
9659
9660 #[test]
9661 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9662 use std::io::{Read, Write};
9663 use std::os::fd::AsRawFd;
9664
9665 let descriptor = std::fs::File::open("/dev/null").unwrap();
9666 let mut command = std::process::Command::new("/bin/sleep");
9667 command.arg("30");
9668 let (mut ready, mut release) =
9669 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9670 .unwrap();
9671 let spawn = std::thread::spawn(move || command.spawn());
9672 let mut message = [0u8; 8];
9673 ready.read_exact(&mut message).unwrap();
9674 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9675 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9676 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9677 let (events, received) = std::sync::mpsc::channel();
9678 let observer = std::thread::spawn(move || {
9679 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9680 let image = observe_spawned_image(pid);
9681 let _ = events.send(ExecEvent::Image(image));
9682 });
9683 let first = received.recv_timeout(Duration::from_secs(5));
9687 release.write_all(&[1]).unwrap();
9688 let mut child = spawn.join().unwrap().unwrap();
9689 let after = match first {
9690 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9691 Ok(ExecEvent::Image(image)) => image,
9692 _ => None,
9693 },
9694 Ok(ExecEvent::Image(image)) => image,
9695 Err(_) => None,
9696 };
9697 observer.join().unwrap();
9698 child.kill().unwrap();
9699 child.wait().unwrap();
9700
9701 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9702 assert_eq!(
9703 inherited.unwrap().executable,
9704 subc_os::file_identity(&std::env::current_exe().unwrap())
9705 );
9706 assert_eq!(
9707 after.unwrap().executable,
9708 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9709 );
9710 }
9711
9712 #[test]
9713 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9714 let committed = Cell::new(false);
9715 let inherited = subc_os::Observation {
9716 start_time: 23,
9717 executable: Some(subc_os::FileIdentity {
9718 device: 29,
9719 inode: 31,
9720 }),
9721 };
9722 let module = subc_os::Observation {
9723 start_time: 23,
9724 executable: Some(subc_os::FileIdentity {
9725 device: 29,
9726 inode: 37,
9727 }),
9728 };
9729 let polls = Cell::new(0);
9730 let image = sample_after_linux_exec(
9731 || {
9732 polls.set(polls.get() + 1);
9733 committed.set(polls.get() == 2);
9736 Some(committed.get())
9737 },
9738 || Some(if committed.get() { module } else { inherited }),
9739 );
9740 assert_eq!(image, Some(module));
9741 assert_eq!(polls.get(), 2);
9742 }
9743
9744 #[test]
9745 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9746 assert_eq!(
9747 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9748 Some(false)
9749 );
9750 assert_eq!(
9751 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9752 Some(true)
9753 );
9754 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9755 assert_eq!(linux_exec_committed("17 (a) R"), None);
9756 }
9757
9758 #[test]
9759 fn unreadable_exec_state_never_samples_an_inherited_image() {
9760 assert_eq!(
9761 sample_after_linux_exec(
9762 || None,
9763 || panic!("must not sample before exec confirmation")
9764 ),
9765 None
9766 );
9767 }
9768
9769 #[test]
9770 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9771 assert_eq!(
9772 sample_after_linux_exec(
9773 || Some(false),
9774 || panic!("must not sample after an exec timeout")
9775 ),
9776 None
9777 );
9778 }
9779}
9780
9781#[cfg(windows)]
9784fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9785 let mut program = spec.program.clone();
9786 if program.is_absolute() || program.components().count() > 1 {
9787 match subc_os::ExecutableCapture::open(&program) {
9790 Ok(capture) => return Ok(capture),
9791 Err(error)
9792 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9793 Err(error) => return Err(error),
9794 }
9795 program.set_extension("exe");
9796 return subc_os::ExecutableCapture::open(&program);
9797 }
9798 if program.extension().is_none() {
9799 program.set_extension("exe");
9800 }
9801 let custom_path = spec
9802 .env
9803 .iter()
9804 .rev()
9805 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9806 let mut directories = Vec::new();
9807 if let Some(parent) = std::env::current_exe()?.parent() {
9810 directories.push(parent.to_path_buf());
9811 }
9812 directories.push(std::env::current_dir()?);
9813 if let Some(windows) = std::env::var_os("SystemRoot") {
9814 let windows = PathBuf::from(windows);
9815 directories.push(windows.join("System32"));
9816 directories.push(windows.clone());
9817 }
9818 let path = custom_path
9819 .map(|(_, value)| std::ffi::OsString::from(value))
9820 .or_else(|| std::env::var_os("PATH"));
9821 if let Some(path) = path {
9822 directories.extend(std::env::split_paths(&path));
9823 }
9824 for directory in directories {
9825 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9826 Ok(capture) => return Ok(capture),
9827 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9828 Err(error) => return Err(error),
9829 }
9830 }
9831 Err(io::Error::new(
9832 io::ErrorKind::NotFound,
9833 "resolved executable not found",
9834 ))
9835}
9836
9837fn spawn_and_mark_running(
9838 spec: &ModuleSpec,
9839 runtime: &SupervisorRuntimeConfig,
9840 snapshot: &SharedSnapshot,
9841) -> Result<SupervisedChild, SuperviseError> {
9842 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9843 let child = spawn_child(
9844 spec,
9845 runtime.connection_file_path.as_deref(),
9846 runtime.supervisor_handle.as_ref(),
9847 &runtime.stderr_ring,
9848 runtime.capture_logs_dir.as_deref(),
9849 &runtime.child_roster,
9850 #[cfg(target_os = "linux")]
9851 runtime.cgroup_placement.as_ref(),
9852 )?;
9853 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9854 if let Some(spawn) = activity_spawn {
9855 spawn.complete(snapshot);
9856 }
9857 Ok(child)
9858}
9859
9860enum RegistrationWaitOutcome {
9861 Registered,
9862 Exited(ExitReport),
9863 TimedOut,
9864}
9865
9866struct ReloadRegistrationFailure {
9867 exit_report: ExitReport,
9868 reason: String,
9869}
9870
9871#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9872enum BusyGaugeObservation {
9873 Quiescent,
9874 Busy,
9875 Omitted,
9876}
9877
9878#[derive(Debug, Clone, PartialEq, Eq)]
9880enum GaugeVerdict {
9881 Zero,
9883 NonZero(u64),
9885 NotACount(String),
9889 Missing,
9891 ProbeFailed(&'static str),
9894}
9895
9896impl GaugeVerdict {
9897 fn reads_quiet(&self) -> bool {
9898 matches!(self, Self::Zero)
9899 }
9900}
9901
9902impl fmt::Display for GaugeVerdict {
9903 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9904 match self {
9905 Self::Zero => f.write_str("zero"),
9906 Self::NonZero(value) => write!(f, "non_zero({value})"),
9907 Self::NotACount(value) => write!(f, "not_a_count({value})"),
9908 Self::Missing => f.write_str("missing_from_health_body"),
9909 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9910 }
9911 }
9912}
9913
9914const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9917
9918fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9920 let metrics = metrics.and_then(Value::as_object);
9921 gauges
9922 .iter()
9923 .map(
9924 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9925 None => GaugeVerdict::Missing,
9926 Some(value) => match value.as_u64() {
9927 Some(0) => GaugeVerdict::Zero,
9928 Some(count) => GaugeVerdict::NonZero(count),
9929 None => {
9930 let mut text = value.to_string();
9931 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9932 let mut end = GAUGE_VALUE_TEXT_LIMIT;
9933 while !text.is_char_boundary(end) {
9934 end -= 1;
9935 }
9936 text.truncate(end);
9937 text.push_str("...");
9938 }
9939 GaugeVerdict::NotACount(text)
9940 }
9941 },
9942 },
9943 )
9944 .collect()
9945}
9946
9947fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9950 if verdicts
9951 .iter()
9952 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9953 {
9954 BusyGaugeObservation::Omitted
9955 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9956 BusyGaugeObservation::Quiescent
9957 } else {
9958 BusyGaugeObservation::Busy
9959 }
9960}
9961
9962#[derive(Debug, Clone, Default, PartialEq, Eq)]
9967struct GaugeTrace {
9968 latest: Option<(GaugeVerdict, Duration)>,
9969 last_value: Option<(GaugeVerdict, Duration)>,
9970}
9971
9972#[derive(Debug, Clone, Default, PartialEq, Eq)]
9978struct DrainGaugeTrace {
9979 probes: u32,
9981 gauges: Vec<(String, GaugeTrace)>,
9983}
9984
9985impl DrainGaugeTrace {
9986 fn new(gauges: &[String]) -> Self {
9987 Self {
9988 probes: 0,
9989 gauges: gauges
9990 .iter()
9991 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9992 .collect(),
9993 }
9994 }
9995
9996 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9998 self.probes = self.probes.saturating_add(1);
9999 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10000 trace.latest = Some((verdict.clone(), at));
10001 if matches!(
10002 verdict,
10003 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10004 ) {
10005 trace.last_value = Some((verdict.clone(), at));
10006 }
10007 }
10008 }
10009}
10010
10011impl fmt::Display for DrainGaugeTrace {
10016 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10017 if self.gauges.is_empty() {
10018 return f.write_str("none declared");
10019 }
10020 for (index, (name, trace)) in self.gauges.iter().enumerate() {
10021 if index > 0 {
10022 f.write_str(" | ")?;
10023 }
10024 write!(f, "{name}: ")?;
10025 match &trace.latest {
10026 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10027 None => f.write_str("never probed")?,
10028 }
10029 match &trace.last_value {
10030 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10031 None => f.write_str("; no value ever read")?,
10032 }
10033 }
10034 Ok(())
10035 }
10036}
10037
10038pub(crate) fn declared_busy_gauges(
10039 registry: &Registry,
10040 module_id: &str,
10041) -> Result<Vec<String>, SuperviseError> {
10042 busy_gauges_of(
10043 registry
10044 .get_module(module_id)
10045 .map_err(SuperviseError::Registry)?,
10046 )
10047}
10048
10049fn declared_busy_gauges_for_connection(
10053 registry: &Registry,
10054 connection_id: ConnectionId,
10055) -> Result<Vec<String>, SuperviseError> {
10056 busy_gauges_of(
10057 registry
10058 .get_module_by_connection(connection_id)
10059 .map_err(SuperviseError::Registry)?,
10060 )
10061}
10062
10063fn busy_gauges_of(
10064 registration: Option<crate::registry::ModuleRegistration>,
10065) -> Result<Vec<String>, SuperviseError> {
10066 let Some(registration) = registration else {
10067 return Ok(Vec::new());
10068 };
10069 let Some(self_signals) = registration.manifest.self_signals else {
10070 return Ok(Vec::new());
10071 };
10072
10073 let mut gauges = Vec::new();
10074 for declaration in self_signals {
10075 if declaration.kind != SelfSignalKind::Busy {
10076 continue;
10077 }
10078 match declaration.anchored_to {
10079 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10080 gauges.extend(declared)
10081 }
10082 _ => {
10083 gauges.push(String::new());
10086 }
10087 }
10088 }
10089 Ok(gauges)
10090}
10091
10092#[allow(clippy::too_many_arguments)]
10101async fn wait_for_forwarding_quiescence(
10102 forwarding: &ForwardingTable,
10103 module_id: &str,
10104 runtime: &SupervisorRuntimeConfig,
10105 endpoint: crate::ModuleEndpointId,
10106 deadline: Instant,
10107 busy_gauges: &[String],
10108 scope: DrainScope,
10109 trace: &mut DrainGaugeTrace,
10110) -> Result<bool, SuperviseError> {
10111 let wait_started = Instant::now();
10112 let mut gauges_quiescent = busy_gauges.is_empty();
10113 let mut next_probe_at = wait_started;
10114 let mut omission_reported = false;
10115
10116 loop {
10117 let now = Instant::now();
10118 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10119 let report = match scope {
10120 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10121 DrainScope::Endpoint(endpoint) => {
10122 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10123 }
10124 };
10125 let verdicts = match &report {
10126 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10127 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10128 };
10129 trace.record(&verdicts, wait_started.elapsed());
10130 gauges_quiescent = match report {
10131 Ok(_) => match busy_gauge_observation(&verdicts) {
10132 BusyGaugeObservation::Quiescent => true,
10133 BusyGaugeObservation::Busy => false,
10134 BusyGaugeObservation::Omitted => {
10135 if !omission_reported {
10138 forwarding
10139 .counters()
10140 .increment_drains_with_undeclared_gauge();
10141 let missing = busy_gauges
10142 .iter()
10143 .zip(&verdicts)
10144 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10145 .map(|(gauge, _)| gauge.as_str())
10146 .collect::<Vec<_>>()
10147 .join(",");
10148 warn!(
10149 module_id,
10150 missing_gauges = %missing,
10151 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10152 );
10153 omission_reported = true;
10154 }
10155 false
10156 }
10157 },
10158 Err(err) => {
10159 warn!(
10160 module_id,
10161 error = %err,
10162 "drain health.check did not produce declared busy gauges; treating module as busy"
10163 );
10164 false
10165 }
10166 };
10167 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10170 }
10171
10172 let in_flight = forwarding
10173 .endpoint_in_flight_count(endpoint)
10174 .map_err(SuperviseError::Forwarding)?;
10175 if in_flight == 0 && gauges_quiescent {
10176 return Ok(true);
10177 }
10178
10179 let now = Instant::now();
10180 if now >= deadline {
10181 return Ok(false);
10182 }
10183 let mut wait = deadline
10184 .saturating_duration_since(now)
10185 .min(REGISTRY_RELEASE_POLL);
10186 if !busy_gauges.is_empty() {
10187 wait = wait.min(next_probe_at.saturating_duration_since(now));
10188 }
10189 sleep(wait).await;
10190 }
10191}
10192
10193fn drain_busy_reprobe_interval() -> Duration {
10197 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10198}
10199
10200fn warn_drain_timed_out(
10215 module_id: &str,
10216 drain_timeout: Duration,
10217 reason: RouteCloseReason,
10218 holdouts: &crate::forwarding::DrainHoldouts,
10219 gauges: &DrainGaugeTrace,
10220) {
10221 warn!(
10222 module_id = %module_id,
10223 waited = ?drain_timeout,
10224 ?reason,
10225 held_requests = holdouts.requests,
10226 held_routes = holdouts.routes,
10227 total_routes = holdouts.total_routes,
10228 top_connections = ?holdouts.top_connections,
10229 held = %holdouts
10232 .held
10233 .iter()
10234 .map(|(channel, corr)| format!("{channel}:{corr}"))
10235 .collect::<Vec<_>>()
10236 .join(","),
10237 gauge_probes = gauges.probes,
10238 busy_gauges = %gauges,
10239 "route drain timed out before request quiescence; forcing teardown"
10240 );
10241}
10242
10243fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10251 match wait_result {
10252 Ok(drained) => *drained,
10253 Err(_) => false,
10254 }
10255}
10256
10257fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10258 for released in released_routes {
10259 let frame = match Frame::build_with_version(
10260 released.negotiated_ver,
10261 FrameType::Goodbye,
10262 control_flags(),
10263 released.channel,
10264 released.epoch,
10265 0,
10266 Vec::new(),
10267 ) {
10268 Ok(frame) => frame,
10269 Err(err) => {
10270 warn!(
10271 route_channel = released.channel,
10272 error = %err,
10273 "failed to build supervisor drain route GOODBYE frame"
10274 );
10275 continue;
10276 }
10277 };
10278 if !released.close_on_delivery_failure() {
10279 crate::forwarding::send_module_route_goodbye(
10280 &forwarding.counters(),
10281 &released.sink,
10282 frame,
10283 released.module_id.as_deref(),
10284 "supervisor drain",
10285 );
10286 continue;
10287 }
10288 if let Err(err) = released.sink.try_send(frame) {
10289 warn!(
10290 target_connection_id = released.connection_id.get(),
10291 route_channel = released.channel,
10292 error = %err,
10293 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10294 );
10295 let _ = forwarding.escalate_client_delivery_failure(
10296 released.connection_id,
10297 released.channel,
10298 released.epoch,
10299 CloseReason::new(
10300 "route_goodbye_delivery_failed",
10301 format!(
10302 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10303 released.channel
10304 ),
10305 ),
10306 crate::forwarding::UndeliveredFrame {
10307 module_id: released.module_id.as_deref(),
10308 sink: &released.sink,
10309 },
10310 );
10311 }
10312 }
10313}
10314
10315fn send_module_draining(
10316 module_id: &str,
10317 reason: RouteCloseReason,
10318 deadline_ms: u64,
10319 target: &ModuleDrainTarget,
10320) {
10321 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10322 reason,
10323 deadline_ms,
10324 }) {
10325 Ok(body) => body,
10326 Err(err) => {
10327 warn!(
10328 module_id,
10329 error = %err,
10330 "failed to encode module draining command"
10331 );
10332 return;
10333 }
10334 };
10335 let frame = match Frame::build_with_version(
10336 target.negotiated_ver,
10337 FrameType::Push,
10338 control_flags(),
10339 0,
10340 0,
10341 0,
10342 body,
10343 ) {
10344 Ok(frame) => frame,
10345 Err(err) => {
10346 warn!(
10347 module_id,
10348 error = %err,
10349 "failed to build module draining command frame"
10350 );
10351 return;
10352 }
10353 };
10354 if let Err(err) = target.sink.try_send(frame) {
10355 warn!(
10356 module_id,
10357 target_connection_id = target.endpoint.connection_id.get(),
10358 error = %err,
10359 "module draining command was not delivered to peer"
10360 );
10361 }
10362}
10363
10364fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10366 match Frame::build_with_version(
10367 negotiated_ver,
10368 FrameType::Goodbye,
10369 control_flags(),
10370 0,
10371 0,
10372 0,
10373 Vec::new(),
10374 ) {
10375 Ok(frame) => Some(frame),
10376 Err(err) => {
10377 warn!(
10378 module_id,
10379 error = %err,
10380 "failed to build module GOODBYE frame"
10381 );
10382 None
10383 }
10384 }
10385}
10386
10387#[cfg(unix)]
10401async fn send_module_goodbyes_for_daemon_shutdown(
10402 forwarding: &Arc<ForwardingTable>,
10403 reason: &CloseReason,
10404 wait_for_flush: bool,
10405) {
10406 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10407 let targets = match forwarding.module_connections() {
10408 Ok(targets) => targets,
10409 Err(err) => {
10410 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10411 return;
10412 }
10413 };
10414 let deadline = Instant::now() + GOODBYE_BUDGET;
10415 let mut sends = tokio::task::JoinSet::new();
10416 for target in targets {
10417 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10418 continue;
10419 };
10420 if !wait_for_flush {
10421 if let Err(err) = target.sink.try_send(frame) {
10422 debug!(
10423 module_id = %target.module_id,
10424 error = %err,
10425 "shutdown module GOODBYE was not queued"
10426 );
10427 }
10428 continue;
10429 }
10430 let forwarding = Arc::clone(forwarding);
10431 let reason = reason.clone();
10432 sends.spawn(async move {
10433 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10434 Ok(Ok(())) => {}
10435 Ok(Err(err)) => debug!(
10436 module_id = %target.module_id,
10437 error = %err,
10438 "module connection closed before its shutdown GOODBYE was written"
10439 ),
10440 Err(_) => warn!(
10441 module_id = %target.module_id,
10442 budget = ?GOODBYE_BUDGET,
10443 "shutdown module GOODBYE was not written within its budget; closing anyway"
10444 ),
10445 }
10446 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10447 });
10448 }
10449 while sends.join_next().await.is_some() {}
10451}
10452
10453fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10454 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10455 return;
10456 };
10457 if let Err(err) = target.sink.try_send(frame) {
10458 warn!(
10459 module_id,
10460 target_connection_id = target.endpoint.connection_id.get(),
10461 error = %err,
10462 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10463 );
10464 forwarding.request_connection_close(
10465 target.endpoint.connection_id,
10466 CloseReason::new(
10467 "module_goodbye_delivery_failed",
10468 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10469 ),
10470 );
10471 }
10472}
10473
10474#[derive(Clone, Copy)]
10475struct ForwardingDrainContext<'a> {
10476 spec: &'a ModuleSpec,
10477 runtime: &'a SupervisorRuntimeConfig,
10478 registry: &'a Registry,
10479 scope: DrainScope,
10480}
10481
10482#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10484enum DrainScope {
10485 Active,
10488 Endpoint(crate::ModuleEndpointId),
10493}
10494
10495#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10503enum StopNotice {
10504 SentOverConnection,
10507 NoConnection,
10511 NotSent,
10515}
10516
10517async fn begin_forwarding_drain(
10518 spec: &ModuleSpec,
10519 runtime: &SupervisorRuntimeConfig,
10520 registry: &Registry,
10521 snapshot: &SharedSnapshot,
10522 enabled: Option<bool>,
10523 reason: RouteCloseReason,
10524) -> Result<StopNotice, SuperviseError> {
10525 let Some(forwarding) = runtime.forwarding.as_ref() else {
10526 return Err(SuperviseError::ReloadUnavailable {
10527 module_id: spec.module_id.clone(),
10528 reason: "supervisor was not configured with a forwarding table".to_string(),
10529 });
10530 };
10531
10532 begin_forwarding_drain_with(
10533 forwarding,
10534 ForwardingDrainContext {
10535 spec,
10536 runtime,
10537 registry,
10538 scope: DrainScope::Active,
10539 },
10540 snapshot,
10541 enabled,
10542 reason,
10543 runtime.drain_timeout,
10544 )
10545 .await
10546}
10547
10548async fn begin_forwarding_drain_if_configured(
10549 spec: &ModuleSpec,
10550 runtime: &SupervisorRuntimeConfig,
10551 registry: &Registry,
10552 snapshot: &SharedSnapshot,
10553 enabled: Option<bool>,
10554 reason: RouteCloseReason,
10555) -> Result<StopNotice, SuperviseError> {
10556 begin_forwarding_drain_with_timeout(
10557 spec,
10558 runtime,
10559 registry,
10560 snapshot,
10561 enabled,
10562 reason,
10563 runtime.drain_timeout,
10564 )
10565 .await
10566}
10567
10568async fn begin_forwarding_drain_with_timeout(
10572 spec: &ModuleSpec,
10573 runtime: &SupervisorRuntimeConfig,
10574 registry: &Registry,
10575 snapshot: &SharedSnapshot,
10576 enabled: Option<bool>,
10577 reason: RouteCloseReason,
10578 drain_timeout: Duration,
10579) -> Result<StopNotice, SuperviseError> {
10580 let Some(forwarding) = runtime.forwarding.as_ref() else {
10581 return Ok(StopNotice::NotSent);
10582 };
10583
10584 begin_forwarding_drain_with(
10585 forwarding,
10586 ForwardingDrainContext {
10587 spec,
10588 runtime,
10589 registry,
10590 scope: DrainScope::Active,
10591 },
10592 snapshot,
10593 enabled,
10594 reason,
10595 drain_timeout,
10596 )
10597 .await
10598}
10599
10600async fn begin_forwarding_drain_with(
10601 forwarding: &ForwardingTable,
10602 context: ForwardingDrainContext<'_>,
10603 snapshot: &SharedSnapshot,
10604 enabled: Option<bool>,
10605 reason: RouteCloseReason,
10606 drain_timeout: Duration,
10607) -> Result<StopNotice, SuperviseError> {
10608 let ForwardingDrainContext {
10609 spec,
10610 runtime,
10611 registry,
10612 scope,
10613 } = context;
10614 debug_assert_ne!(reason, RouteCloseReason::Crash);
10615 let terminal = matches!(reason, RouteCloseReason::Disable);
10616 let drain_started_at = Instant::now();
10617 let drain_deadline = drain_started_at + drain_timeout;
10618 let deadline_ms =
10619 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10620 let busy_gauges = match scope {
10621 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10622 DrainScope::Endpoint(endpoint) => {
10623 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10624 }
10625 };
10626
10627 let gate_started = Instant::now();
10630 let drain_target = match scope {
10631 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10632 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10633 }
10634 .map_err(SuperviseError::Forwarding)?;
10635 info!(
10640 module_id = %spec.module_id,
10641 ?reason,
10642 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10643 connected = drain_target.is_some(),
10644 "module drain began; route admission closed"
10645 );
10646 if scope == DrainScope::Active {
10647 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10648 state.state = ModuleState::Draining;
10649 state.draining_to_replace =
10650 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10651 if let Some(enabled) = enabled {
10652 state.enabled = enabled;
10653 }
10654 })?;
10655 }
10656
10657 let Some(target) = drain_target.as_ref() else {
10658 return Ok(StopNotice::NoConnection);
10662 };
10663 {
10664 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10665 #[cfg(feature = "test-support")]
10666 forwarding.wait_before_route_closing_for_test().await;
10667 let routes = forwarding
10668 .endpoint_routes(target.endpoint)
10669 .map_err(SuperviseError::Forwarding)?;
10670 let routes_notified = routes.len();
10671 crate::control::send_route_control_pushes(
10672 forwarding,
10673 routes.clone(),
10674 ClientControlPush::RouteClosing {
10675 module_id: spec.module_id.clone(),
10676 channels: Vec::new(),
10677 reason,
10678 },
10679 );
10680 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10681
10682 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10688 let wait_result = wait_for_forwarding_quiescence(
10689 forwarding,
10690 &spec.module_id,
10691 runtime,
10692 target.endpoint,
10693 drain_deadline,
10694 &busy_gauges,
10695 scope,
10696 &mut gauge_trace,
10697 )
10698 .await;
10699 let drained = drained_after_quiescence_wait(&wait_result);
10700 if let Err(err) = &wait_result {
10701 error!(
10702 module_id = %spec.module_id,
10703 ?reason,
10704 error = %err,
10705 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10706 );
10707 } else if !drained {
10708 let holdouts = forwarding
10710 .endpoint_drain_holdouts(target.endpoint)
10711 .unwrap_or_default();
10712 warn_drain_timed_out(
10713 &spec.module_id,
10714 drain_timeout,
10715 reason,
10716 &holdouts,
10717 &gauge_trace,
10718 );
10719 }
10720 crate::control::send_route_control_pushes(
10721 forwarding,
10722 routes,
10723 ClientControlPush::RouteClosed {
10724 module_id: spec.module_id.clone(),
10725 channels: Vec::new(),
10726 reason,
10727 drained,
10728 abandoned: target.abandoned_bindings.len() as u32,
10729 excluded_subscriptions: target.excluded_subscriptions,
10730 terminal: Some(terminal),
10731 },
10732 );
10733 wait_result?;
10734
10735 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10741 Ok(routes) => routes,
10742 Err(err) => {
10743 warn!(
10744 module_id = %spec.module_id,
10745 ?reason,
10746 error = %err,
10747 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10748 );
10749 send_module_goodbye(&spec.module_id, forwarding, target);
10750 return Err(SuperviseError::Forwarding(err));
10751 }
10752 };
10753 let route_goodbye_count = released_routes.len();
10754 send_route_goodbyes(forwarding, released_routes);
10755 send_module_goodbye(&spec.module_id, forwarding, target);
10756
10757 info!(
10763 module_id = %spec.module_id,
10764 ?reason,
10765 routes_notified,
10766 route_goodbyes = route_goodbye_count,
10767 abandoned_reservations = target.abandoned_bindings.len(),
10768 excluded_subscriptions = target.excluded_subscriptions,
10769 drained,
10770 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10771 );
10772 }
10773
10774 Ok(StopNotice::SentOverConnection)
10775}
10776
10777async fn wait_for_registration_after_reload(
10780 registry: &Registry,
10781 module_id: &str,
10782 snapshot: &SharedSnapshot,
10783 child: &mut SupervisedChild,
10784 wait: Duration,
10785) -> Result<RegistrationWaitOutcome, SuperviseError> {
10786 wait_for_slot_registration(
10787 registry,
10788 crate::registry::RegistrationSlot::Active(module_id),
10789 module_id,
10790 snapshot,
10791 child,
10792 wait,
10793 )
10794 .await
10795}
10796
10797async fn wait_for_slot_registration(
10805 registry: &Registry,
10806 slot: crate::registry::RegistrationSlot<'_>,
10807 module_id: &str,
10808 snapshot: &SharedSnapshot,
10809 child: &mut SupervisedChild,
10810 wait: Duration,
10811) -> Result<RegistrationWaitOutcome, SuperviseError> {
10812 let deadline = Instant::now() + wait;
10813 loop {
10814 if registry
10815 .registration(slot)
10816 .map_err(SuperviseError::Registry)?
10817 .is_some()
10818 {
10819 return Ok(RegistrationWaitOutcome::Registered);
10820 }
10821
10822 let now = Instant::now();
10823 if now >= deadline {
10824 return Ok(RegistrationWaitOutcome::TimedOut);
10825 }
10826 let remaining = deadline.saturating_duration_since(now);
10827 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10828
10829 tokio::select! {
10830 wait_result = child.wait() => {
10831 let status = wait_result.map_err(|source| SuperviseError::Wait {
10832 module_id: module_id.to_string(),
10833 source,
10834 })?;
10835 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10836 snapshot,
10837 child,
10838 &status,
10839 )));
10840 }
10841 _ = sleep(poll) => {}
10842 }
10843 }
10844}
10845
10846fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10847 if exit_report.kind != ExitKind::DeliberateSeverance {
10850 exit_report.kind = ExitKind::Crash;
10851 }
10852 exit_report
10853}
10854
10855async fn handle_reload_child_registration_failure(
10856 spec: &ModuleSpec,
10857 runtime: &SupervisorRuntimeConfig,
10858 registry: &Registry,
10859 process_liveness: &SupervisorProcessLiveness,
10860 snapshot: &SharedSnapshot,
10861 _child: &mut Option<SupervisedChild>,
10862 failure: ReloadRegistrationFailure,
10863) -> Result<(), SuperviseError> {
10864 let ReloadRegistrationFailure {
10865 exit_report,
10866 reason,
10867 } = failure;
10868 match on_child_exit(
10869 spec,
10870 runtime.restart_policy,
10871 registry,
10872 snapshot,
10873 &runtime.terminal_ring,
10874 &runtime.spawn_events,
10875 &runtime.child_roster,
10876 exit_report,
10877 runtime.recovery_is_daemon,
10878 )
10879 .await
10880 {
10881 NextAction::Stop {
10882 registration_released,
10883 } => {
10884 if registration_released {
10885 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10886 }
10887 }
10888 NextAction::Restart { schedule } => {
10889 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10890 schedule.delay
10891 });
10892 if let Some(schedule) = schedule {
10893 log_crash_respawn(&spec.module_id, schedule);
10894 }
10895 schedule_respawn(
10896 runtime,
10897 snapshot,
10898 &spec.module_id,
10899 delay,
10900 RespawnKind::Spawn,
10901 true,
10902 )?;
10903 }
10904 }
10905 Err(SuperviseError::ReloadFailed {
10906 module_id: spec.module_id.clone(),
10907 reason,
10908 })
10909}
10910
10911async fn handle_reload_spawn_failure(
10912 spec: &ModuleSpec,
10913 runtime: &SupervisorRuntimeConfig,
10914 process_liveness: &SupervisorProcessLiveness,
10915 snapshot: &SharedSnapshot,
10916 _child: &mut Option<SupervisedChild>,
10917 reason: String,
10918) -> Result<(), SuperviseError> {
10919 let now = Instant::now();
10920 let mut schedule = None;
10921 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10922 clear_current_process_facts(state);
10923 if state.enabled {
10924 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10925 state.state = if schedule.is_some() {
10926 ModuleState::Restarting
10927 } else {
10928 if runtime
10929 .failure_context(runtime.recovery_is_daemon)
10930 .may_retry()
10931 {
10932 state.failed_retry.schedule(&spec.module_id);
10933 }
10934 ModuleState::Failed
10935 };
10936 } else {
10937 state.state = ModuleState::Disabled;
10938 }
10939 })?;
10940 if let Some(schedule) = schedule {
10941 schedule_respawn(
10942 runtime,
10943 snapshot,
10944 &spec.module_id,
10945 schedule.delay,
10946 RespawnKind::Spawn,
10947 true,
10948 )?;
10949 } else {
10950 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10951 }
10952 Err(SuperviseError::ReloadFailed {
10953 module_id: spec.module_id.clone(),
10954 reason,
10955 })
10956}
10957
10958fn control_flags() -> Flags {
10959 Flags::new(false, Priority::Passive, false)
10960}
10961
10962#[allow(clippy::too_many_arguments)]
10963async fn drain_optional_child(
10964 module_id: &str,
10965 protocol: ModuleProtocol,
10966 stop_notice: StopNotice,
10967 registry: &Registry,
10968 forwarding: Option<&ForwardingTable>,
10969 snapshot: &SharedSnapshot,
10970 terminal_ring: &Arc<Mutex<TerminalRing>>,
10971 spawn_events: &SpawnEventFeed,
10972 child: &mut Option<SupervisedChild>,
10973 drain_timeout: Duration,
10974 final_state: ModuleState,
10975 enabled: Option<bool>,
10976 failure: &FailureContext,
10977) -> Result<(), SuperviseError> {
10978 if let Some(child) = child.take() {
10979 drain_child_to_state(
10980 module_id,
10981 protocol,
10982 stop_notice,
10983 registry,
10984 forwarding,
10985 snapshot,
10986 terminal_ring,
10987 spawn_events,
10988 child,
10989 drain_timeout,
10990 final_state,
10991 enabled,
10992 failure,
10993 )
10994 .await
10995 } else {
10996 update_snapshot(snapshot, Some(module_id), |state| {
10997 state.state = final_state;
10998 if final_state == ModuleState::Failed {
10999 state.failed_retry.failed_at = Some(Instant::now());
11000 }
11001 if let Some(enabled) = enabled {
11002 state.enabled = enabled;
11003 }
11004 clear_current_process_facts(state);
11005 })?;
11006 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11007 }
11008}
11009
11010#[allow(clippy::too_many_arguments)]
11011async fn drain_child_to_state(
11012 module_id: &str,
11013 _protocol: ModuleProtocol,
11014 stop_notice: StopNotice,
11015 registry: &Registry,
11016 forwarding: Option<&ForwardingTable>,
11017 snapshot: &SharedSnapshot,
11018 terminal_ring: &Arc<Mutex<TerminalRing>>,
11019 spawn_events: &SpawnEventFeed,
11020 mut child: SupervisedChild,
11021 drain_timeout: Duration,
11022 final_state: ModuleState,
11023 enabled: Option<bool>,
11024 failure: &FailureContext,
11025) -> Result<(), SuperviseError> {
11026 let protocol = child.protocol;
11027 update_snapshot(snapshot, Some(module_id), |state| {
11028 state.state = ModuleState::Draining;
11029 state.draining_to_replace = final_state == ModuleState::Restarting;
11030 if let Some(enabled) = enabled {
11031 state.enabled = enabled;
11032 }
11033 })?;
11034
11035 if stop_notice != StopNotice::SentOverConnection {
11046 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11047 info!(
11048 module_id,
11049 pid = child.pid,
11050 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11051 "module has no connection yet; requesting stop by signal"
11052 );
11053 }
11054 request_graceful_stop(module_id, &child);
11055 }
11056
11057 let exit_report = match timeout(drain_timeout, child.wait()).await {
11058 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11059 Ok(Err(source)) => {
11060 fail_after_wait_error(
11061 module_id,
11062 snapshot,
11063 terminal_ring,
11064 spawn_events,
11065 format!("failed to wait during drain: {source}"),
11066 failure,
11067 );
11068 return Err(SuperviseError::Wait {
11069 module_id: module_id.to_string(),
11070 source,
11071 });
11072 }
11073 Err(_) => {
11074 warn!(
11087 module_id,
11088 pid = child.pid,
11089 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11090 reason = ?final_state,
11091 ?stop_notice,
11092 "drain budget expired before the module exited; killing it"
11093 );
11094 child.start_kill().map_err(|source| {
11095 fail_after_wait_error(
11096 module_id,
11097 snapshot,
11098 terminal_ring,
11099 spawn_events,
11100 format!("failed to kill during drain: {source}"),
11101 failure,
11102 );
11103 SuperviseError::Kill {
11104 module_id: module_id.to_string(),
11105 source,
11106 }
11107 })?;
11108 let status = child.wait().await.map_err(|source| {
11109 fail_after_wait_error(
11110 module_id,
11111 snapshot,
11112 terminal_ring,
11113 spawn_events,
11114 format!("failed to wait after drain kill: {source}"),
11115 failure,
11116 );
11117 SuperviseError::Wait {
11118 module_id: module_id.to_string(),
11119 source,
11120 }
11121 })?;
11122 classify_reaped_child_exit(snapshot, &child, &status)
11123 }
11124 };
11125
11126 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11127 publish_after_terminal_record(
11128 module_id,
11129 || {
11130 record_terminal_with_detail(
11131 module_id,
11132 terminal_ring,
11133 spawn_events,
11134 &exit_report,
11135 terminal_disposition(final_state),
11136 detail,
11137 )
11138 },
11139 || {
11140 update_snapshot(snapshot, Some(module_id), |state| {
11141 state.state = final_state;
11142 if final_state == ModuleState::Failed {
11143 state.failed_retry.failed_at = Some(Instant::now());
11144 }
11145 if let Some(enabled) = enabled {
11146 state.enabled = enabled;
11147 }
11148 clear_current_process_facts(state);
11149 state.last_exit = Some(exit_report.clone());
11150 if exit_report.kind == ExitKind::DeliberateSeverance {
11151 state.lifetime_restarts += 1;
11152 }
11153 })
11154 },
11155 )?;
11156 child.drain_stderr(module_id).await;
11157
11158 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11159}
11160
11161#[cfg(unix)]
11181fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
11182 let Some(pid) = child
11183 .id()
11184 .and_then(|pid| i32::try_from(pid).ok())
11185 .and_then(rustix::process::Pid::from_raw)
11186 else {
11187 debug!(
11188 module_id,
11189 "no pid to signal for teardown; falling through to the drain wait"
11190 );
11191 return;
11192 };
11193 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
11194 Ok(()) => debug!(
11195 module_id,
11196 "sent SIGTERM to a module nothing else asked to stop"
11197 ),
11198 Err(err) => debug!(
11199 module_id,
11200 error = %err,
11201 "SIGTERM to module failed; the drain wait and kill still apply"
11202 ),
11203 }
11204}
11205
11206#[cfg(not(unix))]
11214fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11215 debug!(
11216 module_id,
11217 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11218 );
11219}
11220
11221fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11222 match final_state {
11223 ModuleState::Stopped => TerminalDisposition::Stopped,
11224 ModuleState::Disabled => TerminalDisposition::Disabled,
11225 ModuleState::Restarting => TerminalDisposition::Restarting,
11226 ModuleState::Failed => TerminalDisposition::Failed,
11227 ModuleState::Starting
11228 | ModuleState::Running
11229 | ModuleState::Unresponsive
11230 | ModuleState::Draining => {
11231 unreachable!("terminal exits only finish in terminal or restarting states")
11232 }
11233 }
11234}
11235
11236async fn release_dead_registration(
11244 registry: &Registry,
11245 forwarding: Option<&ForwardingTable>,
11246 snapshot: &SharedSnapshot,
11247 module_id: &str,
11248 failure: &FailureContext,
11249) -> Result<(), SuperviseError> {
11250 let result = async {
11251 let registration = registry
11252 .get_module(module_id)
11253 .map_err(SuperviseError::Registry)?;
11254 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11255 Ok(()) => return Ok(()),
11256 Err(SuperviseError::RegistrationStillActive { .. }) => {}
11257 Err(err) => return Err(err),
11258 }
11259 let pid = lock_snapshot(snapshot)?.reaped_pid;
11260 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11261 warn!(
11262 module_id,
11263 pid,
11264 connection_id = registration.connection_id.get(),
11265 "reaped module registration outlived release grace; closing dead connection"
11266 );
11267 forwarding.request_connection_close(
11268 registration.connection_id,
11269 CloseReason::new(
11270 "supervised_process_reaped",
11271 format!("module '{module_id}' pid {pid} exited"),
11272 ),
11273 );
11274 wait_for_slot_registration_release(
11275 registry,
11276 crate::registry::RegistrationSlot::Connection(registration.connection_id),
11277 REGISTRY_RELEASE_TIMEOUT,
11278 )
11279 .await?;
11280 }
11281 wait_for_registration_release(registry, module_id, Duration::ZERO).await
11282 }
11283 .await;
11284 if let Err(err) = &result {
11285 fail_snapshot(snapshot, Some(module_id), None, failure);
11286 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11287 }
11288 result
11289}
11290
11291async fn wait_for_registration_release(
11294 registry: &Registry,
11295 module_id: &str,
11296 wait: Duration,
11297) -> Result<(), SuperviseError> {
11298 wait_for_slot_registration_release(
11299 registry,
11300 crate::registry::RegistrationSlot::Active(module_id),
11301 wait,
11302 )
11303 .await
11304}
11305
11306async fn wait_for_slot_registration_release(
11314 registry: &Registry,
11315 slot: crate::registry::RegistrationSlot<'_>,
11316 wait: Duration,
11317) -> Result<(), SuperviseError> {
11318 let deadline = Instant::now() + wait;
11319 let mut release_events = registration_release_events().subscribe();
11320 let still_active = |registration: &crate::registry::ModuleRegistration| {
11321 SuperviseError::RegistrationStillActive {
11322 module_id: registration.manifest.module_id.clone(),
11323 waited: wait,
11324 }
11325 };
11326 loop {
11327 let _observed_generation = *release_events.borrow_and_update();
11328 let Some(registration) = registry
11329 .registration(slot)
11330 .map_err(SuperviseError::Registry)?
11331 else {
11332 return Ok(());
11333 };
11334
11335 let now = Instant::now();
11336 if now >= deadline {
11337 return Err(still_active(®istration));
11338 }
11339
11340 let remaining = deadline.saturating_duration_since(now);
11341 match timeout(remaining, release_events.changed()).await {
11342 Ok(Ok(())) | Ok(Err(_)) => {}
11343 Err(_) => return Err(still_active(®istration)),
11344 }
11345 }
11346}
11347
11348#[cfg(test)]
11349mod slot_registration_wait_tests {
11350 use super::*;
11351 use crate::registry::{ConnectionId, RegistrationSlot};
11352 use subc_protocol::manifest::ModuleManifest;
11353
11354 #[tokio::test]
11355 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11356 let registry = Arc::new(Registry::default());
11357 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
11358 let runtime = supervisor.runtime_config();
11359 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11360 let spec = ModuleSpec {
11361 module_id: "enable-stale-registration".to_string(),
11362 program: PathBuf::from("/missing/enable-retry-test"),
11363 args: Vec::new(),
11364 env: Vec::new(),
11365 reserved: false,
11366 reserved_prefixes: Vec::new(),
11367 protocol: ModuleProtocol::Subc,
11368 overlap: Default::default(),
11369 };
11370 let connection = ConnectionId::new(90);
11371 registry
11372 .register_with_control_ops(
11373 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11374 1,
11375 connection,
11376 Vec::new(),
11377 )
11378 .unwrap();
11379 let mut child = None;
11380 let err = set_child_enabled(
11381 &spec,
11382 &runtime,
11383 ®istry,
11384 &supervisor.process_liveness,
11385 &snapshot,
11386 &mut child,
11387 true,
11388 &runtime.failure_context(false),
11389 )
11390 .await
11391 .unwrap_err();
11392 assert!(matches!(
11393 err,
11394 SuperviseError::RegistrationStillActive { .. }
11395 ));
11396 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11397 assert!(child.is_none());
11398 registry.deregister_connection(connection).unwrap();
11399 let err = set_child_enabled(
11400 &spec,
11401 &runtime,
11402 ®istry,
11403 &supervisor.process_liveness,
11404 &snapshot,
11405 &mut child,
11406 true,
11407 &runtime.failure_context(false),
11408 )
11409 .await
11410 .unwrap_err();
11411 assert!(
11412 matches!(err, SuperviseError::Spawn { .. }),
11413 "second enable must attempt a spawn: {err}"
11414 );
11415 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11416 }
11417
11418 const INCUMBENT: u64 = 1;
11419 const CANDIDATE: u64 = 2;
11420
11421 fn swapped_registry() -> Arc<Registry> {
11422 let registry = Arc::new(Registry::default());
11423 let manifest = ModuleManifest::builder("m", "0.1.0").build();
11424 registry
11425 .register_with_control_ops(
11426 manifest.clone(),
11427 1,
11428 ConnectionId::new(INCUMBENT),
11429 Vec::new(),
11430 )
11431 .unwrap();
11432 registry
11433 .register_candidate_with_control_ops(
11434 manifest,
11435 1,
11436 ConnectionId::new(CANDIDATE),
11437 Vec::new(),
11438 )
11439 .unwrap();
11440 registry
11441 }
11442
11443 #[tokio::test]
11447 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11448 let registry = swapped_registry();
11449 registry.promote_candidate("m").unwrap().unwrap();
11450
11451 assert!(matches!(
11452 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
11453 Err(SuperviseError::RegistrationStillActive { .. })
11454 ));
11455
11456 assert!(matches!(
11458 wait_for_slot_registration_release(
11459 ®istry,
11460 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11461 Duration::from_millis(50),
11462 )
11463 .await,
11464 Err(SuperviseError::RegistrationStillActive { .. })
11465 ));
11466
11467 let releaser = Arc::clone(®istry);
11468 let release = tokio::spawn(async move {
11469 sleep(Duration::from_millis(20)).await;
11470 releaser
11471 .deregister_connection(ConnectionId::new(INCUMBENT))
11472 .unwrap();
11473 notify_registration_release();
11474 });
11475 wait_for_slot_registration_release(
11476 ®istry,
11477 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11478 Duration::from_secs(5),
11479 )
11480 .await
11481 .expect("the incumbent's own registration is released");
11482 release.await.unwrap();
11483 assert!(registry.get_module("m").unwrap().is_some());
11484 }
11485
11486 #[tokio::test]
11489 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11490 let registry = swapped_registry();
11491 assert!(matches!(
11492 wait_for_slot_registration_release(
11493 ®istry,
11494 RegistrationSlot::Candidate("m"),
11495 Duration::from_millis(50),
11496 )
11497 .await,
11498 Err(SuperviseError::RegistrationStillActive { .. })
11499 ));
11500 registry
11501 .deregister_connection(ConnectionId::new(CANDIDATE))
11502 .unwrap();
11503 wait_for_slot_registration_release(
11504 ®istry,
11505 RegistrationSlot::Candidate("m"),
11506 Duration::from_millis(50),
11507 )
11508 .await
11509 .expect("a candidate slot with no candidate is released");
11510 assert!(registry
11511 .registration(RegistrationSlot::Active("m"))
11512 .unwrap()
11513 .is_some());
11514 }
11515}
11516
11517fn classify_exit(status: &ExitStatus) -> ExitReport {
11518 ExitReport {
11519 kind: if status.success() {
11520 ExitKind::Clean
11521 } else {
11522 ExitKind::Crash
11523 },
11524 code: status.code(),
11525 signal: exit_signal(status),
11526 at_ms: unix_ms_now(),
11527 }
11528}
11529
11530fn wait_error_exit_report() -> ExitReport {
11536 ExitReport {
11537 kind: ExitKind::Crash,
11538 code: None,
11539 signal: None,
11540 at_ms: unix_ms_now(),
11541 }
11542}
11543
11544#[cfg(unix)]
11545fn exit_signal(status: &ExitStatus) -> Option<i32> {
11546 use std::os::unix::process::ExitStatusExt;
11547
11548 status.signal()
11549}
11550
11551#[cfg(not(unix))]
11552fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11553 None
11554}
11555
11556fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11562 update_snapshot(snapshot, Some(module_id), |state| {
11563 state.clear_crash_restarts();
11564 })
11565}
11566
11567fn set_running(
11568 snapshot: &SharedSnapshot,
11569 child: &SupervisedChild,
11570 module_id: &str,
11571 spawn_events: &SpawnEventFeed,
11572) -> Result<(), SuperviseError> {
11573 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11574 module_id: Some(module_id.to_string()),
11575 })?;
11576 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11577 if std::mem::take(&mut state.coalesced_restart_pending) {
11578 let generation = state.spawn_generation;
11579 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11580 }
11581 state.drain_disposition_detail = None;
11582 state.spawn_failure = None;
11583 state.waiting_for_previous_pid = None;
11584 state.waiting_for_previous_detail = None;
11585 state.in_alternate_slot = false;
11588 state.configuration_updated_since_spawn = false;
11589 state.spawned_protocol = Some(child.protocol);
11590 state.state = ModuleState::Running;
11591 state.enabled = true;
11592 state.process_alive = true;
11593 state.pid = child.id();
11594 state.health.status = SupervisorHealthStatus::Unknown;
11597 state.health.consecutive_failures = 0;
11598 state.health.detail = None;
11599 state.health.metrics = None;
11600 state.health.last_probe_ms = None;
11601 state.failed_retry.healthy_since = None;
11602 state.failed_retry.failed_at = None;
11603 state.failed_retry.running_since = Some(child.spawned_at);
11604 #[cfg(windows)]
11605 {
11606 state.pipe_consumed = child
11607 .nonce_delivery
11608 .as_ref()
11609 .map(|delivery| delivery.consumed());
11610 }
11611 #[cfg(target_os = "macos")]
11612 {
11613 state.report_ready = Some(Arc::clone(&child.report_ready));
11614 }
11615 state.spawned_at_ms = Some(child.spawned_at_ms);
11616 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11617 state.spawned_from = Some(child.spawned_from.clone());
11618 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11619 state.spawned_file_identity = child.spawned_file_identity.clone();
11620 state.process_start_time = child.process_start_time;
11621 Ok(())
11622}
11623
11624fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11625 state.failed_retry.healthy_since = None;
11626 state.failed_retry.running_since = None;
11627 state.waiting_for_previous_pid = None;
11628 state.waiting_for_previous_detail = None;
11629 state.process_alive = false;
11630 state.spawned_protocol = None;
11631 state.pid = None;
11632 #[cfg(windows)]
11633 {
11634 state.pipe_consumed = None;
11635 }
11636 #[cfg(target_os = "macos")]
11637 {
11638 state.report_ready = None;
11639 }
11640 state.spawned_at_ms = None;
11641 state.exec_outcome = None;
11642 state.spawned_from = None;
11643 state.launch_spec_digest = None;
11644 state.spawned_file_identity = None;
11645 state.process_start_time = None;
11646 state.deliberate_severance = None;
11647}
11648
11649#[cfg(test)]
11650fn record_deliberate_severance(
11651 snapshot: &SharedSnapshot,
11652 identity: ProcessIdentity,
11653) -> Result<(), SuperviseError> {
11654 update_snapshot(snapshot, None, |state| {
11655 state.deliberate_severance = Some(identity);
11656 })
11657}
11658
11659fn apply_deliberate_severance_marker(
11660 snapshot: &SharedSnapshot,
11661 exited_identity: Option<ProcessIdentity>,
11662 mut exit_report: ExitReport,
11663) -> ExitReport {
11664 let marker = lock_snapshot(snapshot)
11665 .ok()
11666 .and_then(|mut state| state.deliberate_severance.take());
11667 if marker.is_some() && marker == exited_identity {
11668 exit_report.kind = ExitKind::DeliberateSeverance;
11669 }
11670 exit_report
11671}
11672
11673fn classify_reaped_child_exit(
11674 snapshot: &SharedSnapshot,
11675 child: &SupervisedChild,
11676 status: &ExitStatus,
11677) -> ExitReport {
11678 let _ = update_snapshot(snapshot, None, |state| {
11679 state.reaped_pid = Some(child.pid);
11680 state.spawn_failure = child.spawn_failure.clone();
11681 });
11682 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11683}
11684
11685fn fail_snapshot(
11686 snapshot: &SharedSnapshot,
11687 module_id: Option<&str>,
11688 last_exit: Option<ExitReport>,
11689 failure: &FailureContext,
11690) {
11691 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11692 state.state = ModuleState::Failed;
11693 clear_current_process_facts(state);
11694 if failure.may_retry() && state.enabled {
11695 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
11696 } else if failure.shutdown.is_set() {
11697 state.failed_retry.cancel();
11698 }
11699 if let Some(last_exit) = last_exit {
11700 state.last_exit = Some(last_exit);
11701 }
11702 }) {
11703 error!(error = %err, "failed to mark supervisor state failed");
11704 }
11705}
11706
11707fn update_snapshot(
11708 snapshot: &SharedSnapshot,
11709 module_id: Option<&str>,
11710 update: impl FnOnce(&mut SupervisorSnapshot),
11711) -> Result<(), SuperviseError> {
11712 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11713 module_id: module_id.map(ToOwned::to_owned),
11714 })?;
11715 update(&mut state);
11716 Ok(())
11717}
11718
11719const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11720
11721fn lock_snapshot_for_control<'a>(
11722 snapshot: &'a SharedSnapshot,
11723 module_id: &str,
11724 caller: &'static str,
11725) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11726 let started_at = Instant::now();
11727 let guard = lock_snapshot(snapshot)?;
11728 let waited = started_at.elapsed();
11729 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11730 warn!(
11731 module_id = %module_id,
11732 waited_ms = waited.as_millis() as u64,
11733 caller = %caller,
11734 "slow snapshot lock"
11735 );
11736 }
11737 Ok(guard)
11738}
11739
11740fn lock_snapshot(
11741 snapshot: &SharedSnapshot,
11742) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11743 snapshot
11744 .lock()
11745 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11746}
11747
11748#[cfg(test)]
11749mod terminal_history_tests {
11750 use std::{
11751 path::PathBuf,
11752 sync::Arc,
11753 time::{Duration, Instant},
11754 };
11755
11756 use tokio::{sync::mpsc, time::sleep};
11757
11758 use super::{
11759 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11760 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11761 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11762 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11763 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11764 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11765 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11766 SupervisorHealthStatus, SupervisorSnapshot,
11767 };
11768 use super::Instant as ClockInstant;
11773 use crate::{
11774 registry::Registry,
11775 terminal_ring::{TerminalRing, TerminalRingConfig},
11776 };
11777 use std::sync::Mutex;
11778 use subc_control::TerminalDisposition;
11779
11780 pub(super) fn fake_aft_stub_path() -> PathBuf {
11785 let mut path = std::env::current_exe().expect("current_exe available in tests");
11786 path.pop();
11787 path.pop();
11788 path.push(if cfg!(windows) {
11789 "fake-aft-stub.exe"
11790 } else {
11791 "fake-aft-stub"
11792 });
11793 assert!(
11794 path.exists(),
11795 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11796 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11797 path.display()
11798 );
11799 path
11800 }
11801
11802 #[test]
11803 fn reserved_never_spawned_refuses_every_hello() {
11804 let supervisor = SupervisorHandle::default();
11809 supervisor.apply_identity_configuration(&ModuleSpec {
11810 module_id: "never-spawned".to_string(),
11811 program: PathBuf::from("/usr/bin/false"),
11812 args: Vec::new(),
11813 env: Vec::new(),
11814 reserved: true,
11815 reserved_prefixes: Vec::new(),
11816 protocol: ModuleProtocol::Subc,
11817 overlap: Default::default(),
11818 });
11819 assert!(
11820 supervisor
11821 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11822 .is_err(),
11823 "forged nonce must refuse on a reserved never-spawned id"
11824 );
11825 assert!(
11826 supervisor
11827 .reserved_hello_verification("never-spawned", None)
11828 .is_err(),
11829 "absent nonce must refuse on a reserved never-spawned id"
11830 );
11831 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11833 supervisor.apply_identity_configuration(&ModuleSpec {
11834 module_id: "never-spawned".to_string(),
11835 program: PathBuf::from("/usr/bin/false"),
11836 args: Vec::new(),
11837 env: Vec::new(),
11838 reserved: true,
11839 reserved_prefixes: Vec::new(),
11840 protocol: ModuleProtocol::Subc,
11841 overlap: Default::default(),
11842 });
11843 assert!(supervisor
11844 .reserved_hello_verification("never-spawned", Some("minted"))
11845 .is_ok());
11846 assert!(supervisor
11847 .reserved_hello_verification("never-spawned", Some("forged"))
11848 .is_err());
11849 }
11850
11851 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11854 let now = ClockInstant::now();
11855 for _ in 0..count {
11856 state.crash_restarts.push_back(now);
11857 }
11858 }
11859
11860 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11864 let aged = state
11865 .crash_restarts
11866 .front()
11867 .expect("a crash restart must be recorded before it can be aged")
11868 .checked_sub(window + Duration::from_secs(1))
11869 .expect("the test clock is far enough from its origin to age an instant");
11870 state.crash_restarts[0] = aged;
11871 }
11872
11873 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11874 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11875 seed_crash_restarts(&mut state, count);
11876 state
11877 }
11878
11879 #[test]
11880 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11881 let policy = RestartPolicy::new(3, Duration::ZERO);
11882 let now = ClockInstant::now();
11883 assert!(daemon_will_restart(
11884 &mut snapshot_with_restarts(true, 2),
11885 &policy,
11886 now
11887 ));
11888 assert!(!daemon_will_restart(
11889 &mut snapshot_with_restarts(true, 3),
11890 &policy,
11891 now
11892 ));
11893 assert!(!daemon_will_restart(
11894 &mut snapshot_with_restarts(false, 0),
11895 &policy,
11896 now
11897 ));
11898 }
11899
11900 #[test]
11901 fn crash_restart_backoff_escalates_with_in_window_count() {
11902 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11903 .with_max_backoff(Duration::from_secs(30));
11904 let now = ClockInstant::now();
11905 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11906 let schedules = (0..4)
11907 .map(|_| {
11908 state
11909 .next_crash_restart(&policy, now)
11910 .expect("the test policy allows four crash restarts")
11911 })
11912 .collect::<Vec<_>>();
11913
11914 assert_eq!(
11915 schedules
11916 .iter()
11917 .map(|schedule| schedule.restart_in_window)
11918 .collect::<Vec<_>>(),
11919 vec![0, 1, 2, 3]
11920 );
11921 assert_eq!(
11922 schedules
11923 .iter()
11924 .map(|schedule| schedule.delay)
11925 .collect::<Vec<_>>(),
11926 vec![
11927 Duration::from_millis(100),
11928 Duration::from_secs(1),
11929 Duration::from_secs(10),
11930 Duration::from_secs(30),
11931 ]
11932 );
11933 }
11934
11935 #[test]
11936 fn crash_restart_backoff_resets_after_ring_clear() {
11937 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11938 let now = ClockInstant::now();
11939 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11940 assert_eq!(
11941 state.next_crash_restart(&policy, now).unwrap().delay,
11942 Duration::from_millis(100)
11943 );
11944 assert_eq!(
11945 state.next_crash_restart(&policy, now).unwrap().delay,
11946 Duration::from_secs(1)
11947 );
11948
11949 state.clear_crash_restarts();
11950 let schedule = state
11951 .next_crash_restart(&policy, now)
11952 .expect("a cleared ring must allow another restart");
11953 assert_eq!(schedule.restart_in_window, 0);
11954 assert_eq!(schedule.delay, Duration::from_millis(100));
11955 }
11956
11957 #[test]
11958 fn crash_restart_backoff_ignores_aged_restarts() {
11959 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11960 let now = ClockInstant::now();
11961 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11962 state
11963 .next_crash_restart(&policy, now)
11964 .expect("the first restart is allowed");
11965 state
11966 .next_crash_restart(&policy, now)
11967 .expect("the second restart is allowed");
11968 state.crash_restarts[0] = now
11969 .checked_sub(policy.window + Duration::from_secs(1))
11970 .expect("the fake clock can age a restart past the window");
11971
11972 let schedule = state
11973 .next_crash_restart(&policy, now)
11974 .expect("an aged restart must release its slot");
11975 assert_eq!(schedule.restart_in_window, 1);
11976 assert_eq!(schedule.delay, Duration::from_secs(1));
11977 assert_eq!(state.crash_restarts.len(), 2);
11978 }
11979
11980 #[test]
11984 fn a_budget_spent_before_the_window_no_longer_refuses() {
11985 let policy = RestartPolicy::new(3, Duration::ZERO);
11986 let mut state = snapshot_with_restarts(true, 3);
11987 let now = ClockInstant::now();
11988 assert!(!daemon_will_restart(&mut state, &policy, now));
11989
11990 assert!(daemon_will_restart(
11991 &mut state,
11992 &policy,
11993 now + policy.window + Duration::from_secs(1)
11994 ));
11995 assert!(
11996 state.crash_restarts.is_empty(),
11997 "reading the budget must drop the instants that left the window"
11998 );
11999 }
12000
12001 fn module_with_recovery_snapshot(
12002 state: ModuleState,
12003 enabled: bool,
12004 restart_count: u32,
12005 ) -> SupervisedModule {
12006 let registry = Arc::new(Registry::default());
12007 let supervisor =
12008 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
12009 let runtime = supervisor.runtime_config();
12010 let spec = ModuleSpec {
12011 module_id: "recovery-snapshot".to_string(),
12012 program: fake_aft_stub_path(),
12013 args: Vec::new(),
12014 env: Vec::new(),
12015 reserved: false,
12016 reserved_prefixes: Vec::new(),
12017 protocol: ModuleProtocol::Subc,
12018 overlap: Default::default(),
12019 };
12020 let mut snapshot = SupervisorSnapshot::new(state, enabled);
12021 seed_crash_restarts(&mut snapshot, restart_count);
12022 let (commands, _rx) = mpsc::channel(4);
12025 SupervisedModule {
12026 inner: Arc::new(SupervisedModuleInner {
12027 module_id: spec.module_id.clone(),
12028 registry,
12029 snapshot: Arc::new(Mutex::new(snapshot)),
12030 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12031 spec,
12032 health: runtime.health,
12033 })),
12034 stderr_ring: runtime.stderr_ring,
12035 terminal_ring: runtime.terminal_ring,
12036 commands,
12037 monitor: Mutex::new(None),
12038 restart_policy: runtime.restart_policy,
12039 effective_drain_timeout: runtime.effective_drain_timeout,
12040 provenance_probe: supervisor.provenance_probe.clone(),
12041 }),
12042 }
12043 }
12044
12045 #[cfg(target_os = "linux")]
12046 #[tokio::test]
12047 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12048 let supervisor =
12049 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12050 .with_cgroup_placement(None);
12051 let result = supervisor.spawn(ModuleSpec {
12052 module_id: "no-cgroup-placement".to_string(),
12053 program: fake_aft_stub_path(),
12054 args: Vec::new(),
12055 env: Vec::new(),
12056 reserved: false,
12057 reserved_prefixes: Vec::new(),
12058 protocol: ModuleProtocol::Subc,
12059 overlap: Default::default(),
12060 });
12061
12062 assert!(
12063 result.is_ok(),
12064 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12065 );
12066 }
12067
12068 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12069 async fn undecided_snapshot_uses_shared_restart_predicate() {
12070 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12071 .will_recover_after_connection_loss()
12072 .unwrap());
12073 assert!(
12074 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12075 .will_recover_after_connection_loss()
12076 .unwrap()
12077 );
12078 }
12079
12080 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12081 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12082 assert!(
12083 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12084 .will_recover_after_connection_loss()
12085 .unwrap()
12086 );
12087 }
12088
12089 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12090 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12091 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12092 .will_recover_after_connection_loss()
12093 .unwrap());
12094 assert!(
12095 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12096 .will_recover_after_connection_loss()
12097 .unwrap()
12098 );
12099 }
12100
12101 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12102 async fn warming_snapshot_is_limited_to_startup_phases() {
12103 for state in [
12104 ModuleState::Starting,
12105 ModuleState::Running,
12106 ModuleState::Restarting,
12107 ] {
12108 assert!(
12109 module_with_recovery_snapshot(state, true, 0)
12110 .is_warming()
12111 .unwrap(),
12112 "{state:?} should be warming"
12113 );
12114 }
12115 for state in [
12116 ModuleState::Unresponsive,
12117 ModuleState::Draining,
12118 ModuleState::Stopped,
12119 ModuleState::Failed,
12120 ModuleState::Disabled,
12121 ] {
12122 assert!(
12123 !module_with_recovery_snapshot(state, true, 0)
12124 .is_warming()
12125 .unwrap(),
12126 "{state:?} should not be warming"
12127 );
12128 }
12129 }
12130
12131 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12132 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12133 let registry = Arc::new(Registry::default());
12134 let supervisor =
12135 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
12136 let module = supervisor
12137 .spawn(ModuleSpec {
12138 module_id: "terminal-history".to_string(),
12139 program: fake_aft_stub_path(),
12140 args: Vec::new(),
12141 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12142 reserved: false,
12143 reserved_prefixes: Vec::new(),
12144 protocol: ModuleProtocol::Subc,
12145 overlap: Default::default(),
12146 })
12147 .unwrap();
12148
12149 let deadline = Instant::now() + Duration::from_secs(5);
12150 loop {
12151 let history = module.terminal_history();
12152 if history.entries.len() == 2 {
12153 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12154 assert_eq!(history.dropped, 0);
12155 assert_eq!(
12156 history
12157 .entries
12158 .iter()
12159 .map(|entry| entry.exit_code)
12160 .collect::<Vec<_>>(),
12161 vec![Some(23), Some(23)]
12162 );
12163 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
12164 return;
12165 }
12166 assert!(
12167 Instant::now() < deadline,
12168 "module did not retain two terminal exits: {history:?}"
12169 );
12170 sleep(Duration::from_millis(10)).await;
12171 }
12172 }
12173
12174 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12178 async fn disable_during_crash_backoff_cancels_pending_respawn() {
12179 let backoff = Duration::from_secs(2);
12180 let supervisor = Supervisor::new_for_test(
12181 Arc::new(Registry::default()),
12182 RestartPolicy::new(10, backoff),
12183 );
12184 let module = supervisor
12185 .spawn(ModuleSpec {
12186 module_id: "disable-during-backoff".to_string(),
12187 program: fake_aft_stub_path(),
12188 args: Vec::new(),
12189 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12190 reserved: false,
12191 reserved_prefixes: Vec::new(),
12192 protocol: ModuleProtocol::Subc,
12193 overlap: Default::default(),
12194 })
12195 .unwrap();
12196
12197 let deadline = Instant::now() + Duration::from_secs(5);
12199 loop {
12200 if module.status().unwrap().state == ModuleState::Restarting {
12201 break;
12202 }
12203 assert!(
12204 Instant::now() < deadline,
12205 "module never entered the crash backoff"
12206 );
12207 sleep(Duration::from_millis(10)).await;
12208 }
12209
12210 let started = Instant::now();
12211 module.set_enabled(false).await.unwrap();
12212 let waited = started.elapsed();
12213
12214 assert!(
12215 waited < backoff / 2,
12216 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12217 );
12218 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12219
12220 sleep(backoff + Duration::from_millis(500)).await;
12222 let status = module.status().unwrap();
12223 assert_eq!(status.state, ModuleState::Disabled);
12224 assert_eq!(
12225 status.spawn_generation, 1,
12226 "module respawned after the operator disabled it"
12227 );
12228 }
12229
12230 #[cfg(unix)]
12233 fn protocol_none_sigterm_exits_clean_spec(
12234 module_id: &str,
12235 dir: &std::path::Path,
12236 ) -> (ModuleSpec, PathBuf, PathBuf) {
12237 let ready = dir.join("ready");
12238 let marker = dir.join("sigterm");
12239 let spec = ModuleSpec {
12240 module_id: module_id.to_string(),
12241 program: fake_aft_stub_path(),
12242 args: Vec::new(),
12243 env: vec![
12244 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12245 (
12246 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12247 marker.display().to_string(),
12248 ),
12249 (
12250 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12251 ready.display().to_string(),
12252 ),
12253 ],
12254 reserved: false,
12255 reserved_prefixes: Vec::new(),
12256 protocol: ModuleProtocol::None,
12257 overlap: Default::default(),
12258 };
12259 (spec, ready, marker)
12260 }
12261
12262 #[cfg(unix)]
12266 async fn wait_for_file(path: &std::path::Path) {
12267 let deadline = Instant::now() + Duration::from_secs(10);
12268 while !path.exists() {
12269 assert!(
12270 Instant::now() < deadline,
12271 "{} never appeared",
12272 path.display()
12273 );
12274 sleep(Duration::from_millis(10)).await;
12275 }
12276 }
12277
12278 #[cfg(unix)]
12282 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12283 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12284 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12285 let (spec, ready, marker) =
12286 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12287 let supervisor = Supervisor::new_for_test(
12288 Arc::new(Registry::default()),
12289 RestartPolicy::new(3, Duration::ZERO),
12290 );
12291 let module = supervisor.spawn(spec).unwrap();
12292 wait_for_file(&ready).await;
12293 let deadline = Instant::now() + Duration::from_secs(10);
12297 let first_pid = loop {
12298 let status = module.status().unwrap();
12299 if status.state == ModuleState::Running {
12300 if let Some(pid) = status.pid {
12301 break pid;
12302 }
12303 }
12304 assert!(
12305 Instant::now() < deadline,
12306 "a running module must report its pid after exec confirmation: {status:?}"
12307 );
12308 sleep(Duration::from_millis(10)).await;
12309 };
12310
12311 rustix::process::kill_process(
12312 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12313 rustix::process::Signal::TERM,
12314 )
12315 .unwrap();
12316
12317 let deadline = Instant::now() + Duration::from_secs(10);
12318 let respawned = loop {
12319 let status = module.status().unwrap();
12320 if status.state == ModuleState::Running
12321 && status.pid.is_some_and(|pid| pid != first_pid)
12322 {
12323 break status;
12324 }
12325 assert!(
12326 Instant::now() < deadline,
12327 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12328 );
12329 sleep(Duration::from_millis(10)).await;
12330 };
12331 assert_eq!(respawned.spawn_generation, 2);
12332 assert!(
12333 marker.exists(),
12334 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12335 );
12336
12337 let history = module.terminal_history();
12338 assert_eq!(history.entries.len(), 1, "{history:?}");
12339 let entry = &history.entries[0];
12340 assert_eq!(entry.exit_code, Some(0));
12341 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12342 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12343
12344 module.stop().await.unwrap();
12345 }
12346
12347 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12351 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12352 let supervisor = Supervisor::new_for_test(
12353 Arc::new(Registry::default()),
12354 RestartPolicy::new(1, Duration::ZERO),
12355 );
12356 let module = supervisor
12357 .spawn(ModuleSpec {
12358 module_id: "none-clean-exit-budget".to_string(),
12359 program: fake_aft_stub_path(),
12360 args: Vec::new(),
12361 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12362 reserved: false,
12363 reserved_prefixes: Vec::new(),
12364 protocol: ModuleProtocol::None,
12365 overlap: Default::default(),
12366 })
12367 .unwrap();
12368
12369 let deadline = Instant::now() + Duration::from_secs(10);
12372 loop {
12373 let status = module.status().unwrap();
12374 if status.state == ModuleState::Failed {
12375 break;
12376 }
12377 assert!(
12378 Instant::now() < deadline,
12379 "module never exhausted its budget: {status:?} {:?}",
12380 module.terminal_history()
12381 );
12382 sleep(Duration::from_millis(10)).await;
12383 }
12384 let history = module.terminal_history();
12385 assert_eq!(
12386 history
12387 .entries
12388 .iter()
12389 .map(|entry| (entry.exit_code, entry.disposition.clone()))
12390 .collect::<Vec<_>>(),
12391 vec![
12392 (Some(0), TerminalDisposition::Restarting),
12393 (Some(0), TerminalDisposition::Failed),
12394 ]
12395 );
12396 let detail = history.entries[1]
12397 .disposition_detail
12398 .as_deref()
12399 .expect("a budget failure names the budget");
12400 assert!(detail.contains("max_restarts=1"), "{detail}");
12401 assert_eq!(module.status().unwrap().spawn_generation, 2);
12402 }
12403
12404 #[test]
12405 fn restart_budget_failure_is_published_after_its_terminal_record() {
12406 use std::io::{Read, Write};
12407 let supervisor = Supervisor::new_for_test(
12408 Arc::new(Registry::default()),
12409 RestartPolicy::new(0, Duration::ZERO),
12410 );
12411 let runtime = supervisor.runtime_config();
12412 let spec = ModuleSpec {
12413 protocol: ModuleProtocol::None,
12414 ..windowed_crash_spec("budget-publication-order")
12415 };
12416 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12417 ModuleState::Running,
12418 true,
12419 )));
12420 lock_snapshot(&snapshot).unwrap().process_alive = true;
12421 let reaped_snapshot = snapshot.clone();
12422 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12423 let history = runtime.terminal_ring.clone();
12424 let reap = std::thread::spawn(move || {
12425 tokio::runtime::Builder::new_current_thread()
12426 .enable_all()
12427 .build()
12428 .unwrap()
12429 .block_on(on_child_exit(
12430 &spec,
12431 runtime.restart_policy,
12432 &supervisor.registry,
12433 &reaped_snapshot,
12434 &runtime.terminal_ring,
12435 &runtime.spawn_events,
12436 &runtime.child_roster,
12437 ExitReport {
12438 kind: ExitKind::Clean,
12439 code: Some(0),
12440 signal: None,
12441 at_ms: 1,
12442 },
12443 true,
12444 ))
12445 });
12446 let mut reached = [0];
12447 barrier.read_exact(&mut reached).unwrap();
12448 assert_eq!(&reached, b"R");
12449 let (before_state, before_alive) = {
12450 let state = lock_snapshot(&snapshot).unwrap();
12451 (state.state, state.process_alive)
12452 };
12453 let before_history = history.lock().unwrap().snapshot();
12454 barrier.write_all(b"X").unwrap();
12455 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12456 assert!(before_history.entries.is_empty());
12457 assert_ne!(
12458 before_state,
12459 ModuleState::Failed,
12460 "Failed was visible before its terminal record could be written"
12461 );
12462 assert!(before_alive);
12463 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12464 let history = history.lock().unwrap().snapshot();
12465 assert_eq!(history.entries.len(), 1);
12466 assert_eq!(history.entries[0].exit_code, Some(0));
12467 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12468 }
12469
12470 #[test]
12471 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12472 use std::io::{Read, Write};
12473
12474 let supervisor = Supervisor::new_for_test(
12475 Arc::new(Registry::default()),
12476 RestartPolicy::new(3, Duration::ZERO),
12477 );
12478 let mut runtime = supervisor.runtime_config();
12479 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12480 let snapshot = module.inner.snapshot.clone();
12481 lock_snapshot(&snapshot).unwrap().process_alive = true;
12482 runtime.terminal_ring = module.inner.terminal_ring.clone();
12483 let spec = windowed_crash_spec(module.module_id());
12484 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12485 let reaped_snapshot = snapshot.clone();
12486 let reap = std::thread::spawn(move || {
12487 tokio::runtime::Builder::new_current_thread()
12488 .enable_all()
12489 .build()
12490 .unwrap()
12491 .block_on(on_child_exit(
12492 &spec,
12493 runtime.restart_policy,
12494 &supervisor.registry,
12495 &reaped_snapshot,
12496 &runtime.terminal_ring,
12497 &runtime.spawn_events,
12498 &runtime.child_roster,
12499 ExitReport {
12500 kind: ExitKind::Crash,
12501 code: Some(2),
12502 signal: None,
12503 at_ms: 1,
12504 },
12505 true,
12506 ))
12507 });
12508
12509 let mut reached = [0];
12513 barrier.read_exact(&mut reached).unwrap();
12514 assert_eq!(&reached, b"R");
12515 let before_status = module.status().unwrap();
12516 let before_history = module.terminal_history();
12517 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12518 barrier.write_all(b"X").unwrap();
12519 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12520
12521 assert_eq!(before_status.state, ModuleState::Running);
12522 assert!(before_status.process_alive);
12523 assert_eq!(before_status.restart_count, 3);
12524 assert!(before_history.entries.is_empty());
12525 assert!(
12526 before_recovers,
12527 "connection cleanup would report a terminal close for a scheduled crash restart"
12528 );
12529 let restarting = module.status().unwrap();
12530 assert_eq!(restarting.state, ModuleState::Restarting);
12531 assert_eq!(restarting.restart_count, 3);
12532 assert!(module.will_recover_after_connection_loss().unwrap());
12533 let history = module.terminal_history();
12534 assert_eq!(history.entries.len(), 1);
12535 assert_eq!(
12536 history.entries[0].disposition,
12537 TerminalDisposition::Restarting
12538 );
12539
12540 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12543 state.state = ModuleState::Running;
12544 state.respawn_pending = false;
12545 })
12546 .unwrap();
12547 assert!(!module.will_recover_after_connection_loss().unwrap());
12548 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12549 state.enabled = false;
12550 state.respawn_pending = true;
12551 })
12552 .unwrap();
12553 assert!(!module.will_recover_after_connection_loss().unwrap());
12554 }
12555
12556 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12557 use std::net::{TcpListener, TcpStream};
12558 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12559 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12560 let (barrier, _) = listener.accept().unwrap();
12561 for stream in [&writer, &barrier] {
12562 stream
12563 .set_read_timeout(Some(Duration::from_secs(10)))
12564 .unwrap();
12565 stream
12566 .set_write_timeout(Some(Duration::from_secs(10)))
12567 .unwrap();
12568 }
12569 *events.1.lock().unwrap() = Some(writer);
12570 barrier
12571 }
12572
12573 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12574 use std::io::{Read, Write};
12575 let barrier = events.1.lock().unwrap().take();
12576 if let Some(mut barrier) = barrier {
12577 barrier.write_all(b"R").unwrap();
12578 let mut release = [0];
12579 barrier.read_exact(&mut release).unwrap();
12580 assert_eq!(&release, b"X");
12581 }
12582 }
12583
12584 #[test]
12585 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12586 use std::io::{Read, Write};
12587
12588 let supervisor = Supervisor::new_for_test(
12589 Arc::new(Registry::default()),
12590 RestartPolicy::new(0, Duration::ZERO),
12591 );
12592 let mut runtime = supervisor.runtime_config();
12593 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12594 let snapshot = module.inner.snapshot.clone();
12595 runtime.terminal_ring = module.inner.terminal_ring.clone();
12596 let spec = ModuleSpec {
12597 module_id: module.module_id().into(),
12598 program: fake_aft_stub_path(),
12599 protocol: ModuleProtocol::None,
12600 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12601 ..windowed_crash_spec(module.module_id())
12602 };
12603 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12604 let reap = std::thread::spawn(move || {
12605 tokio::runtime::Builder::new_current_thread()
12606 .enable_all()
12607 .build()
12608 .unwrap()
12609 .block_on(async {
12610 let mut child =
12611 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12612 health_restart_child(
12613 &spec,
12614 &runtime,
12615 &supervisor.registry,
12616 &supervisor.process_liveness,
12617 &snapshot,
12618 &mut child,
12619 SupervisorHealthStatus::Failing,
12620 Some("health probe did not answer"),
12621 1,
12622 )
12623 .await
12624 .unwrap();
12625 assert!(child.is_none());
12626 });
12627 });
12628 let mut reached = [0];
12633 barrier.read_exact(&mut reached).unwrap();
12634 assert_eq!(&reached, b"R");
12635 let before_status = module.status().unwrap();
12636 let before_history = module.terminal_history();
12637 barrier.write_all(b"X").unwrap();
12638 reap.join().unwrap();
12639 assert!(before_history.entries.is_empty());
12640 assert_ne!(
12641 before_status.state,
12642 ModuleState::Failed,
12643 "Failed was visible before its terminal record could be written"
12644 );
12645 assert!(
12646 before_status.process_alive,
12647 "process death was visible before its terminal record could be written"
12648 );
12649 let failed = module.status().unwrap();
12650 assert_eq!(failed.state, ModuleState::Failed);
12651 assert!(!failed.process_alive);
12652 let history = module.terminal_history();
12653 assert_eq!(history.entries.len(), 1);
12654 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12655 assert_eq!(
12656 history.entries[0].disposition_detail.as_deref(),
12657 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12658 );
12659 }
12660
12661 #[test]
12662 fn observed_exits_publish_process_death_after_terminal_history() {
12663 use std::io::{Read, Write};
12664 for (kind, enabled, shutdown, wait_error, expected_state) in [
12665 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12666 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12667 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12668 (
12669 ExitKind::DeliberateSeverance,
12670 true,
12671 false,
12672 false,
12673 ModuleState::Restarting,
12674 ),
12675 #[cfg(unix)]
12676 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12677 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12678 ] {
12679 let supervisor = Supervisor::new_for_test(
12680 Arc::new(Registry::default()),
12681 RestartPolicy::new(1, Duration::ZERO),
12682 );
12683 let runtime = supervisor.runtime_config();
12684 #[cfg(unix)]
12685 if shutdown {
12686 runtime.child_roster.close();
12687 }
12688 #[cfg(not(unix))]
12689 let _ = shutdown;
12690 let spec = windowed_crash_spec("exit-publication-order");
12691 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12692 ModuleState::Running,
12693 enabled,
12694 )));
12695 lock_snapshot(&snapshot).unwrap().process_alive = true;
12696 let reaped_snapshot = snapshot.clone();
12697 let history = runtime.terminal_ring.clone();
12698 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12699 let reap = std::thread::spawn(move || {
12700 if wait_error {
12701 super::fail_after_wait_error(
12702 &spec.module_id,
12703 &reaped_snapshot,
12704 &runtime.terminal_ring,
12705 &runtime.spawn_events,
12706 "wait failed".into(),
12707 &runtime.failure_context(true),
12708 );
12709 } else {
12710 tokio::runtime::Builder::new_current_thread()
12711 .enable_all()
12712 .build()
12713 .unwrap()
12714 .block_on(on_child_exit(
12715 &spec,
12716 runtime.restart_policy,
12717 &supervisor.registry,
12718 &reaped_snapshot,
12719 &runtime.terminal_ring,
12720 &runtime.spawn_events,
12721 &runtime.child_roster,
12722 ExitReport {
12723 kind,
12724 code: Some(0),
12725 signal: None,
12726 at_ms: 1,
12727 },
12728 true,
12729 ));
12730 }
12731 });
12732 let mut reached = [0];
12733 barrier.read_exact(&mut reached).unwrap();
12734 assert_eq!(&reached, b"R");
12735 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12736 let before_history = history.lock().unwrap().snapshot();
12737 barrier.write_all(b"X").unwrap();
12738 reap.join().unwrap();
12739 assert!(
12740 before_alive,
12741 "{kind:?}: death was published before recording"
12742 );
12743 assert!(before_history.entries.is_empty());
12744 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12745 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12746 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12747 }
12748 }
12749
12750 #[tokio::test]
12751 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12752 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12753 let path = dir.join("terminals.jsonl");
12754 std::fs::create_dir(&path).unwrap();
12755 let supervisor = Supervisor::new_for_test(
12756 Arc::new(Registry::default()),
12757 RestartPolicy::new(0, Duration::ZERO),
12758 )
12759 .with_terminal_journal(path, "health-budget-journal-failure".into());
12760 let runtime = supervisor.runtime_config();
12761 let spec = ModuleSpec {
12762 program: fake_aft_stub_path(),
12763 protocol: ModuleProtocol::None,
12764 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12765 ..windowed_crash_spec("health-budget-journal-failure")
12766 };
12767 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12768 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12769 health_restart_child(
12770 &spec,
12771 &runtime,
12772 &supervisor.registry,
12773 &supervisor.process_liveness,
12774 &snapshot,
12775 &mut child,
12776 SupervisorHealthStatus::Failing,
12777 None,
12778 1,
12779 )
12780 .await
12781 .unwrap();
12782 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12783 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12784 let history = runtime
12785 .terminal_ring
12786 .lock()
12787 .unwrap()
12788 .durable_history(&spec.module_id);
12789 assert!(history.journal_write_failures > 0);
12790 assert_eq!(history.entries.len(), 1);
12791 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12792 }
12793
12794 #[test]
12795 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12796 let supervisor = Supervisor::new_for_test(
12797 Arc::new(Registry::default()),
12798 RestartPolicy::new(0, Duration::ZERO),
12799 );
12800 let runtime = supervisor.runtime_config();
12801 let spec = ModuleSpec {
12802 program: fake_aft_stub_path(),
12803 protocol: ModuleProtocol::None,
12804 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12805 ..windowed_crash_spec("health-budget-recording-panic")
12806 };
12807 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12808 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12809 tokio::runtime::Builder::new_current_thread()
12810 .enable_all()
12811 .build()
12812 .unwrap()
12813 .block_on(async {
12814 let mut child =
12815 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12816 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12817 health_restart_child(
12818 &spec,
12819 &runtime,
12820 &supervisor.registry,
12821 &supervisor.process_liveness,
12822 &snapshot,
12823 &mut child,
12824 SupervisorHealthStatus::Failing,
12825 None,
12826 1,
12827 )
12828 .await
12829 .unwrap();
12830 });
12831 }));
12832 let panic = result.expect_err("recording must still unwind");
12833 assert_eq!(
12834 panic.downcast_ref::<String>().map(String::as_str),
12835 Some("spawn event sequence exhausted")
12836 );
12837 let state = lock_snapshot(&snapshot).unwrap();
12838 assert_eq!(state.state, ModuleState::Failed);
12839 assert!(!state.process_alive);
12840 assert!(state.last_exit.is_some());
12841 }
12842
12843 #[tokio::test]
12844 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12845 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12846 let path = dir.join("terminals.jsonl");
12847 std::fs::create_dir(&path).unwrap();
12848 let supervisor = Supervisor::new_for_test(
12849 Arc::new(Registry::default()),
12850 RestartPolicy::new(0, Duration::ZERO),
12851 )
12852 .with_terminal_journal(path, "budget-journal-failure".into());
12853 let runtime = supervisor.runtime_config();
12854 let spec = windowed_crash_spec("budget-journal-failure");
12855 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12856 assert!(matches!(
12857 on_child_exit(
12858 &spec,
12859 runtime.restart_policy,
12860 &supervisor.registry,
12861 &snapshot,
12862 &runtime.terminal_ring,
12863 &runtime.spawn_events,
12864 &runtime.child_roster,
12865 crash_exit_report(1),
12866 true,
12867 )
12868 .await,
12869 NextAction::Stop { .. }
12870 ));
12871 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12872 let history = runtime
12873 .terminal_ring
12874 .lock()
12875 .unwrap()
12876 .durable_history(&spec.module_id);
12877 assert!(history.journal_write_failures > 0);
12878 assert_eq!(history.entries.len(), 1);
12879 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12880 }
12881
12882 #[test]
12883 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12884 let supervisor = Supervisor::new_for_test(
12885 Arc::new(Registry::default()),
12886 RestartPolicy::new(0, Duration::ZERO),
12887 );
12888 let runtime = supervisor.runtime_config();
12889 let spec = windowed_crash_spec("budget-recording-panic");
12890 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12891 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12892 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12895 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12896 tokio::runtime::Builder::new_current_thread()
12897 .enable_all()
12898 .build()
12899 .unwrap()
12900 .block_on(on_child_exit(
12901 &spec,
12902 runtime.restart_policy,
12903 &supervisor.registry,
12904 &snapshot,
12905 &runtime.terminal_ring,
12906 &runtime.spawn_events,
12907 &runtime.child_roster,
12908 crash_exit_report(1),
12909 true,
12910 ))
12911 }));
12912 let panic = result.err().expect("recording must still unwind");
12913 assert_eq!(
12914 panic.downcast_ref::<String>().map(String::as_str),
12915 Some("spawn event sequence exhausted")
12916 );
12917 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12918 }
12919
12920 #[cfg(unix)]
12923 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12924 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12925 for disable in [false, true] {
12926 let label = if disable {
12927 "none-requested-disable"
12928 } else {
12929 "none-requested-stop"
12930 };
12931 let dir = cortexkit_test_support::ScratchDir::new(label);
12932 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12933 let supervisor = Supervisor::new_for_test(
12934 Arc::new(Registry::default()),
12935 RestartPolicy::new(3, Duration::ZERO),
12936 );
12937 let module = supervisor.spawn(spec).unwrap();
12938 wait_for_file(&ready).await;
12939
12940 if disable {
12941 module.set_enabled(false).await.unwrap();
12942 } else {
12943 module.stop().await.unwrap();
12944 }
12945 assert!(
12946 marker.exists(),
12947 "{label}: the child must have left through its SIGTERM handler with exit 0"
12948 );
12949
12950 sleep(Duration::from_millis(500)).await;
12953 let status = module.status().unwrap();
12954 let expected = if disable {
12955 ModuleState::Disabled
12956 } else {
12957 ModuleState::Stopped
12958 };
12959 assert_eq!(status.state, expected, "{label}");
12960 assert_eq!(
12961 status.spawn_generation, 1,
12962 "{label}: respawned after a requested stop"
12963 );
12964 let history = module.terminal_history();
12965 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12966 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12967 assert_ne!(
12968 history.entries[0].disposition,
12969 TerminalDisposition::Restarting,
12970 "{label}"
12971 );
12972 }
12973 }
12974
12975 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12978 async fn subc_wire_clean_exit_is_still_a_stop() {
12979 let supervisor = Supervisor::new_for_test(
12980 Arc::new(Registry::default()),
12981 RestartPolicy::new(3, Duration::ZERO),
12982 );
12983 let module = supervisor
12984 .spawn(ModuleSpec {
12985 module_id: "wire-clean-exit".to_string(),
12986 program: fake_aft_stub_path(),
12987 args: Vec::new(),
12988 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12989 reserved: false,
12990 reserved_prefixes: Vec::new(),
12991 protocol: ModuleProtocol::Subc,
12992 overlap: Default::default(),
12993 })
12994 .unwrap();
12995
12996 let deadline = Instant::now() + Duration::from_secs(10);
12997 while module.terminal_history().entries.is_empty() {
12998 assert!(Instant::now() < deadline, "module never exited");
12999 sleep(Duration::from_millis(10)).await;
13000 }
13001 sleep(Duration::from_millis(500)).await;
13003 let status = module.status().unwrap();
13004 assert_eq!(status.state, ModuleState::Stopped);
13005 assert_eq!(status.spawn_generation, 1);
13006 let history = module.terminal_history();
13007 assert_eq!(history.entries.len(), 1, "{history:?}");
13008 assert_eq!(history.entries[0].exit_code, Some(0));
13009 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13010 }
13011
13012 #[cfg(unix)]
13013 #[tokio::test]
13014 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13015 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13016 let record = dir.join("live-children.json");
13017 let supervisor = Supervisor::new_for_test(
13018 Arc::new(Registry::default()),
13019 RestartPolicy::new(0, Duration::ZERO),
13020 );
13021 let mut runtime = supervisor.runtime_config();
13022 runtime.child_roster.record_to(record.clone());
13023 let gate = Arc::new(super::ReloadExitRecordGate::default());
13024 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13025 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13026 let spec = ModuleSpec {
13027 module_id: "reload-exit-roster".into(),
13028 program: fake_aft_stub_path(),
13029 args: Vec::new(),
13030 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13031 reserved: false,
13032 reserved_prefixes: Vec::new(),
13033 protocol: ModuleProtocol::Subc,
13034 overlap: Default::default(),
13035 };
13036 let mut child = None;
13037 let reload = super::finish_reload_child(
13038 &spec,
13039 &runtime,
13040 &supervisor.registry,
13041 &supervisor.process_liveness,
13042 &snapshot,
13043 &mut child,
13044 );
13045 tokio::pin!(reload);
13046 tokio::select! {
13047 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13048 _ = gate.reached.notified() => {}
13049 }
13050 assert!(runtime
13051 .terminal_ring
13052 .lock()
13053 .unwrap()
13054 .snapshot()
13055 .entries
13056 .is_empty());
13057 assert_eq!(
13058 crate::live_children::read_record(&record).unwrap().len(),
13059 1,
13060 "shutdown must still wait for the reaped child until its terminal record exists"
13061 );
13062 runtime.child_roster.close();
13063 gate.resume.notify_one();
13064 assert!(reload.await.is_err());
13065 assert!(crate::live_children::read_record(&record)
13066 .unwrap()
13067 .is_empty());
13068 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13069 assert_eq!(history.entries.len(), 1);
13070 assert_eq!(
13071 history.entries[0].disposition,
13072 TerminalDisposition::DaemonShutdown
13073 );
13074 }
13075
13076 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13080 async fn every_restart_increment_path_advances_lifetime_count() {
13081 let supervisor = Supervisor::new_for_test(
13082 Arc::new(Registry::default()),
13083 RestartPolicy::new(1, Duration::ZERO),
13084 );
13085 let runtime = supervisor.runtime_config();
13086 let spec = ModuleSpec {
13087 module_id: "lifetime-increment-path".to_string(),
13088 program: PathBuf::from("/unused/lifetime-increment-path"),
13089 args: Vec::new(),
13090 env: Vec::new(),
13091 reserved: false,
13092 reserved_prefixes: Vec::new(),
13093 protocol: ModuleProtocol::Subc,
13094 overlap: Default::default(),
13095 };
13096
13097 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13098 assert!(matches!(
13099 on_child_exit(
13100 &spec,
13101 runtime.restart_policy,
13102 &supervisor.registry,
13103 &crash_snapshot,
13104 &runtime.terminal_ring,
13105 &runtime.spawn_events,
13106 &runtime.child_roster,
13107 ExitReport {
13108 kind: ExitKind::Crash,
13109 code: Some(1),
13110 signal: None,
13111 at_ms: 1,
13112 },
13113 true,
13114 )
13115 .await,
13116 NextAction::Restart { schedule: _ }
13117 ));
13118 let (crash_restarts, crash_lifetime) = {
13119 let state = lock_snapshot(&crash_snapshot).unwrap();
13120 (state.crash_restarts.len(), state.lifetime_restarts)
13121 };
13122 assert_eq!(crash_restarts, 1);
13123 assert_eq!(crash_lifetime, 1);
13124
13125 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13126 let mut health_child = None;
13127 assert!(matches!(
13128 health_restart_child(
13129 &spec,
13130 &runtime,
13131 &supervisor.registry,
13132 &supervisor.process_liveness,
13133 &health_snapshot,
13134 &mut health_child,
13135 SupervisorHealthStatus::Failing,
13136 None,
13137 2,
13138 )
13139 .await,
13140 Ok(())
13141 ));
13142 assert!(health_child.is_none());
13143 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13144 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13145 let (health_restarts, health_lifetime) = {
13146 let state = lock_snapshot(&health_snapshot).unwrap();
13147 (state.crash_restarts.len(), state.lifetime_restarts)
13148 };
13149 assert_eq!(health_restarts, 1);
13150 assert_eq!(health_lifetime, 1);
13151
13152 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13153 let mut reload_child = None;
13154 assert!(matches!(
13155 handle_reload_spawn_failure(
13156 &spec,
13157 &runtime,
13158 &supervisor.process_liveness,
13159 &reload_snapshot,
13160 &mut reload_child,
13161 "forced reload spawn failure".to_string(),
13162 )
13163 .await,
13164 Err(SuperviseError::ReloadFailed { .. })
13165 ));
13166 let (reload_restarts, reload_lifetime) = {
13167 let state = lock_snapshot(&reload_snapshot).unwrap();
13168 (state.crash_restarts.len(), state.lifetime_restarts)
13169 };
13170 assert_eq!(reload_restarts, 1);
13171 assert_eq!(reload_lifetime, 1);
13172 }
13173
13174 #[tokio::test]
13175 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
13176 let supervisor = Supervisor::new_for_test(
13177 Arc::new(Registry::default()),
13178 RestartPolicy::new(3, Duration::ZERO),
13179 );
13180 let runtime = supervisor.runtime_config();
13181 let spec = ModuleSpec {
13182 module_id: "deliberately-severed".to_string(),
13183 program: PathBuf::from("/unused/deliberately-severed"),
13184 args: Vec::new(),
13185 env: Vec::new(),
13186 reserved: false,
13187 reserved_prefixes: Vec::new(),
13188 protocol: ModuleProtocol::Subc,
13189 overlap: Default::default(),
13190 };
13191 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13192 let process = ProcessIdentity {
13193 pid: 41,
13194 start_time: 101,
13195 };
13196 record_deliberate_severance(&snapshot, process).unwrap();
13197 let exit_report = apply_deliberate_severance_marker(
13198 &snapshot,
13199 Some(process),
13200 ExitReport {
13201 kind: ExitKind::Crash,
13202 code: Some(1),
13203 signal: None,
13204 at_ms: 1,
13205 },
13206 );
13207 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
13208
13209 assert!(matches!(
13210 on_child_exit(
13211 &spec,
13212 runtime.restart_policy,
13213 &supervisor.registry,
13214 &snapshot,
13215 &runtime.terminal_ring,
13216 &runtime.spawn_events,
13217 &runtime.child_roster,
13218 exit_report,
13219 true,
13220 )
13221 .await,
13222 NextAction::Restart { schedule: _ }
13223 ));
13224 let state = lock_snapshot(&snapshot).unwrap();
13225 assert_eq!(state.lifetime_restarts, 1);
13226 assert_eq!(state.crash_restarts.len(), 0);
13227 }
13228
13229 #[tokio::test]
13230 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13231 let supervisor = Supervisor::new_for_test(
13232 Arc::new(Registry::default()),
13233 RestartPolicy::new(3, Duration::ZERO),
13234 );
13235 let runtime = supervisor.runtime_config();
13236 let spec = ModuleSpec {
13237 module_id: "genuine-crash".to_string(),
13238 program: PathBuf::from("/unused/genuine-crash"),
13239 args: Vec::new(),
13240 env: Vec::new(),
13241 reserved: false,
13242 reserved_prefixes: Vec::new(),
13243 protocol: ModuleProtocol::Subc,
13244 overlap: Default::default(),
13245 };
13246 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13247
13248 assert!(matches!(
13249 on_child_exit(
13250 &spec,
13251 runtime.restart_policy,
13252 &supervisor.registry,
13253 &snapshot,
13254 &runtime.terminal_ring,
13255 &runtime.spawn_events,
13256 &runtime.child_roster,
13257 ExitReport {
13258 kind: ExitKind::Crash,
13259 code: Some(1),
13260 signal: None,
13261 at_ms: 1,
13262 },
13263 true,
13264 )
13265 .await,
13266 NextAction::Restart { schedule: _ }
13267 ));
13268 let state = lock_snapshot(&snapshot).unwrap();
13269 assert_eq!(state.lifetime_restarts, 1);
13270 assert_eq!(state.crash_restarts.len(), 1);
13271 }
13272
13273 fn crash_exit_report(at_ms: u64) -> ExitReport {
13274 ExitReport {
13275 kind: ExitKind::Crash,
13276 code: Some(1),
13277 signal: None,
13278 at_ms,
13279 }
13280 }
13281
13282 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13283 ModuleSpec {
13284 module_id: module_id.to_string(),
13285 program: PathBuf::from("/unused").join(module_id),
13286 args: Vec::new(),
13287 env: Vec::new(),
13288 reserved: false,
13289 reserved_prefixes: Vec::new(),
13290 protocol: ModuleProtocol::Subc,
13291 overlap: Default::default(),
13292 }
13293 }
13294
13295 #[tokio::test]
13301 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13302 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13303 let supervisor = Supervisor::new_for_test(
13304 Arc::new(Registry::default()),
13305 RestartPolicy::new(2, Duration::ZERO),
13306 );
13307 let runtime = supervisor.runtime_config();
13308 let spec = windowed_crash_spec("crash-loop-in-window");
13309 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13310
13311 for attempt in 1..=2 {
13312 assert!(
13313 matches!(
13314 on_child_exit(
13315 &spec,
13316 runtime.restart_policy,
13317 &supervisor.registry,
13318 &snapshot,
13319 &runtime.terminal_ring,
13320 &runtime.spawn_events,
13321 &runtime.child_roster,
13322 crash_exit_report(attempt),
13323 true,
13324 )
13325 .await,
13326 NextAction::Restart { schedule: _ }
13327 ),
13328 "crash {attempt} is inside the budget and must respawn"
13329 );
13330 }
13331
13332 assert!(matches!(
13333 on_child_exit(
13334 &spec,
13335 runtime.restart_policy,
13336 &supervisor.registry,
13337 &snapshot,
13338 &runtime.terminal_ring,
13339 &runtime.spawn_events,
13340 &runtime.child_roster,
13341 crash_exit_report(3),
13342 true,
13343 )
13344 .await,
13345 NextAction::Stop { .. }
13346 ));
13347
13348 {
13349 let state = lock_snapshot(&snapshot).unwrap();
13350 assert_eq!(state.state, ModuleState::Failed);
13351 assert_eq!(state.crash_restarts.len(), 2);
13352 assert_eq!(state.lifetime_restarts, 2);
13353 }
13354
13355 let history = runtime
13356 .terminal_ring
13357 .lock()
13358 .expect("terminal ring is not poisoned")
13359 .snapshot();
13360 let last = history
13361 .entries
13362 .last()
13363 .expect("the refused crash is retained");
13364 assert_eq!(last.disposition, TerminalDisposition::Failed);
13365 assert_eq!(
13366 last.disposition_detail.as_deref(),
13367 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13368 );
13369
13370 let captured = crate::router::test_log::captured_logs(&logs);
13371 assert!(
13372 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13373 "the stop must be logged with its window: {captured}"
13374 );
13375 }
13376
13377 #[tokio::test]
13385 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13386 let supervisor = Supervisor::new_for_test(
13387 Arc::new(Registry::default()),
13388 RestartPolicy::new(2, Duration::ZERO),
13389 );
13390 let runtime = supervisor.runtime_config();
13391 let spec = windowed_crash_spec("crash-across-windows");
13392 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13393
13394 for attempt in 1..=2 {
13395 assert!(matches!(
13396 on_child_exit(
13397 &spec,
13398 runtime.restart_policy,
13399 &supervisor.registry,
13400 &snapshot,
13401 &runtime.terminal_ring,
13402 &runtime.spawn_events,
13403 &runtime.child_roster,
13404 crash_exit_report(attempt),
13405 true,
13406 )
13407 .await,
13408 NextAction::Restart { schedule: _ }
13409 ));
13410 }
13411
13412 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13415 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13416 })
13417 .unwrap();
13418
13419 assert!(
13420 matches!(
13421 on_child_exit(
13422 &spec,
13423 runtime.restart_policy,
13424 &supervisor.registry,
13425 &snapshot,
13426 &runtime.terminal_ring,
13427 &runtime.spawn_events,
13428 &runtime.child_roster,
13429 crash_exit_report(3),
13430 true,
13431 )
13432 .await,
13433 NextAction::Restart { schedule: _ }
13434 ),
13435 "a crash older than the window must not hold a budget slot"
13436 );
13437
13438 let state = lock_snapshot(&snapshot).unwrap();
13439 assert_eq!(state.state, ModuleState::Restarting);
13440 assert_eq!(
13441 state.crash_restarts.len(),
13442 2,
13443 "the aged instant is dropped and the new one takes its place"
13444 );
13445 assert_eq!(
13446 state.lifetime_restarts, 3,
13447 "the ledger counts every restart, including the ones the window forgot"
13448 );
13449 }
13450
13451 #[tokio::test]
13456 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13457 let supervisor = Supervisor::new_for_test(
13458 Arc::new(Registry::default()),
13459 RestartPolicy::new(2, Duration::ZERO),
13460 );
13461 let runtime = supervisor.runtime_config();
13462 let spec = windowed_crash_spec("operator-cleared-budget");
13463 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13464
13465 for attempt in 1..=2 {
13466 assert!(matches!(
13467 on_child_exit(
13468 &spec,
13469 runtime.restart_policy,
13470 &supervisor.registry,
13471 &snapshot,
13472 &runtime.terminal_ring,
13473 &runtime.spawn_events,
13474 &runtime.child_roster,
13475 crash_exit_report(attempt),
13476 true,
13477 )
13478 .await,
13479 NextAction::Restart { schedule: _ }
13480 ));
13481 }
13482
13483 reset_restart_count(&snapshot, &spec.module_id).unwrap();
13484 {
13485 let state = lock_snapshot(&snapshot).unwrap();
13486 assert!(
13487 state.crash_restarts.is_empty(),
13488 "an operator restart returns the full budget"
13489 );
13490 assert_eq!(
13491 state.lifetime_restarts, 2,
13492 "clearing the budget must not unmake the crashes"
13493 );
13494 }
13495
13496 assert!(
13497 matches!(
13498 on_child_exit(
13499 &spec,
13500 runtime.restart_policy,
13501 &supervisor.registry,
13502 &snapshot,
13503 &runtime.terminal_ring,
13504 &runtime.spawn_events,
13505 &runtime.child_roster,
13506 crash_exit_report(3),
13507 true,
13508 )
13509 .await,
13510 NextAction::Restart { schedule: _ }
13511 ),
13512 "the cleared budget must be spendable again"
13513 );
13514 let state = lock_snapshot(&snapshot).unwrap();
13515 assert_eq!(state.crash_restarts.len(), 1);
13516 assert_eq!(state.lifetime_restarts, 3);
13517 }
13518
13519 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13520 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13521 let severed = ProcessIdentity {
13522 pid: 41,
13523 start_time: 101,
13524 };
13525 let successor = ProcessIdentity {
13526 pid: 41,
13527 start_time: 202,
13528 };
13529 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13530 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13531 state.pid = Some(successor.pid);
13532 state.process_start_time = Some(successor.start_time);
13533 })
13534 .unwrap();
13535 assert!(!module.record_deliberate_severance(severed).unwrap());
13536
13537 let exit_report = apply_deliberate_severance_marker(
13538 &module.inner.snapshot,
13539 Some(successor),
13540 ExitReport {
13541 kind: ExitKind::Crash,
13542 code: Some(1),
13543 signal: None,
13544 at_ms: 1,
13545 },
13546 );
13547
13548 assert_eq!(exit_report.kind, ExitKind::Crash);
13549 }
13550
13551 #[tokio::test]
13552 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13553 let registry = Registry::default();
13554 let supervisor = Supervisor::new_for_test(
13555 Arc::new(Registry::default()),
13556 RestartPolicy::new(3, Duration::ZERO),
13557 );
13558 let runtime = supervisor.runtime_config();
13559 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13560 let spec = ModuleSpec {
13561 module_id: "drain-deliberate-severance".to_string(),
13562 program: fake_aft_stub_path(),
13563 args: Vec::new(),
13564 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13565 reserved: false,
13566 reserved_prefixes: Vec::new(),
13567 protocol: ModuleProtocol::Subc,
13568 overlap: Default::default(),
13569 };
13570 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13571 let process = ProcessIdentity {
13572 pid: 41,
13573 start_time: 101,
13574 };
13575 child.process_identity = Some(process);
13576 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13577 state.pid = Some(process.pid);
13578 state.process_start_time = Some(process.start_time);
13579 })
13580 .unwrap();
13581 record_deliberate_severance(&snapshot, process).unwrap();
13582
13583 drain_child_to_state(
13584 &spec.module_id,
13585 spec.protocol,
13586 StopNotice::SentOverConnection,
13589 ®istry,
13590 None,
13591 &snapshot,
13592 &runtime.terminal_ring,
13593 &runtime.spawn_events,
13594 child,
13595 Duration::from_secs(1),
13596 ModuleState::Stopped,
13597 Some(false),
13598 &super::FailureContext::operator(),
13599 )
13600 .await
13601 .unwrap();
13602
13603 let state = lock_snapshot(&snapshot).unwrap();
13604 assert_eq!(
13605 state.last_exit.as_ref().map(|exit| exit.kind),
13606 Some(ExitKind::DeliberateSeverance)
13607 );
13608 assert_eq!(state.lifetime_restarts, 1);
13609 assert_eq!(state.crash_restarts.len(), 0);
13610 drop(state);
13611 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13612 assert_eq!(
13613 history.entries[0].exit_kind,
13614 subc_control::TerminalExitKind::DeliberateSeverance
13615 );
13616 }
13617
13618 #[tokio::test]
13619 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13620 let registry = Registry::default();
13621 let supervisor = Supervisor::new_for_test(
13622 Arc::new(Registry::default()),
13623 RestartPolicy::new(3, Duration::ZERO),
13624 );
13625 let runtime = supervisor.runtime_config();
13626 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13627 let spec = ModuleSpec {
13628 module_id: "ordinary-drain".to_string(),
13629 program: fake_aft_stub_path(),
13630 args: Vec::new(),
13631 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13632 reserved: false,
13633 reserved_prefixes: Vec::new(),
13634 protocol: ModuleProtocol::Subc,
13635 overlap: Default::default(),
13636 };
13637 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13638
13639 drain_child_to_state(
13640 &spec.module_id,
13641 spec.protocol,
13642 StopNotice::SentOverConnection,
13645 ®istry,
13646 None,
13647 &snapshot,
13648 &runtime.terminal_ring,
13649 &runtime.spawn_events,
13650 child,
13651 Duration::from_secs(1),
13652 ModuleState::Stopped,
13653 Some(false),
13654 &super::FailureContext::operator(),
13655 )
13656 .await
13657 .unwrap();
13658
13659 let state = lock_snapshot(&snapshot).unwrap();
13660 assert_eq!(
13661 state.last_exit.as_ref().map(|exit| exit.kind),
13662 Some(ExitKind::Crash)
13663 );
13664 assert_eq!(state.lifetime_restarts, 0);
13665 assert_eq!(state.crash_restarts.len(), 0);
13666 }
13667
13668 #[test]
13669 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13670 assert!(!include_str!("server.rs")
13676 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13677 }
13678
13679 #[test]
13686 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13687 assert!(drained_after_quiescence_wait(&Ok(true)));
13688 assert!(!drained_after_quiescence_wait(&Ok(false)));
13689 assert!(!drained_after_quiescence_wait(&Err(
13690 SuperviseError::StatePoisoned { module_id: None }
13691 )));
13692 }
13693
13694 #[test]
13703 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13704 let ring = Arc::new(Mutex::new(TerminalRing::new(
13705 TerminalRingConfig::default(),
13706 0,
13707 )));
13708 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13709
13710 let snapshot = ring.lock().unwrap().snapshot();
13711 assert_eq!(snapshot.entries.len(), 1);
13712 let entry = &snapshot.entries[0];
13713 assert_eq!(entry.exit_code, None);
13714 assert_eq!(entry.exit_signal, None);
13715 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13716 }
13717
13718 #[test]
13719 fn wait_error_exit_path_preserves_spawn_event_density() {
13720 let feed = super::SpawnEventFeed::default();
13721 feed.configure_incarnation("wait-error-density".to_string());
13722 feed.emit_spawned("wait-error", 41, 1);
13723 let ring = Arc::new(Mutex::new(TerminalRing::new(
13724 TerminalRingConfig::default(),
13725 0,
13726 )));
13727
13728 record_wait_error_terminal("wait-error", &ring, &feed);
13729 feed.emit_spawned("after-wait-error", 42, 2);
13730
13731 let state = feed.0.lock().unwrap();
13732 let sequences = state
13733 .events
13734 .iter()
13735 .map(|event| event.cursor.seq)
13736 .collect::<Vec<_>>();
13737 assert_eq!(sequences, vec![1, 2, 3]);
13738 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13739 assert_eq!(state.events[1].exit_code, None);
13740 assert_eq!(state.events[1].exit_signal, None);
13741 }
13742
13743 #[test]
13747 fn wait_error_exit_report_is_classified_as_a_crash() {
13748 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13749 }
13750}
13751
13752#[cfg(all(test, unix))]
13753mod restart_ordering_tests {
13754 use super::*;
13755 use std::io::Write;
13756 use tokio::io::AsyncReadExt;
13757
13758 #[test]
13759 #[ignore = "child process fixture"]
13760 fn serving_child() {
13761 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13762 return;
13763 };
13764 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13765 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13766 for stream in listener.incoming() {
13767 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13768 }
13769 }
13770
13771 async fn restart_before_publication(requests: usize) {
13772 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13773 let address_file = home.path().join("address");
13774 let spec = ModuleSpec {
13775 module_id: "restart-serving-child".into(),
13776 program: std::env::current_exe().unwrap(),
13777 args: vec![
13778 "--exact".into(),
13779 "supervise::restart_ordering_tests::serving_child".into(),
13780 "--ignored".into(),
13781 "--nocapture".into(),
13782 ],
13783 env: vec![(
13784 "RESTART_TEST_ADDRESS_FILE".into(),
13785 address_file.display().to_string(),
13786 )],
13787 reserved: false,
13788 reserved_prefixes: vec![],
13789 protocol: ModuleProtocol::None,
13790 overlap: Default::default(),
13791 };
13792 let mut supervisor =
13793 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13794 let (spawned_tx, spawned_rx) = oneshot::channel();
13795 let spawned_tx = Mutex::new(Some(spawned_tx));
13796 let (release_tx, release_rx) = std::sync::mpsc::channel();
13797 let release_rx = Mutex::new(release_rx);
13798 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13801 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13802 release_rx
13803 .lock()
13804 .unwrap()
13805 .recv_timeout(Duration::from_secs(20))
13806 .unwrap();
13807 })));
13808 let registered = supervisor
13809 .register_configured_with_health(
13810 spec.clone(),
13811 true,
13812 HealthConfig::default(),
13813 None,
13814 RestartPolicy::new(3, Duration::ZERO),
13815 )
13816 .unwrap();
13817 let module = registered.module.clone();
13818 let startup = tokio::task::spawn_blocking(move || {
13819 supervisor.start_registered_configured(registered, None, true)
13820 });
13821 spawned_rx.await.unwrap();
13822 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13823 let address = loop {
13824 if let Ok(address) = std::fs::read_to_string(&address_file) {
13825 if !address.is_empty() {
13826 break address;
13827 }
13828 }
13829 tokio::time::sleep(Duration::from_millis(1)).await;
13830 };
13831 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13832 let mut reply = String::new();
13833 stream.read_to_string(&mut reply).await.unwrap();
13834 reply.trim().parse::<u32>().unwrap()
13835 })
13836 .await
13837 .expect("child must serve before generation publication");
13838 assert_eq!(
13839 lock_snapshot(&module.inner.snapshot)
13840 .unwrap()
13841 .spawn_generation,
13842 0
13843 );
13844 let mut replies = Vec::new();
13845 for _ in 0..requests {
13846 let mut restart = Box::pin(module.restart(Some(0)));
13847 std::future::poll_fn(|cx| {
13850 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13851 std::task::Poll::Ready(())
13852 })
13853 .await;
13854 replies.push(restart);
13855 }
13856 release_tx.send(()).unwrap();
13857 startup.await.unwrap().unwrap();
13858 for reply in replies {
13859 reply.await.unwrap();
13860 }
13861 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13864 loop {
13865 if lock_snapshot(&module.inner.snapshot)
13866 .unwrap()
13867 .spawn_generation
13868 > 1
13869 {
13870 break;
13871 }
13872 tokio::time::sleep(Duration::from_millis(1)).await;
13873 }
13874 })
13875 .await;
13876 module.update_spec_for_test(spec).await.unwrap();
13880 let (generation, pid) = {
13881 let state = lock_snapshot(&module.inner.snapshot).unwrap();
13882 (state.spawn_generation, state.pid)
13883 };
13884 module.stop().await.unwrap();
13885 replacement.expect("restart must publish a replacement for the already-serving child");
13886 assert_ne!(
13887 pid,
13888 Some(old_pid),
13889 "a serving child predating the restart must be replaced"
13890 );
13891 assert_eq!(generation, 2, "queued restarts must share one replacement");
13892 }
13893
13894 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13895 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13896 restart_before_publication(1).await;
13897 }
13898
13899 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13900 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13901 restart_before_publication(2).await;
13902 }
13903
13904 #[tokio::test]
13905 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13906 let supervisor =
13907 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13908 let registered = supervisor
13909 .register_configured_with_health(
13910 ModuleSpec {
13911 module_id: "queued-before-replacement".into(),
13912 program: "/bin/sleep".into(),
13917 args: vec!["600".into()],
13918 env: vec![],
13919 reserved: false,
13920 reserved_prefixes: vec![],
13921 protocol: ModuleProtocol::None,
13922 overlap: Default::default(),
13923 },
13924 true,
13925 HealthConfig::default(),
13926 None,
13927 RestartPolicy::default(),
13928 )
13929 .unwrap();
13930 let RegisteredConfiguredModule {
13931 mut spec,
13932 mut runtime,
13933 module,
13934 mut commands,
13935 } = registered;
13936 let snapshot = &module.inner.snapshot;
13937 let mut restart = Box::pin(module.restart(Some(0)));
13938 std::future::poll_fn(|cx| {
13939 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13940 std::task::Poll::Ready(())
13941 })
13942 .await;
13943 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13946 let replacement_pid = child.as_ref().unwrap().id();
13947 let command = commands.recv().await.unwrap();
13948 assert!(
13949 handle_supervisor_command(
13950 command,
13951 &mut spec,
13952 &mut runtime,
13953 &supervisor.registry,
13954 &supervisor.process_liveness,
13955 snapshot,
13956 &mut child,
13957 &mut commands,
13958 &mut VecDeque::new(),
13959 )
13960 .await
13961 );
13962 restart.await.unwrap();
13963 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13964 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13965 if let Some(mut child) = child {
13966 child.start_kill().unwrap();
13967 child.wait().await.unwrap();
13968 }
13969 assert_eq!(
13970 kept_pid, replacement_pid,
13971 "a newer replacement must survive"
13972 );
13973 assert!(
13974 !pending,
13975 "a satisfied restart must not schedule another spawn"
13976 );
13977 }
13978}
13979
13980#[cfg(all(test, unix))]
13981mod health_event_tests {
13982 use super::*;
13983 use cortexkit_test_support::ScratchDir;
13984 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13985
13986 struct Actor {
13987 module: SupervisedModule,
13988 registry: Arc<Registry>,
13989 forwarding: Arc<ForwardingTable>,
13990 spec: ModuleSpec,
13991 health: HealthConfig,
13992 rx: mpsc::Receiver<crate::router::OutboundFrame>,
13993 _home: ScratchDir,
13994 }
13995
13996 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13997 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14001 let tick = Instant::now();
14002 loop {
14003 if let Some(value) = observe() {
14004 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14005 return value;
14006 }
14007 assert!(
14008 std::time::Instant::now() < deadline,
14009 "timed out waiting for {reason}"
14010 );
14011 tokio::task::yield_now().await;
14012 }
14013 }
14014
14015 impl Actor {
14016 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14017 let home = ScratchDir::new("health-events");
14018 let registry = Arc::new(Registry::default());
14019 let forwarding = Arc::new(ForwardingTable::default());
14020 let health = HealthConfig {
14021 cadence,
14022 deadline: Duration::from_secs(3600),
14023 ..HealthConfig::default()
14024 };
14025 let supervisor =
14026 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
14027 .with_forwarding(forwarding.clone())
14028 .with_health_config(health.clone());
14029 let spec = ModuleSpec {
14030 module_id: "event-health-child".into(),
14031 program: PathBuf::from("/bin/sleep"),
14039 args: vec!["600".into()],
14042 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14043 .into_iter()
14044 .map(|key| (key.into(), home.path().display().to_string()))
14045 .collect(),
14046 reserved: false,
14047 reserved_prefixes: vec![],
14048 protocol,
14049 overlap: Default::default(),
14050 };
14051 let module = supervisor.spawn(spec.clone()).unwrap();
14052 let (_, rx) = mpsc::channel(8);
14053 let actor = Self {
14054 module,
14055 registry,
14056 forwarding,
14057 spec,
14058 health,
14059 rx,
14060 _home: home,
14061 };
14062 actor
14063 .wait_for_select(|checkpoint| checkpoint.generation > 0)
14064 .await;
14065 actor
14066 }
14067
14068 fn turns(&self) -> u64 {
14069 lock_snapshot(&self.module.inner.snapshot)
14070 .unwrap()
14071 .actor_turns
14072 }
14073
14074 async fn wait_for_select(
14075 &self,
14076 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14077 ) -> ActorSelectCheckpoint {
14078 wait_for("actor select checkpoint", || {
14079 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14080 state.actor_select.filter(|checkpoint| {
14081 checkpoint.generation == state.spawn_generation
14082 && state.state == ModuleState::Running
14083 && state.process_alive
14084 && state.reported_pid().is_some()
14085 && expected(checkpoint)
14086 })
14087 })
14088 .await
14089 }
14090
14091 fn hello(&mut self, connection: u64, health: bool) {
14092 let manifest =
14093 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14094 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14095 .build();
14096 let connection = ConnectionId::new(connection);
14097 let (tx, rx) = mpsc::channel(8);
14098 self.rx = rx;
14099 self.forwarding
14100 .register_module_connection(
14101 connection,
14102 self.spec.module_id.clone(),
14103 subc_protocol::PROTOCOL_VERSION,
14104 Concurrency::ModuleManaged,
14105 FrameSink::new(tx),
14106 )
14107 .unwrap();
14108 self.registry
14109 .register_with_control_ops(
14110 manifest,
14111 subc_protocol::PROTOCOL_VERSION,
14112 connection,
14113 if health {
14114 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
14115 } else {
14116 vec![]
14117 },
14118 )
14119 .unwrap();
14120 }
14121
14122 async fn probe(&mut self) -> crate::router::OutboundFrame {
14123 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
14124 Ok(frame) => Some(frame),
14125 Err(mpsc::error::TryRecvError::Empty) => None,
14126 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
14127 })
14128 .await;
14129 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14130 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14131 frame
14132 }
14133
14134 fn no_probe(&mut self) {
14135 assert!(matches!(
14136 self.rx.try_recv(),
14137 Err(mpsc::error::TryRecvError::Empty)
14138 ));
14139 }
14140
14141 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
14142 self.forwarding
14143 .complete_module_control_rpc(
14144 ConnectionId::new(connection),
14145 frame.header.corr,
14146 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14147 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14148 status: HealthStatus::Ok,
14149 detail: None,
14150 metrics: None,
14151 }),
14152 )
14153 .unwrap();
14154 }
14155 }
14156
14157 #[tokio::test(start_paused = true)]
14158 async fn no_health_children_park_for_sixty_seconds() {
14159 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14160 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14161 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14162 unadvertised.hello(1, false);
14163 for _ in 0..6000 {
14168 tokio::time::advance(Duration::from_millis(10)).await;
14169 tokio::task::yield_now().await;
14170 }
14171 for actor in [&none, &unregistered, &unadvertised] {
14172 assert!(
14173 actor.turns() <= 5,
14174 "no-health actor woke {} times",
14175 actor.turns()
14176 );
14177 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
14178 }
14179 }
14180
14181 #[tokio::test(start_paused = true)]
14182 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
14183 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
14187 let tick = Instant::now();
14188 actor.hello(1, true);
14189 actor.probe().await;
14190 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
14191 }
14192
14193 #[tokio::test(start_paused = true)]
14194 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
14195 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14196 let hello_tick = Instant::now();
14197 actor.hello(1, true);
14198 let armed = actor
14199 .wait_for_select(|checkpoint| {
14200 checkpoint.registered_connection == Some(ConnectionId::new(1))
14201 && checkpoint.next_probe_at.is_some()
14202 })
14203 .await;
14204 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14205 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
14206 let first_deadline = hello_tick + delay;
14207 assert_eq!(armed.next_probe_at, Some(first_deadline));
14208 tokio::time::advance(Duration::from_secs(29)).await;
14209 let turns = actor.turns();
14210 actor
14211 .registry
14212 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
14213 .unwrap();
14214 let updated = actor
14215 .wait_for_select(|checkpoint| checkpoint.turn > turns)
14216 .await;
14217 assert_eq!(
14218 updated.next_probe_at,
14219 Some(first_deadline),
14220 "catalog update must not reset cadence"
14221 );
14222 actor.no_probe();
14223 tokio::time::advance(first_deadline - Instant::now()).await;
14224 let frame = actor.probe().await;
14225 actor.answer(1, frame);
14226 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
14227 let next_deadline = Instant::now() + next;
14228 let rearmed = actor
14229 .wait_for_select(|checkpoint| {
14230 checkpoint
14231 .next_probe_at
14232 .is_some_and(|deadline| deadline > first_deadline)
14233 })
14234 .await;
14235 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14236 tokio::time::advance(next - Duration::from_nanos(1)).await;
14237 actor.no_probe();
14238 tokio::time::advance(Duration::from_nanos(1)).await;
14239 actor.probe().await;
14240 }
14241
14242 #[tokio::test(start_paused = true)]
14243 async fn disconnect_disarms_and_reconnect_rearms_health() {
14244 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14245 actor.hello(1, true);
14246 actor
14247 .wait_for_select(|checkpoint| {
14248 checkpoint.registered_connection == Some(ConnectionId::new(1))
14249 && checkpoint.next_probe_at.is_some()
14250 })
14251 .await;
14252 let turns = actor.turns();
14253 let tick = Instant::now();
14254 actor
14255 .registry
14256 .deregister_connection(ConnectionId::new(1))
14257 .unwrap();
14258 let disarmed = actor
14259 .wait_for_select(|checkpoint| {
14260 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14261 })
14262 .await;
14263 assert_eq!(disarmed.next_probe_at, None);
14264 assert_eq!(disarmed.wake_after, None);
14265 assert_eq!(
14266 Instant::now(),
14267 tick,
14268 "disconnect must disarm in the notification tick"
14269 );
14270 tokio::time::advance(Duration::from_secs(60)).await;
14271 actor.no_probe();
14272 let reconnect_tick = Instant::now();
14273 actor.hello(2, true);
14274 let rearmed = actor
14275 .wait_for_select(|checkpoint| {
14276 checkpoint.registered_connection == Some(ConnectionId::new(2))
14277 && checkpoint.next_probe_at.is_some()
14278 })
14279 .await;
14280 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14281 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14282 tokio::time::advance(delay).await;
14283 actor.probe().await;
14284 }
14285
14286 #[tokio::test(start_paused = true)]
14287 async fn reregistration_without_health_disarms_the_old_schedule() {
14288 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14289 actor.hello(1, true);
14290 actor
14291 .wait_for_select(|checkpoint| {
14292 checkpoint.registered_connection == Some(ConnectionId::new(1))
14293 && checkpoint.next_probe_at.is_some()
14294 })
14295 .await;
14296 actor
14297 .registry
14298 .deregister_connection(ConnectionId::new(1))
14299 .unwrap();
14300 actor.hello(2, false);
14301 let disarmed = actor
14302 .wait_for_select(|checkpoint| {
14303 checkpoint.registered_connection == Some(ConnectionId::new(2))
14304 })
14305 .await;
14306 assert_eq!(disarmed.next_probe_at, None);
14307 assert_eq!(disarmed.wake_after, None);
14308 tokio::time::advance(Duration::from_secs(60)).await;
14309 actor.no_probe();
14310 }
14311
14312 #[tokio::test(start_paused = true)]
14313 async fn restart_waits_for_its_own_hello_before_rearming_health() {
14314 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14315 actor.hello(1, true);
14316 let armed = actor
14317 .wait_for_select(|checkpoint| {
14318 checkpoint.registered_connection == Some(ConnectionId::new(1))
14319 && checkpoint.next_probe_at.is_some()
14320 })
14321 .await;
14322 actor
14326 .registry
14327 .deregister_connection(ConnectionId::new(1))
14328 .unwrap();
14329 actor.module.restart(Some(0)).await.unwrap();
14330 let parked = actor
14331 .wait_for_select(|checkpoint| {
14332 checkpoint.generation > armed.generation
14333 && checkpoint.registered_connection.is_none()
14334 && checkpoint.next_probe_at.is_none()
14335 })
14336 .await;
14337 assert_eq!(parked.wake_after, None);
14338 actor.rx.close();
14339 while actor.rx.try_recv().is_ok() {}
14342 let turns = actor.turns();
14343 tokio::time::advance(Duration::from_secs(60)).await;
14344 let after = actor.turns();
14345 eprintln!(
14346 "restart park turns: before={turns} after={after} delta={}",
14347 after - turns
14348 );
14349 assert_eq!(after, turns, "replacement must park before HELLO");
14350 let hello_tick = Instant::now();
14351 actor.hello(2, true);
14352 let rearmed = actor
14353 .wait_for_select(|checkpoint| {
14354 checkpoint.registered_connection == Some(ConnectionId::new(2))
14355 && checkpoint.next_probe_at.is_some()
14356 })
14357 .await;
14358 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14359 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14360 tokio::time::advance(delay).await;
14361 actor.probe().await;
14362 }
14363
14364 #[tokio::test(start_paused = true)]
14365 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14366 use tokio::{
14367 io::{AsyncReadExt, AsyncWriteExt},
14368 net::TcpListener,
14369 };
14370 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14371 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14372 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14373 let (started_tx, mut started_rx) = mpsc::channel(8);
14374 let peer = tokio::spawn(async move {
14375 loop {
14376 let (mut socket, _) = listener.accept().await.unwrap();
14377 let mut request = [0; 1024];
14378 assert!(socket.read(&mut request).await.unwrap() > 0);
14379 started_tx.send(Instant::now()).await.unwrap();
14380 socket
14381 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14382 .await
14383 .unwrap();
14384 }
14385 });
14386 actor
14387 .module
14388 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14389 .await
14390 .unwrap();
14391 let first_tick = Instant::now();
14392 let first = actor
14393 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
14394 .await;
14395 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
14396 assert!(started_rx.try_recv().is_err());
14397
14398 tokio::time::advance(
14401 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
14402 )
14403 .await;
14404 actor.module.restart(Some(0)).await.unwrap();
14405 let spawn_tick = Instant::now();
14406 let replacement = actor
14407 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
14408 .await;
14409 let deadline = replacement.next_probe_at.unwrap();
14410 assert!(
14411 deadline >= spawn_tick + actor.health.cadence,
14412 "a replacement must not inherit its predecessor's probe deadline"
14413 );
14414 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
14415 assert!(
14416 started_rx.try_recv().is_err(),
14417 "no HTTP probe during startup grace"
14418 );
14419 tokio::time::advance(deadline - Instant::now()).await;
14420 let landed = wait_for("first replacement HTTP probe", || {
14421 started_rx.try_recv().ok()
14422 })
14423 .await;
14424 assert!(landed >= spawn_tick + actor.health.cadence);
14425 wait_for("healthy replacement", || {
14426 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14427 .then_some(())
14428 })
14429 .await;
14430 peer.abort();
14431 }
14432
14433 #[tokio::test(start_paused = true)]
14434 async fn rescan_adds_and_removes_http_health_without_polling() {
14435 use tokio::{
14436 io::{AsyncReadExt, AsyncWriteExt},
14437 net::TcpListener,
14438 };
14439 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14440 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14441 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
14442 actor.health.cadence = Duration::from_millis(1);
14444 let (started_tx, mut started_rx) = mpsc::channel(8);
14445 let peer = tokio::spawn(async move {
14446 loop {
14447 let (mut socket, _) = listener.accept().await.unwrap();
14448 let mut request = [0; 1024];
14449 assert!(socket.read(&mut request).await.unwrap() > 0);
14450 started_tx.send(()).await.unwrap();
14451 socket
14452 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14453 .await
14454 .unwrap();
14455 }
14456 });
14457 let tick = Instant::now();
14458 let turns = actor.turns();
14459 actor
14460 .module
14461 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14462 .await
14463 .unwrap();
14464 let armed = actor
14465 .wait_for_select(|checkpoint| {
14466 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
14467 })
14468 .await;
14469 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
14470 assert_eq!(Instant::now(), tick);
14471 tokio::time::advance(Duration::from_millis(1)).await;
14472 let mut started = false;
14473 wait_for("successful HTTP probe", || {
14474 started |= started_rx.try_recv().is_ok();
14475 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14476 .then_some(())
14477 })
14478 .await;
14479 assert!(started, "added HTTP check must start at its first deadline");
14480 assert_eq!(
14481 actor.module.status().unwrap().health.status,
14482 SupervisorHealthStatus::Ok
14483 );
14484 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
14485 actor.health.http = None;
14486 let turns = actor.turns();
14487 actor
14488 .module
14489 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14490 .await
14491 .unwrap();
14492 let parked = actor
14493 .wait_for_select(|checkpoint| {
14494 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
14495 })
14496 .await;
14497 assert_eq!(parked.wake_after, None);
14498 let turns = actor.turns();
14499 tokio::time::advance(Duration::from_secs(60)).await;
14500 assert_eq!(
14501 actor.turns(),
14502 turns,
14503 "removed HTTP check must park the actor"
14504 );
14505 assert!(
14506 started_rx.try_recv().is_err(),
14507 "removed HTTP check must stay disarmed"
14508 );
14509 assert_eq!(
14510 actor.module.status().unwrap().health,
14511 ModuleHealthStatus::default()
14512 );
14513 peer.abort();
14514 }
14515}
14516
14517#[cfg(test)]
14518mod drain_busy_reprobe_tests {
14519 use super::*;
14523 use std::collections::BTreeMap;
14524 use std::sync::Mutex as StdMutex;
14525 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14526
14527 const MODULE_ID: &str = "drain-reprobe-module";
14528 const CONNECTION: u64 = 7;
14529 const GAUGE: &str = "runs_in_flight";
14530 const CADENCE: Duration = Duration::from_secs(30);
14533 const CEILING: Duration = Duration::from_secs(30);
14534
14535 #[derive(Clone, Copy)]
14537 enum Answer {
14538 BusyUntil { until: Instant, latency: Duration },
14541 OmitsGauge,
14543 Silent,
14545 }
14546
14547 struct Drain {
14548 forwarding: Arc<ForwardingTable>,
14549 runtime: SupervisorRuntimeConfig,
14550 endpoint: crate::ModuleEndpointId,
14551 probes: Arc<StdMutex<Vec<Instant>>>,
14553 responder: tokio::task::JoinHandle<()>,
14554 }
14555
14556 impl Drain {
14557 fn start(answer: Answer) -> Self {
14558 let registry = Arc::new(Registry::default());
14559 let forwarding = Arc::new(ForwardingTable::default());
14560 let health = HealthConfig {
14561 cadence: CADENCE,
14562 ..HealthConfig::default()
14563 };
14564 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14565 .with_forwarding(forwarding.clone())
14566 .with_health_config(health)
14567 .runtime_config();
14568 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14569 let endpoint = forwarding
14570 .register_module_connection(
14571 ConnectionId::new(CONNECTION),
14572 MODULE_ID.to_string(),
14573 subc_protocol::PROTOCOL_VERSION,
14574 Concurrency::ModuleManaged,
14575 FrameSink::new(tx),
14576 )
14577 .unwrap();
14578 let probes = Arc::new(StdMutex::new(Vec::new()));
14579 let responder = tokio::spawn({
14580 let forwarding = forwarding.clone();
14581 let probes = probes.clone();
14582 async move {
14583 while let Some(frame) = rx.recv().await {
14584 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14585 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14586 probes.lock().unwrap().push(Instant::now());
14587 let metrics = match answer {
14588 Answer::BusyUntil { until, latency } => {
14589 sleep(latency).await;
14590 let busy = u64::from(Instant::now() < until);
14591 serde_json::json!({ GAUGE: busy })
14592 }
14593 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14594 Answer::Silent => continue,
14595 };
14596 let _ = forwarding.complete_module_control_rpc(
14599 ConnectionId::new(CONNECTION),
14600 frame.header.corr,
14601 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14602 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14603 status: HealthStatus::Ok,
14604 detail: None,
14605 metrics: Some(metrics),
14606 }),
14607 );
14608 }
14609 }
14610 });
14611 Self {
14612 forwarding,
14613 runtime,
14614 endpoint,
14615 probes,
14616 responder,
14617 }
14618 }
14619
14620 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14623 let (drained, elapsed, _) = self.run_traced(ceiling).await;
14624 (drained, elapsed)
14625 }
14626
14627 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14629 let started = Instant::now();
14630 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14631 let drained = wait_for_forwarding_quiescence(
14632 &self.forwarding,
14633 MODULE_ID,
14634 &self.runtime,
14635 self.endpoint,
14636 started + ceiling,
14637 &[GAUGE.to_string()],
14638 DrainScope::Active,
14639 &mut trace,
14640 )
14641 .await
14642 .unwrap();
14643 (drained, started.elapsed(), trace)
14644 }
14645
14646 fn probes(&self) -> Vec<Instant> {
14647 self.probes.lock().unwrap().clone()
14648 }
14649 }
14650
14651 impl Drop for Drain {
14652 fn drop(&mut self) {
14653 self.responder.abort();
14654 }
14655 }
14656
14657 #[tokio::test(start_paused = true)]
14658 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14659 let drain = Drain::start(Answer::BusyUntil {
14660 until: Instant::now() + Duration::from_secs(1),
14661 latency: Duration::ZERO,
14662 });
14663 let (drained, elapsed) = drain.run(CEILING).await;
14664 assert!(
14665 drained,
14666 "gauges that read zero during the drain must let it finish drained, \
14667 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14668 );
14669 assert!(
14670 elapsed < Duration::from_secs(3),
14671 "drain took {elapsed:?} after the gauges went quiet at 1s"
14672 );
14673 let probes = drain.probes();
14674 assert!(
14675 probes.len() >= 2,
14676 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14677 );
14678 }
14679
14680 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14685 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14686 let drain = Drain::start(Answer::BusyUntil {
14687 until: Instant::now() + Duration::from_millis(600),
14688 latency: Duration::ZERO,
14689 });
14690 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14691 assert!(drained, "the gauge fell at 600ms: {trace}");
14692 assert!(
14693 elapsed < Duration::from_secs(3),
14694 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14695 );
14696 let probes = drain.probes();
14697 assert!(
14698 probes.len() >= 2 && trace.probes as usize == probes.len(),
14699 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14700 );
14701 }
14702
14703 #[tokio::test(start_paused = true)]
14704 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14705 let started = Instant::now();
14706 let drain = Drain::start(Answer::BusyUntil {
14707 until: started + Duration::from_secs(3600),
14708 latency: Duration::ZERO,
14709 });
14710 let (drained, elapsed) = drain.run(CEILING).await;
14711 assert!(!drained, "a module busy throughout must not drain");
14712 assert!(
14713 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14714 "a busy module is held until the ceiling and no longer: {elapsed:?}"
14715 );
14716 let probes = drain.probes();
14719 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14720 assert!(
14721 probes.len() as u128 <= max_probes,
14722 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14723 probes.len()
14724 );
14725 for pair in probes.windows(2) {
14726 assert!(
14727 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14728 "consecutive probes {:?} apart",
14729 pair[1] - pair[0]
14730 );
14731 }
14732 }
14733
14734 #[tokio::test(start_paused = true)]
14735 async fn a_slow_probe_is_never_overlapped_by_the_next() {
14736 let latency = Duration::from_secs(1);
14739 let drain = Drain::start(Answer::BusyUntil {
14740 until: Instant::now() + Duration::from_secs(3600),
14741 latency,
14742 });
14743 let (drained, _) = drain.run(Duration::from_secs(5)).await;
14744 assert!(!drained);
14745 let probes = drain.probes();
14746 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14747 for pair in probes.windows(2) {
14748 assert!(
14749 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14750 "a probe started {:?} after the previous one, before it could finish",
14751 pair[1] - pair[0]
14752 );
14753 }
14754 }
14755
14756 #[tokio::test(start_paused = true)]
14757 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14758 let drain = Drain::start(Answer::Silent);
14759 let ceiling = Duration::from_secs(2);
14760 let (drained, elapsed) = drain.run(ceiling).await;
14761 assert!(!drained, "an unanswered probe counts as busy");
14762 assert!(
14763 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14764 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14765 );
14766 assert_eq!(
14767 drain.probes().len(),
14768 1,
14769 "the unanswered probe holds the only slot until the ceiling"
14770 );
14771 }
14772
14773 #[derive(Clone, Default)]
14775 struct LogCapture {
14776 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14777 }
14778
14779 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14780 fn on_event(
14781 &self,
14782 event: &tracing::Event<'_>,
14783 _context: tracing_subscriber::layer::Context<'_, S>,
14784 ) {
14785 struct Fields(BTreeMap<String, String>);
14786 impl tracing::field::Visit for Fields {
14787 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14788 self.0
14789 .insert(field.name().to_string(), format!("{value:?}"));
14790 }
14791 }
14792 let mut fields = Fields(BTreeMap::new());
14793 event.record(&mut fields);
14794 self.events.lock().unwrap().push(fields.0);
14795 }
14796 }
14797
14798 impl LogCapture {
14799 fn install(&self) -> tracing::subscriber::DefaultGuard {
14800 use tracing_subscriber::layer::SubscriberExt;
14801 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14802 }
14803
14804 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14805 self.events
14806 .lock()
14807 .unwrap()
14808 .iter()
14809 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14810 .cloned()
14811 .collect()
14812 }
14813 }
14814
14815 #[tokio::test(start_paused = true)]
14820 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14821 let capture = LogCapture::default();
14822 let _guard = capture.install();
14823 let drain = Drain::start(Answer::OmitsGauge);
14824 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14825 assert!(!drained, "a gauge never reported cannot read quiet");
14826 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14827
14828 let missing = capture.with_message("lacks a declared busy gauge");
14829 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14830 assert_eq!(
14831 missing[0].get("missing_gauges").map(String::as_str),
14832 Some(GAUGE)
14833 );
14834
14835 warn_drain_timed_out(
14836 MODULE_ID,
14837 Duration::from_secs(2),
14838 RouteCloseReason::Restart,
14839 &Default::default(),
14840 &trace,
14841 );
14842 let timeouts = capture.with_message("route drain timed out");
14843 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14844 let gauges = &timeouts[0]["busy_gauges"];
14845 assert!(
14846 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14847 "the timeout line must name the gauge and why it read busy: {gauges}"
14848 );
14849 assert!(
14850 gauges.ends_with("; no value ever read"),
14851 "no value was ever read for the gauge: {gauges}"
14852 );
14853 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14854 }
14855
14856 #[tokio::test(start_paused = true)]
14860 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14861 let busy = Drain::start(Answer::BusyUntil {
14862 until: Instant::now() + Duration::from_secs(3600),
14863 latency: Duration::ZERO,
14864 });
14865 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14866 let rendered = trace.to_string();
14867 assert!(
14868 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14869 && rendered.contains("; last value non_zero(1) @"),
14870 "{rendered}"
14871 );
14872
14873 let silent = Drain::start(Answer::Silent);
14874 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14875 assert_eq!(
14878 trace.to_string(),
14879 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14880 );
14881 assert_eq!(trace.probes, 1);
14882 }
14883
14884 #[test]
14886 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14887 let gauges = [GAUGE.to_string()];
14888 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14889 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14890 assert_eq!(
14891 busy_gauge_observation(&verdicts),
14892 BusyGaugeObservation::Busy
14893 );
14894 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14895 assert_eq!(
14896 busy_gauge_observation(&zero),
14897 BusyGaugeObservation::Quiescent
14898 );
14899 let none = gauge_verdicts(None, &gauges);
14900 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14901 }
14902}
14903
14904#[cfg(test)]
14905mod health_evidence_tests {
14906 use super::{HealthProbeError, HealthProbeEvidence};
14907 use std::collections::HashSet;
14908
14909 #[test]
14917 fn only_a_dead_lane_is_proof_of_death() {
14918 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14919 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14923 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14924 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14925 }
14926
14927 #[test]
14933 fn every_evidence_class_has_a_distinct_label() {
14934 let labels = [
14935 HealthProbeError::lane_dead("").label(),
14936 HealthProbeError::no_answer("").label(),
14937 HealthProbeError::bad_answer("").label(),
14938 HealthProbeError::misconfigured("").label(),
14939 ];
14940 let unique: HashSet<_> = labels.iter().collect();
14941 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14942 }
14943
14944 #[test]
14950 fn classification_preserves_the_original_message() {
14951 let err = HealthProbeError::no_answer("module did not answer within 5s");
14952 assert_eq!(err.to_string(), "module did not answer within 5s");
14953 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14954 }
14955}
14956
14957#[cfg(test)]
14958mod health_tombstone_tests {
14959 use std::{path::PathBuf, sync::Arc, time::Duration};
14960
14961 use subc_protocol::{
14962 manifest::Concurrency,
14963 session::{HealthStatus, ModuleControlResponse},
14964 };
14965 use tokio::sync::mpsc;
14966
14967 use super::{
14968 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14969 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14970 };
14971 use crate::{
14972 control::ControlHandler,
14973 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14974 registry::{ConnectionId, Registry},
14975 router::FrameSink,
14976 };
14977
14978 struct ProbeHarness {
14979 spec: ModuleSpec,
14980 runtime: SupervisorRuntimeConfig,
14981 forwarding: Arc<ForwardingTable>,
14982 module_connection: ConnectionId,
14983 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14984 handler: ControlHandler,
14985 module: super::SupervisedModule,
14986 }
14987
14988 fn probe_harness() -> ProbeHarness {
14989 let registry = Arc::new(Registry::default());
14990 let forwarding = Arc::new(ForwardingTable::default());
14991 let supervisor_handle = super::SupervisorHandle::new();
14992 let health = HealthConfig {
14993 http: None,
14994 cadence: Duration::from_secs(30),
14995 deadline: Duration::from_secs(5),
14996 failure_threshold: 3,
14997 on_degraded: HealthAction::Report,
14998 on_failing: HealthAction::Report,
14999 critical: false,
15000 };
15001 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15002 .with_forwarding(Arc::clone(&forwarding))
15003 .with_handle(supervisor_handle.clone())
15004 .with_health_config(health);
15005 let spec = ModuleSpec {
15006 module_id: "late-health-module".to_string(),
15007 program: PathBuf::from("disabled-module"),
15008 args: Vec::new(),
15009 env: Vec::new(),
15010 reserved: false,
15011 reserved_prefixes: Vec::new(),
15012 protocol: ModuleProtocol::Subc,
15013 overlap: Default::default(),
15014 };
15015 let module = supervisor
15016 .supervise_configured(spec.clone(), false)
15017 .unwrap();
15018 let runtime = supervisor.runtime_config();
15019 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
15020 .with_supervisor(supervisor_handle);
15021 let module_connection = ConnectionId::new(700);
15022 let (module_tx, module_rx) = mpsc::channel(8);
15023 forwarding
15024 .register_module_connection(
15025 module_connection,
15026 spec.module_id.clone(),
15027 subc_protocol::PROTOCOL_VERSION,
15028 Concurrency::ModuleManaged,
15029 FrameSink::new(module_tx),
15030 )
15031 .unwrap();
15032
15033 ProbeHarness {
15034 spec,
15035 runtime,
15036 forwarding,
15037 module_connection,
15038 module_rx,
15039 handler,
15040 module,
15041 }
15042 }
15043
15044 async fn finish_after(
15045 harness: &mut ProbeHarness,
15046 stall: Duration,
15047 ) -> ModuleControlRpcCompletion {
15048 assert!(stall > harness.runtime.health.deadline);
15049 let deadline = harness.runtime.health.deadline;
15050 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15051 let answer = async {
15052 let frame = harness.module_rx.recv().await.expect("health.check frame");
15053 tokio::time::advance(deadline).await;
15054 tokio::task::yield_now().await;
15055 tokio::time::advance(stall - deadline).await;
15056 harness
15057 .forwarding
15058 .complete_module_control_rpc(
15059 harness.module_connection,
15060 frame.header.corr,
15061 Some("health.check"),
15062 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15063 status: HealthStatus::Ok,
15064 detail: None,
15065 metrics: None,
15066 }),
15067 )
15068 .unwrap()
15069 };
15070 let (probe_result, completion) = tokio::join!(probe, answer);
15071 let err = probe_result.expect_err("probe must miss its deadline");
15072 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15073 completion
15074 }
15075
15076 async fn time_out_without_answer(harness: &mut ProbeHarness) {
15077 let deadline = harness.runtime.health.deadline;
15078 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15079 let exhaust_deadline = async {
15080 let _frame = harness.module_rx.recv().await.expect("health.check frame");
15081 tokio::time::advance(deadline).await;
15082 tokio::task::yield_now().await;
15083 };
15084 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
15085 let err = probe_result.expect_err("probe must miss its deadline");
15086 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15087 }
15088
15089 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
15090 let registry = Arc::clone(&harness.module.inner.registry);
15091 let snapshot = Arc::clone(&harness.module.inner.snapshot);
15092 let process_liveness = super::SupervisorProcessLiveness::default();
15093 let mut child = None;
15094 let cycle = super::run_health_probe_cycle(
15095 &harness.spec,
15096 &harness.runtime,
15097 ®istry,
15098 &process_liveness,
15099 &snapshot,
15100 &mut child,
15101 );
15102 let peer = async {
15103 let frame = harness.module_rx.recv().await.expect("health.check frame");
15104 if answer {
15105 harness
15106 .forwarding
15107 .complete_module_control_rpc(
15108 harness.module_connection,
15109 frame.header.corr,
15110 Some("health.check"),
15111 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15112 status: HealthStatus::Ok,
15113 detail: None,
15114 metrics: Some(serde_json::json!({"ready": true})),
15115 }),
15116 )
15117 .unwrap();
15118 } else {
15119 tokio::time::advance(harness.runtime.health.deadline).await;
15120 tokio::task::yield_now().await;
15121 }
15122 };
15123 tokio::join!(cycle, peer);
15124 }
15125
15126 #[tokio::test(start_paused = true)]
15127 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
15128 let mut harness = probe_harness();
15129 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
15133 monitor.abort();
15134 let _ = monitor.await;
15135 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
15136 state.enabled = true;
15137 state.state = super::ModuleState::Running;
15138 state.process_alive = true;
15139 })
15140 .unwrap();
15141
15142 run_probe_cycle(&mut harness, true).await;
15143 assert_eq!(
15144 harness.module.status().unwrap().health.status,
15145 super::SupervisorHealthStatus::Ok
15146 );
15147
15148 for failures in 1..harness.runtime.health.failure_threshold {
15149 run_probe_cycle(&mut harness, false).await;
15150 let status = harness.module.status().unwrap();
15151 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15152 assert_eq!(status.health.consecutive_failures, failures);
15153 assert!(status.health.last_probe_ms.is_some());
15154 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
15155 assert!(status.health.metrics.is_none());
15156 assert_eq!(status.state, super::ModuleState::Running);
15157 assert!(status.process_alive);
15158 assert_eq!(status.restart_count, 0);
15159 assert_eq!(status.lifetime_restarts, 0);
15160 assert!(status.health.last_action.is_none());
15161 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15162 }
15163
15164 run_probe_cycle(&mut harness, true).await;
15165 let recovered = harness.module.status().unwrap();
15166 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
15167 assert_eq!(recovered.health.consecutive_failures, 0);
15168 assert!(recovered.health.detail.is_none());
15169 assert_eq!(
15170 recovered.health.metrics,
15171 Some(serde_json::json!({"ready": true}))
15172 );
15173 assert_eq!(recovered.lifetime_restarts, 0);
15174
15175 for failures in 1..=harness.runtime.health.failure_threshold {
15176 run_probe_cycle(&mut harness, false).await;
15177 let status = harness.module.status().unwrap();
15178 assert_eq!(status.health.consecutive_failures, failures);
15179 if failures < harness.runtime.health.failure_threshold {
15180 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15181 assert_eq!(status.state, super::ModuleState::Running);
15182 assert_eq!(status.lifetime_restarts, 0);
15183 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15184 } else {
15185 assert_eq!(
15186 status.health.status,
15187 super::SupervisorHealthStatus::Unresponsive
15188 );
15189 assert_eq!(status.state, super::ModuleState::Restarting);
15190 assert_eq!(status.restart_count, 1);
15191 assert_eq!(status.lifetime_restarts, 1);
15192 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
15193 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
15194 }
15195 }
15196 }
15197
15198 #[tokio::test(start_paused = true)]
15199 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
15200 let mut harness = probe_harness();
15201
15202 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
15203 let first_latency = match &first {
15204 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15205 other => panic!("late answer was not retained: {other:?}"),
15206 };
15207 assert!(harness.handler.observe_module_control_completion(first));
15208
15209 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
15210 let second_latency = match &second {
15211 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15212 other => panic!("late answer was not retained: {other:?}"),
15213 };
15214 assert!(harness.handler.observe_module_control_completion(second));
15215
15216 assert_eq!(first_latency, Duration::from_secs(8));
15217 assert_eq!(
15218 second_latency - first_latency,
15219 Duration::from_secs(3),
15220 "latency must grow linearly with the additional stall"
15221 );
15222 let health = harness.module.status().unwrap().health;
15223 assert_eq!(health.late_answer_count, 2);
15224 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
15225 }
15226
15227 #[tokio::test(start_paused = true)]
15235 async fn late_answer_clears_the_consecutive_failure_streak() {
15236 let mut harness = probe_harness();
15237
15238 time_out_without_answer(&mut harness).await;
15240 harness
15241 .module
15242 .record_health_probe_failure_for_test("[no-answer] test miss")
15243 .unwrap();
15244 assert_eq!(
15245 harness.module.status().unwrap().health.consecutive_failures,
15246 1,
15247 "precondition: the miss must be on the streak before the late answer"
15248 );
15249
15250 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15252 assert!(matches!(
15253 late,
15254 ModuleControlRpcCompletion::LateHealthAnswer { .. }
15255 ));
15256 assert!(harness.handler.observe_module_control_completion(late));
15257
15258 let health = harness.module.status().unwrap().health;
15259 assert_eq!(
15260 health.consecutive_failures, 0,
15261 "a late answer is an answer: the streak must reset"
15262 );
15263 assert_eq!(health.late_answer_count, 1);
15264 }
15265
15266 #[tokio::test(start_paused = true)]
15267 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15268 let mut harness = probe_harness();
15269
15270 for _ in 0..20 {
15271 time_out_without_answer(&mut harness).await;
15272 assert_eq!(
15273 harness.forwarding.health_probe_tombstone_count().unwrap(),
15274 1
15275 );
15276 }
15277 }
15278}
15279
15280#[cfg(test)]
15281mod child_env_tests {
15282 #[cfg(windows)]
15283 use super::{
15284 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15285 };
15286 #[cfg(windows)]
15287 use std::sync::{Arc, OnceLock};
15288 #[cfg(windows)]
15289 #[test]
15290 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15291 let spec = spec(Vec::new());
15292 let mut command = Command::new("unused");
15293 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15294 .unwrap()
15295 .unwrap();
15296 let name = command
15297 .as_std()
15298 .get_envs()
15299 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15300 .and_then(|(_, value)| value)
15301 .unwrap();
15302 assert_eq!(name, OsStr::new(handoff.name()));
15303 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15304 assert!(command.as_std().get_envs().any(|(key, value)| key
15305 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15306 && value == Some(OsStr::new("env"))));
15307 assert!(command
15308 .as_std()
15309 .get_envs()
15310 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15311 }
15312
15313 #[cfg(windows)]
15314 #[test]
15315 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15316 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15317 snapshot.process_alive = true;
15318 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15319 assert_eq!(
15320 observed_launch_nonce_source(&snapshot).as_deref(),
15321 Some("env")
15322 );
15323 let consumed = Arc::new(OnceLock::new());
15324 snapshot.pipe_consumed = Some(consumed.clone());
15325 assert_eq!(
15326 observed_launch_nonce_source(&snapshot).as_deref(),
15327 Some("env")
15328 );
15329 consumed.set(()).unwrap();
15330 assert_eq!(
15331 observed_launch_nonce_source(&snapshot).as_deref(),
15332 Some("pipe")
15333 );
15334 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15335 assert_eq!(
15336 observed_launch_nonce_source(&snapshot).as_deref(),
15337 Some("env")
15338 );
15339 clear_current_process_facts(&mut snapshot);
15340 assert_eq!(observed_launch_nonce_source(&snapshot), None);
15341 }
15342 use super::{
15343 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15344 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15345 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15346 };
15347 use std::{ffi::OsStr, path::PathBuf};
15348 use tokio::process::Command;
15349
15350 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15351 ModuleSpec {
15352 module_id: "env-plan".to_string(),
15353 program: PathBuf::from("/nonexistent"),
15354 args: Vec::new(),
15355 env,
15356 reserved: false,
15357 reserved_prefixes: Vec::new(),
15358 protocol: ModuleProtocol::Subc,
15359 overlap: Default::default(),
15360 }
15361 }
15362
15363 #[test]
15364 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15365 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15366 let digest = LaunchSpecDigest::from_spec(&base);
15367 let mut reordered = base.clone();
15368 reordered.env.reverse();
15369 reordered.module_id = "different-id".into();
15370 reordered.reserved = true;
15371 reordered.overlap = Default::default();
15372 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15373 let mut delimiter_collision = base.clone();
15374 delimiter_collision.env[0] = ("A=B".into(), "C".into());
15375 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15376 let mut changed = base.clone();
15377 changed.protocol = ModuleProtocol::None;
15378 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15379 changed = base.clone();
15380 changed.args.push("--flag".into());
15381 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15382 changed = base.clone();
15383 changed.program = PathBuf::from("/other");
15384 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15385 changed = base.clone();
15386 changed.env[1].1 = "other".into();
15387 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15388 assert!(!format!("{digest:?}").contains("B=C"));
15389 assert!(!format!("{digest:?}").contains("[u8"));
15390 }
15391
15392 #[cfg(unix)]
15393 #[test]
15394 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
15395 use std::os::unix::ffi::OsStringExt;
15396 let mut left = spec(Vec::new());
15397 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
15398 let mut right = left.clone();
15399 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
15400 assert_eq!(
15401 left.program.to_string_lossy(),
15402 right.program.to_string_lossy()
15403 );
15404 assert_ne!(
15405 LaunchSpecDigest::from_spec(&left),
15406 LaunchSpecDigest::from_spec(&right)
15407 );
15408 }
15409
15410 #[test]
15424 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
15425 let mut command = Command::new("/nonexistent");
15426 apply_child_env(&mut command, &spec(Vec::new()));
15427 let removed = command
15428 .as_std()
15429 .get_envs()
15430 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
15431 assert!(
15432 removed,
15433 "ambient CK_LOG must be explicitly removed for an unconfigured module"
15434 );
15435
15436 let mut configured = Command::new("/nonexistent");
15437 apply_child_env(
15438 &mut configured,
15439 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
15440 );
15441 let effective = configured
15442 .as_std()
15443 .get_envs()
15444 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
15445 .last()
15446 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15447 assert_eq!(
15448 effective,
15449 Some(Some("debug".to_string())),
15450 "a module's configured CK_LOG must survive the ambient removal"
15451 );
15452 }
15453
15454 #[test]
15463 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
15464 let connection_file = std::path::Path::new("/run/subc-connection.json");
15465 let handle = SupervisorHandle::new();
15466
15467 let mut none_spec = spec(Vec::new());
15468 none_spec.protocol = ModuleProtocol::None;
15469 let mut none = Command::new("/nonexistent");
15470 let none_handoff =
15471 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
15472 .expect("protocol-none spawn args apply");
15473 assert!(
15474 none_handoff.is_none(),
15475 "protocol:none spawn must not receive a nonce descriptor"
15476 );
15477 #[cfg(windows)]
15478 assert!(!none.as_std().get_envs().any(|(key, value)| key
15479 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15480 && value.is_some()));
15481 assert!(
15482 !none.as_std().get_envs().any(|(key, value)| key
15483 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
15484 && value.is_some()),
15485 "protocol:none spawn must not name a nonce descriptor"
15486 );
15487 let none_args: Vec<String> = none
15488 .as_std()
15489 .get_args()
15490 .map(|a| a.to_string_lossy().into_owned())
15491 .collect();
15492 assert!(
15493 !none_args.iter().any(|a| a == SUBC_ARG),
15494 "protocol:none argv must not carry --subc; got {none_args:?}"
15495 );
15496 let none_has_nonce = none
15497 .as_std()
15498 .get_envs()
15499 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
15500 assert!(
15501 !none_has_nonce,
15502 "protocol:none spawn must not receive a launch nonce"
15503 );
15504 let none_has_module_id = none
15505 .as_std()
15506 .get_envs()
15507 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
15508 assert!(
15509 none_has_module_id,
15510 "SUBC_MODULE_ID is inert and stays on every path"
15511 );
15512 assert!(
15513 handle.spawn_nonce(&none_spec.module_id).is_none(),
15514 "no nonce record for a process that will never present one"
15515 );
15516
15517 let wire_spec = spec(Vec::new());
15519 let mut wire = Command::new("/nonexistent");
15520 let wire_handoff =
15521 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15522 .expect("subc-wire spawn args apply");
15523 let wire_fd_env = wire
15524 .as_std()
15525 .get_envs()
15526 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15527 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15528 #[cfg(unix)]
15529 assert_eq!(
15530 wire_fd_env,
15531 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15532 "a subc-wire spawn names the pipe it will receive at descriptor 3"
15533 );
15534 #[cfg(windows)]
15535 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15536 #[cfg(not(any(unix, windows)))]
15537 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15538 let wire_args: Vec<String> = wire
15539 .as_std()
15540 .get_args()
15541 .map(|a| a.to_string_lossy().into_owned())
15542 .collect();
15543 assert_eq!(
15544 wire_args,
15545 vec![
15546 SUBC_ARG.to_string(),
15547 connection_file.to_string_lossy().into_owned()
15548 ],
15549 "a subc-wire spawn still carries --subc <path>"
15550 );
15551 assert_eq!(
15552 wire.as_std()
15553 .get_envs()
15554 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15555 !cfg!(unix),
15556 "only Windows supplies the environment nonce"
15557 );
15558 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15559 }
15560
15561 #[test]
15571 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15572 let role = |command: &Command| {
15573 command
15574 .as_std()
15575 .get_envs()
15576 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15577 .last()
15578 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15579 };
15580 let forged = spec(vec![(
15581 SUBC_SPAWN_ROLE_ENV.to_string(),
15582 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15583 )]);
15584
15585 let mut plain = Command::new("/nonexistent");
15586 apply_child_env(&mut plain, &forged);
15587 apply_spawn_role(&mut plain, SpawnRole::Plain);
15588 assert_eq!(
15589 role(&plain),
15590 Some(None),
15591 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15592 );
15593
15594 let mut candidate = Command::new("/nonexistent");
15595 apply_child_env(&mut candidate, &spec(Vec::new()));
15596 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15597 assert_eq!(
15598 role(&candidate),
15599 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15600 );
15601 }
15602
15603 #[test]
15609 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15610 let mut command = Command::new("/nonexistent");
15611 apply_child_env(
15612 &mut command,
15613 &spec(vec![
15614 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15615 ("KEPT".to_string(), "yes".to_string()),
15616 ]),
15617 );
15618 let keys: Vec<String> = command
15619 .as_std()
15620 .get_envs()
15621 .filter(|(_, value)| value.is_some())
15622 .map(|(key, _)| key.to_string_lossy().into_owned())
15623 .collect();
15624 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15625 assert!(
15626 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15627 "daemon-private capture key leaked to the child: {keys:?}"
15628 );
15629 }
15630}
15631
15632#[cfg(test)]
15633mod jitter_tests {
15634 use super::jittered_health_delay;
15635 use std::{collections::HashSet, time::Duration};
15636
15637 const FLEET: [&str; 14] = [
15646 "aft",
15647 "alfonso-core",
15648 "magic-context",
15649 "broca",
15650 "thalamus",
15651 "quota",
15652 "engram",
15653 "plexus",
15654 "cerebellum",
15655 "astrocyte",
15656 "synapse",
15657 "subc-mcp",
15658 "cortexkit-credentials",
15659 "subc-federation",
15660 ];
15661
15662 #[test]
15670 fn probe_delays_disperse_across_the_fleet() {
15671 let cadence = Duration::from_secs(30);
15672 let delays: HashSet<Duration> = FLEET
15673 .iter()
15674 .map(|id| jittered_health_delay(id, 0, cadence))
15675 .collect();
15676 assert_eq!(
15677 delays.len(),
15678 FLEET.len(),
15679 "every supervised module must land on its own probe offset"
15680 );
15681 }
15682
15683 #[test]
15689 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15690 let cadence = Duration::from_secs(30);
15691 let span = cadence / 10;
15692 for id in FLEET {
15693 for probe_index in 0..8 {
15694 let delay = jittered_health_delay(id, probe_index, cadence);
15695 assert!(
15696 delay >= cadence,
15697 "{id}#{probe_index}: jitter must not shorten the cadence"
15698 );
15699 assert!(
15700 delay < cadence + span,
15701 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15702 );
15703 }
15704 }
15705 }
15706
15707 #[test]
15713 fn a_module_offset_is_stable_across_restarts() {
15714 let cadence = Duration::from_secs(30);
15715 for id in FLEET {
15716 assert_eq!(
15717 jittered_health_delay(id, 0, cadence),
15718 jittered_health_delay(id, 0, cadence),
15719 "{id}: the same module and probe index must produce the same offset"
15720 );
15721 }
15722 }
15723
15724 #[test]
15726 fn zero_cadence_yields_zero_delay() {
15727 assert_eq!(
15728 jittered_health_delay("aft", 0, Duration::ZERO),
15729 Duration::ZERO
15730 );
15731 }
15732}
15733
15734#[cfg(all(test, target_os = "linux"))]
15735mod cgroup_placement_tests {
15736 use super::{
15737 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15738 SuperviseError, SupervisedChild,
15739 };
15740 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15741 use cortexkit_test_support::ScratchDir;
15742 use std::{
15743 fs, io,
15744 path::{Path, PathBuf},
15745 sync::{Arc, Mutex},
15746 };
15747 use tokio::process::Command;
15748
15749 #[tokio::test]
15750 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15751 use super::*;
15752 let dir = ScratchDir::new("unique-spawn-cgroups");
15753 let root = PathBuf::from(format!(
15754 "/sys/fs/cgroup/subc-unique-test-{}-{}",
15755 std::process::id(),
15756 unix_ms_now()
15757 ));
15758 if let Err(error) = fs::create_dir(&root) {
15759 assert!(
15760 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15761 "required cgroup test cannot execute: {error}"
15762 );
15763 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15764 return;
15765 }
15766 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15767 let group_count = || {
15768 fs::read_dir(root.join("subc-modules"))
15769 .unwrap()
15770 .map(|entry| entry.unwrap().file_type().unwrap())
15771 .filter(|kind| kind.is_dir())
15772 .count()
15773 };
15774 let supervisor =
15775 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15776 .with_cgroup_placement(Some(placement.clone()));
15777 let runtime = supervisor.runtime_config();
15778 let mut spec = ModuleSpec {
15779 module_id: "unique-spawn".into(),
15780 program: PathBuf::from("/bin/sleep"),
15781 args: vec!["60".into()],
15782 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15783 .into_iter()
15784 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15785 .collect(),
15786 reserved: false,
15787 reserved_prefixes: vec![],
15788 protocol: ModuleProtocol::None,
15789 overlap: Default::default(),
15790 };
15791 let spawn = |spec: &ModuleSpec| {
15792 spawn_child(
15793 spec,
15794 None,
15795 None,
15796 &runtime.stderr_ring,
15797 None,
15798 &runtime.child_roster,
15799 Some(&placement),
15800 )
15801 .unwrap()
15802 };
15803 let mut live = spawn(&spec);
15804 for _ in 0..3 {
15805 let next = spawn(&spec);
15807 assert_ne!(live.module_id, next.module_id);
15808 live.start_kill().unwrap();
15809 live.wait().await.unwrap();
15810 live = next;
15811 assert!(
15812 live.child.try_wait().unwrap().is_none(),
15813 "retiring the old slot must not kill the replacement"
15814 );
15815 assert_eq!(
15816 group_count(),
15817 1,
15818 "only the live spawn's cgroup should remain"
15819 );
15820 }
15821 supervisor.begin_daemon_shutdown();
15822 let reap = tokio::spawn(async move {
15823 live.wait().await.unwrap();
15824 });
15825 supervisor
15826 .end_children_for_daemon_shutdown(false, std::future::pending())
15827 .await;
15828 reap.await.unwrap();
15829 assert_eq!(group_count(), 0);
15830 spec.program = PathBuf::from("/bin/true");
15832 spec.args.clear();
15833 let fresh_roster = ChildRoster::default();
15834 let mut short = spawn_child(
15835 &spec,
15836 None,
15837 None,
15838 &runtime.stderr_ring,
15839 None,
15840 &fresh_roster,
15841 Some(&placement),
15842 )
15843 .unwrap();
15844 short.wait().await.unwrap();
15845 assert_eq!(group_count(), 0);
15846 spec.module_id = "_".repeat(255);
15847 let mut long_id = spawn_child(
15848 &spec,
15849 None,
15850 None,
15851 &runtime.stderr_ring,
15852 None,
15853 &fresh_roster,
15854 Some(&placement),
15855 )
15856 .unwrap();
15857 long_id.wait().await.unwrap();
15858 assert_eq!(
15859 group_count(),
15860 0,
15861 "valid long module IDs must not exceed cgroup NAME_MAX"
15862 );
15863 fs::remove_dir(root.join("subc-modules")).unwrap();
15864 fs::remove_dir(root).unwrap();
15865 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15866 }
15867
15868 #[test]
15869 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15870 let path = Path::new("/definitely-missing-subc-cgroup");
15871 let mut command = Command::new("true");
15872 let error = apply_cgroup_placement(
15873 &mut command,
15874 &ModuleSpec {
15875 module_id: "broken-cgroup".to_string(),
15876 program: PathBuf::from("true"),
15877 args: Vec::new(),
15878 env: Vec::new(),
15879 reserved: false,
15880 reserved_prefixes: Vec::new(),
15881 protocol: ModuleProtocol::Subc,
15882 overlap: Default::default(),
15883 },
15884 path,
15885 )
15886 .expect_err("a parent cgroup open failure must reject the supervised spawn");
15887 let reason = error.to_string();
15888
15889 assert!(
15890 matches!(error, SuperviseError::Cgroup { .. }),
15891 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15892 );
15893 assert!(
15894 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15895 "parent cgroup open failure must name cgroup.procs: {reason}"
15896 );
15897 }
15898
15899 #[tokio::test]
15900 async fn reaping_a_child_removes_its_empty_module_cgroup() {
15901 let root = ScratchDir::new("supervisor-reap-cgroup");
15902 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15903 let placement = subc_cgroup::prepare_at(&root)
15904 .expect("prepare scratch cgroup root")
15905 .expect("scratch root has a cgroup.procs marker");
15906 let module_id = "reaped-module";
15907 let module = placement
15908 .module_path(module_id)
15909 .expect("create scratch module cgroup");
15910 let child = Command::new("true")
15911 .env("XDG_DATA_HOME", root.path())
15912 .env("XDG_RUNTIME_DIR", root.path())
15913 .env("XDG_CONFIG_HOME", root.path())
15914 .spawn()
15915 .expect("spawn short-lived child");
15916 let pid = child.id().expect("spawned child has pid");
15917 let mut child = SupervisedChild {
15918 child,
15919 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15920 protocol: ModuleProtocol::Subc,
15921 module_id: module_id.to_string(),
15922 cgroup_placement: Some(placement),
15923 stdout_pump: None,
15924 stderr_pump: None,
15925 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15926 spawned_at: tokio::time::Instant::now(),
15927 spawned_at_ms: 0,
15928 spawned_from: PathBuf::from("true"),
15929 launch_spec_digest: LaunchSpecDigest([0; 32]),
15930 spawned_file_identity: None,
15931 process_start_time: None,
15932 process_identity: None,
15933 pid,
15934 roster_guard: None,
15935 #[cfg(target_os = "macos")]
15936 privacy_exec: None,
15937 spawn_failure: None,
15938 };
15939
15940 child.wait().await.expect("reap short-lived child");
15941
15942 assert!(
15943 !module.exists(),
15944 "reaping the supervised child must remove its empty cgroup"
15945 );
15946 }
15947
15948 #[test]
15949 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15950 let root = ScratchDir::new("supervisor-non-empty-cgroup");
15951 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15952 let placement = subc_cgroup::prepare_at(&root)
15953 .expect("prepare scratch cgroup root")
15954 .expect("scratch root has a cgroup.procs marker");
15955 let module = placement
15956 .module_path("surviving-module")
15957 .expect("create scratch module cgroup");
15958 fs::write(module.join("surviving-process"), b"still present")
15959 .expect("make scratch cgroup non-empty");
15960 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15961
15962 remove_module_cgroup(&placement, "surviving-module");
15963
15964 let logs = crate::router::test_log::captured_logs(&logs);
15965 assert!(
15966 module.exists(),
15967 "failed removal must leave the cgroup intact"
15968 );
15969 assert!(
15970 logs.contains("could not remove module cgroup after process exit; continuing teardown")
15971 && logs.contains("surviving-module"),
15972 "best-effort removal must report the failure without returning it: {logs}"
15973 );
15974 }
15975
15976 #[test]
15977 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15978 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15979 let reason = SuperviseError::Spawn {
15980 program: PathBuf::from("/bin/true"),
15981 source: io::Error::from_raw_os_error(13),
15982 cgroup_path: Some(cgroup_path.clone()),
15983 }
15984 .to_string();
15985
15986 assert!(
15987 reason.contains(&cgroup_path.display().to_string()),
15988 "a pre_exec spawn failure must name the cgroup path: {reason}"
15989 );
15990 }
15991}
15992
15993#[cfg(test)]
15994mod spawn_subscriber_lag_tests {
15995 use super::*;
15996
15997 #[tokio::test]
16002 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
16003 let feed = SpawnEventFeed::default();
16004 feed.configure_incarnation("lag-incarnation".to_string());
16005 let (tx, mut rx) = mpsc::channel(1);
16008 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
16009 .expect("subscribe");
16010 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
16011 for index in 0..emitted {
16012 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
16013 tokio::task::yield_now().await;
16016 }
16017 assert_eq!(
16018 feed.subscriber_count(),
16019 0,
16020 "the lagged subscriber must be removed"
16021 );
16022
16023 let mut data = Vec::new();
16024 let mut last = None;
16025 loop {
16026 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16027 .await
16028 .expect("the forwarder must finish once the subscriber is dropped");
16029 let Some(outbound) = next else { break };
16030 let frame = outbound.frame;
16031 if frame.header.ty == FrameType::StreamData {
16032 assert!(last.is_none(), "no data may follow the terminal frame");
16033 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
16034 data.push(event.cursor.seq);
16035 } else {
16036 assert!(last.is_none(), "exactly one terminal frame");
16037 last = Some(frame);
16038 }
16039 }
16040 assert!(!data.is_empty(), "queued frames drain before the terminal");
16041 for pair in data.windows(2) {
16042 assert_eq!(
16043 pair[1],
16044 pair[0] + 1,
16045 "queued frames arrive dense and in order"
16046 );
16047 }
16048 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
16049 assert_eq!(terminal.header.ty, FrameType::Error);
16050 assert_eq!(terminal.header.corr, 7);
16051 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
16052 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
16053 let detail = body.detail.expect("lagged error carries detail");
16054 assert_eq!(
16055 detail["first_undelivered_cursor"]["seq"],
16056 data.last().unwrap() + 1,
16057 "the named cursor is the first event the subscriber did not receive"
16058 );
16059 assert_eq!(
16060 detail["first_undelivered_cursor"]["daemon_incarnation"],
16061 "lag-incarnation"
16062 );
16063 }
16064}
16065
16066#[cfg(test)]
16067mod terminal_history_read_concurrency_tests {
16068 use super::*;
16069 use crate::terminal_journal::read_pause;
16070 use cortexkit_test_support::ScratchDir;
16071 use std::sync::mpsc as std_mpsc;
16072
16073 fn journaled_ring(
16074 journal: &Arc<crate::terminal_journal::TerminalJournal>,
16075 ) -> Arc<Mutex<TerminalRing>> {
16076 Arc::new(Mutex::new(
16077 TerminalRing::new(TerminalRingConfig::default(), 1)
16078 .with_journal(Some(Arc::clone(journal))),
16079 ))
16080 }
16081
16082 fn crash(at_ms: u64) -> ExitReport {
16083 ExitReport {
16084 kind: ExitKind::Crash,
16085 code: Some(1),
16086 signal: None,
16087 at_ms,
16088 }
16089 }
16090
16091 fn record_within(
16094 module_id: &'static str,
16095 ring: &Arc<Mutex<TerminalRing>>,
16096 at_ms: u64,
16097 bound: Duration,
16098 ) -> bool {
16099 let ring = Arc::clone(ring);
16100 let (done, done_rx) = std_mpsc::channel();
16101 std::thread::spawn(move || {
16102 record_terminal(
16103 module_id,
16104 &ring,
16105 &SpawnEventFeed::default(),
16106 &crash(at_ms),
16107 TerminalDisposition::Restarting,
16108 );
16109 let _ = done.send(());
16110 });
16111 done_rx.recv_timeout(bound).is_ok()
16112 }
16113
16114 #[test]
16119 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
16120 let dir = ScratchDir::new("terminal-history-concurrent-read");
16121 let path = dir.join("terminals.jsonl");
16122 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
16123 path.clone(),
16124 "daemon".into(),
16125 ));
16126 let reader_ring = journaled_ring(&journal);
16127 let other_ring = journaled_ring(&journal);
16128 assert!(record_within(
16129 "reader-module",
16130 &reader_ring,
16131 10,
16132 Duration::from_secs(5)
16133 ));
16134
16135 let (started, release) = read_pause::install(&path);
16136 let reading = {
16137 let ring = Arc::clone(&reader_ring);
16138 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
16139 };
16140 started
16141 .recv_timeout(Duration::from_secs(5))
16142 .expect("the history read reached its pause");
16143
16144 let bound = Duration::from_secs(1);
16145 assert!(
16146 record_within("other-module", &other_ring, 20, bound),
16147 "another module's exit waited on a history read (journal writer held)"
16148 );
16149 assert!(
16150 record_within("reader-module", &reader_ring, 30, bound),
16151 "the read module's own exit waited on its history read (ring held)"
16152 );
16153
16154 drop(release);
16155 let paused = reading.join().unwrap();
16156 assert_eq!(
16157 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16158 vec![10],
16159 "an exit recorded after the read began lands in neither half of it"
16160 );
16161 assert_eq!(paused.journal_skipped_lines, 0);
16162 assert_eq!(paused.journal_read_errors, 0);
16163
16164 let after = durable_terminal_history_of(&reader_ring, "reader-module");
16165 assert_eq!(
16166 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16167 vec![10, 30],
16168 "the next read merges ring and journal with no duplicate"
16169 );
16170 assert_eq!(after.journal_skipped_lines, 0);
16171 }
16172}
16173
16174#[cfg(test)]
16179mod stderr_settle_tests {
16180 use std::{
16181 future::Future,
16182 io,
16183 pin::Pin,
16184 sync::{Arc, Mutex},
16185 task::{Context, Poll},
16186 time::Duration,
16187 };
16188
16189 use tokio::{
16190 io::{AsyncRead, ReadBuf},
16191 sync::oneshot,
16192 time::Instant,
16193 };
16194
16195 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
16196 use crate::stderr_tail::{
16197 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
16198 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
16199 };
16200
16201 const BOUND: Duration = Duration::from_millis(250);
16202
16203 struct HeldReader {
16207 before: Option<Vec<u8>>,
16208 gate: Option<oneshot::Receiver<()>>,
16209 after: io::Cursor<Vec<u8>>,
16210 }
16211
16212 impl AsyncRead for HeldReader {
16213 fn poll_read(
16214 mut self: Pin<&mut Self>,
16215 cx: &mut Context<'_>,
16216 buf: &mut ReadBuf<'_>,
16217 ) -> Poll<io::Result<()>> {
16218 if let Some(bytes) = self.before.take() {
16219 buf.put_slice(&bytes);
16220 return Poll::Ready(Ok(()));
16221 }
16222 if let Some(gate) = self.gate.as_mut() {
16223 match Pin::new(gate).poll(cx) {
16224 Poll::Pending => return Poll::Pending,
16225 Poll::Ready(_) => self.gate = None,
16226 }
16227 }
16228 Pin::new(&mut self.after).poll_read(cx, buf)
16229 }
16230 }
16231
16232 struct DiscardSink;
16233
16234 impl OutputSink for DiscardSink {
16235 fn write_line(&mut self, _line: &[u8]) {}
16236 }
16237
16238 fn line(text: &str) -> TailEntry {
16239 TailEntry::Line {
16240 text: text.to_string(),
16241 truncated: false,
16242 at_ms: None,
16243 }
16244 }
16245
16246 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16247 ring.lock().unwrap()
16248 }
16249
16250 fn held_pump(
16254 ring: &Arc<Mutex<StderrRing>>,
16255 before: &str,
16256 after: &str,
16257 ) -> (StderrPump, oneshot::Sender<()>) {
16258 let generation = lock(ring).begin_process();
16259 let (release, gate) = oneshot::channel();
16260 let reader = HeldReader {
16261 before: Some(before.as_bytes().to_vec()),
16262 gate: Some(gate),
16263 after: io::Cursor::new(after.as_bytes().to_vec()),
16264 };
16265 let task = tokio::spawn(pump_stderr_to(
16266 reader,
16267 Arc::clone(ring),
16268 generation,
16269 DiscardSink,
16270 ));
16271 (StderrPump { task, generation }, release)
16272 }
16273
16274 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16275 for _ in 0..1000 {
16276 if done(&lock(ring)) {
16277 return;
16278 }
16279 tokio::time::sleep(Duration::from_millis(1)).await;
16280 }
16281 panic!(
16282 "ring never reached the expected state: {:?}",
16283 lock(ring).snapshot(None, None)
16284 );
16285 }
16286
16287 #[tokio::test(start_paused = true)]
16288 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16289 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16290 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16291
16292 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16293 let before_release = lock(&ring).snapshot(None, None);
16294 assert!(
16295 matches!(before_release.capture, CaptureState::Incomplete { .. }),
16296 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16297 );
16298
16299 let next = lock(&ring).begin_process();
16302 lock(&ring).push_line_from(next, "next process booting");
16303 release.send(()).unwrap();
16304 wait_until(&ring, |ring| {
16305 ring.snapshot(None, None).capture == CaptureState::Captured
16306 })
16307 .await;
16308
16309 assert_eq!(
16310 untimed(lock(&ring).snapshot(None, None).entries),
16311 vec![
16312 line("booting"),
16313 line("config error: missing storage"),
16314 TailEntry::ProcessStart,
16315 line("next process booting"),
16316 ],
16317 "the crash's last line must survive a slow reader and stay in the crashed process's section"
16318 );
16319 }
16320
16321 #[tokio::test(start_paused = true)]
16322 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16323 ) {
16324 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16325 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16328
16329 let started = Instant::now();
16330 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16331 assert_eq!(
16332 started.elapsed(),
16333 BOUND,
16334 "the restart must wait exactly the bound for a pipe that stays open, no longer"
16335 );
16336
16337 let next = lock(&ring).begin_process();
16338 lock(&ring).push_line_from(next, "next process booting");
16339 tokio::time::sleep(Duration::from_secs(60)).await;
16340
16341 let snapshot = lock(&ring).snapshot(None, None);
16342 match &snapshot.capture {
16343 CaptureState::Incomplete { reason } => assert!(
16344 reason.contains("had not reached EOF") && reason.contains("250ms"),
16345 "the reason must say what is missing and after how long: {reason}"
16346 ),
16347 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16348 }
16349 assert_eq!(
16350 untimed(snapshot.entries),
16351 vec![
16352 line("parent exiting"),
16353 TailEntry::ProcessStart,
16354 line("next process booting"),
16355 ]
16356 );
16357 }
16358
16359 #[tokio::test(start_paused = true)]
16360 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16361 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16362 let (pump, release) = held_pump(&ring, "one\n", "two\n");
16363 release.send(()).unwrap();
16364
16365 settle_stderr_pump("clean", &ring, pump, BOUND).await;
16366
16367 let snapshot = lock(&ring).snapshot(None, None);
16368 assert_eq!(snapshot.capture, CaptureState::Captured);
16369 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16370 }
16371
16372 #[tokio::test(start_paused = true)]
16373 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16374 const BURST: usize = 150;
16375 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16376 let path = capture.join("two-pipe.stderr.log");
16377 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16378 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16379 let generation = lock(&ring).begin_process();
16380 let stdout = (0..BURST)
16381 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16382 .collect::<String>();
16383 let stderr = (0..BURST)
16384 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16385 .collect::<String>();
16386 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
16391 let (release, gate) = oneshot::channel();
16392 let reader = HeldReader {
16393 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
16394 gate: Some(gate),
16395 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
16396 };
16397 let (finished, completion) = oneshot::channel();
16398 let stdout_sink = sink.clone();
16399 let pump = tokio::spawn(async move {
16400 pump_stdout_to(reader, stdout_sink).await;
16401 finished.send(()).unwrap();
16402 });
16403 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
16404
16405 let started = Instant::now();
16406 settle_stdout_pump("two-pipe", pump, BOUND).await;
16407 assert_eq!(
16408 started.elapsed(),
16409 BOUND,
16410 "a late reader must not hold up restart"
16411 );
16412 assert_eq!(
16413 std::fs::read_to_string(&path).unwrap().lines().count(),
16414 BURST + 1
16415 );
16416
16417 release
16418 .send(())
16419 .expect("stdout reader must survive the exit drain bound");
16420 tokio::time::timeout(Duration::from_secs(1), completion)
16421 .await
16422 .expect("released stdout reader must reach EOF")
16423 .expect("stdout capture must finish at EOF, not be cancelled");
16424 let contents = std::fs::read_to_string(&path).unwrap();
16425 let mut lines = contents
16426 .lines()
16427 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
16428 .collect::<Vec<_>>();
16429 let mut expected = (0..BURST)
16430 .flat_map(|i| {
16431 [
16432 format!("out-{i:04}-{}", "o".repeat(64)),
16433 format!("err-{i:04}-{}", "e".repeat(64)),
16434 ]
16435 })
16436 .collect::<Vec<_>>();
16437 lines.sort_unstable();
16438 expected.sort_unstable();
16439 assert_eq!(
16440 lines, expected,
16441 "both pipes must retain every complete line exactly once"
16442 );
16443 }
16444}
16445
16446#[cfg(all(test, windows))]
16460mod job_containment_tests {
16461 use super::*;
16462 use cortexkit_test_support::ScratchDir;
16463 use std::{
16464 path::{Path, PathBuf},
16465 sync::{Arc, Mutex},
16466 time::{Duration, Instant},
16467 };
16468
16469 fn stub_path() -> PathBuf {
16475 let mut path = std::env::current_exe().expect("current_exe available in tests");
16476 path.pop();
16477 path.pop();
16478 path.push("fake-aft-stub.exe");
16479 assert!(
16480 path.exists(),
16481 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16482 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16483 path.display()
16484 );
16485 path
16486 }
16487
16488 fn read_grandchild_pid(path: &Path) -> u32 {
16490 let deadline = Instant::now() + Duration::from_secs(10);
16491 loop {
16492 if let Ok(contents) = std::fs::read_to_string(path) {
16493 if let Ok(pid) = contents.trim().parse() {
16494 return pid;
16495 }
16496 }
16497 assert!(
16498 Instant::now() < deadline,
16499 "the stub never recorded a grandchild pid at {}",
16500 path.display()
16501 );
16502 std::thread::sleep(Duration::from_millis(10));
16503 }
16504 }
16505
16506 struct Fixture {
16509 _dir: ScratchDir,
16510 module_id: String,
16511 grandchild: u32,
16512 child: Option<SupervisedChild>,
16513 registry: Arc<Registry>,
16514 snapshot: Arc<Mutex<SupervisorSnapshot>>,
16515 terminal_ring: Arc<Mutex<TerminalRing>>,
16516 spawn_events: SpawnEventFeed,
16517 }
16518
16519 fn fixture(label: &str, module_id: &str) -> Fixture {
16520 let dir = ScratchDir::new(label);
16521 let pid_file = dir.join("grandchild.pid");
16522 let supervisor = Supervisor::new_for_test(
16523 Arc::new(Registry::default()),
16524 RestartPolicy::new(3, Duration::ZERO),
16525 );
16526 let runtime = supervisor.runtime_config();
16527 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16528 let spec = ModuleSpec {
16529 module_id: module_id.to_string(),
16530 program: stub_path(),
16531 args: Vec::new(),
16535 env: vec![
16536 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16537 (
16538 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16539 pid_file.display().to_string(),
16540 ),
16541 ],
16542 reserved: false,
16543 reserved_prefixes: Vec::new(),
16544 protocol: ModuleProtocol::Subc,
16545 overlap: Default::default(),
16546 };
16547 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16548 .expect("spawn the supervised fixture");
16549 let grandchild = read_grandchild_pid(&pid_file);
16550 Fixture {
16551 _dir: dir,
16552 module_id: module_id.to_string(),
16553 grandchild,
16554 child: Some(child),
16555 registry: Arc::new(Registry::default()),
16556 snapshot,
16557 terminal_ring: Arc::clone(&runtime.terminal_ring),
16558 spawn_events: SpawnEventFeed::default(),
16559 }
16560 }
16561
16562 impl Fixture {
16563 async fn drain(&mut self) {
16565 let child = self
16566 .child
16567 .take()
16568 .expect("the fixture child is still present");
16569 drain_child_to_state(
16570 &self.module_id,
16571 ModuleProtocol::Subc,
16572 StopNotice::NotSent,
16575 &self.registry,
16576 None,
16577 &self.snapshot,
16578 &self.terminal_ring,
16579 &self.spawn_events,
16580 child,
16581 Duration::from_millis(500),
16582 ModuleState::Stopped,
16583 Some(false),
16584 &FailureContext::operator(),
16585 )
16586 .await
16587 .expect("drain the supervised fixture");
16588 }
16589 }
16590
16591 #[tokio::test]
16597 async fn teardown_reaps_the_grandchild() {
16598 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16599 let grandchild = fixture.grandchild;
16600
16601 assert!(
16602 subc_jobobject::process_exists(grandchild),
16603 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16604 );
16605
16606 fixture.drain().await;
16607
16608 assert!(
16609 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16610 "grandchild {grandchild} outlived module teardown: the tree was not contained"
16611 );
16612 }
16613
16614 #[test]
16627 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16628 let dir = ScratchDir::new("teardown-uncontained");
16629 let pid_file = dir.join("grandchild.pid");
16630 let mut child = std::process::Command::new(stub_path())
16631 .env("FAKE_AFT_NEVER_CONNECT", "1")
16632 .env(
16633 "FAKE_AFT_GRANDCHILD_PID_FILE",
16634 pid_file.display().to_string(),
16635 )
16636 .stdin(std::process::Stdio::null())
16637 .stdout(std::process::Stdio::null())
16638 .stderr(std::process::Stdio::null())
16639 .spawn()
16640 .expect("spawn the uncontained fixture");
16641 let grandchild = read_grandchild_pid(&pid_file);
16642
16643 child.kill().expect("kill the direct child");
16645 let _ = child.wait();
16646
16647 assert!(
16648 subc_jobobject::process_exists(grandchild),
16649 "grandchild {grandchild} died with the direct child, so this control no longer \
16650 distinguishes contained from uncontained teardown and the regression test is \
16651 passing vacuously"
16652 );
16653
16654 kill_tree(grandchild);
16657 }
16658
16659 #[tokio::test]
16668 async fn dropping_containment_reaps_the_grandchild() {
16669 let mut fixture = fixture("drop-containment", "tree-drop");
16670 let grandchild = fixture.grandchild;
16671
16672 assert!(subc_jobobject::process_exists(grandchild));
16673
16674 fixture.child.as_mut().expect("child present").job = None;
16676
16677 assert!(
16678 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16679 "grandchild {grandchild} survived the containment handle closing, so a daemon \
16680 crash would leave the tree behind"
16681 );
16682 }
16683
16684 fn kill_tree(pid: u32) {
16686 let _ = std::process::Command::new("taskkill.exe")
16687 .args(["/PID", &pid.to_string(), "/T", "/F"])
16688 .stdin(std::process::Stdio::null())
16689 .stdout(std::process::Stdio::null())
16690 .stderr(std::process::Stdio::null())
16691 .status();
16692 assert!(
16693 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16694 "could not clean up grandchild {pid}"
16695 );
16696 }
16697}
16698
16699#[cfg(test)]
16700mod privacy_trampoline_configuration_tests {
16701 #[tokio::test]
16702 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16703 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16704 #[cfg(target_os = "macos")]
16705 {
16706 let supervisor = super::Supervisor::new(
16707 std::sync::Arc::new(crate::Registry::default()),
16708 super::RestartPolicy::default(),
16709 );
16710 let error = supervisor.spawn(spec()).unwrap_err();
16711 assert!(
16712 error
16713 .to_string()
16714 .contains("no privacy trampoline configured"),
16715 "{error}"
16716 );
16717 }
16718 }
16719
16720 #[tokio::test]
16721 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16722 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16723 #[cfg(target_os = "macos")]
16724 {
16725 let supervisor = super::Supervisor::new(
16726 std::sync::Arc::new(crate::Registry::default()),
16727 super::RestartPolicy::default(),
16728 )
16729 .with_privacy_trampoline(std::env::current_exe().unwrap());
16730 let error = supervisor.spawn(spec()).unwrap_err();
16731 assert!(
16732 error
16733 .to_string()
16734 .contains("binary does not implement the privacy trampoline protocol"),
16735 "{error}"
16736 );
16737 }
16738 }
16739
16740 #[cfg(target_os = "macos")]
16741 fn spec() -> super::ModuleSpec {
16742 super::ModuleSpec {
16743 module_id: "privacy-configuration".into(),
16744 program: "/bin/sleep".into(),
16745 args: vec!["30".into()],
16746 env: vec![],
16747 reserved: false,
16748 reserved_prefixes: vec![],
16749 protocol: subc_control::ModuleProtocol::None,
16750 overlap: super::ModuleOverlap::Exclusive,
16751 }
16752 }
16753}
16754
16755#[cfg(test)]
16756mod privacy_exec_boundary_tests {
16757 #[cfg(target_os = "macos")]
16758 use super::*;
16759 #[cfg(target_os = "macos")]
16760 use std::{
16761 io::{Read, Write},
16762 net::{TcpListener, TcpStream},
16763 };
16764
16765 #[cfg(target_os = "macos")]
16768 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16769 if let Some((_, path)) = spec
16770 .env
16771 .iter()
16772 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16773 {
16774 let mut barrier = TcpStream::connect(path).unwrap();
16775 barrier
16776 .set_read_timeout(Some(Duration::from_secs(30)))
16777 .unwrap();
16778 barrier.write_all(&pid.to_ne_bytes()).unwrap();
16779 let mut release = [0];
16780 barrier.read_exact(&mut release).unwrap();
16781 assert_eq!(&release, b"X");
16782 }
16783 }
16784
16785 #[cfg(target_os = "macos")]
16786 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16787 ModuleSpec {
16788 module_id: "privacy-boundary".into(),
16789 program: program.into(),
16790 args: args.iter().map(|arg| (*arg).into()).collect(),
16791 env: vec![],
16792 reserved: false,
16793 reserved_prefixes: vec![],
16794 protocol: ModuleProtocol::None,
16795 overlap: ModuleOverlap::Exclusive,
16796 }
16797 }
16798
16799 #[cfg(target_os = "macos")]
16800 async fn accept(listener: TcpListener) -> TcpStream {
16801 let listener = tokio::net::TcpListener::from_std({
16803 listener.set_nonblocking(true).unwrap();
16804 listener
16805 })
16806 .unwrap();
16807 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16808 .await
16809 .unwrap()
16810 .unwrap();
16811 let stream = stream.into_std().unwrap();
16812 stream.set_nonblocking(false).unwrap();
16813 stream
16814 .set_read_timeout(Some(Duration::from_secs(30)))
16815 .unwrap();
16816 stream
16817 }
16818
16819 #[tokio::test]
16820 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16821 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16822 #[cfg(target_os = "macos")]
16823 {
16824 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16825 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16828 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16829 let record = root.join("live-children.json");
16830 let supervisor =
16831 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16832 .with_live_children_record(&record);
16833 let runtime = supervisor.runtime_config();
16834 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16835 let mut spec = spec("/bin/sleep", &["30"]);
16836 spec.env = vec![
16837 (
16838 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16839 exec_listener.local_addr().unwrap().to_string(),
16840 ),
16841 (
16842 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16843 sample_listener.local_addr().unwrap().to_string(),
16844 ),
16845 ];
16846 let spawn = tokio::task::spawn_blocking(move || {
16847 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16848 });
16849 let mut sample = accept(sample_listener).await;
16850 let mut pid = [0; 4];
16851 sample.read_exact(&mut pid).unwrap();
16852 let pid = u32::from_ne_bytes(pid);
16853 let mut exec = accept(exec_listener).await;
16854 let mut ready = [0];
16855 exec.read_exact(&mut ready).unwrap();
16856 assert_eq!(&ready, b"R");
16857 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16860 assert_eq!(
16861 observe_spawned_image(pid).unwrap().executable,
16862 Some(trampoline)
16863 );
16864 sample.write_all(b"X").unwrap();
16865 let mut child = spawn.await.unwrap();
16866 let early = crate::live_children::read_record(&record).unwrap();
16867 assert_eq!(early.len(), 1);
16868 assert_eq!(early[0].pid, pid);
16869 assert_eq!(
16870 early[0].executable, None,
16871 "unconfirmed trampoline image entered the roster"
16872 );
16873 assert!(child.report_ready.get().is_none());
16874 child.privacy_exec.as_mut().unwrap().deadline =
16877 tokio::time::Instant::now() + Duration::from_secs(30);
16878 exec.write_all(b"X").unwrap();
16879 child.confirm_privacy_exec().await;
16880 assert_eq!(child.spawn_failure, None);
16881 assert!(child.report_ready.get().is_some());
16882 let confirmed = crate::live_children::read_record(&record).unwrap();
16883 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16884 assert_ne!(module, trampoline);
16885 assert_eq!(confirmed[0].executable, Some(module.into()));
16886 child.start_kill().unwrap();
16887 child.wait().await.unwrap();
16888 child.release_roster();
16889 }
16890 }
16891
16892 #[tokio::test]
16893 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16894 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16895 #[cfg(target_os = "macos")]
16896 {
16897 let registry = Arc::new(Registry::default());
16898 let policy = RestartPolicy::new(0, Duration::ZERO);
16899 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
16900 let runtime = supervisor.runtime_config();
16901 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16902 let spec = spec("/bin/sh", &["-c", "exit 121"]);
16903 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16907 let pid = child.pid;
16908 tokio::task::spawn_blocking(move || {
16909 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16910 })
16911 .await
16912 .unwrap()
16913 .unwrap();
16914 child.privacy_exec.as_mut().unwrap().deadline =
16915 tokio::time::Instant::now() + Duration::from_secs(30);
16916 let status = child.wait().await.unwrap();
16917 assert_eq!(status.code(), Some(121));
16918 assert!(child.privacy_exec.is_none());
16919 assert!(
16920 child.report_ready.get().is_none(),
16921 "an exited module must not publish a live pid"
16922 );
16923 let report = classify_reaped_child_exit(&snapshot, &child, &status);
16924 on_child_exit(
16925 &spec,
16926 policy,
16927 ®istry,
16928 &snapshot,
16929 &runtime.terminal_ring,
16930 &runtime.spawn_events,
16931 &runtime.child_roster,
16932 report,
16933 true,
16934 )
16935 .await;
16936 let state = lock_snapshot(&snapshot).unwrap();
16937 assert_eq!(state.state, ModuleState::Failed);
16938 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16939 assert_eq!(state.reported_pid(), None);
16940 drop(state);
16941 let history = runtime.terminal_ring.lock().unwrap().snapshot();
16942 assert_eq!(history.entries.len(), 1);
16943 let terminal = &history.entries[0];
16944 assert_eq!(terminal.exit_code, Some(121));
16945 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16946 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16947 assert_eq!(
16948 terminal.disposition_detail.as_deref(),
16949 Some(policy.budget_exhausted_detail().as_str()),
16950 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16951 );
16952 assert_eq!(child.spawn_failure, None);
16953 child.release_roster();
16954 }
16955 }
16956}
16957
16958#[cfg(all(test, unix))]
16962mod launch_nonce_descriptor_tests {
16963 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16964 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16965 use cortexkit_test_support::ScratchDir;
16966 use std::{
16967 path::PathBuf,
16968 sync::{Arc, Mutex},
16969 time::{Duration, Instant},
16970 };
16971
16972 async fn probe(role: super::SpawnRole) {
16973 let scratch = ScratchDir::new("launch-nonce-descriptor");
16974 let fd_copy = scratch.join("from-descriptor");
16975 let env_copy = scratch.join("environment");
16976 let script = format!(
16977 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16978 fd = fd_copy.display(), env = env_copy.display(),
16979 );
16980 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16981 let spec = ModuleSpec {
16982 module_id: "nonce-descriptor-probe".to_string(),
16983 program: PathBuf::from("/bin/sh"),
16984 args: vec!["-c".to_string(), script],
16985 env: vec![
16986 xdg("XDG_DATA_HOME"),
16987 xdg("XDG_RUNTIME_DIR"),
16988 xdg("XDG_CONFIG_HOME"),
16989 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16990 ],
16991 reserved: true,
16992 reserved_prefixes: Vec::new(),
16993 protocol: ModuleProtocol::Subc,
16994 overlap: Default::default(),
16995 };
16996 let handle = SupervisorHandle::new();
16997 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16998 let roster = ChildRoster::default();
16999 #[cfg(target_os = "macos")]
17000 {
17001 let path = super::test_privacy_trampoline();
17002 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
17003 }
17004 let child = super::spawn_child_in_slot(
17005 &spec,
17006 None,
17007 Some(&handle),
17008 &ring,
17009 None,
17010 &roster,
17011 #[cfg(target_os = "linux")]
17012 None,
17013 role,
17014 matches!(role, super::SpawnRole::SwapCandidate),
17015 )
17016 .expect("spawn probe");
17017 let deadline = Instant::now() + Duration::from_secs(10);
17018 while !(fd_copy.exists() && env_copy.exists()) {
17019 assert!(Instant::now() < deadline, "probe never wrote its copies");
17020 tokio::time::sleep(Duration::from_millis(20)).await;
17021 }
17022 let nonce = std::fs::read_to_string(fd_copy).unwrap();
17023 assert!(!nonce.is_empty());
17024 let environment = std::fs::read_to_string(env_copy).unwrap();
17025 assert!(environment
17026 .lines()
17027 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
17028 let copy = environment
17029 .lines()
17030 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
17031 assert_eq!(
17032 copy, None,
17033 "Unix children must never receive the environment nonce"
17034 );
17035 if matches!(role, super::SpawnRole::Plain) {
17036 assert_eq!(
17037 handle.spawn_nonce(&spec.module_id).as_deref(),
17038 Some(nonce.as_str())
17039 );
17040 }
17041 drop(child);
17042 }
17043
17044 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17045 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
17046 probe(super::SpawnRole::Plain).await;
17047 }
17048
17049 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17050 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
17051 probe(super::SpawnRole::SwapCandidate).await;
17052 }
17053}
17054
17055#[cfg(all(test, target_os = "linux"))]
17056mod cgroup_containment_tests {
17057 use super::*;
17058 use cortexkit_test_support::ScratchDir;
17059
17060 fn running(pid: u32) -> bool {
17061 std::fs::read_to_string(format!("/proc/{pid}/stat"))
17063 .ok()
17064 .and_then(|stat| {
17065 stat.rsplit_once(") ")
17066 .map(|(_, rest)| rest.starts_with('Z'))
17067 })
17068 .is_some_and(|zombie| !zombie)
17069 }
17070
17071 #[tokio::test]
17072 async fn linux_teardown_reaps_the_grandchild() {
17073 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
17074 }
17075
17076 #[tokio::test]
17077 async fn linux_shutdown_straggler_reaps_the_grandchild() {
17078 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
17079 }
17080
17081 async fn teardown_tree(test_name: &str, shutdown: bool) {
17082 let dir = ScratchDir::new(test_name);
17083 let root = PathBuf::from(format!(
17084 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
17085 std::process::id(),
17086 unix_ms_now()
17087 ));
17088 if let Err(error) = std::fs::create_dir(&root) {
17089 assert!(
17090 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17091 "required cgroup test cannot execute: {error}"
17092 );
17093 eprintln!(
17094 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
17095 root.display()
17096 );
17097 return;
17098 }
17099 let placement = subc_cgroup::prepare_at(&root)
17100 .expect("prepare isolated kernel cgroup")
17101 .expect("isolated cgroup is delegated");
17102 let module_id = "tree-teardown";
17103 let module = placement
17104 .module_path(module_id)
17105 .expect("create isolated module cgroup");
17106 if !module.join("cgroup.kill").exists() {
17107 std::fs::remove_dir(&module).unwrap();
17108 std::fs::remove_dir(root.join("subc-modules")).unwrap();
17109 std::fs::remove_dir(&root).unwrap();
17110 assert!(
17111 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17112 "required cgroup.kill interface unavailable"
17113 );
17114 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
17115 return;
17116 }
17117 let supervisor = Supervisor::new_for_test(
17118 Arc::new(Registry::default()),
17119 RestartPolicy::new(3, Duration::ZERO),
17120 )
17121 .with_cgroup_placement(Some(placement));
17122 let mut runtime = supervisor.runtime_config();
17123 runtime.child_roster = runtime
17124 .child_roster
17125 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
17126 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17127 let pid_file = dir.join("grandchild.pid");
17128 let spec = ModuleSpec {
17129 module_id: module_id.to_string(),
17130 program: PathBuf::from("/bin/sh"),
17131 args: vec![
17132 "-c".into(),
17133 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
17134 "fixture".into(),
17135 pid_file.display().to_string(),
17136 ],
17137 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17138 .into_iter()
17139 .map(|key| (key.to_string(), dir.display().to_string()))
17140 .collect(),
17141 reserved: false,
17142 reserved_prefixes: Vec::new(),
17143 protocol: ModuleProtocol::None,
17144 overlap: Default::default(),
17145 };
17146 let child =
17147 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
17148 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17149 let grandchild: u32 = loop {
17150 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
17151 if let Ok(pid) = contents.trim().parse() {
17152 break pid;
17153 }
17154 }
17155 assert!(
17156 tokio::time::Instant::now() < deadline,
17157 "grandchild pid was not recorded"
17158 );
17159 tokio::time::sleep(Duration::from_millis(10)).await;
17160 };
17161 assert!(
17162 running(grandchild),
17163 "grandchild must be alive before teardown"
17164 );
17165 if shutdown {
17166 let mut child = child;
17167 crate::child_roster::end_children_for_daemon_shutdown(
17168 &runtime.child_roster,
17169 false,
17170 std::future::pending(),
17171 )
17172 .await;
17173 child.wait().await.expect("reap shutdown straggler");
17174 } else {
17175 drain_child_to_state(
17176 module_id,
17177 ModuleProtocol::None,
17178 StopNotice::NotSent,
17179 &Registry::default(),
17180 None,
17181 &snapshot,
17182 &runtime.terminal_ring,
17183 &SpawnEventFeed::default(),
17184 child,
17185 Duration::from_millis(100),
17186 ModuleState::Stopped,
17187 Some(false),
17188 &FailureContext::operator(),
17189 )
17190 .await
17191 .expect("real supervisor teardown");
17192 }
17193 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
17194 while running(grandchild) && tokio::time::Instant::now() < deadline {
17195 tokio::time::sleep(Duration::from_millis(10)).await;
17196 }
17197 let survived = running(grandchild);
17198 if survived {
17200 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
17201 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
17202 tokio::time::sleep(Duration::from_millis(100)).await;
17203 }
17204 if module.exists() {
17205 std::fs::remove_dir(&module).expect("remove empty module cgroup");
17206 }
17207 std::fs::remove_dir(root.join("subc-modules")).unwrap();
17208 std::fs::remove_dir(&root).unwrap();
17209 assert!(
17210 !survived,
17211 "grandchild {grandchild} outlived module teardown"
17212 );
17213 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
17214 }
17215}
17216
17217#[cfg(all(test, unix))]
17220mod record_write_stall_tests {
17221 use super::*;
17222 use std::sync::atomic::{AtomicBool, Ordering};
17223 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
17224
17225 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
17228 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
17232
17233 fn sleeping_spec() -> ModuleSpec {
17245 ModuleSpec {
17246 module_id: "record-write-stall".into(),
17247 program: "/bin/sleep".into(),
17248 args: vec!["600".into()],
17249 env: vec![],
17250 reserved: false,
17251 reserved_prefixes: vec![],
17252 protocol: ModuleProtocol::None,
17253 overlap: Default::default(),
17254 }
17255 }
17256
17257 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17258 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17259 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17260 let supervisor =
17261 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17262 .with_live_children_record(home.join("live-children.json"));
17263 let module = supervisor.spawn(sleeping_spec()).unwrap();
17264 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17269 loop {
17270 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17271 .unwrap()
17272 .reported_pid()
17273 {
17274 return pid;
17275 }
17276 tokio::time::sleep(Duration::from_millis(10)).await;
17277 }
17278 })
17279 .await
17280 .expect("the first process was admitted as the module");
17281
17282 let key = vec![0x42; 32];
17285 let daemon_id = [0x24; 16];
17286 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17287 let port = listener.local_addr().unwrap().port();
17288 let server = tokio::spawn(crate::server::serve_listener(
17289 listener,
17290 Arc::new(crate::router::Router::with_default_self_handler()),
17291 crate::server::ServerAuth::with_limits(
17292 key.clone(),
17293 daemon_id,
17294 "test",
17295 crate::server::DEFAULT_AUTH_DEADLINE,
17296 4,
17297 ),
17298 ));
17299
17300 let (held_tx, held_rx) = std::sync::mpsc::channel();
17303 let armed = AtomicBool::new(true);
17304 supervisor
17305 .child_roster
17306 .set_record_write_hook_for_test(move |live| {
17307 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17308 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17309 let _ = held_tx.send(());
17310 std::thread::sleep(HELD_RECORD_WRITE);
17311 }
17312 });
17313
17314 let module_process = std::thread::spawn(move || {
17317 held_rx
17318 .recv_timeout(Duration::from_secs(30))
17319 .expect("the restart reached the replacement's record write");
17320 let conn = ConnectionInfo {
17321 schema: SCHEMA_VERSION,
17322 wire_version: None,
17323 endpoints: vec![Endpoint {
17324 host: "127.0.0.1".to_owned(),
17325 port,
17326 }],
17327 key,
17328 daemon_id,
17329 pid: std::process::id(),
17330 daemon_ver: "test".to_owned(),
17331 };
17332 tokio::runtime::Builder::new_current_thread()
17333 .enable_all()
17334 .build()
17335 .unwrap()
17336 .block_on(async {
17337 let started = Instant::now();
17338 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17339 .await
17340 .unwrap();
17341 let result =
17342 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17343 (result, started.elapsed())
17344 })
17345 });
17346
17347 module.restart(Some(0)).await.unwrap();
17348 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17349 .await
17350 .unwrap();
17351 assert!(
17352 auth.is_ok(),
17353 "the daemon must answer a module's handshake while a restart waits on the \
17354 live-children record write; got {auth:?} after {elapsed:?}"
17355 );
17356
17357 tokio::time::timeout(Duration::from_secs(20), async {
17358 while lock_snapshot(&module.inner.snapshot)
17359 .unwrap()
17360 .reported_pid()
17361 .is_none_or(|pid| pid == old_pid)
17362 {
17363 tokio::time::sleep(Duration::from_millis(10)).await;
17364 }
17365 })
17366 .await
17367 .expect("the restart published its replacement");
17368 module.stop().await.unwrap();
17369 server.abort();
17370 }
17371}