1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62mod slow_retry;
63use slow_retry::{FailedRetry, FailureContext};
64
65#[cfg(all(test, unix))]
66mod slow_retry_tests;
67
68pub const SUBC_ARG: &str = "--subc";
74
75const DEFAULT_MAX_RESTARTS: u32 = 3;
76const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
77const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
78const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
82pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
93const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
94const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
95const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
107const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
130pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
132const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
133pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
138
139#[cfg(target_os = "macos")]
140struct PrivacyExec {
141 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
142 deadline: tokio::time::Instant,
143 expected: Option<subc_os::FileIdentity>,
144 trampoline: Option<subc_os::FileIdentity>,
145 script: bool,
146 module_id: String,
147 #[cfg(test)]
148 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
149}
150
151#[cfg(all(test, target_os = "macos"))]
152pub(crate) fn test_privacy_trampoline() -> PathBuf {
153 let path = std::env::current_exe()
156 .unwrap()
157 .parent()
158 .unwrap()
159 .parent()
160 .unwrap()
161 .join("privacy-trampoline-fixture");
162 assert!(
166 path.exists(),
167 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
168 --bins --features test-support` or `cargo test -p subc-daemon` first",
169 path.display()
170 );
171 path
172}
173
174#[cfg(target_os = "macos")]
175fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
176 use std::io::Read;
177 let mut probe = std::process::Command::new(path)
178 .args(["__disclaim-exec", "--probe"])
179 .stdin(Stdio::null())
180 .stdout(Stdio::piped())
181 .stderr(Stdio::piped())
182 .spawn()
183 .map_err(|error| {
184 format!(
185 "privacy trampoline probe failed for {}: {error}",
186 path.display()
187 )
188 })?;
189 let deadline = std::time::Instant::now() + Duration::from_secs(5);
190 let status = loop {
191 match probe.try_wait() {
192 Ok(Some(status)) => break status,
193 Ok(None) if std::time::Instant::now() < deadline => {
194 std::thread::sleep(Duration::from_millis(5))
195 }
196 result => {
197 let _ = probe.kill();
198 let _ = probe.wait();
199 return Err(format!(
200 "privacy trampoline probe failed or timed out for {}: {result:?}",
201 path.display()
202 ));
203 }
204 }
205 };
206 let mut answer = String::new();
207 if let Some(stdout) = probe.stdout.take() {
208 let _ = stdout.take(256).read_to_string(&mut answer);
209 }
210 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
211 return Ok(());
212 }
213 let mut diagnostic = String::new();
214 if let Some(stderr) = probe.stderr.take() {
215 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
216 }
217 let cause = diagnostic
218 .trim()
219 .strip_prefix("ck-subc: own privacy identity refused: ")
220 .unwrap_or("binary does not implement the privacy trampoline protocol");
221 Err(format!(
222 "{cause}: probe of {} exited {status}",
223 path.display()
224 ))
225}
226
227#[cfg(target_os = "macos")]
228fn privacy_command(
229 spec: &ModuleSpec,
230 roster: &ChildRoster,
231) -> Result<
232 (
233 Command,
234 Option<PrivacyExec>,
235 subc_os::privacy_identity::ExecAcknowledgement,
236 ),
237 SuperviseError,
238> {
239 let failure = |cause: String| {
240 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
241 SuperviseError::Spawn {
242 program: spec.program.clone(),
243 source: io::Error::other(cause),
244 cgroup_path: None,
245 }
246 };
247 let trampoline = roster.privacy_trampoline().map_err(failure)?;
248 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
253 let path = spec
254 .env
255 .iter()
256 .find(|(key, _)| key == "PATH")
257 .map(|(_, value)| std::ffi::OsString::from(value))
258 .or_else(|| std::env::var_os("PATH"))
259 .unwrap_or_else(|| "/usr/bin:/bin".into());
260 std::env::split_paths(&path)
261 .map(|dir| dir.join(&spec.program))
262 .find(|path| path.is_file())
263 .unwrap_or_else(|| spec.program.clone())
264 } else {
265 spec.program.clone()
266 };
267 let expected = subc_os::file_identity(&program);
268 let trampoline_image = subc_os::file_identity(&trampoline);
269 let script = {
270 use std::io::Read;
271 let mut prefix = [0u8; 2];
272 std::fs::File::open(&program)
273 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
274 };
275 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
276 return Err(failure(
277 "privacy identity module executable is missing or is the trampoline itself".to_string(),
278 ));
279 }
280 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
281 .map_err(|error| failure(error.to_string()))?;
282 let reader =
283 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
284 let mut command = Command::new(&trampoline);
285 command
286 .arg("__disclaim-exec")
287 .arg(ack.fd().to_string())
288 .arg(&program);
289 ack.install(command.as_std_mut());
290 Ok((
291 command,
292 Some(PrivacyExec {
293 reader,
294 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
295 expected,
296 trampoline: trampoline_image,
297 script,
298 module_id: spec.module_id.clone(),
299 #[cfg(test)]
300 confirmation_pause: spec
301 .env
302 .iter()
303 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
304 .map(|(_, address)| (address.clone(), None)),
305 }),
306 ack,
307 ))
308}
309
310struct SupervisedChild {
311 child: Child,
312 exec: crate::activity::ExecOwner,
313 #[cfg(windows)]
314 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
315 #[cfg(target_os = "macos")]
316 privacy_exec: Option<PrivacyExec>,
317 #[cfg(target_os = "macos")]
324 report_ready: Arc<OnceLock<()>>,
325 spawn_failure: Option<String>,
327 protocol: ModuleProtocol,
332 #[cfg(target_os = "linux")]
338 module_id: String,
339 #[cfg(target_os = "linux")]
340 cgroup_placement: Option<subc_cgroup::Placement>,
341 #[cfg(windows)]
363 job: Option<subc_jobobject::JobObject>,
364 stdout_pump: Option<JoinHandle<()>>,
365 stderr_pump: Option<StderrPump>,
366 stderr_ring: Arc<Mutex<StderrRing>>,
367 spawned_at: Instant,
370 spawned_at_ms: u64,
371 spawned_from: PathBuf,
372 launch_spec_digest: LaunchSpecDigest,
373 spawned_file_identity: Option<SpawnedFileIdentity>,
374 process_start_time: Option<u64>,
375 process_identity: Option<ProcessIdentity>,
376 pid: u32,
377 roster_guard: Option<crate::child_roster::RosterGuard>,
380}
381
382impl SupervisedChild {
383 fn id(&self) -> Option<u32> {
384 Some(self.pid)
385 }
386
387 fn process_identity(&self) -> Option<ProcessIdentity> {
388 self.process_identity
389 }
390
391 async fn wait(&mut self) -> io::Result<ExitStatus> {
392 #[cfg(target_os = "macos")]
393 self.confirm_privacy_exec().await;
394 let result = self.child.wait().await;
402 #[cfg(target_os = "linux")]
403 if result.is_ok() {
404 if let Some(placement) = self.cgroup_placement.as_ref() {
405 cleanup_reaped_cgroup(placement, &self.module_id).await;
406 self.cgroup_placement = None;
409 }
410 }
411 result
412 }
413
414 #[cfg(target_os = "macos")]
415 async fn confirm_privacy_exec(&mut self) {
416 let Some(pending) = &mut self.privacy_exec else {
417 return;
418 };
419 let result = tokio::time::timeout_at(pending.deadline, async {
420 let mut record = Vec::new();
421 loop {
422 let mut ready = pending.reader.readable().await?;
423 let read = ready.try_io(|reader| {
424 use std::io::Read;
425 let mut reader = reader.get_ref();
426 let mut buffer = [0u8; 256];
427 reader.read(&mut buffer).map(|count| (count, buffer))
428 });
429 match read {
430 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
431 Ok(Ok((count, buffer))) => {
432 if record.len() + count > 1024 {
433 return Err(io::Error::other(
434 "privacy exec refusal record is too long",
435 ));
436 }
437 record.extend_from_slice(&buffer[..count]);
438 }
439 Ok(Err(error)) => return Err(error),
440 Err(_) => continue,
441 }
442 }
443 })
444 .await;
445 #[cfg(test)]
446 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
447 use tokio::io::{AsyncReadExt, AsyncWriteExt};
448 if stream.is_none() {
449 *stream = Some(
450 tokio::net::TcpStream::connect(address.as_str())
451 .await
452 .unwrap(),
453 );
454 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
455 stream
456 .as_mut()
457 .unwrap()
458 .write_all(&self.pid.to_ne_bytes())
459 .await
460 .unwrap();
461 }
462 let mut reply = [0];
463 stream
464 .as_mut()
465 .unwrap()
466 .read_exact(&mut reply)
467 .await
468 .unwrap();
469 pending.confirmation_pause = None;
470 if &reply == b"F" {
471 Ok(Err(io::Error::other("injected late exec refusal")))
472 } else {
473 assert_eq!(&reply, b"X");
474 result
475 }
476 } else {
477 result
478 };
479 let pending = self.privacy_exec.as_ref().expect("pending exec");
482 let cause = match result {
483 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
484 Ok(Err(error)) => Some(format!(
485 "privacy identity exec acknowledgement failed: {error}"
486 )),
487 Ok(Ok(record)) if !record.is_empty() => Some(
488 std::str::from_utf8(&record)
489 .ok()
490 .and_then(|record| {
491 record
492 .trim()
493 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
494 })
495 .filter(|cause| !cause.is_empty())
496 .unwrap_or("invalid privacy exec refusal record")
497 .to_string(),
498 ),
499 Ok(Ok(_)) => match self.child.try_wait() {
500 Ok(Some(_status)) => None,
504 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
505 Ok(None) => {
506 let image = observe_spawned_image(self.pid);
507 if let Some(image) = image.filter(|image| {
508 image.executable.is_some()
509 && image.executable != pending.trampoline
510 && (image.executable == pending.expected || pending.script)
511 }) {
512 if let Err(error) = self
513 .roster_guard
514 .as_ref()
515 .map(|guard| guard.confirm_executable(image))
516 .transpose()
517 {
518 Some(format!(
519 "could not durably record confirmed module image: {error}"
520 ))
521 } else {
522 let _ = self.report_ready.set(());
523 info!(module_id = %pending.module_id, pid = self.pid,
524 "module spawned with own privacy identity (responsibility disclaimed)");
525 None
526 }
527 } else if image.is_none()
528 || image.is_some_and(|image| image.executable.is_none())
529 {
530 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
537 Ok(Ok(_status)) => None,
538 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
539 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
540 }
541 } else {
542 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
543 }
544 }
545 },
546 };
547 let pending = self.privacy_exec.take().expect("pending exec");
548 self.exec.0.finish(if cause.is_some() {
551 crate::activity::ExecState::Refused
552 } else {
553 crate::activity::ExecState::CommittedExec
554 });
555 if let Some(cause) = cause {
556 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
557 self.spawn_failure = Some(cause);
558 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
561 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
562 }
563 let _ = self.child.start_kill();
564 }
565 }
566
567 fn release_roster(&mut self) {
571 self.roster_guard = None;
572 }
573
574 fn start_kill(&mut self) -> io::Result<()> {
587 #[cfg(windows)]
588 if let Some(job) = &self.job {
589 if let Err(error) = job.terminate() {
590 debug!(
591 error = %error,
592 "job termination failed; the direct-child kill still owns the outcome"
593 );
594 }
595 }
596 #[cfg(target_os = "linux")]
597 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
598 self.child.start_kill()
599 }
600
601 async fn drain_stderr(&mut self, module_id: &str) {
602 if let Some(pump) = self.stdout_pump.take() {
603 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
604 }
605
606 let Some(pump) = self.stderr_pump.take() else {
607 return;
608 };
609 settle_stderr_pump(
610 module_id,
611 &self.stderr_ring,
612 pump,
613 STDERR_PUMP_DRAIN_TIMEOUT,
614 )
615 .await;
616 }
617}
618
619async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
623 match timeout(bound, &mut pump).await {
624 Ok(Ok(())) => {}
625 Ok(Err(error)) => {
626 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
627 }
628 Err(_) => {
629 drop(pump);
630 warn!(
631 module_id,
632 waited = ?bound,
633 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
634 );
635 }
636 }
637}
638
639struct StderrPump {
642 task: JoinHandle<()>,
643 generation: u64,
644}
645
646async fn settle_stderr_pump(
652 module_id: &str,
653 ring: &Arc<Mutex<StderrRing>>,
654 pump: StderrPump,
655 bound: Duration,
656) {
657 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
658 let StderrPump {
659 mut task,
660 generation,
661 } = pump;
662 lock().retire_pump(generation);
663 match timeout(bound, &mut task).await {
664 Ok(Ok(())) => {}
665 Ok(Err(err)) => {
666 let mut ring = lock();
667 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
668 ring.finish_pump(generation);
669 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
670 }
671 Err(_) => {
672 drop(task);
674 lock().mark_pump_late(
675 generation,
676 format!(
677 "stderr of the exited process had not reached EOF {bound:?} after it was \
678 retired (a descendant may still hold the pipe open); lines it still \
679 writes are kept in that process's section"
680 ),
681 );
682 warn!(
683 module_id,
684 waited = ?bound,
685 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
686 );
687 }
688 }
689}
690
691fn registration_release_events() -> &'static watch::Sender<u64> {
692 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
693 EVENTS.get_or_init(|| {
694 let (sender, _receiver) = watch::channel(0);
695 sender
696 })
697}
698
699pub(crate) fn notify_registration_release() {
700 let events = registration_release_events();
701 let next_generation = (*events.borrow()).wrapping_add(1);
702 events.send_replace(next_generation);
703}
704
705#[derive(Debug, Clone, PartialEq, Eq)]
707pub struct ModuleSpec {
708 pub module_id: String,
709 pub program: PathBuf,
710 pub args: Vec<String>,
711 pub env: Vec<(String, String)>,
712 pub reserved: bool,
717 pub reserved_prefixes: Vec<String>,
722 pub protocol: ModuleProtocol,
741 pub overlap: ModuleOverlap,
746}
747
748#[derive(Clone, PartialEq, Eq)]
751pub(crate) struct LaunchSpecDigest([u8; 32]);
752
753impl fmt::Debug for LaunchSpecDigest {
754 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
755 f.write_str("LaunchSpecDigest([redacted])")
756 }
757}
758
759impl LaunchSpecDigest {
760 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
761 fn field(hash: &mut Sha256, bytes: &[u8]) {
762 hash.update((bytes.len() as u64).to_le_bytes());
763 hash.update(bytes);
764 }
765 let mut hash = Sha256::new();
766 if let Some(program) = spec.program.to_str() {
767 field(&mut hash, b"utf8-program");
768 field(&mut hash, program.as_bytes());
769 } else {
770 #[cfg(unix)]
773 {
774 use std::os::unix::ffi::OsStrExt;
775 field(&mut hash, b"unix-program");
776 field(&mut hash, spec.program.as_os_str().as_bytes());
777 }
778 #[cfg(windows)]
779 {
780 use std::os::windows::ffi::OsStrExt;
781 field(&mut hash, b"windows-program");
782 let bytes = spec
783 .program
784 .as_os_str()
785 .encode_wide()
786 .flat_map(u16::to_le_bytes)
787 .collect::<Vec<_>>();
788 field(&mut hash, &bytes);
789 }
790 #[cfg(not(any(unix, windows)))]
791 field(&mut hash, spec.program.to_string_lossy().as_bytes());
792 }
793 hash.update((spec.args.len() as u64).to_le_bytes());
794 for arg in &spec.args {
795 field(&mut hash, arg.as_bytes());
796 }
797 let mut env = spec.env.iter().collect::<Vec<_>>();
798 env.sort_unstable();
799 hash.update((env.len() as u64).to_le_bytes());
800 for (key, value) in env {
801 field(&mut hash, key.as_bytes());
802 field(&mut hash, value.as_bytes());
803 }
804 field(
805 &mut hash,
806 match spec.protocol {
807 ModuleProtocol::Subc => b"subc",
808 ModuleProtocol::None => b"none",
809 },
810 );
811 Self(hash.finalize().into())
812 }
813}
814
815#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
822pub enum ModuleOverlap {
823 #[default]
825 Exclusive,
826 Safe,
838}
839
840impl ModuleOverlap {
841 pub fn as_str(self) -> &'static str {
842 match self {
843 Self::Exclusive => "exclusive",
844 Self::Safe => "safe",
845 }
846 }
847}
848
849pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
859pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
861pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
866
867#[derive(Debug, Clone, Copy, PartialEq, Eq)]
885pub struct RestartPolicy {
886 pub max_restarts: u32,
887 pub backoff: Duration,
890 pub max_backoff: Duration,
892 pub window: Duration,
896}
897
898impl RestartPolicy {
899 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
903 Self {
904 max_restarts,
905 backoff,
906 max_backoff: DEFAULT_MAX_BACKOFF,
907 window: DEFAULT_RESTART_WINDOW,
908 }
909 }
910
911 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
912 self.max_backoff = max_backoff;
913 self
914 }
915
916 pub fn with_window(mut self, window: Duration) -> Self {
917 self.window = window;
918 self
919 }
920
921 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
926 if self.backoff.is_zero() || self.max_backoff.is_zero() {
927 return Duration::ZERO;
928 }
929
930 let mut delay = self.backoff;
931 for _ in 0..restart_in_window {
932 if delay >= self.max_backoff {
933 return self.max_backoff;
934 }
935 delay = delay
936 .checked_mul(10)
937 .unwrap_or(self.max_backoff)
938 .min(self.max_backoff);
939 }
940 delay.min(self.max_backoff)
941 }
942
943 fn budget_exhausted_detail(&self) -> String {
948 format!(
949 "crash budget exhausted: max_restarts={} within window_secs={}",
950 self.max_restarts,
951 self.window.as_secs()
952 )
953 }
954}
955
956impl Default for RestartPolicy {
957 fn default() -> Self {
958 Self {
959 max_restarts: DEFAULT_MAX_RESTARTS,
960 backoff: DEFAULT_BACKOFF,
961 max_backoff: DEFAULT_MAX_BACKOFF,
962 window: DEFAULT_RESTART_WINDOW,
963 }
964 }
965}
966
967#[derive(Debug, Clone, Copy, PartialEq, Eq)]
968struct CrashRestartSchedule {
969 restart_in_window: u32,
970 delay: Duration,
971}
972
973fn daemon_will_restart(
980 state: &mut SupervisorSnapshot,
981 policy: &RestartPolicy,
982 now: Instant,
983) -> bool {
984 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
985}
986
987const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
988const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
989const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
990const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
991
992#[derive(Debug, Clone, Copy, PartialEq, Eq)]
993pub enum HealthAction {
994 Report,
995 Restart,
996 Alert,
997}
998
999impl fmt::Display for HealthAction {
1000 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1001 f.write_str(match self {
1002 Self::Report => "report",
1003 Self::Restart => "restart",
1004 Self::Alert => "alert",
1005 })
1006 }
1007}
1008
1009#[derive(Debug, Clone, PartialEq, Eq)]
1010pub struct HealthConfig {
1011 pub http: Option<String>,
1015 pub cadence: Duration,
1016 pub deadline: Duration,
1017 pub failure_threshold: u32,
1018 pub on_degraded: HealthAction,
1019 pub on_failing: HealthAction,
1020 pub critical: bool,
1021}
1022
1023impl Default for HealthConfig {
1024 fn default() -> Self {
1025 Self {
1026 http: None,
1027 cadence: DEFAULT_HEALTH_CADENCE,
1028 deadline: DEFAULT_HEALTH_DEADLINE,
1029 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1030 on_degraded: HealthAction::Report,
1031 on_failing: HealthAction::Report,
1032 critical: false,
1033 }
1034 }
1035}
1036
1037#[derive(Debug, Clone, PartialEq)]
1055pub struct ModuleHealthStatus {
1056 pub status: SupervisorHealthStatus,
1057 pub last_probe_ms: Option<u64>,
1058 pub detail: Option<String>,
1059 pub metrics: Option<Value>,
1060 pub consecutive_failures: u32,
1061 pub late_answer_count: u64,
1064 pub last_late_answer_latency_ms: Option<u64>,
1066 pub last_action: Option<String>,
1067 pub last_action_ms: Option<u64>,
1071}
1072
1073impl Default for ModuleHealthStatus {
1074 fn default() -> Self {
1075 Self {
1076 status: SupervisorHealthStatus::Unknown,
1077 last_probe_ms: None,
1078 detail: None,
1079 metrics: None,
1080 consecutive_failures: 0,
1081 late_answer_count: 0,
1082 last_late_answer_latency_ms: None,
1083 last_action: None,
1084 last_action_ms: None,
1085 }
1086 }
1087}
1088
1089#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1091pub enum ModuleState {
1092 Starting,
1093 Running,
1094 Unresponsive,
1095 Restarting,
1096 Draining,
1097 Stopped,
1098 Failed,
1099 Disabled,
1100}
1101
1102impl fmt::Display for ModuleState {
1103 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1104 f.write_str(match self {
1105 Self::Starting => "starting",
1106 Self::Running => "running",
1107 Self::Unresponsive => "unresponsive",
1108 Self::Restarting => "restarting",
1109 Self::Draining => "draining",
1110 Self::Stopped => "stopped",
1111 Self::Failed => "failed",
1112 Self::Disabled => "disabled",
1113 })
1114 }
1115}
1116
1117#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1119pub enum ExitKind {
1120 Clean,
1121 Crash,
1122 DeliberateSeverance,
1123}
1124
1125impl From<ExitKind> for TerminalExitKind {
1126 fn from(kind: ExitKind) -> Self {
1127 match kind {
1128 ExitKind::Clean => Self::Clean,
1129 ExitKind::Crash => Self::Crash,
1130 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1131 }
1132 }
1133}
1134
1135#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1138pub(crate) struct ProcessIdentity {
1139 pub(crate) pid: u32,
1140 pub(crate) start_time: u64,
1141}
1142
1143#[derive(Debug, Clone, PartialEq, Eq)]
1145pub struct ExitReport {
1146 pub kind: ExitKind,
1147 pub code: Option<i32>,
1148 pub signal: Option<i32>,
1149 pub at_ms: u64,
1150}
1151
1152#[derive(Debug, Clone, PartialEq)]
1155pub struct ModuleStatus {
1156 pub module_id: String,
1157 pub waiting_for_previous_pid: Option<u32>,
1159 pub waiting_for_previous_detail: Option<String>,
1161 pub launch_nonce_source: Option<String>,
1166 pub state: ModuleState,
1167 pub enabled: bool,
1168 pub process_alive: bool,
1169 pub registration_active: bool,
1170 pub protocol: ModuleProtocol,
1175 pub live: bool,
1186 pub restart_count: u32,
1190 pub lifetime_restarts: u32,
1194 pub spawn_generation: u64,
1195 pub max_restarts: u32,
1200 pub restart_window: Duration,
1204 pub drain_timeout: Duration,
1208 pub restart_backoff: Duration,
1209 pub restart_max_backoff: Duration,
1210 pub pid: Option<u32>,
1215 pub spawned_at_ms: Option<u64>,
1216 pub spawned_from: Option<PathBuf>,
1217 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1218 pub process_start_time: Option<u64>,
1219 pub last_exit: Option<ExitReport>,
1220 pub health: ModuleHealthStatus,
1221}
1222
1223#[cfg(test)]
1224#[derive(Debug, Clone, Copy, PartialEq)]
1225struct ActorSelectCheckpoint {
1226 generation: u64,
1227 turn: u64,
1228 registered_connection: Option<ConnectionId>,
1229 next_probe_at: Option<Instant>,
1230 wake_after: Option<Duration>,
1231}
1232
1233#[derive(Debug, Clone, PartialEq)]
1234struct SupervisorSnapshot {
1235 failed_retry: FailedRetry,
1236 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1237 #[cfg(windows)]
1238 pipe_consumed: Option<Arc<OnceLock<()>>>,
1239 #[cfg(test)]
1241 actor_turns: u64,
1242 #[cfg(test)]
1245 actor_select: Option<ActorSelectCheckpoint>,
1246 state: ModuleState,
1247 enabled: bool,
1248 process_alive: bool,
1249 spawned_protocol: Option<ModuleProtocol>,
1250 spawn_failure: Option<String>,
1251 waiting_for_previous_pid: Option<u32>,
1252 waiting_for_previous_detail: Option<String>,
1253 crash_restarts: VecDeque<Instant>,
1259 lifetime_restarts: u32,
1260 spawn_generation: u64,
1269 pid: Option<u32>,
1270 #[cfg(target_os = "macos")]
1271 report_ready: Option<Arc<OnceLock<()>>>,
1272 reaped_pid: Option<u32>,
1274 respawn_pending: bool,
1276 coalesced_restart_pending: bool,
1278 spawned_at_ms: Option<u64>,
1279 spawned_from: Option<PathBuf>,
1280 launch_spec_digest: Option<LaunchSpecDigest>,
1281 spawned_file_identity: Option<SpawnedFileIdentity>,
1282 process_start_time: Option<u64>,
1283 deliberate_severance: Option<ProcessIdentity>,
1284 last_exit: Option<ExitReport>,
1285 drain_disposition_detail: Option<String>,
1287 health: ModuleHealthStatus,
1288 in_alternate_slot: bool,
1293 draining_to_replace: bool,
1300 configuration_updated_since_spawn: bool,
1306}
1307
1308impl SupervisorSnapshot {
1309 fn reported_pid(&self) -> Option<u32> {
1314 #[cfg(target_os = "macos")]
1315 if self
1316 .report_ready
1317 .as_ref()
1318 .is_some_and(|ready| ready.get().is_none())
1319 {
1320 return None;
1321 }
1322 self.pid
1323 }
1324
1325 fn starting() -> Self {
1326 Self::new(ModuleState::Starting, true)
1327 }
1328
1329 fn disabled() -> Self {
1330 Self::new(ModuleState::Disabled, false)
1331 }
1332
1333 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1337 while let Some(oldest) = self.crash_restarts.front() {
1338 if now.duration_since(*oldest) > window {
1339 self.crash_restarts.pop_front();
1340 } else {
1341 break;
1342 }
1343 }
1344 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1345 }
1346
1347 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1353 self.crash_restarts.push_back(now);
1354 while self.crash_restarts.len() > policy.max_restarts as usize {
1355 self.crash_restarts.pop_front();
1356 }
1357 self.lifetime_restarts += 1;
1358 }
1359
1360 fn next_crash_restart(
1364 &mut self,
1365 policy: &RestartPolicy,
1366 now: Instant,
1367 ) -> Option<CrashRestartSchedule> {
1368 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1369 if restart_in_window >= policy.max_restarts {
1370 return None;
1371 }
1372 self.record_crash_restart(policy, now);
1373 Some(CrashRestartSchedule {
1374 restart_in_window,
1375 delay: policy.delay_for_restart(restart_in_window),
1376 })
1377 }
1378
1379 fn clear_crash_restarts(&mut self) {
1384 self.crash_restarts.clear();
1385 }
1386
1387 fn new(state: ModuleState, enabled: bool) -> Self {
1388 Self {
1389 failed_retry: FailedRetry::default(),
1390 exec_outcome: None,
1391 #[cfg(test)]
1392 actor_turns: 0,
1393 #[cfg(test)]
1394 actor_select: None,
1395 state,
1396 enabled,
1397 process_alive: false,
1398 #[cfg(windows)]
1399 pipe_consumed: None,
1400 spawned_protocol: None,
1401 spawn_failure: None,
1402 waiting_for_previous_pid: None,
1403 waiting_for_previous_detail: None,
1404 crash_restarts: VecDeque::new(),
1405 lifetime_restarts: 0,
1406 spawn_generation: 0,
1407 pid: None,
1408 #[cfg(target_os = "macos")]
1409 report_ready: None,
1410 reaped_pid: None,
1411 respawn_pending: false,
1412 coalesced_restart_pending: false,
1413 spawned_at_ms: None,
1414 spawned_from: None,
1415 launch_spec_digest: None,
1416 spawned_file_identity: None,
1417 process_start_time: None,
1418 deliberate_severance: None,
1419 last_exit: None,
1420 drain_disposition_detail: None,
1421 health: ModuleHealthStatus::default(),
1422 in_alternate_slot: false,
1423 draining_to_replace: false,
1424 configuration_updated_since_spawn: false,
1425 }
1426 }
1427}
1428
1429type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1430
1431type SpawnSubscriberKey = (ConnectionId, u64);
1432
1433#[derive(Debug)]
1434struct SpawnSubscriber {
1435 version: u8,
1436 frames: mpsc::Sender<Frame>,
1437 lagged: Option<oneshot::Sender<SpawnCursor>>,
1441}
1442
1443#[derive(Debug)]
1444struct SpawnEventState {
1445 daemon_incarnation: String,
1446 seq: u64,
1447 capacity: usize,
1448 live: HashMap<String, LiveSpawn>,
1449 generations: HashMap<String, u64>,
1450 events: VecDeque<SpawnEvent>,
1451 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1452}
1453
1454impl Default for SpawnEventState {
1455 fn default() -> Self {
1456 Self {
1457 daemon_incarnation: "unconfigured".to_string(),
1458 seq: 0,
1459 capacity: SPAWN_EVENT_RING_CAPACITY,
1460 live: HashMap::new(),
1461 generations: HashMap::new(),
1462 events: VecDeque::new(),
1463 subscribers: HashMap::new(),
1464 }
1465 }
1466}
1467
1468#[derive(Debug, Clone, Default)]
1469struct SpawnEventFeed(
1470 Arc<Mutex<SpawnEventState>>,
1471 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1472);
1473
1474#[derive(Debug, Clone, PartialEq, Eq)]
1475pub(crate) enum SpawnSubscribeRefusal {
1476 ForeignIncarnation { current: String },
1477 TooOld { oldest: SpawnCursor },
1478 Frame(String),
1479}
1480
1481impl SpawnEventFeed {
1482 fn configure_incarnation(&self, daemon_incarnation: String) {
1483 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1484 state.daemon_incarnation = daemon_incarnation;
1485 state.seq = 0;
1486 state.live.clear();
1487 state.generations.clear();
1488 state.events.clear();
1489 state.subscribers.clear();
1490 }
1491
1492 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1493 SpawnCursor {
1494 daemon_incarnation: state.daemon_incarnation.clone(),
1495 seq: state.seq,
1496 }
1497 }
1498
1499 fn snapshot(&self) -> SpawnSnapshot {
1500 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1501 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1502 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1503 SpawnSnapshot {
1504 cursor: Self::cursor(&state),
1505 ring_bound: state.capacity as u64,
1506 live,
1507 }
1508 }
1509
1510 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1511 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1512 let generation = state
1513 .generations
1514 .get(module_id)
1515 .copied()
1516 .unwrap_or(0)
1517 .checked_add(1)
1518 .expect("spawn generation exhausted");
1519 state.generations.insert(module_id.to_string(), generation);
1520 let live = LiveSpawn {
1521 module_id: module_id.to_string(),
1522 spawn_generation: generation,
1523 pid,
1524 spawned_at_ms,
1525 };
1526 state.live.insert(module_id.to_string(), live);
1527 Self::emit_locked(
1528 &mut state,
1529 SpawnEventKind::Spawned,
1530 module_id.to_string(),
1531 generation,
1532 pid,
1533 None,
1534 None,
1535 );
1536 generation
1537 }
1538
1539 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1540 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1541 let Some(live) = state.live.remove(module_id) else {
1542 warn!(
1543 module_id,
1544 "terminal record had no live spawn event identity"
1545 );
1546 return;
1547 };
1548 Self::emit_locked(
1549 &mut state,
1550 SpawnEventKind::Exited,
1551 module_id.to_string(),
1552 live.spawn_generation,
1553 live.pid,
1554 exit_code,
1555 exit_signal,
1556 );
1557 }
1558
1559 fn emit_superseded_exited(
1566 &self,
1567 module_id: &str,
1568 spawn_generation: u64,
1569 pid: u32,
1570 exit_code: Option<i32>,
1571 exit_signal: Option<i32>,
1572 ) {
1573 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1574 if state
1575 .live
1576 .get(module_id)
1577 .is_some_and(|live| live.spawn_generation == spawn_generation)
1578 {
1579 state.live.remove(module_id);
1580 }
1581 Self::emit_locked(
1582 &mut state,
1583 SpawnEventKind::Exited,
1584 module_id.to_string(),
1585 spawn_generation,
1586 pid,
1587 exit_code,
1588 exit_signal,
1589 );
1590 }
1591
1592 #[allow(clippy::too_many_arguments)]
1593 fn emit_locked(
1594 state: &mut SpawnEventState,
1595 kind: SpawnEventKind,
1596 module_id: String,
1597 spawn_generation: u64,
1598 pid: u32,
1599 exit_code: Option<i32>,
1600 exit_signal: Option<i32>,
1601 ) {
1602 state.seq = state
1603 .seq
1604 .checked_add(1)
1605 .expect("spawn event sequence exhausted");
1606 let event = SpawnEvent {
1607 cursor: Self::cursor(state),
1608 kind,
1609 module_id,
1610 spawn_generation,
1611 pid,
1612 exit_code,
1613 exit_signal,
1614 };
1615 state.events.push_back(event.clone());
1616 while state.events.len() > state.capacity {
1617 state.events.pop_front();
1618 }
1619 let body = match serde_json::to_vec(&event) {
1620 Ok(body) => body,
1621 Err(error) => {
1622 error!(%error, "failed to serialize supervisor spawn event");
1623 return;
1624 }
1625 };
1626 state.subscribers.retain(|(connection_id, corr), subscriber| {
1627 let frame = Frame::build_with_version(
1628 subscriber.version,
1629 FrameType::StreamData,
1630 control_flags(),
1631 0,
1632 0,
1633 *corr,
1634 body.clone(),
1635 );
1636 match frame {
1637 Ok(frame) => {
1638 if subscriber.frames.try_send(frame).is_ok() {
1639 true
1640 } else {
1641 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1642 if let Some(lagged) = subscriber.lagged.take() {
1643 let _ = lagged.send(event.cursor.clone());
1644 }
1645 false
1646 }
1647 }
1648 Err(error) => {
1649 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1650 false
1651 }
1652 }
1653 });
1654 }
1655
1656 fn subscribe(
1657 &self,
1658 connection_id: ConnectionId,
1659 corr: u64,
1660 version: u8,
1661 since: Option<SpawnCursor>,
1662 sink: FrameSink,
1663 ) -> Result<(), SpawnSubscribeRefusal> {
1664 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1665 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1666 {
1667 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1668 let replay = if let Some(since) = since {
1669 if since.daemon_incarnation != state.daemon_incarnation {
1670 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1671 current: state.daemon_incarnation.clone(),
1672 });
1673 }
1674 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1675 if since.seq < oldest.seq.saturating_sub(1) {
1676 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1677 }
1678 }
1679 state
1680 .events
1681 .iter()
1682 .filter(|event| event.cursor.seq > since.seq)
1683 .cloned()
1684 .collect::<Vec<_>>()
1685 } else {
1686 Vec::new()
1687 };
1688 for event in replay {
1689 let body = serde_json::to_vec(&event)
1690 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1691 let frame = Frame::build_with_version(
1692 version,
1693 FrameType::StreamData,
1694 control_flags(),
1695 0,
1696 0,
1697 corr,
1698 body,
1699 )
1700 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1701 frames
1702 .try_send(frame)
1703 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1704 }
1705 state.subscribers.insert(
1706 (connection_id, corr),
1707 SpawnSubscriber {
1708 version,
1709 frames,
1710 lagged: Some(lagged),
1711 },
1712 );
1713 }
1714 tokio::spawn(async move {
1725 while let Some(frame) = receiver.recv().await {
1726 if sink.send(frame).await.is_err() {
1727 return;
1728 }
1729 }
1730 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1731 return;
1732 };
1733 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1734 Ok(frame) => {
1735 let _ = sink.send(frame).await;
1736 }
1737 Err(error) => {
1738 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1739 }
1740 }
1741 });
1742 Ok(())
1743 }
1744
1745 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1746 let Some(subscriber) = self
1747 .0
1748 .lock()
1749 .unwrap_or_else(|p| p.into_inner())
1750 .subscribers
1751 .remove(&(connection_id, corr))
1752 else {
1753 return false;
1754 };
1755 if let Ok(frame) = Frame::build_with_version(
1756 subscriber.version,
1757 FrameType::StreamEnd,
1758 control_flags(),
1759 0,
1760 0,
1761 corr,
1762 Vec::new(),
1763 ) {
1764 tokio::spawn(async move {
1765 let _ = subscriber.frames.send(frame).await;
1766 });
1767 }
1768 true
1769 }
1770
1771 fn remove_connection(&self, connection_id: ConnectionId) {
1772 self.0
1773 .lock()
1774 .unwrap_or_else(|p| p.into_inner())
1775 .subscribers
1776 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1777 }
1778
1779 #[cfg(any(test, feature = "test-support"))]
1780 fn set_capacity(&self, capacity: usize) {
1781 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1782 }
1783
1784 #[cfg(any(test, feature = "test-support"))]
1785 fn subscriber_count(&self) -> usize {
1786 self.0
1787 .lock()
1788 .unwrap_or_else(|p| p.into_inner())
1789 .subscribers
1790 .len()
1791 }
1792}
1793
1794fn spawn_subscriber_lagged_frame(
1797 version: u8,
1798 corr: u64,
1799 first_undelivered: SpawnCursor,
1800) -> Result<Frame, String> {
1801 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1802 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1803 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1804 .to_string(),
1805 detail: Some(serde_json::json!({
1806 "first_undelivered_cursor": first_undelivered
1807 })),
1808 })
1809 .map_err(|error| error.to_string())?;
1810 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1811 .map_err(|error| error.to_string())
1812}
1813
1814pub trait ModuleProcessLiveness: Send + Sync {
1815 fn process_live(&self, module_id: &str) -> Option<bool>;
1816
1817 fn process_replacing(&self, _module_id: &str) -> bool {
1823 false
1824 }
1825}
1826
1827#[derive(Debug, Clone, Default)]
1829pub struct SupervisorProcessLiveness {
1830 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1831}
1832
1833impl SupervisorProcessLiveness {
1834 pub fn new() -> Self {
1835 Self::default()
1836 }
1837
1838 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1839 let mut snapshots = self
1840 .snapshots
1841 .lock()
1842 .unwrap_or_else(|poisoned| poisoned.into_inner());
1843 snapshots.insert(module_id, snapshot);
1844 }
1845
1846 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1847 let mut snapshots = self
1848 .snapshots
1849 .lock()
1850 .unwrap_or_else(|poisoned| poisoned.into_inner());
1851 let is_current = snapshots
1852 .get(module_id)
1853 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1854 .unwrap_or(false);
1855 if is_current {
1856 snapshots.remove(module_id);
1857 }
1858 }
1859}
1860
1861impl ModuleProcessLiveness for SupervisorProcessLiveness {
1862 fn process_live(&self, module_id: &str) -> Option<bool> {
1863 let snapshot = {
1864 let snapshots = self
1865 .snapshots
1866 .lock()
1867 .unwrap_or_else(|poisoned| poisoned.into_inner());
1868 snapshots.get(module_id).cloned()
1869 }?;
1870 let snapshot = snapshot
1871 .lock()
1872 .unwrap_or_else(|poisoned| poisoned.into_inner());
1873 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1874 }
1875
1876 fn process_replacing(&self, module_id: &str) -> bool {
1877 let Some(snapshot) = self
1878 .snapshots
1879 .lock()
1880 .unwrap_or_else(|poisoned| poisoned.into_inner())
1881 .get(module_id)
1882 .cloned()
1883 else {
1884 return false;
1885 };
1886 let snapshot = snapshot
1887 .lock()
1888 .unwrap_or_else(|poisoned| poisoned.into_inner());
1889 snapshot.enabled
1890 && match snapshot.state {
1891 ModuleState::Restarting => true,
1892 ModuleState::Draining => snapshot.draining_to_replace,
1893 ModuleState::Starting
1894 | ModuleState::Running
1895 | ModuleState::Unresponsive
1896 | ModuleState::Stopped
1897 | ModuleState::Failed
1898 | ModuleState::Disabled => false,
1899 }
1900 }
1901}
1902
1903#[cfg(test)]
1904#[derive(Debug, Default)]
1905struct ReloadExitRecordGate {
1906 reached: tokio::sync::Notify,
1907 resume: tokio::sync::Notify,
1908}
1909
1910#[derive(Debug, Clone, Copy)]
1911enum RespawnKind {
1912 Spawn,
1913 Reload,
1914}
1915
1916#[derive(Debug, Clone, Copy)]
1917struct PendingRespawn {
1918 deadline: Instant,
1919 kind: RespawnKind,
1920 daemon_initiated: bool,
1921}
1922
1923type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1924
1925#[derive(Debug, Clone)]
1926struct SupervisorRuntimeConfig {
1927 recovery_is_daemon: bool,
1929 start_requester: Option<Principal>,
1930 pending_activity_restart: bool,
1931 provenance_probe: ExecutableIdentityProbe,
1932 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1934 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1936 restart_policy: RestartPolicy,
1937 drain_timeout: Duration,
1940 effective_drain_timeout: Arc<Mutex<Duration>>,
1943 default_drain_timeout: Duration,
1946 health: HealthConfig,
1947 connection_file_path: Option<PathBuf>,
1948 capture_logs_dir: Option<PathBuf>,
1949 forwarding: Option<Arc<ForwardingTable>>,
1950 supervisor_handle: Option<SupervisorHandle>,
1953 stderr_ring: Arc<Mutex<StderrRing>>,
1960 terminal_ring: Arc<Mutex<TerminalRing>>,
1961 spawn_events: SpawnEventFeed,
1962 child_roster: ChildRoster,
1963 #[cfg(target_os = "linux")]
1964 cgroup_placement: Option<subc_cgroup::Placement>,
1965 #[cfg(test)]
1966 test_seed_stale_facts_before_enable_spawn: bool,
1967 #[cfg(test)]
1968 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1969}
1970
1971impl SupervisorRuntimeConfig {
1972 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
1973 FailureContext {
1974 daemon_initiated,
1975 shutdown: self.child_roster.shutdown_flag(),
1976 }
1977 }
1978}
1979
1980#[derive(Debug, Clone, PartialEq, Eq)]
1981struct SupervisedConfiguration {
1982 spec: ModuleSpec,
1983 health: HealthConfig,
1984}
1985
1986#[derive(Debug, Clone, Default)]
1992pub struct SupervisorHandle {
1993 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1994 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1995 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1996 configured_ids: Arc<Mutex<HashSet<String>>>,
2008 spawn_events: SpawnEventFeed,
2009 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2020 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2026 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2030 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2038 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2044 promotion_observer: PromotionObserverSlot,
2046 operation_lock: Arc<AsyncMutex<()>>,
2050}
2051
2052pub(crate) trait SwapPromotionObserver: Send + Sync {
2061 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2062}
2063
2064#[derive(Clone, Default)]
2068struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2069
2070impl fmt::Debug for PromotionObserverSlot {
2071 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2072 f.write_str("PromotionObserverSlot")
2073 }
2074}
2075
2076#[derive(Debug, Clone)]
2078struct OpenSwap {
2079 candidate_nonce: String,
2082 incumbent_nonce: Option<String>,
2087 candidate_admitted: bool,
2091}
2092
2093#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2096pub(crate) enum SwapHelloAdmission {
2097 NotSwapping,
2100 Candidate,
2102 Refused,
2105}
2106
2107#[derive(Debug, Clone, PartialEq, Eq)]
2108pub(crate) enum ReservedHelloRejection {
2109 Exact {
2110 module_id: String,
2111 },
2112 Prefix {
2113 prefix: String,
2114 owner_module_id: String,
2115 },
2116}
2117
2118impl SupervisorHandle {
2119 pub fn new() -> Self {
2120 Self::default()
2121 }
2122
2123 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2124 self.spawn_events.snapshot()
2125 }
2126
2127 pub(crate) fn subscribe_spawns(
2128 &self,
2129 connection_id: ConnectionId,
2130 corr: u64,
2131 version: u8,
2132 since: Option<SpawnCursor>,
2133 sink: FrameSink,
2134 ) -> Result<(), SpawnSubscribeRefusal> {
2135 self.spawn_events
2136 .subscribe(connection_id, corr, version, since, sink)
2137 }
2138
2139 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2140 self.spawn_events.cancel(connection_id, corr)
2141 }
2142
2143 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2144 self.spawn_events.remove_connection(connection_id);
2145 }
2146
2147 #[cfg(any(test, feature = "test-support"))]
2148 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2149 assert!(capacity > 0, "spawn event capacity must be non-zero");
2150 self.spawn_events.set_capacity(capacity);
2151 }
2152
2153 #[cfg(any(test, feature = "test-support"))]
2154 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2155 self.spawn_events.subscriber_count()
2156 }
2157
2158 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2161 self.spawn_nonces
2162 .lock()
2163 .unwrap_or_else(|poisoned| poisoned.into_inner())
2164 .insert(module_id.to_string(), nonce);
2165 }
2166
2167 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2170 self.reserved_nonces
2171 .lock()
2172 .unwrap_or_else(|poisoned| poisoned.into_inner())
2173 .insert(module_id.to_string(), Some(nonce));
2174 }
2175
2176 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2178 let mut owners = self
2179 .reserved_prefix_owners
2180 .lock()
2181 .unwrap_or_else(|poisoned| poisoned.into_inner());
2182 owners.retain(|_, owner| owner != owner_module_id);
2183 for prefix in prefixes {
2184 owners.insert(prefix.clone(), owner_module_id.to_string());
2185 }
2186 }
2187
2188 #[cfg(test)]
2190 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2191 self.spawn_nonces
2192 .lock()
2193 .unwrap_or_else(|poisoned| poisoned.into_inner())
2194 .get(module_id)
2195 .cloned()
2196 }
2197
2198 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2199 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2200 let spawn_nonce = self
2201 .spawn_nonces
2202 .lock()
2203 .unwrap_or_else(|poisoned| poisoned.into_inner())
2204 .get(&spec.module_id)
2205 .cloned();
2206 let mut reserved_nonces = self
2207 .reserved_nonces
2208 .lock()
2209 .unwrap_or_else(|poisoned| poisoned.into_inner());
2210 if spec.reserved {
2211 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2216 }
2217 drop(reserved_nonces);
2218 self.removal_tombstones
2222 .lock()
2223 .unwrap_or_else(|poisoned| poisoned.into_inner())
2224 .remove(&spec.module_id);
2225 }
2226
2227 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2232 self.reserved_hello_verification(module_id, presented)
2233 .is_ok()
2234 }
2235
2236 pub(crate) fn reserved_hello_verification(
2239 &self,
2240 module_id: &str,
2241 presented: Option<&str>,
2242 ) -> Result<bool, ReservedHelloRejection> {
2243 let nonces = self
2244 .reserved_nonces
2245 .lock()
2246 .unwrap_or_else(|poisoned| poisoned.into_inner());
2247 if let Some(expected) = nonces.get(module_id) {
2248 let authorized = match expected {
2252 Some(expected) => {
2253 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2254 }
2255 None => false,
2256 };
2257 if authorized {
2258 return Ok(true);
2259 }
2260 return Err(ReservedHelloRejection::Exact {
2261 module_id: module_id.to_string(),
2262 });
2263 }
2264 drop(nonces);
2265
2266 let matched_prefix = self
2267 .reserved_prefix_owners
2268 .lock()
2269 .unwrap_or_else(|poisoned| poisoned.into_inner())
2270 .iter()
2271 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2272 .max_by_key(|(prefix, _)| prefix.len())
2273 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2274 let Some((prefix, owner_module_id)) = matched_prefix else {
2275 return Ok(false);
2276 };
2277
2278 let authorized = presented.is_some_and(|presented| {
2279 self.spawn_nonces
2280 .lock()
2281 .unwrap_or_else(|poisoned| poisoned.into_inner())
2282 .get(&owner_module_id)
2283 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2284 || self.swap_nonce_matches(&owner_module_id, presented)
2287 });
2288 if authorized {
2289 Ok(true)
2290 } else {
2291 Err(ReservedHelloRejection::Prefix {
2292 prefix,
2293 owner_module_id,
2294 })
2295 }
2296 }
2297
2298 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2303 if presented.is_empty() {
2304 return false;
2305 }
2306 let nonces = self
2307 .spawn_nonces
2308 .lock()
2309 .unwrap_or_else(|poisoned| poisoned.into_inner());
2310 let current = nonces
2311 .get(module_id)
2312 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2313 drop(nonces);
2314 current || self.swap_nonce_matches(module_id, presented)
2319 }
2320
2321 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2323 let swaps = self
2324 .swaps
2325 .lock()
2326 .unwrap_or_else(|poisoned| poisoned.into_inner());
2327 swaps.get(module_id).is_some_and(|swap| {
2328 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2329 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2330 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2331 })
2332 })
2333 }
2334
2335 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2338 let incumbent_nonce = self
2339 .spawn_nonces
2340 .lock()
2341 .unwrap_or_else(|poisoned| poisoned.into_inner())
2342 .get(module_id)
2343 .cloned();
2344 self.swaps
2345 .lock()
2346 .unwrap_or_else(|poisoned| poisoned.into_inner())
2347 .insert(
2348 module_id.to_string(),
2349 OpenSwap {
2350 candidate_nonce,
2351 incumbent_nonce,
2352 candidate_admitted: false,
2353 },
2354 );
2355 }
2356
2357 pub(crate) fn close_swap(&self, module_id: &str) {
2360 self.swaps
2361 .lock()
2362 .unwrap_or_else(|poisoned| poisoned.into_inner())
2363 .remove(module_id);
2364 }
2365
2366 pub(crate) fn set_swap_promotion_observer(
2369 &self,
2370 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2371 ) {
2372 *self
2373 .promotion_observer
2374 .0
2375 .lock()
2376 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2377 }
2378
2379 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2382 let observer = self
2383 .promotion_observer
2384 .0
2385 .lock()
2386 .unwrap_or_else(|poisoned| poisoned.into_inner())
2387 .as_ref()
2388 .and_then(std::sync::Weak::upgrade);
2389 if let Some(observer) = observer {
2390 observer.swap_promoted(registration);
2391 }
2392 }
2393
2394 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2396 self.swaps
2397 .lock()
2398 .unwrap_or_else(|poisoned| poisoned.into_inner())
2399 .contains_key(module_id)
2400 }
2401
2402 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2407 let candidate_nonce = self
2408 .swaps
2409 .lock()
2410 .unwrap_or_else(|poisoned| poisoned.into_inner())
2411 .get(module_id)
2412 .map(|swap| swap.candidate_nonce.clone());
2413 let Some(nonce) = candidate_nonce else {
2414 return;
2415 };
2416 self.set_spawn_nonce(module_id, nonce.clone());
2417 if reserved {
2418 self.set_reserved_nonce(module_id, nonce);
2419 }
2420 }
2421
2422 pub(crate) fn swap_hello_admission(
2437 &self,
2438 module_id: &str,
2439 presented: Option<&str>,
2440 ) -> SwapHelloAdmission {
2441 let swaps = self
2442 .swaps
2443 .lock()
2444 .unwrap_or_else(|poisoned| poisoned.into_inner());
2445 let Some(swap) = swaps.get(module_id) else {
2446 return SwapHelloAdmission::NotSwapping;
2447 };
2448 let Some(presented) = presented else {
2449 return SwapHelloAdmission::Refused;
2450 };
2451 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2452 return if swap.candidate_admitted {
2453 SwapHelloAdmission::Refused
2454 } else {
2455 SwapHelloAdmission::Candidate
2456 };
2457 }
2458 if swap
2459 .incumbent_nonce
2460 .as_deref()
2461 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2462 {
2463 return SwapHelloAdmission::NotSwapping;
2464 }
2465 SwapHelloAdmission::Refused
2466 }
2467
2468 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2471 if let Some(swap) = self
2472 .swaps
2473 .lock()
2474 .unwrap_or_else(|poisoned| poisoned.into_inner())
2475 .get_mut(module_id)
2476 {
2477 swap.candidate_admitted = true;
2478 }
2479 }
2480
2481 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2483 self.spawn_nonces
2484 .lock()
2485 .unwrap_or_else(|poisoned| poisoned.into_inner())
2486 .get(module_id)
2487 .cloned()
2488 }
2489
2490 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2492 self.reserved_nonces
2493 .lock()
2494 .unwrap_or_else(|poisoned| poisoned.into_inner())
2495 .get(module_id)
2496 .cloned()
2497 .flatten()
2498 }
2499
2500 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2501 self.mark_configured(module.module_id());
2505 let mut modules = self
2506 .modules
2507 .lock()
2508 .unwrap_or_else(|poisoned| poisoned.into_inner());
2509 modules.insert(module.module_id().to_string(), module)
2510 }
2511
2512 fn mark_configured(&self, module_id: &str) {
2516 self.configured_ids
2517 .lock()
2518 .unwrap_or_else(|poisoned| poisoned.into_inner())
2519 .insert(module_id.to_string());
2520 }
2521
2522 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2526 let modules = self
2527 .modules
2528 .lock()
2529 .unwrap_or_else(|poisoned| poisoned.into_inner());
2530 if !modules.contains_key(module_id) {
2531 self.configured_ids
2532 .lock()
2533 .unwrap_or_else(|poisoned| poisoned.into_inner())
2534 .remove(module_id);
2535 }
2536 }
2537
2538 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2546 self.configured_ids
2547 .lock()
2548 .unwrap_or_else(|poisoned| poisoned.into_inner())
2549 .contains(module_id)
2550 }
2551
2552 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2553 let modules = self
2554 .modules
2555 .lock()
2556 .unwrap_or_else(|poisoned| poisoned.into_inner());
2557 modules.get(module_id).cloned()
2558 }
2559
2560 pub(crate) fn record_late_health_answer(
2561 &self,
2562 module_id: &str,
2563 latency_ms: u64,
2564 ) -> Result<bool, SuperviseError> {
2565 let Some(module) = self.get(module_id) else {
2566 return Ok(false);
2567 };
2568 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2569 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2570 state.health.last_late_answer_latency_ms = Some(latency_ms);
2571 state.health.consecutive_failures = 0;
2579 })?;
2580 Ok(true)
2581 }
2582
2583 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2589 let Some(module) = self.get(module_id) else {
2590 return Ok(false);
2591 };
2592 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2593 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2594 return Ok(false);
2595 };
2596 drop(snapshot);
2597 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2598 }
2599
2600 pub fn list(&self) -> Vec<SupervisedModule> {
2601 let modules = self
2602 .modules
2603 .lock()
2604 .unwrap_or_else(|poisoned| poisoned.into_inner());
2605 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2606 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2607 modules
2608 }
2609
2610 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2611 self.activity_images
2612 .lock()
2613 .unwrap_or_else(|p| p.into_inner())
2614 .remove(module_id);
2615 self.spawn_nonces
2616 .lock()
2617 .unwrap_or_else(|poisoned| poisoned.into_inner())
2618 .remove(module_id);
2619 self.close_swap(module_id);
2620 let mut reserved_nonces = self
2621 .reserved_nonces
2622 .lock()
2623 .unwrap_or_else(|poisoned| poisoned.into_inner());
2624 if reserved_nonces.contains_key(module_id) {
2625 reserved_nonces.insert(module_id.to_string(), None);
2628 }
2629 drop(reserved_nonces);
2630 self.reserved_prefix_owners
2631 .lock()
2632 .unwrap_or_else(|poisoned| poisoned.into_inner())
2633 .retain(|_, owner| owner != module_id);
2634 let removed = self
2635 .modules
2636 .lock()
2637 .unwrap_or_else(|poisoned| poisoned.into_inner())
2638 .remove(module_id);
2639 self.configured_ids
2640 .lock()
2641 .unwrap_or_else(|poisoned| poisoned.into_inner())
2642 .remove(module_id);
2643 removed
2644 }
2645
2646 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2649 self.removal_tombstones
2650 .lock()
2651 .unwrap_or_else(|poisoned| poisoned.into_inner())
2652 .insert(module_id.to_string(), unix_ms_now());
2653 }
2654
2655 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2657 self.removal_tombstones
2658 .lock()
2659 .unwrap_or_else(|poisoned| poisoned.into_inner())
2660 .get(module_id)
2661 .copied()
2662 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2663 }
2664
2665 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2670 if self.get(module_id).is_some() {
2671 return false;
2672 }
2673 let mut reserved_nonces = self
2674 .reserved_nonces
2675 .lock()
2676 .unwrap_or_else(|poisoned| poisoned.into_inner());
2677 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2678 return false;
2679 }
2680 reserved_nonces.remove(module_id);
2681 true
2682 }
2683
2684 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2685 Arc::clone(&self.operation_lock)
2686 }
2687}
2688
2689#[derive(Debug, Clone)]
2691pub struct Supervisor {
2692 registry: Arc<Registry>,
2693 restart_policy: RestartPolicy,
2694 drain_timeout: Duration,
2695 connection_file_path: Option<PathBuf>,
2696 capture_logs_dir: Option<PathBuf>,
2697 forwarding: Option<Arc<ForwardingTable>>,
2698 process_liveness: Arc<SupervisorProcessLiveness>,
2699 supervisor_handle: Option<SupervisorHandle>,
2700 health: HealthConfig,
2701 daemon_start_clock: crate::clock::StartClock,
2702 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2703 spawn_events: SpawnEventFeed,
2704 provenance_probe: ExecutableIdentityProbe,
2705 child_roster: ChildRoster,
2708 #[cfg(target_os = "linux")]
2709 cgroup_placement: Option<subc_cgroup::Placement>,
2710 #[cfg(test)]
2711 test_after_first_spawn: AfterFirstSpawnHook,
2712}
2713
2714pub(crate) struct RegisteredConfiguredModule {
2718 spec: ModuleSpec,
2719 runtime: SupervisorRuntimeConfig,
2720 module: SupervisedModule,
2721 commands: mpsc::Receiver<SupervisorCommand>,
2722}
2723
2724#[cfg(test)]
2730#[derive(Clone, Default)]
2731struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2732
2733#[cfg(test)]
2734type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2735
2736#[cfg(test)]
2737impl fmt::Debug for AfterFirstSpawnHook {
2738 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2739 f.write_str("AfterFirstSpawnHook")
2740 }
2741}
2742
2743#[cfg(test)]
2744impl AfterFirstSpawnHook {
2745 fn run(&self, module_id: &str) {
2746 if let Some(hook) = &self.0 {
2747 hook(module_id);
2748 }
2749 }
2750}
2751
2752impl Supervisor {
2753 #[cfg(test)]
2754 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2755 let supervisor = Self::new(registry, policy);
2756 #[cfg(target_os = "macos")]
2757 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2758 supervisor
2759 }
2760 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2765 let path = path.into();
2766 #[cfg(target_os = "macos")]
2767 {
2768 let result = probe_privacy_trampoline(&path).map(|()| path);
2769 if let Err(cause) = &result {
2770 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2771 }
2772 self.child_roster.set_privacy_trampoline(result);
2773 }
2774 #[cfg(not(target_os = "macos"))]
2775 let _ = path;
2776 self
2777 }
2778 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2781 self.child_roster.shutdown_flag()
2782 }
2783
2784 #[cfg(unix)]
2795 pub(crate) fn begin_daemon_shutdown(&self) {
2796 self.child_roster.close();
2797 if let Some(journal) = &self.terminal_journal {
2798 journal.stamp_shutdown();
2799 }
2800 }
2801
2802 #[cfg(unix)]
2806 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2807 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2808 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2809 let Some(forwarding) = &self.forwarding else {
2810 return Ok(());
2811 };
2812 let notice_started = Instant::now();
2813 let modules = forwarding
2814 .begin_daemon_drain()
2815 .map_err(SuperviseError::Forwarding)?;
2816 let deadline_ms =
2817 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2818 let mut notices = tokio::task::JoinSet::new();
2819 let mut drains = Vec::new();
2820 for (module_id, endpoint) in modules {
2821 let Some(target) = forwarding
2822 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2823 .map_err(SuperviseError::Forwarding)?
2824 else {
2825 continue;
2826 };
2827 let routes = forwarding
2828 .endpoint_routes(target.endpoint)
2829 .map_err(SuperviseError::Forwarding)?;
2830 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2836 reason: RouteCloseReason::Restart,
2837 deadline_ms,
2838 })
2839 .expect("module draining serializes");
2840 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2841 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2842 for route in routes {
2843 let client = route.goodbye_target;
2844 if let Some((_, channels)) = clients
2845 .iter_mut()
2846 .find(|(existing, _)| existing.connection_id == client.connection_id)
2847 {
2848 channels.push(client.channel);
2849 } else {
2850 let channel = client.channel;
2851 clients.push((client, vec![channel]));
2852 }
2853 }
2854 for (client, mut channels) in clients {
2855 channels.sort_unstable();
2856 channels.dedup();
2857 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2858 module_id: module_id.clone(),
2859 channels,
2860 reason: RouteCloseReason::Restart,
2861 })
2862 .expect("route closing serializes");
2863 recipients.push((client.sink, client.negotiated_ver, closing));
2864 }
2865 for (sink, version, body) in recipients {
2866 notices.spawn(async move {
2867 let frame = Frame::build_with_version(
2868 version,
2869 FrameType::Push,
2870 control_flags(),
2871 0,
2872 0,
2873 0,
2874 body,
2875 )
2876 .expect("bounded lifecycle notice frame builds");
2877 sink.send_flushed(frame).await
2878 });
2879 }
2880 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2881 drains.push((module_id, target.endpoint, gauges));
2882 }
2883 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2886 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2887 if !matches!(result, Ok(Ok(()))) {
2888 warn!(?result, "daemon shutdown notice delivery failed");
2889 }
2890 }
2891 notices.abort_all();
2892 let notice_elapsed = notice_started.elapsed();
2893 let drain_started = Instant::now();
2894 let deadline = drain_started + DRAIN_BUDGET;
2895 let mut waits = tokio::task::JoinSet::new();
2896 for (module_id, endpoint, gauges) in drains {
2897 let forwarding = Arc::clone(forwarding);
2898 let mut runtime = self.runtime_config();
2899 runtime.health.cadence = Duration::from_millis(100);
2900 waits.spawn(async move {
2901 let wait_result = wait_for_forwarding_quiescence(
2902 &forwarding,
2903 &module_id,
2904 &runtime,
2905 endpoint,
2906 deadline,
2907 &gauges,
2908 DrainScope::Endpoint(endpoint),
2909 &mut DrainGaugeTrace::new(&gauges),
2910 )
2911 .await;
2912 forwarding
2916 .record_daemon_drain_result(
2917 endpoint,
2918 drained_after_quiescence_wait(&wait_result),
2919 )
2920 .map_err(SuperviseError::Forwarding)?;
2921 wait_result
2922 });
2923 }
2924 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2925 if !matches!(result, Ok(Ok(true))) {
2926 warn!(?result, "daemon shutdown drain did not reach quiescence");
2927 }
2928 }
2929 info!(
2932 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
2933 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
2934 "daemon shutdown forwarding drain complete"
2935 );
2936 Ok(())
2937 }
2938
2939 #[cfg(unix)]
2951 pub(crate) async fn end_children_for_daemon_shutdown(
2952 &self,
2953 already_escalated: bool,
2954 escalate: impl std::future::Future<Output = ()>,
2955 ) {
2956 tokio::pin!(escalate);
2957 let mut escalated = already_escalated;
2958 if let Some(forwarding) = &self.forwarding {
2959 let reason = CloseReason::new(
2960 "daemon_shutdown",
2961 "the daemon is exiting after its shutdown notice and drain",
2962 );
2963 if escalated {
2964 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2967 } else {
2968 tokio::select! {
2969 biased;
2970 _ = escalate.as_mut() => {
2971 info!("second SIGTERM: abandoning module GOODBYE delivery");
2972 escalated = true;
2973 }
2974 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2975 }
2976 }
2977 let closed = forwarding.close_all_connections(&reason);
2978 debug!(closed, "closed established connections for daemon shutdown");
2979 }
2980 let escalated_here = escalated && !already_escalated;
2984 let remaining_escalate = async move {
2985 if escalated_here {
2986 std::future::pending::<()>().await;
2987 } else {
2988 escalate.await;
2989 }
2990 };
2991 crate::child_roster::end_children_for_daemon_shutdown(
2992 &self.child_roster,
2993 escalated,
2994 remaining_escalate,
2995 )
2996 .await;
2997 }
2998
2999 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3000 Self {
3001 registry,
3002 restart_policy,
3003 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3004 connection_file_path: None,
3005 capture_logs_dir: None,
3006 forwarding: None,
3007 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3008 supervisor_handle: None,
3009 health: HealthConfig::default(),
3010 daemon_start_clock: crate::clock::StartClock::capture(),
3011 terminal_journal: None,
3012 spawn_events: SpawnEventFeed::default(),
3013 provenance_probe: ExecutableIdentityProbe::default(),
3014 child_roster: ChildRoster::default(),
3015 #[cfg(target_os = "linux")]
3016 cgroup_placement: None,
3017 #[cfg(test)]
3018 test_after_first_spawn: AfterFirstSpawnHook::default(),
3019 }
3020 }
3021
3022 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3023 self.drain_timeout = drain_timeout;
3024 self
3025 }
3026
3027 pub(crate) fn default_drain_timeout(&self) -> Duration {
3029 self.drain_timeout
3030 }
3031
3032 pub fn with_process_liveness(
3033 mut self,
3034 process_liveness: Arc<SupervisorProcessLiveness>,
3035 ) -> Self {
3036 self.process_liveness = process_liveness;
3037 self
3038 }
3039
3040 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3041 self.connection_file_path = Some(connection_file_path.into());
3042 self
3043 }
3044
3045 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3047 self.capture_logs_dir = Some(logs_dir.into());
3048 self
3049 }
3050
3051 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3054 self.spawn_events.configure_incarnation(daemon_incarnation);
3058 self
3059 }
3060
3061 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3064 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3065 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3066 path,
3067 daemon_incarnation,
3068 )));
3069 this
3070 }
3071
3072 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3073 self.forwarding = Some(forwarding);
3074 self
3075 }
3076
3077 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3078 self.spawn_events = supervisor_handle.spawn_events.clone();
3079 self.supervisor_handle = Some(supervisor_handle);
3080 self
3081 }
3082
3083 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3084 self.health = health;
3085 self
3086 }
3087
3088 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3092 self.child_roster.record_to(path.into());
3093 self
3094 }
3095
3096 #[cfg(target_os = "linux")]
3097 pub fn with_cgroup_placement(
3098 mut self,
3099 cgroup_placement: Option<subc_cgroup::Placement>,
3100 ) -> Self {
3101 self.cgroup_placement = cgroup_placement;
3102 self
3103 }
3104
3105 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3111 validate_spec(&spec)?;
3112 self.establish_identity(&spec);
3113
3114 let runtime = self.runtime_config();
3115 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3116 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3117 let spawned = spawn_child(
3118 &spec,
3119 runtime.connection_file_path.as_deref(),
3120 self.supervisor_handle.as_ref(),
3121 &runtime.stderr_ring,
3122 runtime.capture_logs_dir.as_deref(),
3123 &runtime.child_roster,
3124 #[cfg(target_os = "linux")]
3125 runtime.cgroup_placement.as_ref(),
3126 );
3127 #[cfg(test)]
3128 self.test_after_first_spawn.run(&spec.module_id);
3129 let child = match spawned {
3130 Ok(child) => child,
3131 Err(err) => {
3132 self.abandon_unrostered(&spec.module_id);
3135 return Err(err);
3136 }
3137 };
3138 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3139 if let Some(spawn) = activity_spawn {
3140 spawn.complete(&snapshot);
3141 }
3142
3143 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3144 }
3145
3146 fn establish_identity(&self, spec: &ModuleSpec) {
3159 if let Some(supervisor_handle) = &self.supervisor_handle {
3160 supervisor_handle.apply_identity_configuration(spec);
3161 supervisor_handle.mark_configured(&spec.module_id);
3162 }
3163 }
3164
3165 fn abandon_unrostered(&self, module_id: &str) {
3168 if let Some(supervisor_handle) = &self.supervisor_handle {
3169 supervisor_handle.unmark_configured_unless_rostered(module_id);
3170 }
3171 }
3172
3173 fn mark_first_process_running(
3176 &self,
3177 spec: &ModuleSpec,
3178 runtime: &SupervisorRuntimeConfig,
3179 snapshot: &SharedSnapshot,
3180 child: &SupervisedChild,
3181 ) -> Result<(), SuperviseError> {
3182 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3183 self.abandon_unrostered(&spec.module_id);
3184 return Err(err);
3185 }
3186 self.process_liveness
3187 .track(spec.module_id.clone(), Arc::clone(snapshot));
3188 Ok(())
3189 }
3190
3191 pub fn supervise_configured(
3197 &self,
3198 spec: ModuleSpec,
3199 enabled: bool,
3200 ) -> Result<SupervisedModule, SuperviseError> {
3201 self.supervise_configured_requested(
3202 spec,
3203 enabled,
3204 self.health.clone(),
3205 None,
3206 self.restart_policy,
3207 None,
3208 )
3209 }
3210
3211 pub fn supervise_configured_with_health(
3217 &self,
3218 spec: ModuleSpec,
3219 enabled: bool,
3220 health: HealthConfig,
3221 drain_timeout_ms: Option<u64>,
3222 restart_policy: RestartPolicy,
3223 ) -> Result<SupervisedModule, SuperviseError> {
3224 self.supervise_configured_requested(
3225 spec,
3226 enabled,
3227 health,
3228 drain_timeout_ms,
3229 restart_policy,
3230 None,
3231 )
3232 }
3233
3234 pub(crate) fn supervise_configured_requested(
3235 &self,
3236 spec: ModuleSpec,
3237 enabled: bool,
3238 health: HealthConfig,
3239 drain_timeout_ms: Option<u64>,
3240 restart_policy: RestartPolicy,
3241 requester: Option<Principal>,
3242 ) -> Result<SupervisedModule, SuperviseError> {
3243 let registered = self.register_configured_with_health(
3244 spec,
3245 enabled,
3246 health,
3247 drain_timeout_ms,
3248 restart_policy,
3249 )?;
3250 let daemon_initiated = requester.is_none();
3251 self.start_registered_configured(registered, requester, daemon_initiated)
3252 }
3253
3254 pub(crate) fn register_configured_with_health(
3259 &self,
3260 spec: ModuleSpec,
3261 enabled: bool,
3262 health: HealthConfig,
3263 drain_timeout_ms: Option<u64>,
3264 restart_policy: RestartPolicy,
3265 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3266 validate_spec(&spec)?;
3267 self.establish_identity(&spec);
3268
3269 let mut runtime = self.runtime_config();
3270 runtime.health = health;
3271 runtime.restart_policy = restart_policy;
3272 if let Some(ms) = drain_timeout_ms {
3273 runtime.drain_timeout = Duration::from_millis(ms);
3274 *runtime
3275 .effective_drain_timeout
3276 .lock()
3277 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3278 }
3279 let snapshot = if enabled {
3280 SupervisorSnapshot::starting()
3281 } else {
3282 SupervisorSnapshot::disabled()
3283 };
3284 let snapshot = {
3285 let mut snapshot = snapshot;
3286 if enabled {
3287 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3288 {
3289 snapshot.waiting_for_previous_pid = Some(pid);
3290 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3291 }
3292 }
3293 snapshot
3294 };
3295 let (module, commands) =
3296 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3297 Ok(RegisteredConfiguredModule {
3298 spec,
3299 runtime,
3300 module,
3301 commands,
3302 })
3303 }
3304
3305 pub(crate) fn start_registered_configured(
3308 &self,
3309 registered: RegisteredConfiguredModule,
3310 requester: Option<Principal>,
3311 daemon_initiated: bool,
3312 ) -> Result<SupervisedModule, SuperviseError> {
3313 let RegisteredConfiguredModule {
3314 spec,
3315 mut runtime,
3316 module,
3317 commands,
3318 } = registered;
3319 runtime.start_requester = requester;
3320 runtime.recovery_is_daemon = daemon_initiated;
3321 let snapshot = &module.inner.snapshot;
3322 let should_spawn = {
3323 let state = lock_snapshot(snapshot)?;
3324 state.enabled && state.waiting_for_previous_pid.is_none()
3325 };
3326 let child = if should_spawn {
3327 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3328 let spawned = spawn_child(
3329 &spec,
3330 runtime.connection_file_path.as_deref(),
3331 self.supervisor_handle.as_ref(),
3332 &runtime.stderr_ring,
3333 runtime.capture_logs_dir.as_deref(),
3334 &runtime.child_roster,
3335 #[cfg(target_os = "linux")]
3336 runtime.cgroup_placement.as_ref(),
3337 );
3338 #[cfg(test)]
3339 self.test_after_first_spawn.run(&spec.module_id);
3340 match spawned {
3341 Ok(child) => {
3342 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3343 if let Some(spawn) = activity_spawn {
3344 spawn.complete(snapshot);
3345 }
3346 Some(child)
3347 }
3348 Err(err) => {
3349 if runtime.health.critical {
3350 error!(
3351 module_id = %spec.module_id,
3352 program = %spec.program.display(),
3353 error = %err,
3354 "critical configured module failed to spawn; marking failed and alerting"
3355 );
3356 } else {
3357 error!(
3358 module_id = %spec.module_id,
3359 program = %spec.program.display(),
3360 error = %err,
3361 "configured module failed to spawn; marking failed and continuing"
3362 );
3363 }
3364 fail_snapshot(
3365 snapshot,
3366 Some(&spec.module_id),
3367 None,
3368 &runtime.failure_context(daemon_initiated),
3369 );
3370 None
3371 }
3372 }
3373 } else {
3374 None
3375 };
3376 self.start_monitor(spec, runtime, &module, child, commands);
3377 Ok(module)
3378 }
3379
3380 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3381 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3382 SupervisorRuntimeConfig {
3383 recovery_is_daemon: true,
3384 start_requester: None,
3385 pending_activity_restart: false,
3386 provenance_probe: self.provenance_probe.clone(),
3387 scheduled_respawn: Arc::default(),
3388 deferred_reload_reply: Arc::default(),
3389 restart_policy: self.restart_policy,
3390 drain_timeout: self.drain_timeout,
3391 child_roster: self
3394 .child_roster
3395 .for_module(Arc::clone(&effective_drain_timeout)),
3396 effective_drain_timeout,
3397 default_drain_timeout: self.drain_timeout,
3398 health: self.health.clone(),
3399 connection_file_path: self.connection_file_path.clone(),
3400 capture_logs_dir: self.capture_logs_dir.clone(),
3401 forwarding: self.forwarding.clone(),
3402 supervisor_handle: self.supervisor_handle.clone(),
3403 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3404 terminal_ring: Arc::new(Mutex::new(
3405 TerminalRing::new(
3406 TerminalRingConfig::default(),
3407 self.daemon_start_clock.started_at_ms(),
3408 )
3409 .with_start_clock(self.daemon_start_clock)
3410 .with_journal(self.terminal_journal.clone())
3411 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3412 )),
3413 spawn_events: self.spawn_events.clone(),
3414 #[cfg(target_os = "linux")]
3415 cgroup_placement: self.cgroup_placement.clone(),
3416 #[cfg(test)]
3417 test_seed_stale_facts_before_enable_spawn: false,
3418 #[cfg(test)]
3419 test_reload_exit_record_gate: None,
3420 }
3421 }
3422
3423 fn supervised_module(
3424 &self,
3425 spec: ModuleSpec,
3426 runtime: SupervisorRuntimeConfig,
3427 snapshot: SharedSnapshot,
3428 child: Option<SupervisedChild>,
3429 ) -> SupervisedModule {
3430 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3431 self.start_monitor(spec, runtime, &module, child, commands);
3432 module
3433 }
3434
3435 fn register_module(
3436 &self,
3437 spec: &ModuleSpec,
3438 runtime: &SupervisorRuntimeConfig,
3439 snapshot: SharedSnapshot,
3440 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3441 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3442 spec: spec.clone(),
3443 health: runtime.health.clone(),
3444 }));
3445 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3446 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3447 let restart_policy = runtime.restart_policy;
3451 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3452 let (tx, rx) = mpsc::channel(4);
3453 let module_id = spec.module_id.clone();
3454 let module = SupervisedModule {
3455 inner: Arc::new(SupervisedModuleInner {
3456 module_id: module_id.clone(),
3457 registry: Arc::clone(&self.registry),
3458 snapshot,
3459 configuration,
3460 stderr_ring,
3461 terminal_ring,
3462 commands: tx,
3463 monitor: Mutex::new(None),
3464 restart_policy,
3465 effective_drain_timeout,
3466 provenance_probe: self.provenance_probe.clone(),
3467 }),
3468 };
3469 if let Some(supervisor_handle) = &self.supervisor_handle {
3472 supervisor_handle.insert(module.clone());
3473 }
3474 (module, rx)
3475 }
3476
3477 fn start_monitor(
3478 &self,
3479 spec: ModuleSpec,
3480 runtime: SupervisorRuntimeConfig,
3481 module: &SupervisedModule,
3482 child: Option<SupervisedChild>,
3483 commands: mpsc::Receiver<SupervisorCommand>,
3484 ) {
3485 let monitor = tokio::spawn(supervise_loop(
3486 spec,
3487 runtime,
3488 Arc::clone(&self.registry),
3489 Arc::clone(&self.process_liveness),
3490 Arc::clone(&module.inner.snapshot),
3491 child,
3492 commands,
3493 ));
3494 *module
3495 .inner
3496 .monitor
3497 .lock()
3498 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3499 }
3500}
3501
3502impl Default for Supervisor {
3503 fn default() -> Self {
3504 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3505 }
3506}
3507
3508#[derive(Clone)]
3510pub struct SupervisedModule {
3511 inner: Arc<SupervisedModuleInner>,
3512}
3513
3514struct SupervisedModuleInner {
3515 module_id: String,
3516 registry: Arc<Registry>,
3517 snapshot: SharedSnapshot,
3518 configuration: Arc<Mutex<SupervisedConfiguration>>,
3519 stderr_ring: Arc<Mutex<StderrRing>>,
3520 terminal_ring: Arc<Mutex<TerminalRing>>,
3521 commands: mpsc::Sender<SupervisorCommand>,
3522 monitor: Mutex<Option<JoinHandle<()>>>,
3523 restart_policy: RestartPolicy,
3527 effective_drain_timeout: Arc<Mutex<Duration>>,
3528 provenance_probe: ExecutableIdentityProbe,
3529}
3530
3531impl fmt::Debug for SupervisedModule {
3532 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3533 f.debug_struct("SupervisedModule")
3534 .field("module_id", &self.inner.module_id)
3535 .field("status", &self.status())
3536 .finish_non_exhaustive()
3537 }
3538}
3539
3540impl SupervisedModule {
3541 pub fn module_id(&self) -> &str {
3542 &self.inner.module_id
3543 }
3544
3545 #[cfg(test)]
3549 pub(crate) fn record_health_probe_failure_for_test(
3550 &self,
3551 detail: &str,
3552 ) -> Result<(), SuperviseError> {
3553 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3554 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3555 state.health.detail = Some(detail.to_string());
3556 })
3557 }
3558
3559 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3560 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3561 }
3562
3563 pub(crate) fn registration_end_reason(
3564 &self,
3565 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3566 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3567 Ok(match snapshot.state {
3568 ModuleState::Draining if snapshot.draining_to_replace => {
3569 Some(RegistrationEndReason::SupervisorRestart)
3570 }
3571 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3572 ModuleState::Restarting if snapshot.draining_to_replace => {
3573 Some(RegistrationEndReason::SupervisorRestart)
3574 }
3575 _ => None,
3576 })
3577 }
3578
3579 pub fn stderr_tail(
3586 &self,
3587 max_lines: Option<usize>,
3588 max_bytes: Option<usize>,
3589 ) -> StderrTailSnapshot {
3590 self.inner
3591 .stderr_ring
3592 .lock()
3593 .unwrap_or_else(|poisoned| poisoned.into_inner())
3594 .snapshot(max_lines, max_bytes)
3595 }
3596
3597 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3602 self.inner
3603 .terminal_ring
3604 .lock()
3605 .unwrap_or_else(|poisoned| poisoned.into_inner())
3606 .snapshot()
3607 }
3608
3609 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3614 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3615 }
3616
3617 pub(crate) async fn read_durable_terminal_history(
3622 &self,
3623 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3624 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3625 let module_id = self.inner.module_id.clone();
3626 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3627 .await
3628 }
3629
3630 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3631 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3632 .map(|(status, _)| status)
3633 }
3634
3635 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3636 let state = lock_snapshot(&self.inner.snapshot)?;
3637 Ok(state
3638 .failed_retry
3639 .pending
3640 .filter(|_| state.state == ModuleState::Failed && state.enabled)
3641 .map(|retry| {
3642 unix_ms_now().saturating_add(
3643 u64::try_from(
3644 retry
3645 .deadline
3646 .saturating_duration_since(Instant::now())
3647 .as_millis(),
3648 )
3649 .unwrap_or(u64::MAX),
3650 )
3651 }))
3652 }
3653
3654 pub(crate) fn record_deliberate_severance(
3655 &self,
3656 identity: ProcessIdentity,
3657 ) -> Result<bool, SuperviseError> {
3658 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3659 if snapshot.pid != Some(identity.pid)
3660 || snapshot.process_start_time != Some(identity.start_time)
3661 {
3662 return Ok(false);
3663 }
3664 snapshot.deliberate_severance = Some(identity);
3665 Ok(true)
3666 }
3667
3668 pub(crate) fn status_for_control(
3673 &self,
3674 caller: &'static str,
3675 ) -> Result<ModuleStatus, SuperviseError> {
3676 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3677 .map(|(status, _)| status)
3678 }
3679
3680 fn status_with_snapshot_lock(
3681 &self,
3682 snapshot: &SharedSnapshot,
3683 caller: Option<&'static str>,
3684 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3685 let mut guard = match caller {
3686 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3687 None => lock_snapshot(snapshot)?,
3688 };
3689 let restart_count =
3692 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3693 let snapshot = guard.clone();
3694 drop(guard);
3695 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3696 SuperviseError::StatePoisoned {
3697 module_id: Some(self.inner.module_id.clone()),
3698 }
3699 })?;
3700 let registration_active = self
3701 .inner
3702 .registry
3703 .get_module(&self.inner.module_id)
3704 .map_err(SuperviseError::Registry)?
3705 .is_some();
3706 let protocol = snapshot
3707 .spawned_protocol
3708 .unwrap_or(self.declared_protocol()?);
3709 let running_process =
3710 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3711 let live = match protocol {
3717 ModuleProtocol::Subc => running_process && registration_active,
3718 ModuleProtocol::None => running_process,
3719 };
3720
3721 Ok((
3722 ModuleStatus {
3723 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3724 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3725 module_id: self.inner.module_id.clone(),
3726 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3727 state: snapshot.state,
3728 enabled: snapshot.enabled,
3729 process_alive: snapshot.process_alive,
3730 registration_active,
3731 protocol,
3732 live,
3733 restart_count,
3734 lifetime_restarts: snapshot.lifetime_restarts,
3735 spawn_generation: snapshot.spawn_generation,
3736 max_restarts: self.inner.restart_policy.max_restarts,
3737 restart_window: self.inner.restart_policy.window,
3738 drain_timeout,
3739 restart_backoff: self.inner.restart_policy.backoff,
3740 restart_max_backoff: self.inner.restart_policy.max_backoff,
3741 pid: snapshot.reported_pid(),
3742 spawned_at_ms: snapshot.spawned_at_ms,
3743 spawned_from: snapshot.spawned_from,
3744 launch_spec_digest: snapshot.launch_spec_digest,
3745 process_start_time: snapshot.process_start_time,
3746 last_exit: snapshot.last_exit,
3747 health: snapshot.health,
3748 },
3749 snapshot.spawned_file_identity.clone(),
3750 ))
3751 }
3752
3753 #[cfg(test)]
3754 pub(crate) fn hold_snapshot_for_test(
3755 &self,
3756 acquired: std::sync::mpsc::Sender<()>,
3757 hold: Duration,
3758 ) -> std::thread::JoinHandle<()> {
3759 let snapshot = Arc::clone(&self.inner.snapshot);
3760 std::thread::spawn(move || {
3761 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3762 acquired
3763 .send(())
3764 .expect("test receiver waits for snapshot lock");
3765 std::thread::sleep(hold);
3766 })
3767 }
3768
3769 pub(crate) async fn status_and_running_image_agreement(
3775 &self,
3776 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3777 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3778 let image = self
3779 .inner
3780 .provenance_probe
3781 .observe(
3782 status.pid,
3783 status.spawned_from.as_deref(),
3784 identity,
3785 status.process_start_time,
3786 )
3787 .await;
3788 Ok((status, image))
3789 }
3790
3791 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3792 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3793 Ok(snapshot) => snapshot.clone(),
3794 Err(_) => {
3795 return subc_control::RunningImageAgreement::Unavailable {
3796 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3797 };
3798 }
3799 };
3800 self.inner
3801 .provenance_probe
3802 .observe(
3803 snapshot.reported_pid(),
3804 snapshot.spawned_from.as_deref(),
3805 snapshot.spawned_file_identity.clone(),
3806 snapshot.process_start_time,
3807 )
3808 .await
3809 }
3810
3811 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3814 #[cfg(windows)]
3815 {
3816 let image = lock_snapshot(&self.inner.snapshot)
3817 .ok()
3818 .and_then(|snapshot| {
3819 snapshot
3820 .spawned_file_identity
3821 .as_ref()
3822 .and_then(|identity| identity.windows_image.clone())
3823 });
3824 match image {
3825 Some(image) => {
3826 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3827 }
3828 None => subc_control::ChildResourceUsage::Unavailable {
3829 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3830 },
3831 }
3832 }
3833 #[cfg(not(windows))]
3834 {
3835 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3836 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3837 Err(_) => {
3838 return subc_control::ChildResourceUsage::Unavailable {
3839 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3840 }
3841 }
3842 };
3843 crate::child_resources::read(pid, start_time)
3844 }
3845 }
3846
3847 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3848 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3849 Ok(match snapshot.state {
3850 ModuleState::Restarting => true,
3851 ModuleState::Failed | ModuleState::Disabled => false,
3852 _ if snapshot.enabled && snapshot.respawn_pending => true,
3853 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3854 })
3855 }
3856
3857 #[cfg(test)]
3858 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3859 self.is_warming_with_snapshot_lock(None)
3860 }
3861
3862 pub(crate) fn is_warming_for_control(
3863 &self,
3864 caller: &'static str,
3865 ) -> Result<bool, SuperviseError> {
3866 self.is_warming_with_snapshot_lock(Some(caller))
3867 }
3868
3869 fn is_warming_with_snapshot_lock(
3870 &self,
3871 caller: Option<&'static str>,
3872 ) -> Result<bool, SuperviseError> {
3873 let snapshot = match caller {
3874 Some(caller) => {
3875 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3876 }
3877 None => lock_snapshot(&self.inner.snapshot)?,
3878 }
3879 .clone();
3880 Ok(matches!(
3881 snapshot.state,
3882 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3883 ))
3884 }
3885
3886 pub async fn drain(&self) -> Result<(), SuperviseError> {
3888 self.stop().await
3889 }
3890
3891 pub(crate) async fn retire_requested(
3892 &self,
3893 requester: Option<Principal>,
3894 ) -> Result<(), SuperviseError> {
3895 match self.state()? {
3896 ModuleState::Stopped => return Ok(()),
3897 ModuleState::Failed
3898 if lock_snapshot(&self.inner.snapshot)?
3899 .failed_retry
3900 .pending
3901 .is_none() =>
3902 {
3903 return Ok(())
3904 }
3905 ModuleState::Failed
3906 | ModuleState::Starting
3907 | ModuleState::Running
3908 | ModuleState::Unresponsive
3909 | ModuleState::Restarting
3910 | ModuleState::Draining
3911 | ModuleState::Disabled => {}
3912 }
3913
3914 let (reply_tx, reply_rx) = oneshot::channel();
3915 self.inner
3916 .commands
3917 .send(SupervisorCommand::Retire {
3918 requester,
3919 reply: reply_tx,
3920 })
3921 .await
3922 .map_err(|_| SuperviseError::CommandClosed {
3923 module_id: self.inner.module_id.clone(),
3924 })?;
3925 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3926 module_id: self.inner.module_id.clone(),
3927 })?
3928 }
3929
3930 pub async fn stop(&self) -> Result<(), SuperviseError> {
3931 self.stop_requested(None).await
3932 }
3933
3934 pub(crate) async fn stop_requested(
3935 &self,
3936 requester: Option<Principal>,
3937 ) -> Result<(), SuperviseError> {
3938 match self.state()? {
3939 ModuleState::Stopped => return Ok(()),
3940 ModuleState::Failed
3941 if lock_snapshot(&self.inner.snapshot)?
3942 .failed_retry
3943 .pending
3944 .is_none() =>
3945 {
3946 return Ok(())
3947 }
3948 ModuleState::Failed
3949 | ModuleState::Starting
3950 | ModuleState::Running
3951 | ModuleState::Unresponsive
3952 | ModuleState::Restarting
3953 | ModuleState::Draining
3954 | ModuleState::Disabled => {}
3955 }
3956
3957 let (reply_tx, reply_rx) = oneshot::channel();
3958 self.inner
3959 .commands
3960 .send(SupervisorCommand::Drain {
3961 requester,
3962 reply: reply_tx,
3963 })
3964 .await
3965 .map_err(|_| SuperviseError::CommandClosed {
3966 module_id: self.inner.module_id.clone(),
3967 })?;
3968 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3969 module_id: self.inner.module_id.clone(),
3970 })?
3971 }
3972
3973 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3974 self.restart_requested(drain_timeout_ms, None).await
3975 }
3976
3977 pub(crate) async fn restart_requested(
3978 &self,
3979 drain_timeout_ms: Option<u64>,
3980 requester: Option<Principal>,
3981 ) -> Result<(), SuperviseError> {
3982 let queued_at = Instant::now();
3983 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3984 let (reply_tx, reply_rx) = oneshot::channel();
3985 self.inner
3986 .commands
3987 .send(SupervisorCommand::Restart {
3988 requester,
3989 drain_timeout_ms,
3990 received_at_generation,
3991 queued_at,
3992 reply: reply_tx,
3993 })
3994 .await
3995 .map_err(|_| SuperviseError::CommandClosed {
3996 module_id: self.inner.module_id.clone(),
3997 })?;
3998 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3999 module_id: self.inner.module_id.clone(),
4000 })?
4001 }
4002
4003 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4008 self.swap_requested(ready_timeout, None).await
4009 }
4010
4011 pub(crate) async fn swap_requested(
4012 &self,
4013 ready_timeout: Option<Duration>,
4014 requester: Option<Principal>,
4015 ) -> Result<(), SuperviseError> {
4016 let (reply_tx, reply_rx) = oneshot::channel();
4017 self.inner
4018 .commands
4019 .send(SupervisorCommand::Swap {
4020 requester,
4021 ready_timeout,
4022 reply: reply_tx,
4023 })
4024 .await
4025 .map_err(|_| SuperviseError::CommandClosed {
4026 module_id: self.inner.module_id.clone(),
4027 })?;
4028 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4029 module_id: self.inner.module_id.clone(),
4030 })?
4031 }
4032
4033 pub async fn reload(&self) -> Result<(), SuperviseError> {
4034 self.reload_requested(None).await
4035 }
4036
4037 pub(crate) async fn reload_requested(
4038 &self,
4039 requester: Option<Principal>,
4040 ) -> Result<(), SuperviseError> {
4041 let (reply_tx, reply_rx) = oneshot::channel();
4042 self.inner
4043 .commands
4044 .send(SupervisorCommand::Reload {
4045 requester,
4046 reply: reply_tx,
4047 })
4048 .await
4049 .map_err(|_| SuperviseError::CommandClosed {
4050 module_id: self.inner.module_id.clone(),
4051 })?;
4052 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4053 module_id: self.inner.module_id.clone(),
4054 })?
4055 }
4056
4057 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4058 self.set_enabled_requested(enabled, None).await
4059 }
4060
4061 pub(crate) async fn set_enabled_requested(
4062 &self,
4063 enabled: bool,
4064 requester: Option<Principal>,
4065 ) -> Result<bool, SuperviseError> {
4066 let (reply_tx, reply_rx) = oneshot::channel();
4067 self.inner
4068 .commands
4069 .send(SupervisorCommand::SetEnabled {
4070 requester,
4071 enabled,
4072 reply: reply_tx,
4073 })
4074 .await
4075 .map_err(|_| SuperviseError::CommandClosed {
4076 module_id: self.inner.module_id.clone(),
4077 })?;
4078 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4079 module_id: self.inner.module_id.clone(),
4080 })?
4081 }
4082
4083 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4087 let configured = self
4088 .inner
4089 .configuration
4090 .lock()
4091 .map_err(|_| SuperviseError::StatePoisoned {
4092 module_id: Some(self.inner.module_id.clone()),
4093 })?
4094 .spec
4095 .protocol;
4096 let state = lock_snapshot(&self.inner.snapshot)?;
4097 Ok(state.spawned_protocol.unwrap_or(configured))
4098 }
4099
4100 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4101 let configuration =
4102 self.inner
4103 .configuration
4104 .lock()
4105 .map_err(|_| SuperviseError::StatePoisoned {
4106 module_id: Some(self.inner.module_id.clone()),
4107 })?;
4108 Ok((configuration.spec.clone(), configuration.health.clone()))
4109 }
4110
4111 #[cfg(any(test, feature = "test-support"))]
4115 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4116 let (_, health) = self.configuration()?;
4117 let drain_timeout_ms = u64::try_from(
4118 self.inner
4119 .effective_drain_timeout
4120 .lock()
4121 .unwrap_or_else(|poisoned| poisoned.into_inner())
4122 .as_millis(),
4123 )
4124 .ok();
4125 self.update_configuration(spec, health, drain_timeout_ms)
4126 .await
4127 }
4128
4129 pub(crate) async fn update_configuration(
4130 &self,
4131 spec: ModuleSpec,
4132 health: HealthConfig,
4133 drain_timeout_ms: Option<u64>,
4134 ) -> Result<(), SuperviseError> {
4135 if spec.module_id != self.inner.module_id {
4136 return Err(SuperviseError::InvalidSpec {
4137 reason: "a supervised module's module_id cannot be changed".to_string(),
4138 });
4139 }
4140 validate_spec(&spec)?;
4141 let (reply_tx, reply_rx) = oneshot::channel();
4142 self.inner
4143 .commands
4144 .send(SupervisorCommand::UpdateConfiguration {
4145 spec: spec.clone(),
4146 health: health.clone(),
4147 drain_timeout_ms,
4148 reply: reply_tx,
4149 })
4150 .await
4151 .map_err(|_| SuperviseError::CommandClosed {
4152 module_id: self.inner.module_id.clone(),
4153 })?;
4154 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4155 module_id: self.inner.module_id.clone(),
4156 })?;
4157 let mut configuration =
4158 self.inner
4159 .configuration
4160 .lock()
4161 .map_err(|_| SuperviseError::StatePoisoned {
4162 module_id: Some(self.inner.module_id.clone()),
4163 })?;
4164 configuration.spec = spec;
4165 configuration.health = health;
4166 Ok(())
4167 }
4168}
4169
4170impl Drop for SupervisedModuleInner {
4171 fn drop(&mut self) {
4172 let Ok(mut monitor) = self.monitor.lock() else {
4173 return;
4174 };
4175 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4176 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4177 state.state = ModuleState::Stopped;
4178 clear_current_process_facts(state);
4179 });
4180 monitor.abort();
4181 }
4182 let _ = monitor.take();
4183 }
4184}
4185
4186#[derive(Debug)]
4187enum SupervisorCommand {
4188 Drain {
4189 requester: Option<Principal>,
4190 reply: oneshot::Sender<Result<(), SuperviseError>>,
4191 },
4192 Retire {
4193 requester: Option<Principal>,
4194 reply: oneshot::Sender<Result<(), SuperviseError>>,
4195 },
4196 Restart {
4197 requester: Option<Principal>,
4198 drain_timeout_ms: Option<u64>,
4203 received_at_generation: u64,
4207 queued_at: Instant,
4210 reply: oneshot::Sender<Result<(), SuperviseError>>,
4211 },
4212 Reload {
4213 requester: Option<Principal>,
4214 reply: oneshot::Sender<Result<(), SuperviseError>>,
4215 },
4216 SetEnabled {
4217 requester: Option<Principal>,
4218 enabled: bool,
4219 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4220 },
4221 UpdateConfiguration {
4222 spec: ModuleSpec,
4223 health: HealthConfig,
4224 drain_timeout_ms: Option<u64>,
4227 reply: oneshot::Sender<()>,
4228 },
4229 Swap {
4230 requester: Option<Principal>,
4231 ready_timeout: Option<Duration>,
4234 reply: oneshot::Sender<Result<(), SuperviseError>>,
4236 },
4237}
4238
4239#[derive(Debug)]
4240pub enum SuperviseError {
4241 InvalidSpec {
4242 reason: String,
4243 },
4244 Spawn {
4245 program: PathBuf,
4246 source: io::Error,
4247 cgroup_path: Option<PathBuf>,
4248 },
4249 Cgroup {
4250 module_id: String,
4251 source: io::Error,
4252 },
4253 LaunchNonce {
4256 reason: String,
4257 },
4258 Wait {
4259 module_id: String,
4260 source: io::Error,
4261 },
4262 Kill {
4263 module_id: String,
4264 source: io::Error,
4265 },
4266 Forwarding(ForwardingError),
4267 Registry(RegistryError),
4268 ReloadUnavailable {
4269 module_id: String,
4270 reason: String,
4271 },
4272 Disabled {
4277 module_id: String,
4278 },
4279 ReloadFailed {
4280 module_id: String,
4281 reason: String,
4282 },
4283 RegistrationStillActive {
4284 module_id: String,
4285 waited: Duration,
4286 },
4287 StatePoisoned {
4288 module_id: Option<String>,
4289 },
4290 CommandClosed {
4291 module_id: String,
4292 },
4293 SwapInProgress {
4297 module_id: String,
4298 },
4299 SwapRefused {
4301 module_id: String,
4302 reason: SwapRefusal,
4303 },
4304 SwapFailed {
4308 module_id: String,
4309 arm: SwapFailureArm,
4310 detail: String,
4311 candidate_exit: Option<ExitReport>,
4314 },
4315}
4316
4317#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4319pub enum SwapRefusal {
4320 OverlapExclusive,
4322 NotRegistered,
4325 ProtocolNone,
4328 NotConfigured,
4331 AlreadySwapping,
4333}
4334
4335impl SwapRefusal {
4336 pub fn as_str(self) -> &'static str {
4337 match self {
4338 Self::OverlapExclusive => "overlap_exclusive",
4339 Self::NotRegistered => "not_registered",
4340 Self::ProtocolNone => "protocol_none",
4341 Self::NotConfigured => "not_configured",
4342 Self::AlreadySwapping => "already_swapping",
4343 }
4344 }
4345}
4346
4347#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4350pub enum SwapFailureArm {
4351 SpawnFailed,
4353 NeverRegistered,
4355 NeverReady,
4357 CandidateExited,
4359 CandidateUnhealthy,
4361 Interrupted,
4365 CutoverLost,
4370}
4371
4372impl SwapFailureArm {
4373 pub fn as_str(self) -> &'static str {
4374 match self {
4375 Self::SpawnFailed => "spawn_failed",
4376 Self::NeverRegistered => "never_registered",
4377 Self::NeverReady => "never_ready",
4378 Self::CandidateExited => "candidate_exited",
4379 Self::CandidateUnhealthy => "candidate_unhealthy",
4380 Self::Interrupted => "interrupted",
4381 Self::CutoverLost => "cutover_lost",
4382 }
4383 }
4384}
4385
4386impl fmt::Display for SuperviseError {
4387 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4388 match self {
4389 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4390 Self::Spawn {
4391 program,
4392 source,
4393 cgroup_path: Some(cgroup_path),
4394 } => write!(
4395 f,
4396 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4397 cgroup_path.display(),
4398 program.display()
4399 ),
4400 Self::Spawn {
4401 program,
4402 source,
4403 cgroup_path: None,
4404 } => write!(
4405 f,
4406 "failed to spawn module '{}': {source}",
4407 program.display()
4408 ),
4409 Self::Cgroup { module_id, source } => {
4410 write!(
4411 f,
4412 "failed to prepare cgroup for module '{module_id}': {source}"
4413 )
4414 }
4415 Self::LaunchNonce { reason } => {
4416 write!(
4417 f,
4418 "failed to generate reserved-module launch nonce: {reason}"
4419 )
4420 }
4421 Self::Wait { module_id, source } => {
4422 write!(f, "failed to wait for module '{module_id}': {source}")
4423 }
4424 Self::Kill { module_id, source } => {
4425 write!(f, "failed to kill module '{module_id}': {source}")
4426 }
4427 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4428 Self::Registry(err) => write!(f, "registry error: {err}"),
4429 Self::ReloadUnavailable { module_id, reason } => {
4430 write!(f, "reload unavailable for module '{module_id}': {reason}")
4431 }
4432 Self::Disabled { module_id } => {
4433 write!(
4434 f,
4435 "module '{module_id}' is disabled; enable it before restart or reload"
4436 )
4437 }
4438 Self::ReloadFailed { module_id, reason } => {
4439 write!(f, "reload failed for module '{module_id}': {reason}")
4440 }
4441 Self::RegistrationStillActive { module_id, waited } => write!(
4442 f,
4443 "module '{module_id}' registration remained active after waiting {waited:?}"
4444 ),
4445 Self::StatePoisoned { module_id } => match module_id {
4446 Some(module_id) => {
4447 write!(f, "supervisor state for module '{module_id}' was poisoned")
4448 }
4449 None => write!(f, "supervisor state was poisoned"),
4450 },
4451 Self::CommandClosed { module_id } => {
4452 write!(
4453 f,
4454 "supervisor command channel for module '{module_id}' is closed"
4455 )
4456 }
4457 Self::SwapInProgress { module_id } => write!(
4458 f,
4459 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4460 ),
4461 Self::SwapRefused { module_id, reason } => match reason {
4462 SwapRefusal::OverlapExclusive => write!(
4463 f,
4464 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4465 ),
4466 SwapRefusal::NotRegistered => write!(
4467 f,
4468 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4469 ),
4470 SwapRefusal::ProtocolNone => write!(
4471 f,
4472 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4473 ),
4474 SwapRefusal::NotConfigured => write!(
4475 f,
4476 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4477 ),
4478 SwapRefusal::AlreadySwapping => {
4479 write!(f, "module '{module_id}' is already being swapped")
4480 }
4481 },
4482 Self::SwapFailed {
4483 module_id,
4484 arm,
4485 detail,
4486 ..
4487 } => write!(
4488 f,
4489 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4490 arm.as_str()
4491 ),
4492 }
4493 }
4494}
4495
4496impl Error for SuperviseError {
4497 fn source(&self) -> Option<&(dyn Error + 'static)> {
4498 match self {
4499 Self::Spawn { source, .. }
4500 | Self::Cgroup { source, .. }
4501 | Self::Wait { source, .. }
4502 | Self::Kill { source, .. } => Some(source),
4503 Self::Forwarding(err) => Some(err),
4504 Self::Registry(err) => Some(err),
4505 Self::LaunchNonce { .. }
4506 | Self::InvalidSpec { .. }
4507 | Self::ReloadUnavailable { .. }
4508 | Self::Disabled { .. }
4509 | Self::ReloadFailed { .. }
4510 | Self::RegistrationStillActive { .. }
4511 | Self::StatePoisoned { .. }
4512 | Self::CommandClosed { .. }
4513 | Self::SwapInProgress { .. }
4514 | Self::SwapRefused { .. }
4515 | Self::SwapFailed { .. } => None,
4516 }
4517 }
4518}
4519
4520pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4521 if spec.module_id.trim().is_empty() {
4522 return Err(SuperviseError::InvalidSpec {
4523 reason: "module_id must not be empty".to_string(),
4524 });
4525 }
4526
4527 Ok(())
4528}
4529
4530#[derive(Debug, Default)]
4531struct HealthProbeRuntime {
4532 spawn_generation: Option<u64>,
4533 configured_health: Option<HealthConfig>,
4534 registered_connection: Option<crate::ConnectionId>,
4535 advertised: bool,
4536 next_probe_at: Option<Instant>,
4537 probe_index: u64,
4538}
4539
4540fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4541 lock_snapshot(snapshot)
4542 .ok()
4543 .and_then(|state| state.spawned_protocol)
4544 .unwrap_or(spec.protocol)
4545}
4546
4547impl HealthProbeRuntime {
4548 fn refresh_registration(
4549 &mut self,
4550 spec: &ModuleSpec,
4551 runtime: &SupervisorRuntimeConfig,
4552 registry: &Registry,
4553 snapshot: &SharedSnapshot,
4554 ) {
4555 let generation = lock_snapshot(snapshot)
4556 .ok()
4557 .map(|state| state.spawn_generation);
4558 if self.spawn_generation != generation {
4559 *self = Self {
4563 spawn_generation: generation,
4564 ..Self::default()
4565 };
4566 }
4567 if self.configured_health.as_ref() != Some(&runtime.health) {
4568 self.configured_health = Some(runtime.health.clone());
4569 self.next_probe_at = None;
4570 self.registered_connection = None;
4571 self.probe_index = 0;
4572 }
4573 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4577 self.registered_connection = None;
4578 self.advertised = runtime.health.http.is_some();
4579 if !self.advertised {
4580 self.next_probe_at = None;
4581 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4582 let unknown = ModuleHealthStatus::default();
4583 if state.health != unknown {
4584 state.health = unknown;
4585 }
4586 });
4587 } else if self.next_probe_at.is_none() {
4588 self.next_probe_at = Some(
4589 Instant::now()
4590 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4591 );
4592 }
4593 return;
4594 }
4595
4596 let registration = match registry.get_module(&spec.module_id) {
4597 Ok(registration) => registration,
4598 Err(err) => {
4599 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4600 self.advertised = false;
4601 self.next_probe_at = None;
4602 return;
4603 }
4604 };
4605
4606 let Some(registration) = registration else {
4607 self.registered_connection = None;
4608 self.advertised = false;
4609 self.next_probe_at = None;
4610 return;
4611 };
4612
4613 let advertised = registration
4614 .control_ops
4615 .iter()
4616 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4617 if !advertised {
4618 self.registered_connection = Some(registration.connection_id);
4619 self.advertised = false;
4620 self.next_probe_at = None;
4621 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4622 if state.health.status != SupervisorHealthStatus::Unknown
4623 || state.health.consecutive_failures != 0
4624 || state.health.last_probe_ms.is_some()
4625 || state.health.detail.is_some()
4626 || state.health.metrics.is_some()
4627 {
4628 state.health.status = SupervisorHealthStatus::Unknown;
4629 state.health.consecutive_failures = 0;
4630 state.health.last_probe_ms = None;
4631 state.health.detail = None;
4632 state.health.metrics = None;
4633 }
4634 });
4635 return;
4636 }
4637
4638 let reregistered = self.registered_connection != Some(registration.connection_id);
4639 self.registered_connection = Some(registration.connection_id);
4640 self.advertised = true;
4641 if reregistered || self.next_probe_at.is_none() {
4642 self.probe_index = 0;
4643 self.next_probe_at = Some(
4644 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4645 );
4646 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4647 state.health.status = SupervisorHealthStatus::Unknown;
4648 state.health.consecutive_failures = 0;
4649 state.health.detail = None;
4650 state.health.metrics = None;
4651 });
4652 }
4653 }
4654
4655 fn wake_at(&self) -> Option<Instant> {
4656 if !self.advertised {
4657 return None;
4658 }
4659 self.next_probe_at
4660 }
4661
4662 fn due(&self) -> bool {
4663 self.advertised
4664 && self
4665 .next_probe_at
4666 .is_some_and(|next| Instant::now() >= next)
4667 }
4668
4669 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4670 self.probe_index = self.probe_index.wrapping_add(1);
4671 self.next_probe_at = Some(
4672 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4673 );
4674 }
4675}
4676
4677#[derive(Debug)]
4712enum HealthProbeEvidence {
4713 LaneDead,
4715 NoAnswer,
4717 BadAnswer,
4719 Misconfigured,
4721}
4722
4723#[derive(Debug)]
4724struct HealthProbeError {
4725 evidence: HealthProbeEvidence,
4726 message: String,
4727}
4728
4729impl HealthProbeError {
4730 fn lane_dead(message: impl Into<String>) -> Self {
4731 Self::with(HealthProbeEvidence::LaneDead, message)
4732 }
4733
4734 fn no_answer(message: impl Into<String>) -> Self {
4735 Self::with(HealthProbeEvidence::NoAnswer, message)
4736 }
4737
4738 fn bad_answer(message: impl Into<String>) -> Self {
4739 Self::with(HealthProbeEvidence::BadAnswer, message)
4740 }
4741
4742 fn misconfigured(message: impl Into<String>) -> Self {
4743 Self::with(HealthProbeEvidence::Misconfigured, message)
4744 }
4745
4746 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4747 Self {
4748 evidence,
4749 message: message.into(),
4750 }
4751 }
4752
4753 #[allow(dead_code)]
4767 fn is_proof_of_death(&self) -> bool {
4768 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4769 }
4770
4771 fn label(&self) -> &'static str {
4779 match self.evidence {
4780 HealthProbeEvidence::LaneDead => "lane-dead",
4781 HealthProbeEvidence::NoAnswer => "no-answer",
4782 HealthProbeEvidence::BadAnswer => "bad-answer",
4783 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4784 }
4785 }
4786}
4787
4788impl fmt::Display for HealthProbeError {
4789 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4790 f.write_str(&self.message)
4791 }
4792}
4793
4794async fn run_health_probe_cycle(
4795 spec: &ModuleSpec,
4796 runtime: &SupervisorRuntimeConfig,
4797 registry: &Registry,
4798 process_liveness: &SupervisorProcessLiveness,
4799 snapshot: &SharedSnapshot,
4800 child: &mut Option<SupervisedChild>,
4801) {
4802 let now_ms = unix_ms_now();
4803 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4804 .then_some(runtime.health.http.as_deref())
4805 .flatten();
4806 let result = match http {
4807 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4808 None => probe_module_health(&spec.module_id, runtime, None).await,
4809 };
4810 match result {
4811 Ok(report) => {
4812 handle_health_report(
4813 spec,
4814 runtime,
4815 registry,
4816 process_liveness,
4817 snapshot,
4818 child,
4819 report,
4820 now_ms,
4821 )
4822 .await;
4823 }
4824 Err(err) => {
4825 if http.is_some() {
4826 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4827 state.health.status = SupervisorHealthStatus::Failing;
4828 });
4829 }
4830 handle_health_probe_failure(
4831 spec,
4832 runtime,
4833 registry,
4834 process_liveness,
4835 snapshot,
4836 child,
4837 err,
4838 now_ms,
4839 )
4840 .await;
4841 }
4842 }
4843}
4844
4845pub(crate) struct HttpProbeTarget<'a> {
4846 address: std::net::SocketAddr,
4847 localhost: bool,
4848 authority: &'a str,
4849 path: String,
4850}
4851
4852pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4855 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4856 return Err("must not contain whitespace, controls, or a fragment".into());
4857 }
4858 let rest = url
4859 .strip_prefix("http://")
4860 .ok_or("must use plain http://")?;
4861 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4862 let (authority, suffix) = rest.split_at(split);
4863 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4864 ("::1", rest)
4865 } else {
4866 let split = authority.find(':').unwrap_or(authority.len());
4867 authority.split_at(split)
4868 };
4869 let ip: std::net::IpAddr = match host {
4870 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4871 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4872 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4873 };
4874 let port = if port.is_empty() {
4875 80
4876 } else {
4877 port.strip_prefix(':')
4878 .and_then(|p| p.parse::<u16>().ok())
4879 .filter(|p| *p > 0)
4880 .ok_or("must have a valid nonzero TCP port")?
4881 };
4882 let path = if suffix.is_empty() {
4883 "/".into()
4884 } else if suffix.starts_with('?') {
4885 format!("/{suffix}")
4886 } else {
4887 suffix.into()
4888 };
4889 Ok(HttpProbeTarget {
4890 address: std::net::SocketAddr::new(ip, port),
4891 localhost: host == "localhost",
4892 authority,
4893 path,
4894 })
4895}
4896
4897async fn probe_http_health(
4898 url: &str,
4899 deadline: Duration,
4900) -> Result<HealthReport, HealthProbeError> {
4901 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4902 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4903 let mut response_status = String::new();
4906 let mut body = Vec::new();
4907 let probe = async {
4908 let connection = match tokio::net::TcpStream::connect(target.address).await {
4911 Err(_) if target.localhost => {
4912 tokio::net::TcpStream::connect((
4913 std::net::Ipv6Addr::LOCALHOST,
4914 target.address.port(),
4915 ))
4916 .await
4917 }
4918 result => result,
4919 };
4920 let mut stream = connection.map_err(|error| {
4921 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4922 })?;
4923 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4924 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4925 let mut reader = BufReader::new(stream);
4926 let mut budget = 16 * 1024;
4927 let status = http_line(&mut reader, &mut budget).await?;
4928 let mut words = status.split_ascii_whitespace();
4929 let version = words.next();
4930 let code = words
4931 .next()
4932 .filter(|word| word.len() == 3)
4933 .and_then(|word| word.parse::<u16>().ok());
4934 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4935 || !code.is_some_and(|code| (100..600).contains(&code))
4936 {
4937 return Err(HealthProbeError::bad_answer(format!(
4938 "invalid HTTP status: {status}"
4939 )));
4940 }
4941 let code = code.expect("validated status code");
4942 response_status = status.clone();
4943 let mut length = None;
4944 let mut chunked = false;
4945 loop {
4946 let line = http_line(&mut reader, &mut budget).await?;
4947 if line.is_empty() {
4948 break;
4949 }
4950 if let Some((name, value)) = line.split_once(':') {
4951 if name.eq_ignore_ascii_case("content-length") {
4952 length = Some(value.trim().parse::<u64>().map_err(|_| {
4953 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4954 })?);
4955 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4956 chunked = value.trim().eq_ignore_ascii_case("chunked");
4957 }
4958 }
4959 }
4960 if chunked {
4961 while body.len() < 200 {
4962 let line = http_line(&mut reader, &mut budget).await?;
4963 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4964 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4965 if size == 0 {
4966 break;
4967 }
4968 let count = size.min((200 - body.len()) as u64) as usize;
4969 let start = body.len();
4970 (&mut reader)
4971 .take(count as u64)
4972 .read_to_end(&mut body)
4973 .await
4974 .map_err(|error| {
4975 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4976 })?;
4977 if body.len() - start != count {
4978 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4979 }
4980 if size > count as u64 || body.len() == 200 {
4981 break;
4982 }
4983 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4984 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4985 }
4986 }
4987 } else {
4988 reader
4989 .take(length.unwrap_or(200).min(200))
4990 .read_to_end(&mut body)
4991 .await
4992 .map_err(|error| {
4993 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4994 })?;
4995 }
4996 if (200..300).contains(&code) {
4997 Ok(HealthReport::ok())
4998 } else {
4999 Err(HealthProbeError::bad_answer(
5000 "HTTP health endpoint returned non-2xx",
5001 ))
5002 }
5003 };
5004 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5005 Err(HealthProbeError::no_answer(format!(
5006 "HTTP probe timed out after {deadline:?}"
5007 )))
5008 });
5009 if let Err(error) = &mut result {
5010 if !response_status.is_empty() {
5011 error.message = format!(
5012 "{}; {response_status}: {}",
5013 error.message,
5014 String::from_utf8_lossy(&body)
5015 );
5016 }
5017 }
5018 result
5019}
5020
5021async fn http_line(
5022 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5023 remaining: &mut usize,
5024) -> Result<String, HealthProbeError> {
5025 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5026 let mut line = Vec::new();
5027 (&mut *reader)
5028 .take(*remaining as u64)
5029 .read_until(b'\n', &mut line)
5030 .await
5031 .map_err(|error| {
5032 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5033 })?;
5034 *remaining -= line.len();
5035 if !line.ends_with(b"\r\n") {
5036 return Err(HealthProbeError::bad_answer(
5037 "HTTP headers are incomplete or exceed 16 KiB",
5038 ));
5039 }
5040 line.truncate(line.len() - 2);
5041 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5042}
5043
5044async fn probe_module_health(
5045 module_id: &str,
5046 runtime: &SupervisorRuntimeConfig,
5047 drain_deadline: Option<Instant>,
5048) -> Result<HealthReport, HealthProbeError> {
5049 let Some(forwarding) = runtime.forwarding.as_ref() else {
5050 return Err(HealthProbeError::misconfigured(
5051 "supervisor was not configured with a forwarding table",
5052 ));
5053 };
5054 let probe_started_at = Instant::now();
5055 let mut deadline = probe_started_at + runtime.health.deadline;
5056 if let Some(drain_deadline) = drain_deadline {
5057 deadline = deadline.min(drain_deadline);
5058 }
5059 let pending = if drain_deadline.is_some() {
5060 forwarding.begin_drain_health_probe_rpc_for(
5061 module_id,
5062 MODULE_CONTROL_OP_HEALTH_CHECK,
5063 probe_started_at,
5064 deadline,
5065 )
5066 } else {
5067 forwarding.begin_health_probe_rpc_for(
5068 module_id,
5069 MODULE_CONTROL_OP_HEALTH_CHECK,
5070 probe_started_at,
5071 deadline,
5072 )
5073 }
5074 .map_err(|err| {
5075 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5078 })?;
5079 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5080}
5081
5082async fn probe_endpoint_health(
5089 endpoint: crate::ModuleEndpointId,
5090 runtime: &SupervisorRuntimeConfig,
5091 deadline_cap: Option<Instant>,
5092) -> Result<HealthReport, HealthProbeError> {
5093 let Some(forwarding) = runtime.forwarding.as_ref() else {
5094 return Err(HealthProbeError::misconfigured(
5095 "supervisor was not configured with a forwarding table",
5096 ));
5097 };
5098 let probe_started_at = Instant::now();
5099 let mut deadline = probe_started_at + runtime.health.deadline;
5100 if let Some(cap) = deadline_cap {
5101 deadline = deadline.min(cap);
5102 }
5103 let pending = forwarding
5104 .begin_endpoint_health_probe_rpc_for(
5105 endpoint,
5106 MODULE_CONTROL_OP_HEALTH_CHECK,
5107 probe_started_at,
5108 deadline,
5109 )
5110 .map_err(|err| {
5111 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5112 })?;
5113 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5114}
5115
5116async fn await_health_probe(
5118 forwarding: &ForwardingTable,
5119 pending: PendingModuleControlRpc,
5120 deadline: Instant,
5121 probe_budget: Duration,
5122) -> Result<HealthReport, HealthProbeError> {
5123 let PendingModuleControlRpc {
5124 endpoint,
5125 module_sink,
5126 negotiated_ver,
5127 corr,
5128 receiver,
5129 } = pending;
5130 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5131 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5132 })?;
5133 let frame = Frame::build_with_version(
5134 negotiated_ver,
5135 FrameType::Request,
5136 control_flags(),
5137 0,
5138 0,
5139 corr,
5140 body,
5141 )
5142 .map_err(|err| {
5143 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5144 })?;
5145
5146 match timeout_at(deadline, module_sink.send(frame)).await {
5152 Ok(Ok(())) => {}
5153 Ok(Err(err)) => {
5154 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5155 return Err(HealthProbeError::lane_dead(format!(
5158 "failed to send health.check: {err}"
5159 )));
5160 }
5161 Err(_elapsed) => {
5162 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5163 return Err(HealthProbeError::no_answer(
5167 "health.check send timed out before enqueue (module egress full)",
5168 ));
5169 }
5170 }
5171
5172 match timeout_at(deadline, receiver).await {
5173 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5177 response.health_report().ok_or_else(|| {
5178 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5179 })
5180 }
5181 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5182 format!("health.check rejected: {}", body.message),
5183 )),
5184 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5185 Err(HealthProbeError::lane_dead(message))
5186 }
5187 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5188 Err(HealthProbeError::bad_answer(message))
5189 }
5190 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5191 Err(HealthProbeError::bad_answer(format!(
5192 "expected module-control op '{expected}', got '{actual}'"
5193 )))
5194 }
5195 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5199 "module answered health.check after its daemon deadline",
5200 )),
5201 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5202 "health.check waiter was canceled before the module responded",
5203 )),
5204 Err(_) => {
5205 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5206 Err(HealthProbeError::no_answer(format!(
5207 "module did not answer health.check within {probe_budget:?}"
5208 )))
5209 }
5210 }
5211}
5212
5213#[allow(clippy::too_many_arguments)]
5214async fn handle_health_report(
5215 spec: &ModuleSpec,
5216 runtime: &SupervisorRuntimeConfig,
5217 registry: &Registry,
5218 process_liveness: &SupervisorProcessLiveness,
5219 snapshot: &SharedSnapshot,
5220 child: &mut Option<SupervisedChild>,
5221 report: HealthReport,
5222 now_ms: u64,
5223) {
5224 let status = supervisor_health_status(report.status);
5225 let detail = report.detail.clone();
5226 let metrics = truncate_health_metrics(report.metrics);
5227 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5228 state.health.status = status;
5229 state.health.last_probe_ms = Some(now_ms);
5230 state.health.detail = detail.clone();
5231 state.health.metrics = metrics.clone();
5232 state.health.consecutive_failures = 0;
5233 state.failed_retry.observe_health(
5234 report.status == HealthStatus::Ok,
5235 runtime.restart_policy.window,
5236 );
5237 });
5238
5239 let action = match report.status {
5240 HealthStatus::Ok => return,
5241 HealthStatus::Degraded => runtime.health.on_degraded,
5242 HealthStatus::Failing => runtime.health.on_failing,
5243 };
5244 apply_l3_health_action(
5245 spec,
5246 runtime,
5247 registry,
5248 process_liveness,
5249 snapshot,
5250 child,
5251 status,
5252 detail.as_deref(),
5253 action,
5254 now_ms,
5255 )
5256 .await;
5257}
5258
5259#[allow(clippy::too_many_arguments)]
5260async fn handle_health_probe_failure(
5261 spec: &ModuleSpec,
5262 runtime: &SupervisorRuntimeConfig,
5263 registry: &Registry,
5264 process_liveness: &SupervisorProcessLiveness,
5265 snapshot: &SharedSnapshot,
5266 child: &mut Option<SupervisedChild>,
5267 err: HealthProbeError,
5268 now_ms: u64,
5269) {
5270 let threshold = runtime.health.failure_threshold.max(1);
5271 let mut failures = 0;
5272 let detail = format!("[{}] {err}", err.label());
5277 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5278 state
5279 .failed_retry
5280 .observe_health(false, runtime.restart_policy.window);
5281 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5284 state.health.status = SupervisorHealthStatus::Unknown;
5285 }
5286 state.health.last_probe_ms = Some(now_ms);
5287 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5288 state.health.detail = Some(detail.clone());
5289 state.health.metrics = None;
5290 failures = state.health.consecutive_failures;
5291 });
5292
5293 if failures < threshold {
5294 warn!(
5295 module_id = %spec.module_id,
5296 consecutive_failures = failures,
5297 threshold,
5298 evidence = err.label(),
5299 detail = %detail,
5300 "health.check probe failed"
5301 );
5302 return;
5303 }
5304
5305 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5306 state.state = ModuleState::Unresponsive;
5307 state.health.status = SupervisorHealthStatus::Unresponsive;
5308 });
5309 if runtime.health.critical {
5313 error!(
5314 module_id = %spec.module_id,
5315 status = "unresponsive",
5316 evidence = err.label(),
5317 detail = %detail,
5318 "critical module health alert"
5319 );
5320 } else {
5321 warn!(
5322 module_id = %spec.module_id,
5323 status = "unresponsive",
5324 evidence = err.label(),
5325 detail = %detail,
5326 "module health threshold breached"
5327 );
5328 }
5329 if let Err(err) = health_restart_child(
5330 spec,
5331 runtime,
5332 registry,
5333 process_liveness,
5334 snapshot,
5335 child,
5336 SupervisorHealthStatus::Unresponsive,
5337 Some(&detail),
5338 now_ms,
5339 )
5340 .await
5341 {
5342 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5343 }
5344}
5345
5346#[allow(clippy::too_many_arguments)]
5347async fn apply_l3_health_action(
5348 spec: &ModuleSpec,
5349 runtime: &SupervisorRuntimeConfig,
5350 registry: &Registry,
5351 process_liveness: &SupervisorProcessLiveness,
5352 snapshot: &SharedSnapshot,
5353 child: &mut Option<SupervisedChild>,
5354 status: SupervisorHealthStatus,
5355 detail: Option<&str>,
5356 action: HealthAction,
5357 now_ms: u64,
5358) {
5359 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5360 match action {
5361 HealthAction::Report => {
5362 info!(
5363 module_id = %spec.module_id,
5364 status = ?status,
5365 detail,
5366 "module reported non-ok health"
5367 );
5368 }
5369 HealthAction::Alert => {
5370 error!(
5371 module_id = %spec.module_id,
5372 status = ?status,
5373 detail,
5374 "module health alert"
5375 );
5376 }
5377 HealthAction::Restart => {
5378 if let Err(err) = health_restart_child(
5379 spec,
5380 runtime,
5381 registry,
5382 process_liveness,
5383 snapshot,
5384 child,
5385 status,
5386 detail,
5387 now_ms,
5388 )
5389 .await
5390 {
5391 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5392 }
5393 }
5394 }
5395}
5396
5397#[allow(clippy::too_many_arguments)]
5398async fn health_restart_child(
5399 spec: &ModuleSpec,
5400 runtime: &SupervisorRuntimeConfig,
5401 registry: &Registry,
5402 process_liveness: &SupervisorProcessLiveness,
5403 snapshot: &SharedSnapshot,
5404 child: &mut Option<SupervisedChild>,
5405 status: SupervisorHealthStatus,
5406 detail: Option<&str>,
5407 now_ms: u64,
5408) -> Result<(), SuperviseError> {
5409 let (enabled, schedule) = {
5410 let mut state = lock_snapshot(snapshot)?;
5411 let enabled = state.enabled;
5412 let schedule = if enabled {
5413 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5414 } else {
5415 None
5416 };
5417 (enabled, schedule)
5418 };
5419
5420 if !enabled {
5421 return Err(SuperviseError::Disabled {
5422 module_id: spec.module_id.clone(),
5423 });
5424 }
5425
5426 if schedule.is_none() {
5427 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5428 error!(
5429 module_id = %spec.module_id,
5430 status = ?status,
5431 detail,
5432 max_restarts = runtime.restart_policy.max_restarts,
5433 window_secs = runtime.restart_policy.window.as_secs(),
5434 reason = %runtime.restart_policy.budget_exhausted_detail(),
5435 "health restart budget exhausted; marking module failed"
5436 );
5437 let stop_notice = begin_forwarding_drain_if_configured(
5438 spec,
5439 runtime,
5440 registry,
5441 snapshot,
5442 Some(true),
5443 RouteCloseReason::Disable,
5444 )
5445 .await?;
5446 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5447 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5448 })?;
5449 let result = drain_optional_child(
5450 &spec.module_id,
5451 spec.protocol,
5452 stop_notice,
5453 registry,
5454 runtime.forwarding.as_deref(),
5455 snapshot,
5456 &runtime.terminal_ring,
5457 &runtime.spawn_events,
5458 child,
5459 runtime.drain_timeout,
5460 ModuleState::Failed,
5461 Some(true),
5462 &runtime.failure_context(true),
5463 )
5464 .await;
5465 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5466 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5467 state.failed_retry.schedule(&spec.module_id);
5468 }
5469 })?;
5470 result?;
5471 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5472 return Ok(());
5473 }
5474
5475 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5476 let mut restart_count = 0;
5477 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5478 restart_count = state.crash_restarts.len();
5479 state.state = ModuleState::Unresponsive;
5480 state.health.status = status;
5481 state.health.last_action = Some(HealthAction::Restart.to_string());
5482 state.health.last_action_ms = Some(now_ms);
5483 })?;
5484 warn!(
5485 module_id = %spec.module_id,
5486 status = ?status,
5487 detail,
5488 restart_count,
5489 restart_in_window = schedule.restart_in_window,
5490 delay_ms = schedule.delay.as_millis() as u64,
5491 "health-triggered module restart"
5492 );
5493
5494 let stop_notice = begin_forwarding_drain_if_configured(
5495 spec,
5496 runtime,
5497 registry,
5498 snapshot,
5499 Some(true),
5500 RouteCloseReason::Restart,
5501 )
5502 .await?;
5503 drain_optional_child(
5504 &spec.module_id,
5505 spec.protocol,
5506 stop_notice,
5507 registry,
5508 runtime.forwarding.as_deref(),
5509 snapshot,
5510 &runtime.terminal_ring,
5511 &runtime.spawn_events,
5512 child,
5513 runtime.drain_timeout,
5514 ModuleState::Restarting,
5515 Some(true),
5516 &runtime.failure_context(true),
5517 )
5518 .await?;
5519 schedule_respawn(
5520 runtime,
5521 snapshot,
5522 &spec.module_id,
5523 schedule.delay,
5524 RespawnKind::Spawn,
5525 true,
5526 )
5527}
5528
5529fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5530 if let Some(reply) = runtime
5531 .deferred_reload_reply
5532 .lock()
5533 .unwrap_or_else(|p| p.into_inner())
5534 .take()
5535 {
5536 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5537 module_id: module_id.to_string(),
5538 reason: reason.to_string(),
5539 }));
5540 }
5541}
5542
5543fn schedule_respawn(
5544 runtime: &SupervisorRuntimeConfig,
5545 snapshot: &SharedSnapshot,
5546 module_id: &str,
5547 delay: Duration,
5548 kind: RespawnKind,
5549 daemon_initiated: bool,
5550) -> Result<(), SuperviseError> {
5551 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5552 update_snapshot(snapshot, Some(module_id), |state| {
5553 state.respawn_pending = true
5554 })?;
5555 *runtime
5556 .scheduled_respawn
5557 .lock()
5558 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5559 deadline: Instant::now() + delay,
5560 kind,
5561 daemon_initiated,
5562 });
5563 Ok(())
5564}
5565
5566fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5567 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5568 state.health.last_action = Some(action);
5569 state.health.last_action_ms = Some(now_ms);
5570 });
5571}
5572
5573fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5574 match status {
5575 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5576 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5577 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5578 }
5579}
5580
5581fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5593 let metrics = metrics?;
5594 match serde_json::to_vec(&metrics) {
5595 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5596 "truncated": true,
5597 "original_bytes": encoded.len(),
5598 })),
5599 Ok(_) | Err(_) => Some(metrics),
5600 }
5601}
5602
5603fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5609 if cadence.is_zero() {
5610 return Duration::ZERO;
5611 }
5612 let cadence_ms = cadence.as_millis() as u64;
5613 if cadence_ms == 0 {
5629 return cadence;
5630 }
5631 let jitter_span = (cadence_ms / 10).max(1);
5646 let hash = module_id.as_bytes().iter().fold(
5647 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5648 |acc, byte| {
5649 acc.wrapping_mul(1099511628211)
5650 .wrapping_add(u64::from(*byte))
5651 },
5652 );
5653 cadence + Duration::from_millis(hash % jitter_span)
5654}
5655
5656#[cfg(test)]
5657mod tests {
5658 use super::*;
5659
5660 #[test]
5661 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5662 let handle = SupervisorHandle::new();
5663 let module_id = "readded-tombstone";
5664 handle.record_rescan_removal(module_id);
5665 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5666
5667 handle.apply_identity_configuration(&ModuleSpec {
5668 module_id: module_id.to_string(),
5669 program: PathBuf::from("/test/module"),
5670 args: Vec::new(),
5671 env: Vec::new(),
5672 reserved: false,
5673 reserved_prefixes: Vec::new(),
5674 protocol: ModuleProtocol::Subc,
5675 overlap: Default::default(),
5676 });
5677
5678 assert!(
5679 handle.removal_tombstone_age_ms(module_id).is_none(),
5680 "a re-added module must not retain a stale removal tombstone"
5681 );
5682 }
5683
5684 #[derive(Debug, PartialEq, Eq)]
5687 struct OwnerInSpawnWindow {
5688 module_id: String,
5689 configured: bool,
5690 on_roster: bool,
5691 admission_refusal: Option<&'static str>,
5692 }
5693
5694 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5705 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5706 use crate::scopes::ScopeTable;
5707 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5708
5709 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5710 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5711 module_id: module_id.to_string(),
5712 program,
5713 args: Vec::new(),
5714 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5715 .into_iter()
5716 .map(|key| (key.to_string(), dir.path().display().to_string()))
5717 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5718 .collect(),
5719 reserved: false,
5720 reserved_prefixes: Vec::new(),
5721 protocol: ModuleProtocol::Subc,
5722 overlap: Default::default(),
5723 };
5724 let live = super::terminal_history_tests::fake_aft_stub_path();
5725 let missing = dir.path().join("definitely-missing-module");
5726
5727 let handle = SupervisorHandle::new();
5728 let mut supervisor =
5729 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5730 .with_handle(handle.clone());
5731 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5732 let hook_handle = handle.clone();
5733 let hook_observed = Arc::clone(&observed);
5734 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5735 let configured = hook_handle.is_configured(module_id);
5738 let selector = ScopeSelector::new(
5739 Principal::Reserved {
5740 module_id: module_id.to_string(),
5741 },
5742 "s".to_string(),
5743 Some(1),
5744 );
5745 let carrier = Principal::Reserved {
5746 module_id: "carrier".to_string(),
5747 };
5748 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5749 .admit(&carrier, module_id, &selector, configured)
5750 {
5751 Ok(_) => None,
5752 Err(refusal) => Some(refusal.code),
5753 };
5754 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5755 module_id: module_id.to_string(),
5756 configured,
5757 on_roster: hook_handle.get(module_id).is_some(),
5758 admission_refusal,
5759 });
5760 })));
5761
5762 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5763 let configured = supervisor
5764 .supervise_configured(stub("configured", live.clone()), true)
5765 .unwrap();
5766 let with_health = supervisor
5767 .supervise_configured_with_health(
5768 stub("with-health", live.clone()),
5769 true,
5770 HealthConfig::default(),
5771 None,
5772 RestartPolicy::default(),
5773 )
5774 .unwrap();
5775 let failed = supervisor
5778 .supervise_configured_with_health(
5779 stub("failed-spawn", missing.clone()),
5780 true,
5781 HealthConfig::default(),
5782 None,
5783 RestartPolicy::default(),
5784 )
5785 .unwrap();
5786 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5789
5790 let expected = [
5791 "plain",
5792 "configured",
5793 "with-health",
5794 "failed-spawn",
5795 "spawn-error",
5796 ]
5797 .into_iter()
5798 .map(|module_id| OwnerInSpawnWindow {
5799 module_id: module_id.to_string(),
5800 configured: true,
5801 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5802 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5803 })
5804 .collect::<Vec<_>>();
5805 assert_eq!(*observed.lock().unwrap(), expected);
5806
5807 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5808 assert!(
5809 handle.get(module_id).is_some(),
5810 "{module_id} is on the roster"
5811 );
5812 assert!(
5813 handle.is_configured(module_id),
5814 "{module_id} stays configured"
5815 );
5816 }
5817 assert!(handle.get("spawn-error").is_none());
5818 assert!(
5819 !handle.is_configured("spawn-error"),
5820 "a plain spawn that failed must not leave its module marked configured"
5821 );
5822
5823 handle.retire("failed-spawn");
5825 assert!(!handle.is_configured("failed-spawn"));
5826
5827 for module in [plain, configured, with_health] {
5828 module.stop().await.unwrap();
5829 }
5830 drop(failed);
5831 }
5832
5833 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5840 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5841 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5842 let spec = ModuleSpec {
5843 module_id: "spawned-not-admitted".to_string(),
5844 program: super::terminal_history_tests::fake_aft_stub_path(),
5845 args: Vec::new(),
5846 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5847 .into_iter()
5848 .map(|key| (key.to_string(), dir.path().display().to_string()))
5849 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5850 .collect(),
5851 reserved: false,
5852 reserved_prefixes: Vec::new(),
5853 protocol: ModuleProtocol::Subc,
5854 overlap: Default::default(),
5855 };
5856 let handle = SupervisorHandle::new();
5857 let mut supervisor =
5858 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5859 .with_handle(handle.clone());
5860 let observed = Arc::new(Mutex::new(None));
5861 let hook_handle = handle.clone();
5862 let hook_observed = Arc::clone(&observed);
5863 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5864 let state = hook_handle
5865 .get(module_id)
5866 .expect("a configured module is on the roster before it is spawned")
5867 .state()
5868 .unwrap();
5869 *hook_observed.lock().unwrap() = Some(state);
5870 })));
5871
5872 let module = supervisor.supervise_configured(spec, true).unwrap();
5873 assert_eq!(
5874 *observed.lock().unwrap(),
5875 Some(ModuleState::Starting),
5876 "between the spawn and its admission the supervisor still reports starting"
5877 );
5878 assert_eq!(module.state().unwrap(), ModuleState::Running);
5879 module.stop().await.unwrap();
5880 }
5881
5882 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5883 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5884 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5885 snapshot.process_alive = true;
5886 snapshot.pid = Some(41);
5887 snapshot.spawned_at_ms = Some(42);
5888 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5889 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5890 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5891 device: 43,
5892 inode: 44,
5893 #[cfg(windows)]
5894 windows_image: None,
5895 });
5896 })
5897 .unwrap();
5898 snapshot
5899 }
5900
5901 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5902 let snapshot = lock_snapshot(snapshot).unwrap();
5903 assert!(!snapshot.process_alive);
5904 assert_eq!(snapshot.pid, None);
5905 assert_eq!(snapshot.spawned_at_ms, None);
5906 assert_eq!(snapshot.spawned_from, None);
5907 assert_eq!(snapshot.launch_spec_digest, None);
5908 assert_eq!(snapshot.spawned_file_identity, None);
5909 }
5910
5911 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5912 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5913 let supervisor =
5914 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5915 let mut runtime = supervisor.runtime_config();
5916 runtime.test_seed_stale_facts_before_enable_spawn = true;
5917 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5918 let mut child = None;
5919 let spec = ModuleSpec {
5920 module_id: "failed-enable-clears-facts".to_string(),
5921 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5922 args: Vec::new(),
5923 env: Vec::new(),
5924 reserved: false,
5925 reserved_prefixes: Vec::new(),
5926 protocol: ModuleProtocol::Subc,
5927 overlap: Default::default(),
5928 };
5929
5930 let result = set_child_enabled(
5931 &spec,
5932 &runtime,
5933 &supervisor.registry,
5934 &supervisor.process_liveness,
5935 &snapshot,
5936 &mut child,
5937 true,
5938 &runtime.failure_context(false),
5939 )
5940 .await;
5941
5942 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5943 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5944 assert_snapshot_process_facts_cleared(&snapshot);
5945 }
5946
5947 #[tokio::test]
5948 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5949 let supervisor =
5950 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5951 let runtime = supervisor.runtime_config();
5952 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5953 ModuleState::Restarting,
5954 true,
5955 )));
5956 let spec = ModuleSpec {
5957 module_id: "start-stranded-restarting".to_string(),
5958 program: super::terminal_history_tests::fake_aft_stub_path(),
5959 args: Vec::new(),
5960 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5961 reserved: false,
5962 reserved_prefixes: Vec::new(),
5963 protocol: ModuleProtocol::None,
5964 overlap: Default::default(),
5965 };
5966 let mut child = None;
5967 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5968 assert!(!super::set_child_enabled(
5969 &spec,
5970 &runtime,
5971 &Registry::default(),
5972 &supervisor.process_liveness,
5973 &snapshot,
5974 &mut child,
5975 true,
5976 &runtime.failure_context(false),
5977 )
5978 .await
5979 .unwrap());
5980 assert!(child.is_none());
5981 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5982 assert!(super::set_child_enabled(
5983 &spec,
5984 &runtime,
5985 &Registry::default(),
5986 &supervisor.process_liveness,
5987 &snapshot,
5988 &mut child,
5989 true,
5990 &runtime.failure_context(false),
5991 )
5992 .await
5993 .unwrap());
5994 assert_eq!(
5995 lock_snapshot(&snapshot).unwrap().state,
5996 ModuleState::Running
5997 );
5998 let mut child = child.unwrap();
5999 child.start_kill().unwrap();
6000 child.wait().await.unwrap();
6001 }
6002
6003 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6004 async fn failed_reload_spawn_clears_current_process_facts() {
6005 let supervisor =
6006 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6007 let mut runtime = supervisor.runtime_config();
6008 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6009 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6010 let mut child = None;
6011 let spec = ModuleSpec {
6012 module_id: "failed-reload-clears-facts".to_string(),
6013 program: PathBuf::from("/unused/failed-reload-module"),
6014 args: Vec::new(),
6015 env: Vec::new(),
6016 reserved: false,
6017 reserved_prefixes: Vec::new(),
6018 protocol: ModuleProtocol::Subc,
6019 overlap: Default::default(),
6020 };
6021
6022 let result = handle_reload_spawn_failure(
6023 &spec,
6024 &runtime,
6025 &supervisor.process_liveness,
6026 &snapshot,
6027 &mut child,
6028 "forced reload spawn failure".to_string(),
6029 )
6030 .await;
6031
6032 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6033 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6034 assert_snapshot_process_facts_cleared(&snapshot);
6035 }
6036
6037 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6038 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6039 let supervisor =
6040 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6041 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6042 let module = supervisor.supervised_module(
6043 ModuleSpec {
6044 module_id: "drop-clears-facts".to_string(),
6045 program: PathBuf::from("/unused/drop-module"),
6046 args: Vec::new(),
6047 env: Vec::new(),
6048 reserved: false,
6049 reserved_prefixes: Vec::new(),
6050 protocol: ModuleProtocol::Subc,
6051 overlap: Default::default(),
6052 },
6053 supervisor.runtime_config(),
6054 Arc::clone(&snapshot),
6055 None,
6056 );
6057 assert!(!module
6058 .inner
6059 .monitor
6060 .lock()
6061 .unwrap()
6062 .as_ref()
6063 .unwrap()
6064 .is_finished());
6065
6066 drop(module);
6067
6068 assert_eq!(
6069 lock_snapshot(&snapshot).unwrap().state,
6070 ModuleState::Stopped
6071 );
6072 assert_snapshot_process_facts_cleared(&snapshot);
6073 }
6074
6075 #[cfg(unix)]
6076 #[tokio::test]
6077 async fn rescan_preserves_running_protocol_until_respawn() {
6078 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6079 let initial = ModuleSpec {
6080 module_id: "rescan-protocol".into(),
6081 program: PathBuf::from("/bin/sleep"),
6082 args: vec!["60".into()],
6083 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6084 .into_iter()
6085 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6086 .collect(),
6087 reserved: false,
6088 reserved_prefixes: vec![],
6089 protocol: ModuleProtocol::None,
6090 overlap: Default::default(),
6091 };
6092 let supervisor =
6093 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6094 let module = supervisor.spawn(initial.clone()).unwrap();
6095 assert!(module.status().unwrap().live);
6096 let mut next = initial;
6097 next.protocol = ModuleProtocol::Subc;
6098 module
6099 .update_configuration(next.clone(), HealthConfig::default(), None)
6100 .await
6101 .unwrap();
6102 assert!(
6103 module.status().unwrap().live,
6104 "rescan must not require HELLO from the old non-wire process"
6105 );
6106 let runtime = supervisor.runtime_config();
6107 let action = on_child_exit(
6108 &next,
6109 RestartPolicy::default(),
6110 &supervisor.registry,
6111 &module.inner.snapshot,
6112 &runtime.terminal_ring,
6113 &runtime.spawn_events,
6114 &runtime.child_roster,
6115 ExitReport {
6116 kind: ExitKind::Clean,
6117 code: Some(0),
6118 signal: None,
6119 at_ms: unix_ms_now(),
6120 },
6121 true,
6122 )
6123 .await;
6124 assert!(
6125 matches!(action, NextAction::Restart { .. }),
6126 "the old non-wire process's clean exit must restart"
6127 );
6128 module.drain().await.unwrap();
6129 }
6130
6131 #[cfg(unix)]
6132 #[tokio::test]
6133 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6134 use std::os::unix::fs::PermissionsExt;
6135 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6136 let script = dir.join("module.sh");
6137 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6138 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6139 let record_path = dir.join("live-children.json");
6140 let supervisor =
6141 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6142 .with_live_children_record(&record_path);
6143 for (program, args) in [
6144 (PathBuf::from("sleep"), vec!["60".into()]),
6145 (script, vec![]),
6146 ] {
6147 let spec = ModuleSpec {
6148 module_id: "image-identity".into(),
6149 program: program.clone(),
6150 args,
6151 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6152 .into_iter()
6153 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6154 .collect(),
6155 reserved: false,
6156 reserved_prefixes: vec![],
6157 protocol: ModuleProtocol::None,
6158 overlap: Default::default(),
6159 };
6160 let module = supervisor.spawn(spec).unwrap();
6161 #[cfg(target_os = "macos")]
6162 {
6163 let deadline = Instant::now() + Duration::from_secs(5);
6166 while crate::live_children::read_record(&record_path)
6167 .unwrap()
6168 .iter()
6169 .all(|entry| entry.executable.is_none())
6170 {
6171 assert!(Instant::now() < deadline, "module image was not confirmed");
6172 tokio::time::sleep(Duration::from_millis(5)).await;
6173 }
6174 }
6175 let entry = crate::live_children::read_record(&record_path)
6176 .unwrap()
6177 .pop()
6178 .unwrap();
6179 let observed = subc_os::Process::open(entry.pid)
6180 .unwrap()
6181 .unwrap()
6182 .observe()
6183 .unwrap();
6184 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6185 module.drain().await.unwrap();
6186 assert_eq!(
6187 verdict,
6188 crate::live_children::IdentityVerdict::Matches,
6189 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6190 );
6191 }
6192 }
6193
6194 #[cfg(unix)]
6195 fn http_fixture(
6196 dir: &std::path::Path,
6197 url: &str,
6198 threshold: u32,
6199 ) -> crate::daemon_config::ConfiguredModule {
6200 let path = dir.join("subc.jsonc");
6201 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6202 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6203 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6204 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6205 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6206 }}}).to_string()).unwrap();
6207 crate::daemon_config::load(&path)
6208 .unwrap()
6209 .unwrap()
6210 .modules
6211 .pop()
6212 .unwrap()
6213 }
6214
6215 #[cfg(unix)]
6216 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6217 timeout(Duration::from_secs(5), async {
6218 loop {
6219 if module.status().unwrap().health.status == status {
6220 break;
6221 }
6222 sleep(Duration::from_millis(5)).await;
6223 }
6224 })
6225 .await
6226 .unwrap_or_else(|_| {
6227 panic!(
6228 "expected {status:?}, got {:?}",
6229 module.status().unwrap().health
6230 )
6231 });
6232 }
6233
6234 #[cfg(unix)]
6235 #[tokio::test]
6236 async fn http_health_status_flips_ok_failing_ok() {
6237 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6238 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6239 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6240 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6241 let serving_status = status.clone();
6242 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6243 let server = tokio::spawn(async move {
6244 loop {
6245 let (mut stream, _) = listener.accept().await.unwrap();
6246 let mut request = [0u8; 2048];
6247 let count = stream.read(&mut request).await.unwrap();
6248 assert!(count > 0, "a probe must send an HTTP request");
6249 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6250 let body = if code == 200 {
6251 "ready"
6252 } else {
6253 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6254 };
6255 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6256 let _ = stream.write_all(response.as_bytes()).await;
6257 }
6258 });
6259 let configured = http_fixture(&dir, &url, 1000);
6260 let module =
6261 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6262 .supervise_configured_with_health(
6263 configured.module_spec(),
6264 true,
6265 configured.health,
6266 configured.drain_timeout_ms,
6267 configured.restart,
6268 )
6269 .unwrap();
6270 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6271 status.store(503, std::sync::atomic::Ordering::SeqCst);
6272 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6273 assert!(module
6274 .status()
6275 .unwrap()
6276 .health
6277 .detail
6278 .unwrap()
6279 .contains("scratch failure"));
6280 status.store(200, std::sync::atomic::Ordering::SeqCst);
6281 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6282 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6283 let before = module.status().unwrap();
6284 let (spec, mut health) = module.configuration().unwrap();
6285 health.http = None;
6286 module
6287 .update_configuration(spec.clone(), health.clone(), Some(10))
6288 .await
6289 .unwrap();
6290 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6291 health.http = Some(url);
6292 module
6293 .update_configuration(spec, health, Some(10))
6294 .await
6295 .unwrap();
6296 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6297 assert_eq!(
6298 module.status().unwrap().pid,
6299 before.pid,
6300 "changing a probe must apply live, not restart its process"
6301 );
6302 let (spec, mut health) = module.configuration().unwrap();
6303 health.failure_threshold = 2;
6304 module
6305 .update_configuration(spec, health, Some(10))
6306 .await
6307 .unwrap();
6308 status.store(503, std::sync::atomic::Ordering::SeqCst);
6309 timeout(Duration::from_secs(5), async {
6310 while module.status().unwrap().spawn_generation == before.spawn_generation {
6311 sleep(Duration::from_millis(5)).await;
6312 }
6313 })
6314 .await
6315 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6316 module.drain().await.unwrap();
6317 server.abort();
6318 }
6319
6320 #[cfg(unix)]
6321 #[tokio::test]
6322 async fn http_health_replacement_recovers_after_one_restart() {
6323 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6324 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6325 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6326 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6327 let mut configured = http_fixture(&dir, &url, 2);
6328 configured.restart.max_restarts = 2;
6329 let module =
6330 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6331 .supervise_configured_with_health(
6332 configured.module_spec(),
6333 true,
6334 configured.health,
6335 configured.drain_timeout_ms,
6336 configured.restart,
6337 )
6338 .unwrap();
6339 let first_generation = module.status().unwrap().spawn_generation;
6340 let observed = module.clone();
6341 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6342 let server = tokio::spawn(async move {
6343 let mut generation = first_generation;
6344 let mut probes = 0;
6345 loop {
6346 let (mut stream, _) = listener.accept().await.unwrap();
6347 let mut request = [0; 2048];
6348 assert!(stream.read(&mut request).await.unwrap() > 0);
6349 let current = observed.status().unwrap().spawn_generation;
6350 if current != generation {
6351 generation = current;
6352 probes = 0;
6353 }
6354 probes += 1;
6355 let healthy = generation != first_generation && probes > 1;
6358 let code = if healthy { 200 } else { 503 };
6359 let response = format!(
6360 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6361 );
6362 let _ = stream.write_all(response.as_bytes()).await;
6363 if healthy {
6364 healthy_tx.send(()).await.unwrap();
6365 }
6366 }
6367 });
6368 timeout(Duration::from_secs(5), async {
6369 let mut healthy_probes = 0;
6370 loop {
6371 if module.state().unwrap() == ModuleState::Failed {
6372 break;
6373 }
6374 while healthy_rx.try_recv().is_ok() {
6375 healthy_probes += 1;
6376 }
6377 if healthy_probes >= 3
6378 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6379 {
6380 break;
6381 }
6382 sleep(Duration::from_millis(5)).await;
6383 }
6384 })
6385 .await
6386 .expect("the replacement must settle healthy without exhausting its restart budget");
6387 let status = module.status().unwrap();
6388 module.drain().await.unwrap();
6389 server.abort();
6390 assert_eq!(status.state, ModuleState::Running);
6391 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6392 assert_eq!(status.health.consecutive_failures, 0);
6393 assert_eq!(status.lifetime_restarts, 1);
6394 assert_eq!(status.restart_count, 1);
6395 }
6396
6397 #[cfg(unix)]
6398 #[tokio::test]
6399 async fn http_health_sustained_failure_exhausts_restart_budget() {
6400 use std::collections::BTreeMap;
6401 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6402 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6403 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6404 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6405 let mut configured = http_fixture(&dir, &url, 2);
6406 configured.restart.max_restarts = 2;
6407 let module =
6408 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6409 .supervise_configured_with_health(
6410 configured.module_spec(),
6411 true,
6412 configured.health,
6413 configured.drain_timeout_ms,
6414 configured.restart,
6415 )
6416 .unwrap();
6417 let observed = module.clone();
6418 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6419 let served = probes.clone();
6420 let server = tokio::spawn(async move {
6421 loop {
6422 let (mut stream, _) = listener.accept().await.unwrap();
6423 let mut request = [0; 2048];
6424 assert!(stream.read(&mut request).await.unwrap() > 0);
6425 *served
6426 .lock()
6427 .unwrap()
6428 .entry(observed.status().unwrap().spawn_generation)
6429 .or_default() += 1;
6430 let _ = stream
6431 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6432 .await;
6433 }
6434 });
6435 timeout(Duration::from_secs(5), async {
6436 while module.state().unwrap() != ModuleState::Failed {
6437 sleep(Duration::from_millis(5)).await;
6438 }
6439 })
6440 .await
6441 .expect("sustained failures must still exhaust the health restart budget");
6442 let status = module.status().unwrap();
6443 server.abort();
6444 assert_eq!(status.restart_count, 2);
6445 assert_eq!(status.lifetime_restarts, 2);
6446 assert!(!status.process_alive);
6447 assert_eq!(
6448 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6449 vec![2, 2, 2]
6450 );
6451 }
6452
6453 #[cfg(unix)]
6454 #[tokio::test]
6455 async fn spawn_clears_the_previous_process_health() {
6456 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6457 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6458 configured.health.cadence = Duration::from_secs(3600);
6459 let module =
6460 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6461 .supervise_configured_with_health(
6462 configured.module_spec(),
6463 false,
6464 configured.health,
6465 configured.drain_timeout_ms,
6466 configured.restart,
6467 )
6468 .unwrap();
6469 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6470 state.health.status = SupervisorHealthStatus::Unresponsive;
6471 state.health.consecutive_failures = 3;
6472 state.health.detail = Some("old process refused connections".into());
6473 state.health.metrics = Some(serde_json::json!({"old": true}));
6474 state.health.last_probe_ms = Some(123);
6475 state.health.last_action = Some("restart".into());
6476 state.health.last_action_ms = Some(124);
6477 })
6478 .unwrap();
6479 module.set_enabled(true).await.unwrap();
6480 let health = module.status().unwrap().health;
6481 module.drain().await.unwrap();
6482 assert_eq!(health.consecutive_failures, 0);
6483 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6484 assert_eq!(health.detail, None);
6485 assert_eq!(health.metrics, None);
6486 assert_eq!(health.last_probe_ms, None);
6487 assert_eq!(health.last_action.as_deref(), Some("restart"));
6488 assert_eq!(health.last_action_ms, Some(124));
6489 }
6490
6491 #[cfg(unix)]
6492 #[tokio::test]
6493 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6494 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6495 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6496 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6497 drop(unused);
6498 let configured = http_fixture(&dir, &url, 2);
6499 let module =
6500 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6501 .supervise_configured_with_health(
6502 configured.module_spec(),
6503 true,
6504 configured.health,
6505 configured.drain_timeout_ms,
6506 configured.restart,
6507 )
6508 .unwrap();
6509 let before = module.status().unwrap().spawn_generation;
6510 timeout(Duration::from_secs(5), async {
6511 loop {
6512 let status = module.status().unwrap();
6513 if status.spawn_generation > before {
6514 assert!(status.lifetime_restarts > 0);
6515 break;
6516 }
6517 sleep(Duration::from_millis(5)).await;
6518 }
6519 })
6520 .await
6521 .expect("sustained HTTP refusal must trigger the health restart policy");
6522 module.drain().await.unwrap();
6523 }
6524
6525 #[cfg(unix)]
6526 #[tokio::test]
6527 async fn http_health_timeout_honours_deadline() {
6528 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6529 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6530 let server = tokio::spawn(async move {
6531 let _held = listener.accept().await.unwrap();
6532 std::future::pending::<()>().await;
6533 });
6534 let error = timeout(
6535 Duration::from_secs(1),
6536 probe_http_health(&url, Duration::from_millis(10)),
6537 )
6538 .await
6539 .expect("the probe must enforce its own deadline")
6540 .unwrap_err();
6541 server.abort();
6542 assert!(error.to_string().contains("timed out"));
6543 }
6544
6545 #[cfg(unix)]
6546 #[tokio::test]
6547 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6548 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6549 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6550 let url = format!(
6551 "http://localhost:{}/healthz",
6552 listener.local_addr().unwrap().port()
6553 );
6554 let server = tokio::spawn(async move {
6555 let (mut stream, _) = listener.accept().await.unwrap();
6556 let mut request = [0u8; 2048];
6557 assert!(stream.read(&mut request).await.unwrap() > 0);
6558 stream
6559 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6560 .await
6561 .unwrap();
6562 });
6563 assert_eq!(
6568 probe_http_health(&url, Duration::from_secs(10))
6569 .await
6570 .unwrap()
6571 .status,
6572 HealthStatus::Ok
6573 );
6574 server.await.unwrap();
6575 }
6576
6577 #[cfg(unix)]
6578 #[tokio::test]
6579 async fn http_health_timeout_keeps_partial_status_and_body() {
6580 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6581 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6582 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6583 let server = tokio::spawn(async move {
6584 let (mut stream, _) = listener.accept().await.unwrap();
6585 let mut request = [0u8; 2048];
6586 assert!(stream.read(&mut request).await.unwrap() > 0);
6587 stream
6588 .write_all(
6589 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6590 )
6591 .await
6592 .unwrap();
6593 std::future::pending::<()>().await;
6594 });
6595 let error = probe_http_health(&url, Duration::from_secs(1))
6596 .await
6597 .unwrap_err()
6598 .to_string();
6599 server.abort();
6600 assert!(
6601 error.contains("timed out")
6602 && error.contains("503 Unavailable")
6603 && error.contains("partial diagnostic"),
6604 "{error}"
6605 );
6606 }
6607
6608 #[cfg(unix)]
6609 #[tokio::test]
6610 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6611 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6612 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6613 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6614 let server = tokio::spawn(async move {
6615 let (mut stream, _) = listener.accept().await.unwrap();
6616 let mut request = [0u8; 2048];
6617 assert!(stream.read(&mut request).await.unwrap() > 0);
6618 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6619 let response = format!(
6620 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6621 body.len()
6622 );
6623 stream.write_all(response.as_bytes()).await.unwrap();
6624 });
6625 let error = probe_http_health(&url, Duration::from_secs(1))
6626 .await
6627 .unwrap_err()
6628 .to_string();
6629 server.await.unwrap();
6630 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6631 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6632 assert!(!error.contains("not-in-diagnostic"));
6633 }
6634
6635 #[cfg(unix)]
6636 #[tokio::test]
6637 async fn http_health_real_nats_server_monitoring() {
6638 if std::process::Command::new("nats-server")
6639 .arg("--version")
6640 .env("XDG_DATA_HOME", std::env::temp_dir())
6641 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6642 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6643 .output()
6644 .is_err()
6645 {
6646 eprintln!(
6647 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6648 );
6649 return;
6650 }
6651 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6652 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6653 let port = monitor.local_addr().unwrap().port();
6654 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6655 let client_port = client.local_addr().unwrap().port();
6656 let config = dir.join("server.conf");
6657 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6658 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6659 configured.program = PathBuf::from("nats-server");
6660 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6661 drop(monitor);
6662 drop(client);
6663 let module =
6664 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6665 .supervise_configured_with_health(
6666 configured.module_spec(),
6667 true,
6668 configured.health,
6669 configured.drain_timeout_ms,
6670 configured.restart,
6671 )
6672 .unwrap();
6673 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6674 module.drain().await.unwrap();
6675 }
6676
6677 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6678 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6679 let supervisor =
6680 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6681 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6682 let initial = ModuleSpec {
6683 module_id: "rescan-preserves-spawn-facts".to_string(),
6684 program: PathBuf::from("/spawned/module"),
6685 args: Vec::new(),
6686 env: Vec::new(),
6687 reserved: false,
6688 reserved_prefixes: Vec::new(),
6689 protocol: ModuleProtocol::Subc,
6690 overlap: Default::default(),
6691 };
6692 let module = supervisor.supervised_module(
6693 initial.clone(),
6694 supervisor.runtime_config(),
6695 snapshot,
6696 None,
6697 );
6698 let before = module.status().unwrap();
6699 let mut replacement = initial;
6700 replacement.program = PathBuf::from("/rescanned/replacement-module");
6701
6702 module
6703 .update_configuration(replacement, HealthConfig::default(), None)
6704 .await
6705 .unwrap();
6706
6707 let after = module.status().unwrap();
6708 assert_eq!(after.pid, before.pid);
6709 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6710 assert_eq!(after.spawned_from, before.spawned_from);
6711 drop(module);
6712 }
6713}
6714
6715fn unix_ms_now() -> u64 {
6716 SystemTime::now()
6717 .duration_since(UNIX_EPOCH)
6718 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6719 .unwrap_or(0)
6720}
6721
6722async fn supervise_loop(
6723 mut spec: ModuleSpec,
6724 mut runtime: SupervisorRuntimeConfig,
6725 registry: Arc<Registry>,
6726 process_liveness: Arc<SupervisorProcessLiveness>,
6727 snapshot: SharedSnapshot,
6728 mut child: Option<SupervisedChild>,
6729 mut commands: mpsc::Receiver<SupervisorCommand>,
6730) {
6731 let mut health_probe = HealthProbeRuntime::default();
6732 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6736 Ok(changes) => Some(changes),
6737 Err(err) => {
6738 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6741 None
6742 }
6743 };
6744 let mut pending_respawn: Option<PendingRespawn> = None;
6748 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6751 let mut previous_instance_recheck = tokio::time::interval_at(
6752 Instant::now() + Duration::from_secs(5),
6753 Duration::from_secs(5),
6754 );
6755 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6756 loop {
6757 #[cfg(test)]
6758 {
6759 lock_snapshot(&snapshot).unwrap().actor_select = None;
6760 }
6761 #[cfg(target_os = "macos")]
6762 if let Some(active) = child.as_mut() {
6763 active.confirm_privacy_exec().await;
6764 }
6765 if let Some(scheduled) = runtime
6766 .scheduled_respawn
6767 .lock()
6768 .unwrap_or_else(|p| p.into_inner())
6769 .take()
6770 {
6771 runtime.recovery_is_daemon = scheduled.daemon_initiated;
6772 pending_respawn = Some(scheduled);
6773 }
6774 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6775 pending_respawn = None;
6776 cancel_deferred_reload(
6777 &runtime,
6778 &spec.module_id,
6779 "respawn cancelled by a supervisor command",
6780 );
6781 }
6782 if child.is_none()
6783 && pending_respawn.is_none()
6784 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6785 {
6786 cancel_deferred_reload(
6787 &runtime,
6788 &spec.module_id,
6789 "respawn cancelled before a replacement was spawned",
6790 );
6791 let mut stranded = false;
6792 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6793 state.respawn_pending = false;
6794 state.coalesced_restart_pending = false;
6795 if matches!(
6796 state.state,
6797 ModuleState::Restarting
6798 | ModuleState::Starting
6799 | ModuleState::Draining
6800 | ModuleState::Unresponsive
6801 ) {
6802 error!(module_id = %spec.module_id, state = ?state.state,
6803 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6804 stranded = true;
6805 }
6806 });
6807 if stranded {
6808 fail_snapshot(
6809 &snapshot,
6810 Some(&spec.module_id),
6811 None,
6812 &runtime.failure_context(runtime.recovery_is_daemon),
6813 );
6814 }
6815 }
6816 if let Some(command) = requeued.pop_front() {
6817 if !handle_supervisor_command(
6818 command,
6819 &mut spec,
6820 &mut runtime,
6821 ®istry,
6822 &process_liveness,
6823 &snapshot,
6824 &mut child,
6825 &mut commands,
6826 &mut requeued,
6827 )
6828 .await
6829 {
6830 return;
6831 }
6832 if child.is_some() || !respawn_still_pending(&snapshot) {
6833 pending_respawn = None;
6834 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6835 state.respawn_pending = false
6836 });
6837 }
6838 continue;
6839 }
6840 if child.is_some() {
6841 runtime.recovery_is_daemon = true;
6842 #[cfg(test)]
6843 {
6844 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6845 }
6846 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6847 let no_checks = runtime.health.http.is_none()
6848 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
6849 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
6850 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
6851 state
6852 .failed_retry
6853 .uptime_reset_at(no_checks, runtime.restart_policy.window)
6854 });
6855 let wake_at = match (health_probe.wake_at(), uptime_reset) {
6856 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
6857 (probe, None) => probe,
6858 };
6859 #[cfg(test)]
6860 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
6861 let probe_sleep = supervisor_sleep(wake_at);
6862 tokio::pin!(probe_sleep);
6863 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6868 #[cfg(test)]
6869 {
6870 let registration_pending = wire_child
6871 && registration_changes
6872 .as_ref()
6873 .is_some_and(|changes| changes.has_changed().unwrap());
6874 let mut state = lock_snapshot(&snapshot).unwrap();
6875 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6876 generation: state.spawn_generation,
6877 turn: state.actor_turns,
6878 registered_connection: health_probe.registered_connection,
6879 next_probe_at: health_probe.next_probe_at,
6880 wake_after,
6881 });
6882 }
6883 let active_child = child.as_mut().expect("child checked above");
6884 tokio::select! {
6885 wait_result = active_child.wait() => {
6886 if let Ok(mut state) = lock_snapshot(&snapshot) {
6887 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
6888 }
6889 #[cfg(test)]
6890 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6891 let exit_report = match wait_result {
6900 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6901 Err(err) => {
6902 active_child.drain_stderr(&spec.module_id).await;
6903 fail_after_wait_error(
6904 &spec.module_id,
6905 &snapshot,
6906 &runtime.terminal_ring,
6907 &runtime.spawn_events,
6908 format!("failed to wait for supervised module: {err}"),
6909 &runtime.failure_context(true),
6910 );
6911 untrack_if_registration_released(
6912 &process_liveness,
6913 ®istry,
6914 &spec.module_id,
6915 &snapshot,
6916 );
6917 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6918 child = None;
6919 continue;
6920 }
6921 };
6922 active_child.drain_stderr(&spec.module_id).await;
6923
6924 let next = on_child_exit(
6925 &spec,
6926 runtime.restart_policy,
6927 ®istry,
6928 &snapshot,
6929 &runtime.terminal_ring,
6930 &runtime.spawn_events,
6931 &runtime.child_roster,
6932 exit_report,
6933 true,
6934 ).await;
6935 runtime.start_requester = None;
6936 runtime.pending_activity_restart = false;
6937 active_child.release_roster();
6940 match next {
6941 NextAction::Stop { registration_released } => {
6942 if registration_released {
6943 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6944 }
6945 child = None;
6946 }
6947 NextAction::Restart { schedule } => {
6948 let delay = schedule.map_or(
6949 runtime.restart_policy.delay_for_restart(0),
6950 |schedule| schedule.delay,
6951 );
6952 if let Some(schedule) = schedule {
6953 log_crash_respawn(&spec.module_id, schedule);
6954 }
6955 child = None;
6963 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
6964 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6965 }
6966 }
6967 }
6968 command = commands.recv() => {
6969 #[cfg(test)]
6970 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6971 let Some(command) = command else {
6972 return;
6973 };
6974 if !handle_supervisor_command(
6975 command,
6976 &mut spec,
6977 &mut runtime,
6978 ®istry,
6979 &process_liveness,
6980 &snapshot,
6981 &mut child,
6982 &mut commands,
6983 &mut requeued,
6984 ).await {
6985 return;
6986 }
6987 }
6988 _ = async {
6989 match registration_changes.as_mut() {
6990 Some(changes) => { let _ = changes.changed().await; }
6991 None => std::future::pending().await,
6992 }
6993 }, if wire_child => {
6994 #[cfg(test)]
6995 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6996 }
6997 _ = &mut probe_sleep => {
6998 #[cfg(test)]
6999 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7000 if health_probe.due() {
7001 run_health_probe_cycle(
7002 &spec,
7003 &runtime,
7004 ®istry,
7005 &process_liveness,
7006 &snapshot,
7007 &mut child,
7008 ).await;
7009 if child.is_none() {
7010 runtime.start_requester = None;
7011 runtime.pending_activity_restart = false;
7012 }
7013 if child.is_some() {
7014 health_probe.schedule_next(&spec, runtime.health.cadence);
7015 }
7016 }
7017 }
7018 }
7019 } else if let Some(pending) = pending_respawn {
7020 tokio::select! {
7021 _ = sleep_until(pending.deadline) => {
7022 pending_respawn = None;
7023 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7024 if !respawn_still_pending(&snapshot) {
7028 continue;
7029 }
7030 if runtime.child_roster.is_closed() {
7035 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7036 state.state = ModuleState::Stopped;
7037 });
7038 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7039 continue;
7040 }
7041 if let Err(err) = release_dead_registration(
7042 ®istry,
7043 runtime.forwarding.as_deref(),
7044 &snapshot,
7045 &spec.module_id,
7046 &runtime.failure_context(runtime.recovery_is_daemon),
7047 ).await {
7048 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7049 continue;
7050 }
7051
7052 if matches!(pending.kind, RespawnKind::Reload) {
7053 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7054 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
7055 if result.is_ok() && runtime.pending_activity_restart {
7056 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7057 }
7058 runtime.pending_activity_restart = false;
7059 if let Some(reply) = reply { let _ = reply.send(result); }
7060 continue;
7061 }
7062 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7063 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7064 Ok(next_child) => {
7065 child = Some(next_child);
7066 if runtime.pending_activity_restart {
7067 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7068 runtime.pending_activity_restart = false;
7069 }
7070 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7071 }
7072 Err(err) => {
7073 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7074 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7075 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7076 }
7077 }
7078 }
7079 command = commands.recv() => {
7080 let Some(command) = command else {
7081 return;
7082 };
7083 if !handle_supervisor_command(
7084 command,
7085 &mut spec,
7086 &mut runtime,
7087 ®istry,
7088 &process_liveness,
7089 &snapshot,
7090 &mut child,
7091 &mut commands,
7092 &mut requeued,
7093 ).await {
7094 return;
7095 }
7096 if child.is_some() || !respawn_still_pending(&snapshot) {
7101 pending_respawn = None;
7102 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7103 }
7104 }
7105 }
7106 } else {
7107 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7108 (state.enabled && state.state == ModuleState::Failed)
7109 .then_some(state.failed_retry.pending)
7110 .flatten()
7111 });
7112 let command = if lock_snapshot(&snapshot)
7113 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7114 {
7115 tokio::select! {
7116 command = commands.recv() => command,
7117 _ = previous_instance_recheck.tick() => {
7118 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7119 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7120 if enabled && !runtime.child_roster.is_closed() {
7121 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7122 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7123 Ok(next_child) => child = Some(next_child),
7124 Err(error) => {
7125 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7126 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7127 }
7128 }
7129 } else {
7130 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7131 state.waiting_for_previous_pid = None;
7132 state.waiting_for_previous_detail = None;
7133 });
7134 }
7135 }
7136 continue;
7137 }
7138 }
7139 } else {
7140 tokio::select! {
7141 biased;
7142 command = commands.recv() => command,
7143 _ = async {
7144 match slow_retry {
7145 Some(retry) => sleep_until(retry.deadline).await,
7146 None => std::future::pending().await,
7147 }
7148 } => {
7149 let retry = slow_retry.expect("only a scheduled slow retry can wake");
7150 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7151 state.failed_retry.pending = None;
7152 });
7153 if runtime.child_roster.is_closed() {
7154 continue;
7155 }
7156 info!(module_id = %spec.module_id, attempt = retry.attempt,
7157 delay_secs = retry.delay.as_secs(), "starting failed module retry");
7158 if let Err(error) = set_child_enabled(
7159 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
7160 &runtime.failure_context(true),
7161 ).await {
7162 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7163 }
7164 continue;
7165 }
7166 }
7167 };
7168 let Some(command) = command else {
7169 return;
7170 };
7171 if !handle_supervisor_command(
7172 command,
7173 &mut spec,
7174 &mut runtime,
7175 ®istry,
7176 &process_liveness,
7177 &snapshot,
7178 &mut child,
7179 &mut commands,
7180 &mut requeued,
7181 )
7182 .await
7183 {
7184 return;
7185 }
7186 }
7187 }
7188}
7189
7190async fn supervisor_sleep(wake_at: Option<Instant>) {
7193 match wake_at {
7194 Some(deadline) => sleep_until(deadline).await,
7195 None => std::future::pending().await,
7196 }
7197}
7198
7199fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7200 info!(
7201 module_id,
7202 restart_in_window = schedule.restart_in_window,
7203 delay_ms = schedule.delay.as_millis() as u64,
7204 "respawning after crash"
7205 );
7206}
7207
7208fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7214 matches!(
7215 lock_snapshot(snapshot),
7216 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7217 )
7218}
7219
7220enum NextAction {
7221 Stop {
7222 registration_released: bool,
7223 },
7224 Restart {
7225 schedule: Option<CrashRestartSchedule>,
7226 },
7227}
7228
7229#[allow(clippy::too_many_arguments)]
7230async fn handle_supervisor_command(
7231 command: SupervisorCommand,
7232 spec: &mut ModuleSpec,
7233 runtime: &mut SupervisorRuntimeConfig,
7234 registry: &Arc<Registry>,
7235 process_liveness: &SupervisorProcessLiveness,
7236 snapshot: &SharedSnapshot,
7237 child: &mut Option<SupervisedChild>,
7238 commands: &mut mpsc::Receiver<SupervisorCommand>,
7239 requeued: &mut VecDeque<SupervisorCommand>,
7240) -> bool {
7241 if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7242 runtime.recovery_is_daemon = false;
7243 }
7244 if matches!(
7245 command,
7246 SupervisorCommand::Drain { .. }
7247 | SupervisorCommand::Retire { .. }
7248 | SupervisorCommand::SetEnabled { .. }
7249 | SupervisorCommand::Restart { .. }
7250 ) {
7251 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7252 state.failed_retry.cancel()
7253 });
7254 }
7255 let command =
7256 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7257 match command {
7258 SupervisorCommand::Restart { reply, .. }
7259 | SupervisorCommand::Reload { reply, .. }
7260 | SupervisorCommand::Swap { reply, .. } => {
7261 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7262 return true;
7263 }
7264 SupervisorCommand::SetEnabled {
7265 enabled: true,
7266 reply,
7267 ..
7268 } => {
7269 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7270 return true;
7271 }
7272 other => other,
7273 }
7274 } else {
7275 command
7276 };
7277 match command {
7278 SupervisorCommand::Drain { requester, reply } => {
7279 let result = drain_optional_child(
7282 &spec.module_id,
7283 spec.protocol,
7284 StopNotice::NotSent,
7285 registry,
7286 runtime.forwarding.as_deref(),
7287 snapshot,
7288 &runtime.terminal_ring,
7289 &runtime.spawn_events,
7290 child,
7291 runtime.drain_timeout,
7292 ModuleState::Stopped,
7293 None,
7294 &FailureContext::operator(),
7295 )
7296 .await;
7297 let registration_released = result.is_ok();
7298 if registration_released {
7299 runtime.record_activity_transition(
7300 "subc.module_stopped",
7301 &spec.module_id,
7302 requester,
7303 );
7304 }
7305 let _ = reply.send(result);
7306 if registration_released {
7307 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7308 }
7309 false
7310 }
7311 SupervisorCommand::Retire { requester, reply } => {
7312 let result = async {
7313 let stop_notice = begin_forwarding_drain_if_configured(
7314 spec,
7315 runtime,
7316 registry,
7317 snapshot,
7318 None,
7319 RouteCloseReason::Disable,
7320 )
7321 .await?;
7322 drain_optional_child(
7323 &spec.module_id,
7324 spec.protocol,
7325 stop_notice,
7326 registry,
7327 runtime.forwarding.as_deref(),
7328 snapshot,
7329 &runtime.terminal_ring,
7330 &runtime.spawn_events,
7331 child,
7332 runtime.drain_timeout,
7333 ModuleState::Stopped,
7334 None,
7335 &FailureContext::operator(),
7336 )
7337 .await
7338 }
7339 .await;
7340 let registration_released = result.is_ok();
7341 if registration_released {
7342 runtime.record_activity_transition(
7343 "subc.module_stopped",
7344 &spec.module_id,
7345 requester,
7346 );
7347 }
7348 let _ = reply.send(result);
7349 if registration_released {
7350 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7351 }
7352 false
7353 }
7354 SupervisorCommand::Restart {
7355 requester,
7356 drain_timeout_ms,
7357 received_at_generation,
7358 queued_at,
7359 reply,
7360 } => {
7361 info!(
7365 module_id = %spec.module_id,
7366 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7367 "restart command dequeued"
7368 );
7369 let validation = match lock_snapshot(snapshot) {
7381 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7382 module_id: spec.module_id.clone(),
7383 }),
7384 Ok(_) => Ok(()),
7385 Err(err) => Err(err),
7386 };
7387 let initiated = validation.is_ok();
7388 let _ = reply.send(validation);
7389 let satisfied_by_generation = if initiated
7403 && child
7404 .as_ref()
7405 .is_some_and(|child| child.spawned_at > queued_at)
7406 {
7407 lock_snapshot(snapshot).ok().and_then(|state| {
7408 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7409 })
7410 } else {
7411 None
7412 };
7413 let satisfied_by_pending = initiated
7414 && child.is_none()
7415 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7416 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7420 if pending {
7421 state.coalesced_restart_pending = true;
7422 }
7423 pending
7424 });
7425 if satisfied_by_pending {
7426 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7427 } else if let Some(generation) = satisfied_by_generation {
7428 info!(
7429 module_id = %spec.module_id,
7430 received_at_generation,
7431 "restart already satisfied by generation {generation}; not restarting again"
7432 );
7433 } else if initiated {
7434 let drain_timeout = drain_timeout_ms
7437 .map(Duration::from_millis)
7438 .unwrap_or(runtime.drain_timeout);
7439 if let Err(err) = restart_child(
7440 spec,
7441 runtime,
7442 registry,
7443 process_liveness,
7444 snapshot,
7445 child,
7446 drain_timeout,
7447 )
7448 .await
7449 {
7450 warn!(
7451 module_id = %spec.module_id,
7452 error = %err,
7453 "operator restart failed after initiation ack; module state carries the outcome"
7454 );
7455 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7456 state.state = ModuleState::Failed;
7457 clear_current_process_facts(state);
7458 });
7459 } else {
7460 runtime.start_requester = requester;
7461 runtime.pending_activity_restart = true;
7462 }
7463 }
7464 true
7465 }
7466 SupervisorCommand::Reload { requester, reply } => {
7467 let result =
7468 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7469 if result.is_ok() {
7470 runtime.start_requester = requester;
7471 runtime.pending_activity_restart = true;
7472 }
7473 if result.is_ok()
7474 && runtime
7475 .scheduled_respawn
7476 .lock()
7477 .unwrap_or_else(|p| p.into_inner())
7478 .as_ref()
7479 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7480 {
7481 *runtime
7482 .deferred_reload_reply
7483 .lock()
7484 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7485 } else {
7486 let _ = reply.send(result);
7487 }
7488 true
7489 }
7490 SupervisorCommand::SetEnabled {
7491 enabled,
7492 requester,
7493 reply,
7494 } => {
7495 let previous_requester = runtime.start_requester.clone();
7496 if enabled {
7497 runtime.start_requester = requester.clone();
7498 }
7499 let result = set_child_enabled(
7500 spec,
7501 runtime,
7502 registry,
7503 process_liveness,
7504 snapshot,
7505 child,
7506 enabled,
7507 &runtime.failure_context(false),
7508 )
7509 .await;
7510 if matches!(result, Ok(true)) {
7511 runtime.pending_activity_restart = false;
7512 if enabled {
7513 runtime.start_requester = requester;
7514 } else {
7515 runtime.start_requester = None;
7516 runtime.record_activity_transition(
7517 "subc.module_disabled",
7518 &spec.module_id,
7519 requester,
7520 );
7521 }
7522 } else {
7523 runtime.start_requester = previous_requester;
7524 }
7525 let _ = reply.send(result);
7526 true
7527 }
7528 SupervisorCommand::UpdateConfiguration {
7529 spec: next_spec,
7530 health,
7531 drain_timeout_ms,
7532 reply,
7533 } => {
7534 if let Some(handle) = &runtime.supervisor_handle {
7535 handle.apply_identity_configuration(&next_spec);
7536 }
7537 *spec = next_spec;
7538 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7539 state.configuration_updated_since_spawn = true;
7540 });
7541 let health_changed = runtime.health != health;
7542 runtime.health = health;
7543 if health_changed {
7546 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7547 state.health = ModuleHealthStatus::default();
7548 });
7549 }
7550 runtime.drain_timeout = drain_timeout_ms
7551 .map(Duration::from_millis)
7552 .unwrap_or(runtime.default_drain_timeout);
7553 *runtime
7554 .effective_drain_timeout
7555 .lock()
7556 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7557 let _ = reply.send(());
7558 true
7559 }
7560 SupervisorCommand::Swap {
7561 requester,
7562 ready_timeout,
7563 reply,
7564 } => {
7565 let mut swap_runtime = runtime.clone();
7566 swap_runtime.start_requester = requester;
7567 let before = lock_snapshot(snapshot)
7568 .ok()
7569 .map(|state| state.spawn_generation);
7570 let end = swap::run_swap(
7571 spec,
7572 &swap_runtime,
7573 registry,
7574 process_liveness,
7575 snapshot,
7576 child,
7577 commands,
7578 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7579 reply,
7580 )
7581 .await;
7582 if before
7583 != lock_snapshot(snapshot)
7584 .ok()
7585 .map(|state| state.spawn_generation)
7586 {
7587 runtime.start_requester = swap_runtime.start_requester;
7588 }
7589 requeued.extend(end.requeue);
7590 true
7591 }
7592 }
7593}
7594
7595async fn restart_child(
7596 spec: &ModuleSpec,
7597 runtime: &SupervisorRuntimeConfig,
7598 registry: &Registry,
7599 process_liveness: &SupervisorProcessLiveness,
7600 snapshot: &SharedSnapshot,
7601 child: &mut Option<SupervisedChild>,
7602 drain_timeout: Duration,
7603) -> Result<(), SuperviseError> {
7604 if !lock_snapshot(snapshot)?.enabled {
7606 return Err(SuperviseError::Disabled {
7607 module_id: spec.module_id.clone(),
7608 });
7609 }
7610 let stop_notice = begin_forwarding_drain_with_timeout(
7611 spec,
7612 runtime,
7613 registry,
7614 snapshot,
7615 None,
7616 RouteCloseReason::Restart,
7617 drain_timeout,
7618 )
7619 .await?;
7620
7621 if child.is_some() {
7622 drain_optional_child(
7623 &spec.module_id,
7624 spec.protocol,
7625 stop_notice,
7626 registry,
7627 runtime.forwarding.as_deref(),
7628 snapshot,
7629 &runtime.terminal_ring,
7630 &runtime.spawn_events,
7631 child,
7632 drain_timeout,
7633 ModuleState::Restarting,
7634 Some(true),
7635 &FailureContext::operator(),
7636 )
7637 .await?;
7638 } else {
7639 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7640 state.enabled = true;
7641 state.state = ModuleState::Restarting;
7642 clear_current_process_facts(state);
7643 })?;
7644 release_dead_registration(
7645 registry,
7646 runtime.forwarding.as_deref(),
7647 snapshot,
7648 &spec.module_id,
7649 &runtime.failure_context(false),
7650 )
7651 .await?;
7652 }
7653
7654 reset_restart_count(snapshot, &spec.module_id)?;
7655 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7656 schedule_respawn(
7657 runtime,
7658 snapshot,
7659 &spec.module_id,
7660 runtime.restart_policy.backoff,
7661 RespawnKind::Spawn,
7662 false,
7663 )
7664}
7665
7666async fn reload_child(
7667 spec: &ModuleSpec,
7668 runtime: &SupervisorRuntimeConfig,
7669 registry: &Registry,
7670 process_liveness: &SupervisorProcessLiveness,
7671 snapshot: &SharedSnapshot,
7672 child: &mut Option<SupervisedChild>,
7673) -> Result<(), SuperviseError> {
7674 if !lock_snapshot(snapshot)?.enabled {
7676 return Err(SuperviseError::Disabled {
7677 module_id: spec.module_id.clone(),
7678 });
7679 }
7680 let stop_notice = begin_forwarding_drain(
7681 spec,
7682 runtime,
7683 registry,
7684 snapshot,
7685 Some(true),
7686 RouteCloseReason::Reload,
7687 )
7688 .await?;
7689
7690 if child.is_some() {
7691 drain_optional_child(
7692 &spec.module_id,
7693 spec.protocol,
7694 stop_notice,
7695 registry,
7696 runtime.forwarding.as_deref(),
7697 snapshot,
7698 &runtime.terminal_ring,
7699 &runtime.spawn_events,
7700 child,
7701 runtime.drain_timeout,
7702 ModuleState::Restarting,
7703 Some(true),
7704 &FailureContext::operator(),
7705 )
7706 .await?;
7707 } else {
7708 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7709 state.enabled = true;
7710 state.state = ModuleState::Restarting;
7711 clear_current_process_facts(state);
7712 })?;
7713 release_dead_registration(
7714 registry,
7715 runtime.forwarding.as_deref(),
7716 snapshot,
7717 &spec.module_id,
7718 &runtime.failure_context(false),
7719 )
7720 .await?;
7721 }
7722
7723 reset_restart_count(snapshot, &spec.module_id)?;
7724 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7725 schedule_respawn(
7726 runtime,
7727 snapshot,
7728 &spec.module_id,
7729 runtime.restart_policy.backoff,
7730 RespawnKind::Reload,
7731 false,
7732 )
7733}
7734
7735async fn finish_reload_child(
7736 spec: &ModuleSpec,
7737 runtime: &SupervisorRuntimeConfig,
7738 registry: &Registry,
7739 process_liveness: &SupervisorProcessLiveness,
7740 snapshot: &SharedSnapshot,
7741 child: &mut Option<SupervisedChild>,
7742) -> Result<(), SuperviseError> {
7743 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7744 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7745 Ok(next_child) => next_child,
7746 Err(err) => {
7747 return handle_reload_spawn_failure(
7748 spec,
7749 runtime,
7750 process_liveness,
7751 snapshot,
7752 child,
7753 format!("new child failed to spawn: {err}"),
7754 )
7755 .await;
7756 }
7757 };
7758 *child = Some(next_child);
7759
7760 let wait_outcome = {
7761 let active_child = child.as_mut().expect("new reload child was just stored");
7762 wait_for_registration_after_reload(
7763 registry,
7764 &spec.module_id,
7765 snapshot,
7766 active_child,
7767 REGISTRY_RELEASE_TIMEOUT,
7768 )
7769 .await?
7770 };
7771
7772 match wait_outcome {
7773 RegistrationWaitOutcome::Registered => {
7774 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7775 Ok(())
7776 }
7777 RegistrationWaitOutcome::Exited(exit_report) => {
7778 if let Some(active_child) = child.as_mut() {
7779 active_child.drain_stderr(&spec.module_id).await;
7780 }
7781 let mut exited_child = child.take().expect("exited reload child is still stored");
7784 #[cfg(test)]
7785 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7786 gate.reached.notify_one();
7787 gate.resume.notified().await;
7788 }
7789 let result = handle_reload_child_registration_failure(
7790 spec,
7791 runtime,
7792 registry,
7793 process_liveness,
7794 snapshot,
7795 child,
7796 ReloadRegistrationFailure {
7797 exit_report: registration_failure_exit_report(exit_report),
7798 reason: exited_child
7799 .spawn_failure
7800 .clone()
7801 .unwrap_or_else(|| "new child exited before registering".to_string()),
7802 },
7803 )
7804 .await;
7805 exited_child.release_roster();
7806 result
7807 }
7808 RegistrationWaitOutcome::TimedOut => {
7809 let mut timed_out_child = child
7810 .take()
7811 .expect("timed-out reload child is still running");
7812 timed_out_child
7813 .start_kill()
7814 .map_err(|source| SuperviseError::Kill {
7815 module_id: spec.module_id.clone(),
7816 source,
7817 })?;
7818 let status = timed_out_child
7819 .wait()
7820 .await
7821 .map_err(|source| SuperviseError::Wait {
7822 module_id: spec.module_id.clone(),
7823 source,
7824 })?;
7825 timed_out_child.drain_stderr(&spec.module_id).await;
7826 handle_reload_child_registration_failure(
7827 spec,
7828 runtime,
7829 registry,
7830 process_liveness,
7831 snapshot,
7832 child,
7833 ReloadRegistrationFailure {
7834 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7835 snapshot,
7836 &timed_out_child,
7837 &status,
7838 )),
7839 reason: format!(
7840 "new child did not register within {:?}",
7841 REGISTRY_RELEASE_TIMEOUT
7842 ),
7843 },
7844 )
7845 .await
7846 }
7847 }
7848}
7849
7850#[allow(clippy::too_many_arguments)]
7851async fn set_child_enabled(
7852 spec: &ModuleSpec,
7853 runtime: &SupervisorRuntimeConfig,
7854 registry: &Registry,
7855 process_liveness: &SupervisorProcessLiveness,
7856 snapshot: &SharedSnapshot,
7857 child: &mut Option<SupervisedChild>,
7858 enabled: bool,
7859 failure: &FailureContext,
7860) -> Result<bool, SuperviseError> {
7861 let (current_enabled, current_state, respawn_pending) = {
7862 let state = lock_snapshot(snapshot)?;
7863 (state.enabled, state.state, state.respawn_pending)
7864 };
7865 let revive_terminal = enabled
7873 && current_enabled
7874 && child.is_none()
7875 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7876 || (current_state == ModuleState::Restarting && !respawn_pending));
7877 if current_enabled == enabled && !revive_terminal {
7878 return Ok(false);
7879 }
7880
7881 if enabled {
7882 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7883 state.enabled = true;
7884 state.state = ModuleState::Starting;
7885 clear_current_process_facts(state);
7886 })?;
7887 #[cfg(test)]
7888 if runtime.test_seed_stale_facts_before_enable_spawn {
7889 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7890 state.process_alive = true;
7891 state.pid = Some(41);
7892 state.spawned_at_ms = Some(42);
7893 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7894 state.spawned_file_identity = Some(SpawnedFileIdentity {
7895 device: 43,
7896 inode: 44,
7897 #[cfg(windows)]
7898 windows_image: None,
7899 });
7900 })?;
7901 }
7902 release_dead_registration(
7903 registry,
7904 runtime.forwarding.as_deref(),
7905 snapshot,
7906 &spec.module_id,
7907 failure,
7908 )
7909 .await?;
7910 reset_restart_count(snapshot, &spec.module_id)?;
7911 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7912 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7913 Ok(next_child) => next_child,
7914 Err(err) => {
7915 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7916 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7917 return Err(err);
7918 }
7919 };
7920 *child = Some(next_child);
7921 debug!(module_id = %spec.module_id, "supervised module enabled");
7922 Ok(true)
7923 } else {
7924 let stop_notice = begin_forwarding_drain_if_configured(
7925 spec,
7926 runtime,
7927 registry,
7928 snapshot,
7929 Some(false),
7930 RouteCloseReason::Disable,
7931 )
7932 .await?;
7933 drain_optional_child(
7934 &spec.module_id,
7935 spec.protocol,
7936 stop_notice,
7937 registry,
7938 runtime.forwarding.as_deref(),
7939 snapshot,
7940 &runtime.terminal_ring,
7941 &runtime.spawn_events,
7942 child,
7943 runtime.drain_timeout,
7944 ModuleState::Disabled,
7945 Some(false),
7946 failure,
7947 )
7948 .await?;
7949 debug!(module_id = %spec.module_id, "supervised module disabled");
7950 Ok(true)
7951 }
7952}
7953
7954#[allow(clippy::too_many_arguments)]
7955async fn on_child_exit(
7956 spec: &ModuleSpec,
7957 policy: RestartPolicy,
7958 registry: &Registry,
7959 snapshot: &SharedSnapshot,
7960 terminal_ring: &Arc<Mutex<TerminalRing>>,
7961 spawn_events: &SpawnEventFeed,
7962 roster: &ChildRoster,
7963 exit_report: ExitReport,
7964 daemon_initiated: bool,
7965) -> NextAction {
7966 if roster.is_closed() {
7972 return on_child_exit_during_daemon_shutdown(
7973 spec,
7974 registry,
7975 snapshot,
7976 terminal_ring,
7977 spawn_events,
7978 exit_report,
7979 )
7980 .await;
7981 }
7982 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7997 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7998 match exit_report.kind {
7999 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8000 info!(
8001 module_id = %spec.module_id,
8002 exit_code = ?exit_report.code,
8003 exit_signal = ?exit_report.signal,
8004 "supervised module exited cleanly"
8005 );
8006 if let Err(err) = publish_after_terminal_record(
8007 &spec.module_id,
8008 || {
8009 record_terminal(
8010 &spec.module_id,
8011 terminal_ring,
8012 spawn_events,
8013 &exit_report,
8014 TerminalDisposition::Stopped,
8015 )
8016 },
8017 || {
8018 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8019 state.state = ModuleState::Stopped;
8020 clear_current_process_facts(state);
8021 state.last_exit = Some(exit_report.clone());
8022 })
8023 },
8024 ) {
8025 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8026 }
8027 let registration_released = match wait_for_registration_release(
8028 registry,
8029 &spec.module_id,
8030 REGISTRY_RELEASE_TIMEOUT,
8031 )
8032 .await
8033 {
8034 Ok(()) => true,
8035 Err(err) => {
8036 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8037 false
8038 }
8039 };
8040 NextAction::Stop {
8041 registration_released,
8042 }
8043 }
8044 ExitKind::Clean | ExitKind::Crash => {
8045 if unrequested_clean_exit_of_protocol_none {
8046 warn!(
8047 module_id = %spec.module_id,
8048 exit_code = ?exit_report.code,
8049 exit_signal = ?exit_report.signal,
8050 "protocol-none module exited cleanly without a stop request; handling it as a crash"
8051 );
8052 } else {
8053 warn!(
8054 module_id = %spec.module_id,
8055 exit_code = ?exit_report.code,
8056 exit_signal = ?exit_report.signal,
8057 "supervised module exited abnormally (crash)"
8058 );
8059 }
8060 let mut restart_schedule = None;
8061 let mut disposition = TerminalDisposition::Disabled;
8062 let mut final_state = ModuleState::Disabled;
8063 let mut disposition_detail = lock_snapshot(snapshot)
8067 .ok()
8068 .and_then(|mut state| state.spawn_failure.take());
8069 let now = Instant::now();
8070 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8071 if state.enabled {
8072 if let Some(schedule) = state.next_crash_restart(&policy, now) {
8073 state.respawn_pending = true;
8079 final_state = ModuleState::Restarting;
8080 restart_schedule = Some(schedule);
8081 disposition = TerminalDisposition::Restarting;
8082 } else {
8083 final_state = ModuleState::Failed;
8084 disposition = TerminalDisposition::Failed;
8085 let budget = policy.budget_exhausted_detail();
8086 disposition_detail =
8087 Some(disposition_detail.take().map_or_else(
8088 || budget.clone(),
8089 |cause| format!("{cause}; {budget}"),
8090 ));
8091 }
8092 } else {
8093 disposition = TerminalDisposition::Disabled;
8094 }
8095 }) {
8096 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8097 return NextAction::Stop {
8098 registration_released: false,
8099 };
8100 }
8101 if disposition == TerminalDisposition::Failed {
8102 error!(
8107 module_id = %spec.module_id,
8108 max_restarts = policy.max_restarts,
8109 window_secs = policy.window.as_secs(),
8110 "module stopped: {}",
8111 policy.budget_exhausted_detail()
8112 );
8113 }
8114 if let Err(err) = publish_after_terminal_record(
8115 &spec.module_id,
8116 || {
8117 record_terminal_with_detail(
8118 &spec.module_id,
8119 terminal_ring,
8120 spawn_events,
8121 &exit_report,
8122 disposition,
8123 disposition_detail,
8124 )
8125 },
8126 || {
8127 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8128 state.state = final_state;
8129 clear_current_process_facts(state);
8130 if final_state == ModuleState::Failed
8131 && daemon_initiated
8132 && !roster.is_closed()
8133 {
8134 state.failed_retry.schedule(&spec.module_id);
8135 }
8136 state.last_exit = Some(exit_report.clone());
8137 })
8138 },
8139 ) {
8140 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8141 }
8142
8143 if let Some(schedule) = restart_schedule {
8144 NextAction::Restart {
8145 schedule: Some(schedule),
8146 }
8147 } else {
8148 let registration_released = match wait_for_registration_release(
8149 registry,
8150 &spec.module_id,
8151 REGISTRY_RELEASE_TIMEOUT,
8152 )
8153 .await
8154 {
8155 Ok(()) => true,
8156 Err(err) => {
8157 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8158 false
8159 }
8160 };
8161 NextAction::Stop {
8162 registration_released,
8163 }
8164 }
8165 }
8166 ExitKind::DeliberateSeverance => {
8167 warn!(
8168 module_id = %spec.module_id,
8169 exit_code = ?exit_report.code,
8170 exit_signal = ?exit_report.signal,
8171 "supervised module exited after deliberate connection severance"
8172 );
8173 let should_restart = match lock_snapshot(snapshot) {
8174 Ok(state) => state.enabled,
8175 Err(err) => {
8176 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8177 return NextAction::Stop {
8178 registration_released: false,
8179 };
8180 }
8181 };
8182 let final_state = if should_restart {
8183 ModuleState::Restarting
8184 } else {
8185 ModuleState::Disabled
8186 };
8187 if let Err(err) = publish_after_terminal_record(
8188 &spec.module_id,
8189 || {
8190 record_terminal(
8191 &spec.module_id,
8192 terminal_ring,
8193 spawn_events,
8194 &exit_report,
8195 terminal_disposition(final_state),
8196 )
8197 },
8198 || {
8199 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8200 state.state = final_state;
8201 clear_current_process_facts(state);
8202 state.last_exit = Some(exit_report.clone());
8203 state.lifetime_restarts += 1;
8204 })
8205 },
8206 ) {
8207 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8208 return NextAction::Stop {
8209 registration_released: false,
8210 };
8211 }
8212
8213 if should_restart {
8214 NextAction::Restart { schedule: None }
8215 } else {
8216 let registration_released = match wait_for_registration_release(
8217 registry,
8218 &spec.module_id,
8219 REGISTRY_RELEASE_TIMEOUT,
8220 )
8221 .await
8222 {
8223 Ok(()) => true,
8224 Err(err) => {
8225 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8226 false
8227 }
8228 };
8229 NextAction::Stop {
8230 registration_released,
8231 }
8232 }
8233 }
8234 }
8235}
8236
8237async fn on_child_exit_during_daemon_shutdown(
8238 spec: &ModuleSpec,
8239 registry: &Registry,
8240 snapshot: &SharedSnapshot,
8241 terminal_ring: &Arc<Mutex<TerminalRing>>,
8242 spawn_events: &SpawnEventFeed,
8243 exit_report: ExitReport,
8244) -> NextAction {
8245 info!(
8246 module_id = %spec.module_id,
8247 exit_code = ?exit_report.code,
8248 exit_signal = ?exit_report.signal,
8249 exit_kind = ?exit_report.kind,
8250 "supervised module exited during daemon shutdown; not restarting it"
8251 );
8252 if let Err(err) = publish_after_terminal_record(
8253 &spec.module_id,
8254 || {
8255 record_terminal(
8256 &spec.module_id,
8257 terminal_ring,
8258 spawn_events,
8259 &exit_report,
8260 TerminalDisposition::DaemonShutdown,
8261 )
8262 },
8263 || {
8264 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8265 state.state = ModuleState::Stopped;
8266 clear_current_process_facts(state);
8267 state.last_exit = Some(exit_report.clone());
8268 })
8269 },
8270 ) {
8271 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8272 }
8273 let registration_released =
8274 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8275 .await
8276 .is_ok();
8277 NextAction::Stop {
8278 registration_released,
8279 }
8280}
8281
8282fn publish_after_terminal_record<T>(
8287 module_id: &str,
8288 record: impl FnOnce(),
8289 publish: impl FnOnce() -> T,
8290) -> T {
8291 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8292 let published = publish();
8293 if let Err(panic) = recorded {
8294 error!(
8295 module_id,
8296 "terminal recording panicked; published final state before resuming unwind"
8297 );
8298 std::panic::resume_unwind(panic);
8299 }
8300 published
8301}
8302
8303fn fail_after_wait_error(
8304 module_id: &str,
8305 snapshot: &SharedSnapshot,
8306 terminal_ring: &Arc<Mutex<TerminalRing>>,
8307 spawn_events: &SpawnEventFeed,
8308 detail: String,
8309 failure: &FailureContext,
8310) {
8311 let report = wait_error_exit_report();
8312 publish_after_terminal_record(
8313 module_id,
8314 || {
8315 record_terminal_with_detail(
8316 module_id,
8317 terminal_ring,
8318 spawn_events,
8319 &report,
8320 TerminalDisposition::Failed,
8321 Some(detail),
8322 )
8323 },
8324 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8325 );
8326}
8327
8328#[cfg(test)]
8329fn record_wait_error_terminal(
8330 module_id: &str,
8331 terminal_ring: &Arc<Mutex<TerminalRing>>,
8332 spawn_events: &SpawnEventFeed,
8333) {
8334 record_terminal(
8335 module_id,
8336 terminal_ring,
8337 spawn_events,
8338 &wait_error_exit_report(),
8339 TerminalDisposition::Failed,
8340 );
8341}
8342
8343fn record_terminal(
8344 module_id: &str,
8345 terminal_ring: &Arc<Mutex<TerminalRing>>,
8346 spawn_events: &SpawnEventFeed,
8347 exit_report: &ExitReport,
8348 disposition: TerminalDisposition,
8349) {
8350 record_terminal_with_detail(
8351 module_id,
8352 terminal_ring,
8353 spawn_events,
8354 exit_report,
8355 disposition,
8356 None,
8357 );
8358}
8359
8360fn durable_terminal_history_of(
8364 terminal_ring: &Mutex<TerminalRing>,
8365 module_id: &str,
8366) -> subc_control::TerminalHistory {
8367 let read = terminal_ring
8368 .lock()
8369 .unwrap_or_else(|p| p.into_inner())
8370 .capture_durable_history();
8371 read.read(module_id)
8372}
8373
8374fn record_terminal_with_detail(
8375 module_id: &str,
8376 terminal_ring: &Arc<Mutex<TerminalRing>>,
8377 spawn_events: &SpawnEventFeed,
8378 exit_report: &ExitReport,
8379 disposition: TerminalDisposition,
8380 disposition_detail: Option<String>,
8381) {
8382 #[cfg(test)]
8383 terminal_history_tests::before_terminal_record(spawn_events);
8384 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8385 let record = TerminalRecord {
8386 exit_code: exit_report.code,
8387 exit_signal: exit_report.signal,
8388 at_ms: exit_report.at_ms,
8389 disposition,
8390 exit_kind: exit_report.kind.into(),
8391 disposition_detail,
8392 };
8393 terminal_ring
8394 .lock()
8395 .unwrap_or_else(|poisoned| poisoned.into_inner())
8396 .record_exit(module_id, record);
8397}
8398
8399fn untrack_if_registration_released(
8400 process_liveness: &SupervisorProcessLiveness,
8401 registry: &Registry,
8402 module_id: &str,
8403 snapshot: &SharedSnapshot,
8404) {
8405 match registry.get_module(module_id) {
8406 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8407 Ok(Some(_)) => {}
8408 Err(err) => {
8409 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8410 }
8411 }
8412}
8413
8414#[cfg(test)]
8428fn apply_wire_spawn_args(
8429 command: &mut Command,
8430 spec: &ModuleSpec,
8431 connection_file_path: Option<&std::path::Path>,
8432 handle: Option<&SupervisorHandle>,
8433) -> Result<Option<NonceHandoff>, SuperviseError> {
8434 apply_wire_spawn_args_for_role(
8435 command,
8436 spec,
8437 connection_file_path,
8438 handle,
8439 SpawnRole::Plain,
8440 )
8441}
8442
8443#[cfg(unix)]
8448type NonceHandoff = subc_os::LaunchNonceHandoff;
8449#[cfg(windows)]
8450type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8451#[cfg(not(any(unix, windows)))]
8452type NonceHandoff = std::convert::Infallible;
8453
8454fn apply_wire_spawn_args_for_role(
8475 command: &mut Command,
8476 spec: &ModuleSpec,
8477 connection_file_path: Option<&std::path::Path>,
8478 handle: Option<&SupervisorHandle>,
8479 role: SpawnRole,
8480) -> Result<Option<NonceHandoff>, SuperviseError> {
8481 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8482 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8487 #[cfg(windows)]
8488 {
8489 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8490 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8491 }
8492 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8494 if spec.protocol == ModuleProtocol::None {
8495 return Ok(None);
8496 }
8497 if let Some(connection_file_path) = connection_file_path {
8498 command.arg(SUBC_ARG).arg(connection_file_path);
8499 }
8500
8501 let nonce = generate_launch_nonce()?;
8505 if let Some(handle) = handle {
8506 match role {
8507 SpawnRole::Plain => {
8508 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8509 if spec.reserved {
8510 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8511 }
8512 }
8513 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8514 }
8515 }
8516 #[cfg(unix)]
8517 let handoff = {
8518 let handoff =
8519 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8520 program: spec.program.clone(),
8521 source,
8522 cgroup_path: None,
8523 })?;
8524 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8525 Some(handoff)
8526 };
8527 #[cfg(windows)]
8528 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8529 #[cfg(not(any(unix, windows)))]
8530 let handoff = None;
8531 #[cfg(not(unix))]
8535 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8536 Ok(handoff)
8537}
8538
8539fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8540 command.env_remove(CK_LOG_ENV);
8541 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8548 for (key, value) in &spec.env {
8549 if matches!(
8553 key.as_str(),
8554 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8555 ) || key == SUBC_SPAWN_ROLE_ENV
8556 {
8557 continue;
8558 }
8559 command.env(key, value);
8560 }
8561}
8562
8563#[cfg(windows)]
8564fn prepare_windows_nonce_pipe(
8565 command: &mut Command,
8566 spec: &ModuleSpec,
8567 nonce: &str,
8568) -> Option<NonceHandoff> {
8569 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8570 Ok(handoff) => {
8571 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8572 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8577 Some(handoff)
8578 }
8579 Err(error) => {
8580 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8581 None
8582 }
8583 }
8584}
8585
8586#[cfg(windows)]
8587fn start_windows_nonce_pipe(
8588 handoff: Option<NonceHandoff>,
8589 pid: u32,
8590 spec: &ModuleSpec,
8591) -> Option<subc_os::LaunchNoncePipeDelivery> {
8592 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8596 Ok(delivery) => Some(delivery),
8597 Err(error) => {
8598 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8599 None
8600 }
8601 })
8602}
8603
8604fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8605 #[cfg(windows)]
8606 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8607 return Some(
8608 if snapshot
8609 .pipe_consumed
8610 .as_ref()
8611 .is_some_and(|consumed| consumed.get().is_some())
8612 {
8613 "pipe"
8614 } else {
8615 "env"
8616 }
8617 .to_owned(),
8618 );
8619 }
8620 let _ = snapshot;
8621 None
8622}
8623
8624#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8627enum SpawnRole {
8628 Plain,
8629 SwapCandidate,
8630}
8631
8632fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8635 if role == SpawnRole::SwapCandidate {
8636 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8637 }
8638}
8639
8640fn previous_instance_error(
8641 spec: &ModuleSpec,
8642 pid: u32,
8643 reason: crate::live_children::RetainedReason,
8644) -> SuperviseError {
8645 SuperviseError::Spawn {
8646 program: spec.program.clone(),
8647 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8648 cgroup_path: None,
8649 }
8650}
8651
8652fn spawn_child(
8653 spec: &ModuleSpec,
8654 connection_file_path: Option<&std::path::Path>,
8655 handle: Option<&SupervisorHandle>,
8656 ring: &Arc<Mutex<StderrRing>>,
8657 capture_logs_dir: Option<&std::path::Path>,
8658 roster: &ChildRoster,
8659 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8660) -> Result<SupervisedChild, SuperviseError> {
8661 spawn_child_in_slot(
8662 spec,
8663 connection_file_path,
8664 handle,
8665 ring,
8666 capture_logs_dir,
8667 roster,
8668 #[cfg(target_os = "linux")]
8669 cgroup_placement,
8670 SpawnRole::Plain,
8671 false,
8672 )
8673}
8674
8675#[allow(clippy::too_many_arguments)]
8688fn spawn_child_in_slot(
8689 spec: &ModuleSpec,
8690 connection_file_path: Option<&std::path::Path>,
8691 handle: Option<&SupervisorHandle>,
8692 ring: &Arc<Mutex<StderrRing>>,
8693 capture_logs_dir: Option<&std::path::Path>,
8694 roster: &ChildRoster,
8695 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8696 role: SpawnRole,
8697 alternate_slot: bool,
8698) -> Result<SupervisedChild, SuperviseError> {
8699 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8700 return Err(previous_instance_error(spec, pid, reason));
8701 }
8702 if roster.is_closed() {
8703 return Err(SuperviseError::Spawn {
8704 program: spec.program.clone(),
8705 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8706 cgroup_path: None,
8707 });
8708 }
8709 #[cfg(target_os = "linux")]
8710 let cgroup_name = {
8711 if cgroup_placement.is_none() {
8716 swap::cgroup_name(&spec.module_id, alternate_slot)
8717 } else {
8718 let nonce = generate_launch_nonce()?;
8719 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8720 let mut end = spec.module_id.len().min(64);
8723 while !spec.module_id.is_char_boundary(end) {
8724 end -= 1;
8725 }
8726 format!(
8727 "{}_{suffix}",
8728 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8729 )
8730 }
8731 };
8732 #[cfg(not(target_os = "linux"))]
8733 let _ = alternate_slot;
8734 #[cfg(target_os = "macos")]
8735 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8736 #[cfg(windows)]
8737 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8738 program: spec.program.clone(),
8739 source,
8740 cgroup_path: None,
8741 })?;
8742 #[cfg(windows)]
8743 let mut command = Command::new(image_capture.path());
8744 #[cfg(not(any(target_os = "macos", windows)))]
8745 let mut command = Command::new(&spec.program);
8746 command.args(&spec.args);
8747 apply_child_env(&mut command, spec);
8777 apply_spawn_role(&mut command, role);
8778 let nonce_handoff =
8779 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8780
8781 #[cfg(target_os = "linux")]
8782 let cgroup_path = cgroup_placement
8783 .map(|placement| placement.module_path(&cgroup_name))
8784 .transpose()
8785 .map_err(|source| SuperviseError::Cgroup {
8786 module_id: spec.module_id.clone(),
8787 source,
8788 })?;
8789 #[cfg(not(target_os = "linux"))]
8790 let cgroup_path: Option<PathBuf> = None;
8791 #[cfg(target_os = "linux")]
8792 if let Some(path) = &cgroup_path {
8793 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8794 if let Some(placement) = cgroup_placement {
8795 remove_module_cgroup(placement, &cgroup_name);
8796 }
8797 return Err(error);
8798 }
8799 }
8800
8801 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8802 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8803 match ChildOutputSink::open(&path, capture_retention(spec)) {
8804 Ok(sink) => sink,
8805 Err(error) => {
8806 warn!(
8807 module_id = %spec.module_id,
8808 path = %path.display(),
8809 error = %error,
8810 "could not open child output capture file; forwarding to stderr"
8811 );
8812 ChildOutputSink::Stderr
8813 }
8814 }
8815 } else {
8816 ChildOutputSink::Stderr
8817 };
8818
8819 command.stdout(Stdio::piped());
8820 command.stderr(Stdio::piped());
8821 command.kill_on_drop(true);
8822 #[cfg(unix)]
8839 command.process_group(0);
8840 command.stdin(Stdio::null());
8841 #[cfg(unix)]
8845 if let Some(handoff) = nonce_handoff {
8846 handoff.install_last(command.as_std_mut());
8847 }
8848 #[cfg(not(any(unix, windows)))]
8849 let _ = nonce_handoff;
8850
8851 #[cfg(windows)]
8856 {
8857 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8858 program: spec.program.clone(),
8859 source,
8860 cgroup_path: None,
8861 })?;
8862 subc_jobobject::suspend_on_create_async(&mut command);
8863 }
8864 let spawned_at = Instant::now();
8868 let mut child = match command.spawn() {
8869 Ok(child) => child,
8870 Err(source) => {
8871 #[cfg(target_os = "linux")]
8872 if let Some(placement) = cgroup_placement {
8873 remove_module_cgroup(placement, &cgroup_name);
8874 }
8875 return Err(SuperviseError::Spawn {
8876 program: spec.program.clone(),
8877 source,
8878 cgroup_path,
8879 });
8880 }
8881 };
8882 #[cfg(test)]
8883 spawn_record_tests::after_create(&child, spec);
8884 #[cfg(target_os = "macos")]
8889 drop(exec_ack);
8890
8891 #[cfg(windows)]
8896 let windows_image = match image_capture.bind_tokio(&child) {
8897 Ok(image) => Arc::new(image),
8898 Err(source) => {
8899 kill_and_reap_rejected_child(&mut child);
8900 return Err(SuperviseError::Spawn {
8901 program: spec.program.clone(),
8902 source,
8903 cgroup_path: None,
8904 });
8905 }
8906 };
8907 #[cfg(windows)]
8910 let job = contain_spawned_child(&mut child, spec)?;
8911 let spawned_at_ms = unix_ms_now();
8912 #[cfg(not(windows))]
8913 let spawned_from = spec.program.clone();
8914 #[cfg(windows)]
8915 let spawned_from = windows_image.path().to_path_buf();
8916 #[cfg(not(windows))]
8917 let spawned_file_identity = spawned_file_identity(&spawned_from);
8918 #[cfg(windows)]
8919 let spawned_file_identity = Some(SpawnedFileIdentity {
8920 device: 0,
8921 inode: 0,
8922 windows_image: Some(Arc::clone(&windows_image)),
8923 });
8924 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8925 program: spec.program.clone(),
8926 source: io::Error::other("spawned child exposed no live pid"),
8927 cgroup_path: cgroup_path.clone(),
8928 })?;
8929 #[cfg(windows)]
8930 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8931 #[cfg(not(windows))]
8932 let process_start_time = crate::provenance::process_start_time(pid);
8933 #[cfg(windows)]
8934 let process_start_time = Some(windows_image.start_time());
8935 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8936 #[cfg(all(test, target_os = "macos"))]
8937 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8938 #[cfg(not(windows))]
8943 let recorded_image = observe_spawned_image(pid);
8944 #[cfg(target_os = "macos")]
8948 let recorded_image = if privacy_exec.is_some() {
8949 None
8950 } else {
8951 recorded_image
8952 };
8953 #[cfg(target_os = "linux")]
8954 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8955 #[cfg(not(target_os = "linux"))]
8956 let recorded_cgroup_name = None;
8957 let roster_guard = roster.admit(
8958 spec.module_id.clone(),
8959 pid,
8960 spec.protocol,
8961 process_start_time,
8962 crate::child_roster::RecordedIdentity {
8963 #[cfg(not(windows))]
8964 start_time: recorded_image.map(|image| image.start_time),
8965 #[cfg(windows)]
8966 start_time: process_start_time,
8967 #[cfg(not(windows))]
8968 executable: recorded_image
8969 .and_then(|image| image.executable)
8970 .map(crate::live_children::ExecutableIdentity::from),
8971 #[cfg(windows)]
8972 executable: Some(crate::live_children::ExecutableIdentity::from(
8973 windows_image.identity(),
8974 )),
8975 cgroup_name: recorded_cgroup_name,
8976 #[cfg(target_os = "linux")]
8977 cgroup_placement: cgroup_placement.cloned(),
8978 },
8979 );
8980 let roster_guard = match roster_guard {
8981 Ok(guard) => guard,
8982 Err(failure) => {
8983 return failure.with_cleanup(|source| {
8984 #[cfg(windows)]
8985 if let Some(job) = &job {
8986 let _ = job.terminate();
8987 }
8988 #[cfg(target_os = "linux")]
8989 kill_module_cgroup(cgroup_placement, &cgroup_name);
8990 kill_and_reap_rejected_child(&mut child);
8991 #[cfg(target_os = "linux")]
8992 if let Some(placement) = cgroup_placement {
8993 remove_module_cgroup(placement, &cgroup_name);
8994 }
8995 Err(SuperviseError::Spawn {
8996 program: spec.program.clone(),
8997 source: io::Error::other(format!(
8998 "could not durably record module child: {source}"
8999 )),
9000 cgroup_path,
9001 })
9002 });
9003 }
9004 };
9005 if roster.is_closed() {
9010 #[cfg(target_os = "linux")]
9012 kill_module_cgroup(cgroup_placement, &cgroup_name);
9013 if let Err(error) = child.start_kill() {
9014 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9015 }
9016 #[cfg(target_os = "linux")]
9017 if let Some(placement) = cgroup_placement {
9018 while matches!(child.try_wait(), Ok(None)) {
9022 std::thread::yield_now();
9023 }
9024 if matches!(
9025 subc_cgroup::kill_module(Some(placement), &cgroup_name),
9026 subc_cgroup::KillOutcome::Killed
9027 ) {
9028 if let Ok(path) = placement.module_path(&cgroup_name) {
9029 while std::fs::read_to_string(path.join("cgroup.events"))
9030 .ok()
9031 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9032 {
9033 std::thread::yield_now();
9034 }
9035 }
9036 }
9037 remove_module_cgroup(placement, &cgroup_name);
9038 }
9039 drop(roster_guard);
9040 return Err(SuperviseError::Spawn {
9041 program: spec.program.clone(),
9042 source: io::Error::other(
9043 "the daemon began shutting down while this process was starting; ended it",
9044 ),
9045 cgroup_path,
9046 });
9047 }
9048
9049 #[cfg(windows)]
9050 if let Err(error) = resume_suspended_child(pid, spec) {
9051 kill_and_reap_rejected_child(&mut child);
9052 return Err(error);
9053 }
9054
9055 let stdout_pump = match child.stdout.take() {
9056 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9057 None => {
9058 warn!(
9059 module_id = %spec.module_id,
9060 "spawned child exposed no stdout pipe; file capture will be incomplete"
9061 );
9062 None
9063 }
9064 };
9065 let stderr_pump = match child.stderr.take() {
9066 Some(stderr) => {
9067 let generation = ring
9068 .lock()
9069 .unwrap_or_else(|poisoned| poisoned.into_inner())
9070 .begin_process();
9071 Some(StderrPump {
9072 task: tokio::spawn(pump_stderr_to(
9073 stderr,
9074 Arc::clone(ring),
9075 generation,
9076 output_sink,
9077 )),
9078 generation,
9079 })
9080 }
9081 None => {
9082 ring.lock()
9086 .unwrap_or_else(|poisoned| poisoned.into_inner())
9087 .mark_not_captured("stderr pipe was not available on spawn");
9088 warn!(
9089 module_id = %spec.module_id,
9090 "spawned child exposed no stderr pipe; tail will be unavailable"
9091 );
9092 None
9093 }
9094 };
9095
9096 let exec = Arc::new(crate::activity::ExecOutcome::new());
9097 #[cfg(target_os = "macos")]
9098 let pending_exec = privacy_exec.is_some();
9099 #[cfg(not(target_os = "macos"))]
9100 let pending_exec = false;
9101 if !pending_exec {
9102 exec.finish(crate::activity::ExecState::CommittedExec);
9103 }
9104 Ok(SupervisedChild {
9105 child,
9106 exec: crate::activity::ExecOwner(exec),
9107 #[cfg(windows)]
9108 nonce_delivery,
9109 protocol: spec.protocol,
9110 #[cfg(target_os = "linux")]
9111 module_id: cgroup_name,
9112 #[cfg(target_os = "linux")]
9113 cgroup_placement: cgroup_placement.cloned(),
9114 #[cfg(windows)]
9115 job,
9116 stdout_pump,
9117 stderr_pump,
9118 stderr_ring: Arc::clone(ring),
9119 spawned_at,
9120 spawned_at_ms,
9121 spawned_from,
9122 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9123 spawned_file_identity,
9124 process_start_time,
9125 process_identity,
9126 pid,
9127 roster_guard: Some(roster_guard),
9128 #[cfg(target_os = "macos")]
9129 privacy_exec,
9130 #[cfg(target_os = "macos")]
9131 report_ready: Arc::new(OnceLock::new()),
9132 spawn_failure: None,
9133 })
9134}
9135
9136#[cfg(test)]
9137mod spawn_record_tests {
9138 use super::*;
9139
9140 thread_local! {
9141 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9142 }
9143
9144 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9145 LAST_PID.set(child.id());
9146 #[cfg(windows)]
9147 if let Some((_, address)) = _spec
9148 .env
9149 .iter()
9150 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9151 {
9152 use std::io::{Read, Write};
9153 let mut stream = std::net::TcpStream::connect(address).unwrap();
9154 stream
9155 .write_all(&child.id().unwrap().to_ne_bytes())
9156 .unwrap();
9157 let mut release = [0];
9158 stream.read_exact(&mut release).unwrap();
9159 }
9160 }
9161
9162 #[cfg(windows)]
9163 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9164 if spec
9165 .env
9166 .iter()
9167 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9168 {
9169 Err(io::Error::other("injected module job assignment failure"))
9170 } else {
9171 Ok(())
9172 }
9173 }
9174
9175 fn spec() -> ModuleSpec {
9176 ModuleSpec {
9177 module_id: "spawn-record-worker".into(),
9178 #[cfg(unix)]
9179 program: "/bin/sleep".into(),
9180 #[cfg(windows)]
9181 program: std::env::current_exe().unwrap(),
9182 #[cfg(unix)]
9183 args: vec!["60".into()],
9184 #[cfg(windows)]
9185 args: vec![
9186 "--exact".into(),
9187 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9188 "--nocapture".into(),
9189 ],
9190 env: vec![],
9191 reserved: false,
9192 reserved_prefixes: vec![],
9193 protocol: ModuleProtocol::None,
9194 overlap: Default::default(),
9195 }
9196 }
9197
9198 #[tokio::test]
9199 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9200 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9201 let supervisor =
9202 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9203 .with_live_children_record(home.join("live-children.json"));
9204 crate::live_children::fail_next_file_sync();
9205 let result = supervisor.spawn(spec());
9206 let failed = result.is_err();
9207 if let Ok(module) = result {
9208 module.stop().await.unwrap();
9209 }
9210 assert!(failed, "record failure must fail spawn");
9211 let pid = LAST_PID.get().expect("the real OS child was created");
9212 assert!(
9213 subc_os::Process::open(pid).unwrap().is_none(),
9214 "child {pid} survived rejected admission or was not reaped"
9215 );
9216 }
9217
9218 #[cfg(target_os = "macos")]
9219 #[test]
9220 #[ignore = "manual on-device latency measurement"]
9221 fn durable_record_write_latency_50() {
9222 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9223 let path = home.join("live-children.json");
9224 let children = vec![crate::live_children::LiveChild {
9225 module_id: "latency-worker".into(),
9226 pid: std::process::id(),
9227 protocol: ModuleProtocol::None,
9228 start_time: Some(1),
9229 executable: None,
9230 cgroup_name: None,
9231 }];
9232 let mut samples = Vec::new();
9233 for _ in 0..50 {
9234 let start = std::time::Instant::now();
9235 crate::live_children::write_record(&path, &children).unwrap();
9236 samples.push(start.elapsed());
9237 }
9238 samples.sort();
9239 eprintln!(
9240 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9241 (samples[24] + samples[25]) / 2,
9242 samples[47]
9243 );
9244 }
9245
9246 #[cfg(windows)]
9247 #[test]
9248 fn windows_store_touch_fixture() {
9249 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9250 std::fs::write(path, b"module ran").unwrap();
9251 }
9252 }
9253
9254 #[cfg(windows)]
9255 #[tokio::test]
9256 async fn windows_daemon_fixture() {
9257 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9258 return;
9259 };
9260 let mut spec = spec();
9261 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9262 let supervisor =
9263 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9264 let _module = supervisor.spawn(spec).unwrap();
9265 }
9266
9267 #[cfg(windows)]
9268 #[test]
9269 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9270 use std::io::Read;
9271 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9272 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9275 command
9276 .args([
9277 "--exact",
9278 "supervise::spawn_record_tests::windows_daemon_fixture",
9279 "--nocapture",
9280 ])
9281 .env(
9282 "SUBC_TEST_DAEMON_BARRIER",
9283 listener.local_addr().unwrap().to_string(),
9284 );
9285 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9286 listener.set_nonblocking(true).unwrap();
9287 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9288 let mut stream = loop {
9289 if let Ok((stream, _)) = listener.accept() {
9290 break stream;
9291 }
9292 assert!(
9293 daemon.child.try_wait().unwrap().is_none(),
9294 "daemon failed before rendezvous"
9295 );
9296 assert!(
9297 std::time::Instant::now() < deadline,
9298 "daemon did not reach CreateProcess failpoint"
9299 );
9300 std::thread::yield_now();
9301 };
9302 stream
9303 .set_read_timeout(Some(Duration::from_secs(30)))
9304 .unwrap();
9305 let mut pid = [0; 4];
9306 stream.read_exact(&mut pid).unwrap();
9307 let pid = u32::from_ne_bytes(pid);
9308 assert!(subc_jobobject::process_exists(pid));
9309 daemon.child.kill().unwrap();
9312 daemon.child.wait().unwrap();
9313 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9314 if !exited {
9315 daemon.job.terminate().unwrap();
9316 }
9317 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9318 }
9319
9320 #[cfg(windows)]
9321 #[tokio::test]
9322 async fn windows_containment_failure_never_resumes() {
9323 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9324 let sentinel = home.join("store-touched");
9325 let mut spec = spec();
9326 spec.env
9327 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9328 spec.env.push((
9329 "SUBC_TEST_STORE_TOUCH".into(),
9330 sentinel.display().to_string(),
9331 ));
9332 let supervisor =
9333 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9334 let result = supervisor.spawn(spec);
9335 if let Ok(module) = result {
9338 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9339 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9340 tokio::task::yield_now().await;
9341 }
9342 })
9343 .await;
9344 let _ = module.stop().await;
9345 panic!(
9346 "assignment failure resumed a module; store touched={}",
9347 sentinel.exists()
9348 );
9349 }
9350 assert!(
9351 !sentinel.exists(),
9352 "refused module executed store-touch sentinel"
9353 );
9354 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9355 }
9356
9357 #[cfg(windows)]
9358 #[tokio::test]
9359 async fn windows_nested_daemon_fixture() {
9360 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9361 return;
9362 };
9363 let mut spec = spec();
9364 spec.env.push((
9365 "SUBC_TEST_STORE_TOUCH".into(),
9366 sentinel.to_str().unwrap().into(),
9367 ));
9368 let supervisor =
9369 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9370 let module = supervisor
9371 .spawn(spec)
9372 .expect("per-module job must nest below daemon and host jobs");
9373 tokio::time::timeout(Duration::from_secs(10), async {
9374 while !PathBuf::from(&sentinel).exists() {
9375 tokio::task::yield_now().await;
9376 }
9377 })
9378 .await
9379 .unwrap();
9380 module.stop().await.unwrap();
9381 }
9382
9383 #[cfg(windows)]
9384 #[test]
9385 fn windows_module_job_nests_under_host_job() {
9386 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9387 let sentinel = home.join("nested-store-touched");
9388 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9389 command
9390 .args([
9391 "--exact",
9392 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9393 "--nocapture",
9394 ])
9395 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9396 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9397 assert!(daemon.child.wait().unwrap().success());
9398 assert!(sentinel.exists(), "nested protocol:none module never ran");
9399 }
9400}
9401
9402#[cfg(target_os = "linux")]
9403pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9404 use subc_cgroup::KillOutcome;
9405 match subc_cgroup::kill_module(placement, module_id) {
9406 KillOutcome::Killed => {}
9407 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9408 debug!(
9409 module_id,
9410 "cgroup tree kill unavailable; using direct-child kill"
9411 );
9412 }
9413 KillOutcome::IoError { path, error } => {
9414 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9415 }
9416 }
9417}
9418
9419#[cfg(windows)]
9427fn contain_spawned_child(
9428 child: &mut Child,
9429 spec: &ModuleSpec,
9430) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9431 let assignment = || {
9432 let job = subc_jobobject::JobObject::new()?;
9433 #[cfg(test)]
9434 spawn_record_tests::assignment_failpoint(spec)?;
9435 job.assign(child)?;
9436 Ok::<_, io::Error>(job)
9437 };
9438 match assignment() {
9439 Ok(job) => Ok(Some(job)),
9440 Err(source) => {
9441 kill_and_reap_rejected_child(child);
9442 Err(SuperviseError::Spawn {
9443 program: spec.program.clone(),
9444 source: io::Error::other(format!(
9445 "Windows module job containment failed: {source}"
9446 )),
9447 cgroup_path: None,
9448 })
9449 }
9450 }
9451}
9452
9453fn kill_and_reap_rejected_child(child: &mut Child) {
9456 #[cfg(unix)]
9457 if let Some(pid) = child
9458 .id()
9459 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9460 {
9461 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9462 }
9463 let _ = child.start_kill();
9464 while matches!(child.try_wait(), Ok(None)) {
9465 std::thread::yield_now();
9466 }
9467}
9468
9469#[cfg(windows)]
9474fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9475 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9476 let _ = std::process::Command::new("taskkill.exe")
9480 .args(["/PID", &pid.to_string(), "/T", "/F"])
9481 .stdin(Stdio::null())
9482 .stdout(Stdio::null())
9483 .stderr(Stdio::null())
9484 .status();
9485 return Err(SuperviseError::Spawn {
9486 program: spec.program.clone(),
9487 source,
9488 cgroup_path: None,
9489 });
9490 }
9491 Ok(())
9492}
9493
9494#[cfg(target_os = "linux")]
9495fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9496 match placement.remove_module(module_id) {
9497 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9498 Err(error) => warn!(
9499 module_id,
9500 error = %error,
9501 "could not remove module cgroup after process exit; continuing teardown"
9502 ),
9503 }
9504}
9505
9506#[cfg(target_os = "linux")]
9507async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9508 if matches!(
9512 subc_cgroup::kill_module(Some(placement), module_id),
9513 subc_cgroup::KillOutcome::Killed
9514 ) {
9515 if let Ok(path) = placement.module_path(module_id) {
9516 while std::fs::read_to_string(path.join("cgroup.events"))
9517 .ok()
9518 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9519 {
9520 sleep(Duration::from_millis(1)).await;
9521 }
9522 }
9523 }
9524 remove_module_cgroup(placement, module_id);
9525}
9526
9527#[cfg(target_os = "linux")]
9528fn apply_cgroup_placement(
9529 command: &mut Command,
9530 spec: &ModuleSpec,
9531 path: &std::path::Path,
9532) -> Result<(), SuperviseError> {
9533 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9534 module_id: spec.module_id.clone(),
9535 source,
9536 })
9537}
9538
9539fn capture_retention(spec: &ModuleSpec) -> Retention {
9540 let defaults = Retention::default();
9541 let value = |name: &str| {
9542 spec.env
9543 .iter()
9544 .rev()
9545 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9546 };
9547 Retention {
9548 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9549 .and_then(|value| value.parse().ok())
9550 .unwrap_or(defaults.max_file_mb),
9551 keep: value(CAPTURE_KEEP_ENV)
9552 .and_then(|value| value.parse().ok())
9553 .unwrap_or(defaults.keep),
9554 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9555 .and_then(|value| value.parse().ok())
9556 .unwrap_or(defaults.max_age_days),
9557 }
9558}
9559
9560fn generate_launch_nonce() -> Result<String, SuperviseError> {
9563 let mut bytes = [0u8; 32];
9564 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9565 reason: source.to_string(),
9566 })?;
9567 let mut hex = String::with_capacity(64);
9568 for b in bytes {
9569 use std::fmt::Write;
9570 let _ = write!(hex, "{b:02x}");
9571 }
9572 Ok(hex)
9573}
9574
9575fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9578 if a.len() != b.len() {
9579 return false;
9580 }
9581 let mut diff = 0u8;
9582 for (x, y) in a.iter().zip(b.iter()) {
9583 diff |= x ^ y;
9584 }
9585 diff == 0
9586}
9587
9588#[cfg(not(windows))]
9592fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9593 let process = subc_os::Process::open(pid).ok().flatten()?;
9594 #[cfg(target_os = "linux")]
9595 return sample_after_linux_exec(
9596 || {
9597 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9598 let committed = linux_exec_committed(&stat);
9599 #[cfg(test)]
9600 spawn_image_tests::report_exec_state(committed);
9601 committed
9602 },
9603 || process.observe(),
9604 );
9605 #[cfg(not(target_os = "linux"))]
9606 process.observe()
9607}
9608
9609#[cfg(target_os = "linux")]
9613fn linux_exec_committed(stat: &str) -> Option<bool> {
9614 const PF_FORKNOEXEC: u64 = 0x40;
9615 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9616 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9617}
9618
9619#[cfg(target_os = "linux")]
9620fn sample_after_linux_exec(
9621 mut exec_committed: impl FnMut() -> Option<bool>,
9622 observe: impl FnOnce() -> Option<subc_os::Observation>,
9623) -> Option<subc_os::Observation> {
9624 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9625 while !exec_committed()? {
9626 if std::time::Instant::now() >= deadline {
9630 return None;
9631 }
9632 std::thread::sleep(Duration::from_micros(50));
9633 }
9634 observe()
9635}
9636
9637#[cfg(all(test, target_os = "linux"))]
9638mod spawn_image_tests {
9639 use super::*;
9640 use std::cell::{Cell, RefCell};
9641
9642 enum ExecEvent {
9643 Uncommitted,
9644 Image(Option<subc_os::Observation>),
9645 }
9646
9647 thread_local! {
9648 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9649 }
9650
9651 pub(super) fn report_exec_state(committed: Option<bool>) {
9652 if committed == Some(false) {
9653 EXEC_EVENTS.with(|events| {
9654 if let Some(events) = events.borrow_mut().take() {
9655 let _ = events.send(ExecEvent::Uncommitted);
9656 }
9657 });
9658 }
9659 }
9660
9661 #[test]
9662 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9663 use std::io::{Read, Write};
9664 use std::os::fd::AsRawFd;
9665
9666 let descriptor = std::fs::File::open("/dev/null").unwrap();
9667 let mut command = std::process::Command::new("/bin/sleep");
9668 command.arg("30");
9669 let (mut ready, mut release) =
9670 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9671 .unwrap();
9672 let spawn = std::thread::spawn(move || command.spawn());
9673 let mut message = [0u8; 8];
9674 ready.read_exact(&mut message).unwrap();
9675 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9676 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9677 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9678 let (events, received) = std::sync::mpsc::channel();
9679 let observer = std::thread::spawn(move || {
9680 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9681 let image = observe_spawned_image(pid);
9682 let _ = events.send(ExecEvent::Image(image));
9683 });
9684 let first = received.recv_timeout(Duration::from_secs(5));
9688 release.write_all(&[1]).unwrap();
9689 let mut child = spawn.join().unwrap().unwrap();
9690 let after = match first {
9691 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9692 Ok(ExecEvent::Image(image)) => image,
9693 _ => None,
9694 },
9695 Ok(ExecEvent::Image(image)) => image,
9696 Err(_) => None,
9697 };
9698 observer.join().unwrap();
9699 child.kill().unwrap();
9700 child.wait().unwrap();
9701
9702 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9703 assert_eq!(
9704 inherited.unwrap().executable,
9705 subc_os::file_identity(&std::env::current_exe().unwrap())
9706 );
9707 assert_eq!(
9708 after.unwrap().executable,
9709 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9710 );
9711 }
9712
9713 #[test]
9714 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9715 let committed = Cell::new(false);
9716 let inherited = subc_os::Observation {
9717 start_time: 23,
9718 executable: Some(subc_os::FileIdentity {
9719 device: 29,
9720 inode: 31,
9721 }),
9722 };
9723 let module = subc_os::Observation {
9724 start_time: 23,
9725 executable: Some(subc_os::FileIdentity {
9726 device: 29,
9727 inode: 37,
9728 }),
9729 };
9730 let polls = Cell::new(0);
9731 let image = sample_after_linux_exec(
9732 || {
9733 polls.set(polls.get() + 1);
9734 committed.set(polls.get() == 2);
9737 Some(committed.get())
9738 },
9739 || Some(if committed.get() { module } else { inherited }),
9740 );
9741 assert_eq!(image, Some(module));
9742 assert_eq!(polls.get(), 2);
9743 }
9744
9745 #[test]
9746 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9747 assert_eq!(
9748 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9749 Some(false)
9750 );
9751 assert_eq!(
9752 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9753 Some(true)
9754 );
9755 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9756 assert_eq!(linux_exec_committed("17 (a) R"), None);
9757 }
9758
9759 #[test]
9760 fn unreadable_exec_state_never_samples_an_inherited_image() {
9761 assert_eq!(
9762 sample_after_linux_exec(
9763 || None,
9764 || panic!("must not sample before exec confirmation")
9765 ),
9766 None
9767 );
9768 }
9769
9770 #[test]
9771 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9772 assert_eq!(
9773 sample_after_linux_exec(
9774 || Some(false),
9775 || panic!("must not sample after an exec timeout")
9776 ),
9777 None
9778 );
9779 }
9780}
9781
9782#[cfg(windows)]
9785fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9786 let mut program = spec.program.clone();
9787 if program.is_absolute() || program.components().count() > 1 {
9788 match subc_os::ExecutableCapture::open(&program) {
9791 Ok(capture) => return Ok(capture),
9792 Err(error)
9793 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9794 Err(error) => return Err(error),
9795 }
9796 program.set_extension("exe");
9797 return subc_os::ExecutableCapture::open(&program);
9798 }
9799 if program.extension().is_none() {
9800 program.set_extension("exe");
9801 }
9802 let custom_path = spec
9803 .env
9804 .iter()
9805 .rev()
9806 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9807 let mut directories = Vec::new();
9808 if let Some(parent) = std::env::current_exe()?.parent() {
9811 directories.push(parent.to_path_buf());
9812 }
9813 directories.push(std::env::current_dir()?);
9814 if let Some(windows) = std::env::var_os("SystemRoot") {
9815 let windows = PathBuf::from(windows);
9816 directories.push(windows.join("System32"));
9817 directories.push(windows.clone());
9818 }
9819 let path = custom_path
9820 .map(|(_, value)| std::ffi::OsString::from(value))
9821 .or_else(|| std::env::var_os("PATH"));
9822 if let Some(path) = path {
9823 directories.extend(std::env::split_paths(&path));
9824 }
9825 for directory in directories {
9826 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9827 Ok(capture) => return Ok(capture),
9828 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9829 Err(error) => return Err(error),
9830 }
9831 }
9832 Err(io::Error::new(
9833 io::ErrorKind::NotFound,
9834 "resolved executable not found",
9835 ))
9836}
9837
9838fn spawn_and_mark_running(
9839 spec: &ModuleSpec,
9840 runtime: &SupervisorRuntimeConfig,
9841 snapshot: &SharedSnapshot,
9842) -> Result<SupervisedChild, SuperviseError> {
9843 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9844 let child = spawn_child(
9845 spec,
9846 runtime.connection_file_path.as_deref(),
9847 runtime.supervisor_handle.as_ref(),
9848 &runtime.stderr_ring,
9849 runtime.capture_logs_dir.as_deref(),
9850 &runtime.child_roster,
9851 #[cfg(target_os = "linux")]
9852 runtime.cgroup_placement.as_ref(),
9853 )?;
9854 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9855 if let Some(spawn) = activity_spawn {
9856 spawn.complete(snapshot);
9857 }
9858 Ok(child)
9859}
9860
9861enum RegistrationWaitOutcome {
9862 Registered,
9863 Exited(ExitReport),
9864 TimedOut,
9865}
9866
9867struct ReloadRegistrationFailure {
9868 exit_report: ExitReport,
9869 reason: String,
9870}
9871
9872#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9873enum BusyGaugeObservation {
9874 Quiescent,
9875 Busy,
9876 Omitted,
9877}
9878
9879#[derive(Debug, Clone, PartialEq, Eq)]
9881enum GaugeVerdict {
9882 Zero,
9884 NonZero(u64),
9886 NotACount(String),
9890 Missing,
9892 ProbeFailed(&'static str),
9895}
9896
9897impl GaugeVerdict {
9898 fn reads_quiet(&self) -> bool {
9899 matches!(self, Self::Zero)
9900 }
9901}
9902
9903impl fmt::Display for GaugeVerdict {
9904 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9905 match self {
9906 Self::Zero => f.write_str("zero"),
9907 Self::NonZero(value) => write!(f, "non_zero({value})"),
9908 Self::NotACount(value) => write!(f, "not_a_count({value})"),
9909 Self::Missing => f.write_str("missing_from_health_body"),
9910 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9911 }
9912 }
9913}
9914
9915const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9918
9919fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9921 let metrics = metrics.and_then(Value::as_object);
9922 gauges
9923 .iter()
9924 .map(
9925 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9926 None => GaugeVerdict::Missing,
9927 Some(value) => match value.as_u64() {
9928 Some(0) => GaugeVerdict::Zero,
9929 Some(count) => GaugeVerdict::NonZero(count),
9930 None => {
9931 let mut text = value.to_string();
9932 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9933 let mut end = GAUGE_VALUE_TEXT_LIMIT;
9934 while !text.is_char_boundary(end) {
9935 end -= 1;
9936 }
9937 text.truncate(end);
9938 text.push_str("...");
9939 }
9940 GaugeVerdict::NotACount(text)
9941 }
9942 },
9943 },
9944 )
9945 .collect()
9946}
9947
9948fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9951 if verdicts
9952 .iter()
9953 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9954 {
9955 BusyGaugeObservation::Omitted
9956 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9957 BusyGaugeObservation::Quiescent
9958 } else {
9959 BusyGaugeObservation::Busy
9960 }
9961}
9962
9963#[derive(Debug, Clone, Default, PartialEq, Eq)]
9968struct GaugeTrace {
9969 latest: Option<(GaugeVerdict, Duration)>,
9970 last_value: Option<(GaugeVerdict, Duration)>,
9971}
9972
9973#[derive(Debug, Clone, Default, PartialEq, Eq)]
9979struct DrainGaugeTrace {
9980 probes: u32,
9982 gauges: Vec<(String, GaugeTrace)>,
9984}
9985
9986impl DrainGaugeTrace {
9987 fn new(gauges: &[String]) -> Self {
9988 Self {
9989 probes: 0,
9990 gauges: gauges
9991 .iter()
9992 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9993 .collect(),
9994 }
9995 }
9996
9997 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9999 self.probes = self.probes.saturating_add(1);
10000 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10001 trace.latest = Some((verdict.clone(), at));
10002 if matches!(
10003 verdict,
10004 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10005 ) {
10006 trace.last_value = Some((verdict.clone(), at));
10007 }
10008 }
10009 }
10010}
10011
10012impl fmt::Display for DrainGaugeTrace {
10017 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10018 if self.gauges.is_empty() {
10019 return f.write_str("none declared");
10020 }
10021 for (index, (name, trace)) in self.gauges.iter().enumerate() {
10022 if index > 0 {
10023 f.write_str(" | ")?;
10024 }
10025 write!(f, "{name}: ")?;
10026 match &trace.latest {
10027 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10028 None => f.write_str("never probed")?,
10029 }
10030 match &trace.last_value {
10031 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10032 None => f.write_str("; no value ever read")?,
10033 }
10034 }
10035 Ok(())
10036 }
10037}
10038
10039pub(crate) fn declared_busy_gauges(
10040 registry: &Registry,
10041 module_id: &str,
10042) -> Result<Vec<String>, SuperviseError> {
10043 busy_gauges_of(
10044 registry
10045 .get_module(module_id)
10046 .map_err(SuperviseError::Registry)?,
10047 )
10048}
10049
10050fn declared_busy_gauges_for_connection(
10054 registry: &Registry,
10055 connection_id: ConnectionId,
10056) -> Result<Vec<String>, SuperviseError> {
10057 busy_gauges_of(
10058 registry
10059 .get_module_by_connection(connection_id)
10060 .map_err(SuperviseError::Registry)?,
10061 )
10062}
10063
10064fn busy_gauges_of(
10065 registration: Option<crate::registry::ModuleRegistration>,
10066) -> Result<Vec<String>, SuperviseError> {
10067 let Some(registration) = registration else {
10068 return Ok(Vec::new());
10069 };
10070 let Some(self_signals) = registration.manifest.self_signals else {
10071 return Ok(Vec::new());
10072 };
10073
10074 let mut gauges = Vec::new();
10075 for declaration in self_signals {
10076 if declaration.kind != SelfSignalKind::Busy {
10077 continue;
10078 }
10079 match declaration.anchored_to {
10080 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10081 gauges.extend(declared)
10082 }
10083 _ => {
10084 gauges.push(String::new());
10087 }
10088 }
10089 }
10090 Ok(gauges)
10091}
10092
10093#[allow(clippy::too_many_arguments)]
10102async fn wait_for_forwarding_quiescence(
10103 forwarding: &ForwardingTable,
10104 module_id: &str,
10105 runtime: &SupervisorRuntimeConfig,
10106 endpoint: crate::ModuleEndpointId,
10107 deadline: Instant,
10108 busy_gauges: &[String],
10109 scope: DrainScope,
10110 trace: &mut DrainGaugeTrace,
10111) -> Result<bool, SuperviseError> {
10112 let wait_started = Instant::now();
10113 let mut gauges_quiescent = busy_gauges.is_empty();
10114 let mut next_probe_at = wait_started;
10115 let mut omission_reported = false;
10116
10117 loop {
10118 let now = Instant::now();
10119 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10120 let report = match scope {
10121 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10122 DrainScope::Endpoint(endpoint) => {
10123 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10124 }
10125 };
10126 let verdicts = match &report {
10127 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10128 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10129 };
10130 trace.record(&verdicts, wait_started.elapsed());
10131 gauges_quiescent = match report {
10132 Ok(_) => match busy_gauge_observation(&verdicts) {
10133 BusyGaugeObservation::Quiescent => true,
10134 BusyGaugeObservation::Busy => false,
10135 BusyGaugeObservation::Omitted => {
10136 if !omission_reported {
10139 forwarding
10140 .counters()
10141 .increment_drains_with_undeclared_gauge();
10142 let missing = busy_gauges
10143 .iter()
10144 .zip(&verdicts)
10145 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10146 .map(|(gauge, _)| gauge.as_str())
10147 .collect::<Vec<_>>()
10148 .join(",");
10149 warn!(
10150 module_id,
10151 missing_gauges = %missing,
10152 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10153 );
10154 omission_reported = true;
10155 }
10156 false
10157 }
10158 },
10159 Err(err) => {
10160 warn!(
10161 module_id,
10162 error = %err,
10163 "drain health.check did not produce declared busy gauges; treating module as busy"
10164 );
10165 false
10166 }
10167 };
10168 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10171 }
10172
10173 let in_flight = forwarding
10174 .endpoint_in_flight_count(endpoint)
10175 .map_err(SuperviseError::Forwarding)?;
10176 if in_flight == 0 && gauges_quiescent {
10177 return Ok(true);
10178 }
10179
10180 let now = Instant::now();
10181 if now >= deadline {
10182 return Ok(false);
10183 }
10184 let mut wait = deadline
10185 .saturating_duration_since(now)
10186 .min(REGISTRY_RELEASE_POLL);
10187 if !busy_gauges.is_empty() {
10188 wait = wait.min(next_probe_at.saturating_duration_since(now));
10189 }
10190 sleep(wait).await;
10191 }
10192}
10193
10194fn drain_busy_reprobe_interval() -> Duration {
10198 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10199}
10200
10201fn warn_drain_timed_out(
10216 module_id: &str,
10217 drain_timeout: Duration,
10218 reason: RouteCloseReason,
10219 holdouts: &crate::forwarding::DrainHoldouts,
10220 gauges: &DrainGaugeTrace,
10221) {
10222 warn!(
10223 module_id = %module_id,
10224 waited = ?drain_timeout,
10225 ?reason,
10226 held_requests = holdouts.requests,
10227 held_routes = holdouts.routes,
10228 total_routes = holdouts.total_routes,
10229 top_connections = ?holdouts.top_connections,
10230 held = %holdouts
10233 .held
10234 .iter()
10235 .map(|(channel, corr)| format!("{channel}:{corr}"))
10236 .collect::<Vec<_>>()
10237 .join(","),
10238 gauge_probes = gauges.probes,
10239 busy_gauges = %gauges,
10240 "route drain timed out before request quiescence; forcing teardown"
10241 );
10242}
10243
10244fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10252 match wait_result {
10253 Ok(drained) => *drained,
10254 Err(_) => false,
10255 }
10256}
10257
10258fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10259 for released in released_routes {
10260 let frame = match Frame::build_with_version(
10261 released.negotiated_ver,
10262 FrameType::Goodbye,
10263 control_flags(),
10264 released.channel,
10265 released.epoch,
10266 0,
10267 Vec::new(),
10268 ) {
10269 Ok(frame) => frame,
10270 Err(err) => {
10271 warn!(
10272 route_channel = released.channel,
10273 error = %err,
10274 "failed to build supervisor drain route GOODBYE frame"
10275 );
10276 continue;
10277 }
10278 };
10279 if !released.close_on_delivery_failure() {
10280 crate::forwarding::send_module_route_goodbye(
10281 &forwarding.counters(),
10282 &released.sink,
10283 frame,
10284 released.module_id.as_deref(),
10285 "supervisor drain",
10286 );
10287 continue;
10288 }
10289 if let Err(err) = released.sink.try_send(frame) {
10290 warn!(
10291 target_connection_id = released.connection_id.get(),
10292 route_channel = released.channel,
10293 error = %err,
10294 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10295 );
10296 let _ = forwarding.escalate_client_delivery_failure(
10297 released.connection_id,
10298 released.channel,
10299 released.epoch,
10300 CloseReason::new(
10301 "route_goodbye_delivery_failed",
10302 format!(
10303 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10304 released.channel
10305 ),
10306 ),
10307 crate::forwarding::UndeliveredFrame {
10308 module_id: released.module_id.as_deref(),
10309 sink: &released.sink,
10310 },
10311 );
10312 }
10313 }
10314}
10315
10316fn send_module_draining(
10317 module_id: &str,
10318 reason: RouteCloseReason,
10319 deadline_ms: u64,
10320 target: &ModuleDrainTarget,
10321) {
10322 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10323 reason,
10324 deadline_ms,
10325 }) {
10326 Ok(body) => body,
10327 Err(err) => {
10328 warn!(
10329 module_id,
10330 error = %err,
10331 "failed to encode module draining command"
10332 );
10333 return;
10334 }
10335 };
10336 let frame = match Frame::build_with_version(
10337 target.negotiated_ver,
10338 FrameType::Push,
10339 control_flags(),
10340 0,
10341 0,
10342 0,
10343 body,
10344 ) {
10345 Ok(frame) => frame,
10346 Err(err) => {
10347 warn!(
10348 module_id,
10349 error = %err,
10350 "failed to build module draining command frame"
10351 );
10352 return;
10353 }
10354 };
10355 if let Err(err) = target.sink.try_send(frame) {
10356 warn!(
10357 module_id,
10358 target_connection_id = target.endpoint.connection_id.get(),
10359 error = %err,
10360 "module draining command was not delivered to peer"
10361 );
10362 }
10363}
10364
10365fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10367 match Frame::build_with_version(
10368 negotiated_ver,
10369 FrameType::Goodbye,
10370 control_flags(),
10371 0,
10372 0,
10373 0,
10374 Vec::new(),
10375 ) {
10376 Ok(frame) => Some(frame),
10377 Err(err) => {
10378 warn!(
10379 module_id,
10380 error = %err,
10381 "failed to build module GOODBYE frame"
10382 );
10383 None
10384 }
10385 }
10386}
10387
10388#[cfg(unix)]
10402async fn send_module_goodbyes_for_daemon_shutdown(
10403 forwarding: &Arc<ForwardingTable>,
10404 reason: &CloseReason,
10405 wait_for_flush: bool,
10406) {
10407 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10408 let targets = match forwarding.module_connections() {
10409 Ok(targets) => targets,
10410 Err(err) => {
10411 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10412 return;
10413 }
10414 };
10415 let deadline = Instant::now() + GOODBYE_BUDGET;
10416 let mut sends = tokio::task::JoinSet::new();
10417 for target in targets {
10418 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10419 continue;
10420 };
10421 if !wait_for_flush {
10422 if let Err(err) = target.sink.try_send(frame) {
10423 debug!(
10424 module_id = %target.module_id,
10425 error = %err,
10426 "shutdown module GOODBYE was not queued"
10427 );
10428 }
10429 continue;
10430 }
10431 let forwarding = Arc::clone(forwarding);
10432 let reason = reason.clone();
10433 sends.spawn(async move {
10434 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10435 Ok(Ok(())) => {}
10436 Ok(Err(err)) => debug!(
10437 module_id = %target.module_id,
10438 error = %err,
10439 "module connection closed before its shutdown GOODBYE was written"
10440 ),
10441 Err(_) => warn!(
10442 module_id = %target.module_id,
10443 budget = ?GOODBYE_BUDGET,
10444 "shutdown module GOODBYE was not written within its budget; closing anyway"
10445 ),
10446 }
10447 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10448 });
10449 }
10450 while sends.join_next().await.is_some() {}
10452}
10453
10454fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10455 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10456 return;
10457 };
10458 if let Err(err) = target.sink.try_send(frame) {
10459 warn!(
10460 module_id,
10461 target_connection_id = target.endpoint.connection_id.get(),
10462 error = %err,
10463 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10464 );
10465 forwarding.request_connection_close(
10466 target.endpoint.connection_id,
10467 CloseReason::new(
10468 "module_goodbye_delivery_failed",
10469 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10470 ),
10471 );
10472 }
10473}
10474
10475#[derive(Clone, Copy)]
10476struct ForwardingDrainContext<'a> {
10477 spec: &'a ModuleSpec,
10478 runtime: &'a SupervisorRuntimeConfig,
10479 registry: &'a Registry,
10480 scope: DrainScope,
10481}
10482
10483#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10485enum DrainScope {
10486 Active,
10489 Endpoint(crate::ModuleEndpointId),
10494}
10495
10496#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10504enum StopNotice {
10505 SentOverConnection,
10508 NoConnection,
10512 NotSent,
10516}
10517
10518async fn begin_forwarding_drain(
10519 spec: &ModuleSpec,
10520 runtime: &SupervisorRuntimeConfig,
10521 registry: &Registry,
10522 snapshot: &SharedSnapshot,
10523 enabled: Option<bool>,
10524 reason: RouteCloseReason,
10525) -> Result<StopNotice, SuperviseError> {
10526 let Some(forwarding) = runtime.forwarding.as_ref() else {
10527 return Err(SuperviseError::ReloadUnavailable {
10528 module_id: spec.module_id.clone(),
10529 reason: "supervisor was not configured with a forwarding table".to_string(),
10530 });
10531 };
10532
10533 begin_forwarding_drain_with(
10534 forwarding,
10535 ForwardingDrainContext {
10536 spec,
10537 runtime,
10538 registry,
10539 scope: DrainScope::Active,
10540 },
10541 snapshot,
10542 enabled,
10543 reason,
10544 runtime.drain_timeout,
10545 )
10546 .await
10547}
10548
10549async fn begin_forwarding_drain_if_configured(
10550 spec: &ModuleSpec,
10551 runtime: &SupervisorRuntimeConfig,
10552 registry: &Registry,
10553 snapshot: &SharedSnapshot,
10554 enabled: Option<bool>,
10555 reason: RouteCloseReason,
10556) -> Result<StopNotice, SuperviseError> {
10557 begin_forwarding_drain_with_timeout(
10558 spec,
10559 runtime,
10560 registry,
10561 snapshot,
10562 enabled,
10563 reason,
10564 runtime.drain_timeout,
10565 )
10566 .await
10567}
10568
10569async fn begin_forwarding_drain_with_timeout(
10573 spec: &ModuleSpec,
10574 runtime: &SupervisorRuntimeConfig,
10575 registry: &Registry,
10576 snapshot: &SharedSnapshot,
10577 enabled: Option<bool>,
10578 reason: RouteCloseReason,
10579 drain_timeout: Duration,
10580) -> Result<StopNotice, SuperviseError> {
10581 let Some(forwarding) = runtime.forwarding.as_ref() else {
10582 return Ok(StopNotice::NotSent);
10583 };
10584
10585 begin_forwarding_drain_with(
10586 forwarding,
10587 ForwardingDrainContext {
10588 spec,
10589 runtime,
10590 registry,
10591 scope: DrainScope::Active,
10592 },
10593 snapshot,
10594 enabled,
10595 reason,
10596 drain_timeout,
10597 )
10598 .await
10599}
10600
10601async fn begin_forwarding_drain_with(
10602 forwarding: &ForwardingTable,
10603 context: ForwardingDrainContext<'_>,
10604 snapshot: &SharedSnapshot,
10605 enabled: Option<bool>,
10606 reason: RouteCloseReason,
10607 drain_timeout: Duration,
10608) -> Result<StopNotice, SuperviseError> {
10609 let ForwardingDrainContext {
10610 spec,
10611 runtime,
10612 registry,
10613 scope,
10614 } = context;
10615 debug_assert_ne!(reason, RouteCloseReason::Crash);
10616 let terminal = matches!(reason, RouteCloseReason::Disable);
10617 let drain_started_at = Instant::now();
10618 let drain_deadline = drain_started_at + drain_timeout;
10619 let deadline_ms =
10620 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10621 let busy_gauges = match scope {
10622 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10623 DrainScope::Endpoint(endpoint) => {
10624 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10625 }
10626 };
10627
10628 let gate_started = Instant::now();
10631 let drain_target = match scope {
10632 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10633 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10634 }
10635 .map_err(SuperviseError::Forwarding)?;
10636 info!(
10641 module_id = %spec.module_id,
10642 ?reason,
10643 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10644 connected = drain_target.is_some(),
10645 "module drain began; route admission closed"
10646 );
10647 if scope == DrainScope::Active {
10648 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10649 state.state = ModuleState::Draining;
10650 state.draining_to_replace =
10651 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10652 if let Some(enabled) = enabled {
10653 state.enabled = enabled;
10654 }
10655 })?;
10656 }
10657
10658 let Some(target) = drain_target.as_ref() else {
10659 return Ok(StopNotice::NoConnection);
10663 };
10664 {
10665 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10666 #[cfg(feature = "test-support")]
10667 forwarding.wait_before_route_closing_for_test().await;
10668 let routes = forwarding
10669 .endpoint_routes(target.endpoint)
10670 .map_err(SuperviseError::Forwarding)?;
10671 let routes_notified = routes.len();
10672 crate::control::send_route_control_pushes(
10673 forwarding,
10674 routes.clone(),
10675 ClientControlPush::RouteClosing {
10676 module_id: spec.module_id.clone(),
10677 channels: Vec::new(),
10678 reason,
10679 },
10680 );
10681 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10682
10683 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10689 let wait_result = wait_for_forwarding_quiescence(
10690 forwarding,
10691 &spec.module_id,
10692 runtime,
10693 target.endpoint,
10694 drain_deadline,
10695 &busy_gauges,
10696 scope,
10697 &mut gauge_trace,
10698 )
10699 .await;
10700 let drained = drained_after_quiescence_wait(&wait_result);
10701 if let Err(err) = &wait_result {
10702 error!(
10703 module_id = %spec.module_id,
10704 ?reason,
10705 error = %err,
10706 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10707 );
10708 } else if !drained {
10709 let holdouts = forwarding
10711 .endpoint_drain_holdouts(target.endpoint)
10712 .unwrap_or_default();
10713 warn_drain_timed_out(
10714 &spec.module_id,
10715 drain_timeout,
10716 reason,
10717 &holdouts,
10718 &gauge_trace,
10719 );
10720 }
10721 crate::control::send_route_control_pushes(
10722 forwarding,
10723 routes,
10724 ClientControlPush::RouteClosed {
10725 module_id: spec.module_id.clone(),
10726 channels: Vec::new(),
10727 reason,
10728 drained,
10729 abandoned: target.abandoned_bindings.len() as u32,
10730 excluded_subscriptions: target.excluded_subscriptions,
10731 terminal: Some(terminal),
10732 },
10733 );
10734 wait_result?;
10735
10736 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10742 Ok(routes) => routes,
10743 Err(err) => {
10744 warn!(
10745 module_id = %spec.module_id,
10746 ?reason,
10747 error = %err,
10748 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10749 );
10750 send_module_goodbye(&spec.module_id, forwarding, target);
10751 return Err(SuperviseError::Forwarding(err));
10752 }
10753 };
10754 let route_goodbye_count = released_routes.len();
10755 send_route_goodbyes(forwarding, released_routes);
10756 send_module_goodbye(&spec.module_id, forwarding, target);
10757
10758 info!(
10764 module_id = %spec.module_id,
10765 ?reason,
10766 routes_notified,
10767 route_goodbyes = route_goodbye_count,
10768 abandoned_reservations = target.abandoned_bindings.len(),
10769 excluded_subscriptions = target.excluded_subscriptions,
10770 drained,
10771 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10772 );
10773 }
10774
10775 Ok(StopNotice::SentOverConnection)
10776}
10777
10778async fn wait_for_registration_after_reload(
10781 registry: &Registry,
10782 module_id: &str,
10783 snapshot: &SharedSnapshot,
10784 child: &mut SupervisedChild,
10785 wait: Duration,
10786) -> Result<RegistrationWaitOutcome, SuperviseError> {
10787 wait_for_slot_registration(
10788 registry,
10789 crate::registry::RegistrationSlot::Active(module_id),
10790 module_id,
10791 snapshot,
10792 child,
10793 wait,
10794 )
10795 .await
10796}
10797
10798async fn wait_for_slot_registration(
10806 registry: &Registry,
10807 slot: crate::registry::RegistrationSlot<'_>,
10808 module_id: &str,
10809 snapshot: &SharedSnapshot,
10810 child: &mut SupervisedChild,
10811 wait: Duration,
10812) -> Result<RegistrationWaitOutcome, SuperviseError> {
10813 let deadline = Instant::now() + wait;
10814 loop {
10815 if registry
10816 .registration(slot)
10817 .map_err(SuperviseError::Registry)?
10818 .is_some()
10819 {
10820 return Ok(RegistrationWaitOutcome::Registered);
10821 }
10822
10823 let now = Instant::now();
10824 if now >= deadline {
10825 return Ok(RegistrationWaitOutcome::TimedOut);
10826 }
10827 let remaining = deadline.saturating_duration_since(now);
10828 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10829
10830 tokio::select! {
10831 wait_result = child.wait() => {
10832 let status = wait_result.map_err(|source| SuperviseError::Wait {
10833 module_id: module_id.to_string(),
10834 source,
10835 })?;
10836 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10837 snapshot,
10838 child,
10839 &status,
10840 )));
10841 }
10842 _ = sleep(poll) => {}
10843 }
10844 }
10845}
10846
10847fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10848 if exit_report.kind != ExitKind::DeliberateSeverance {
10851 exit_report.kind = ExitKind::Crash;
10852 }
10853 exit_report
10854}
10855
10856async fn handle_reload_child_registration_failure(
10857 spec: &ModuleSpec,
10858 runtime: &SupervisorRuntimeConfig,
10859 registry: &Registry,
10860 process_liveness: &SupervisorProcessLiveness,
10861 snapshot: &SharedSnapshot,
10862 _child: &mut Option<SupervisedChild>,
10863 failure: ReloadRegistrationFailure,
10864) -> Result<(), SuperviseError> {
10865 let ReloadRegistrationFailure {
10866 exit_report,
10867 reason,
10868 } = failure;
10869 match on_child_exit(
10870 spec,
10871 runtime.restart_policy,
10872 registry,
10873 snapshot,
10874 &runtime.terminal_ring,
10875 &runtime.spawn_events,
10876 &runtime.child_roster,
10877 exit_report,
10878 runtime.recovery_is_daemon,
10879 )
10880 .await
10881 {
10882 NextAction::Stop {
10883 registration_released,
10884 } => {
10885 if registration_released {
10886 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10887 }
10888 }
10889 NextAction::Restart { schedule } => {
10890 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10891 schedule.delay
10892 });
10893 if let Some(schedule) = schedule {
10894 log_crash_respawn(&spec.module_id, schedule);
10895 }
10896 schedule_respawn(
10897 runtime,
10898 snapshot,
10899 &spec.module_id,
10900 delay,
10901 RespawnKind::Spawn,
10902 true,
10903 )?;
10904 }
10905 }
10906 Err(SuperviseError::ReloadFailed {
10907 module_id: spec.module_id.clone(),
10908 reason,
10909 })
10910}
10911
10912async fn handle_reload_spawn_failure(
10913 spec: &ModuleSpec,
10914 runtime: &SupervisorRuntimeConfig,
10915 process_liveness: &SupervisorProcessLiveness,
10916 snapshot: &SharedSnapshot,
10917 _child: &mut Option<SupervisedChild>,
10918 reason: String,
10919) -> Result<(), SuperviseError> {
10920 let now = Instant::now();
10921 let mut schedule = None;
10922 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10923 clear_current_process_facts(state);
10924 if state.enabled {
10925 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10926 state.state = if schedule.is_some() {
10927 ModuleState::Restarting
10928 } else {
10929 if runtime
10930 .failure_context(runtime.recovery_is_daemon)
10931 .may_retry()
10932 {
10933 state.failed_retry.schedule(&spec.module_id);
10934 }
10935 ModuleState::Failed
10936 };
10937 } else {
10938 state.state = ModuleState::Disabled;
10939 }
10940 })?;
10941 if let Some(schedule) = schedule {
10942 schedule_respawn(
10943 runtime,
10944 snapshot,
10945 &spec.module_id,
10946 schedule.delay,
10947 RespawnKind::Spawn,
10948 true,
10949 )?;
10950 } else {
10951 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10952 }
10953 Err(SuperviseError::ReloadFailed {
10954 module_id: spec.module_id.clone(),
10955 reason,
10956 })
10957}
10958
10959fn control_flags() -> Flags {
10960 Flags::new(false, Priority::Passive, false)
10961}
10962
10963#[allow(clippy::too_many_arguments)]
10964async fn drain_optional_child(
10965 module_id: &str,
10966 protocol: ModuleProtocol,
10967 stop_notice: StopNotice,
10968 registry: &Registry,
10969 forwarding: Option<&ForwardingTable>,
10970 snapshot: &SharedSnapshot,
10971 terminal_ring: &Arc<Mutex<TerminalRing>>,
10972 spawn_events: &SpawnEventFeed,
10973 child: &mut Option<SupervisedChild>,
10974 drain_timeout: Duration,
10975 final_state: ModuleState,
10976 enabled: Option<bool>,
10977 failure: &FailureContext,
10978) -> Result<(), SuperviseError> {
10979 if let Some(child) = child.take() {
10980 drain_child_to_state(
10981 module_id,
10982 protocol,
10983 stop_notice,
10984 registry,
10985 forwarding,
10986 snapshot,
10987 terminal_ring,
10988 spawn_events,
10989 child,
10990 drain_timeout,
10991 final_state,
10992 enabled,
10993 failure,
10994 )
10995 .await
10996 } else {
10997 update_snapshot(snapshot, Some(module_id), |state| {
10998 state.state = final_state;
10999 if final_state == ModuleState::Failed {
11000 state.failed_retry.failed_at = Some(Instant::now());
11001 }
11002 if let Some(enabled) = enabled {
11003 state.enabled = enabled;
11004 }
11005 clear_current_process_facts(state);
11006 })?;
11007 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11008 }
11009}
11010
11011#[allow(clippy::too_many_arguments)]
11012async fn drain_child_to_state(
11013 module_id: &str,
11014 _protocol: ModuleProtocol,
11015 stop_notice: StopNotice,
11016 registry: &Registry,
11017 forwarding: Option<&ForwardingTable>,
11018 snapshot: &SharedSnapshot,
11019 terminal_ring: &Arc<Mutex<TerminalRing>>,
11020 spawn_events: &SpawnEventFeed,
11021 mut child: SupervisedChild,
11022 drain_timeout: Duration,
11023 final_state: ModuleState,
11024 enabled: Option<bool>,
11025 failure: &FailureContext,
11026) -> Result<(), SuperviseError> {
11027 let protocol = child.protocol;
11028 update_snapshot(snapshot, Some(module_id), |state| {
11029 state.state = ModuleState::Draining;
11030 state.draining_to_replace = final_state == ModuleState::Restarting;
11031 if let Some(enabled) = enabled {
11032 state.enabled = enabled;
11033 }
11034 })?;
11035
11036 if stop_notice != StopNotice::SentOverConnection {
11047 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11048 info!(
11049 module_id,
11050 pid = child.pid,
11051 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11052 "module has no connection yet; requesting stop by signal"
11053 );
11054 }
11055 request_graceful_stop(module_id, &child);
11056 }
11057
11058 let exit_report = match timeout(drain_timeout, child.wait()).await {
11059 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11060 Ok(Err(source)) => {
11061 fail_after_wait_error(
11062 module_id,
11063 snapshot,
11064 terminal_ring,
11065 spawn_events,
11066 format!("failed to wait during drain: {source}"),
11067 failure,
11068 );
11069 return Err(SuperviseError::Wait {
11070 module_id: module_id.to_string(),
11071 source,
11072 });
11073 }
11074 Err(_) => {
11075 warn!(
11088 module_id,
11089 pid = child.pid,
11090 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11091 reason = ?final_state,
11092 ?stop_notice,
11093 "drain budget expired before the module exited; killing it"
11094 );
11095 child.start_kill().map_err(|source| {
11096 fail_after_wait_error(
11097 module_id,
11098 snapshot,
11099 terminal_ring,
11100 spawn_events,
11101 format!("failed to kill during drain: {source}"),
11102 failure,
11103 );
11104 SuperviseError::Kill {
11105 module_id: module_id.to_string(),
11106 source,
11107 }
11108 })?;
11109 let status = child.wait().await.map_err(|source| {
11110 fail_after_wait_error(
11111 module_id,
11112 snapshot,
11113 terminal_ring,
11114 spawn_events,
11115 format!("failed to wait after drain kill: {source}"),
11116 failure,
11117 );
11118 SuperviseError::Wait {
11119 module_id: module_id.to_string(),
11120 source,
11121 }
11122 })?;
11123 classify_reaped_child_exit(snapshot, &child, &status)
11124 }
11125 };
11126
11127 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11128 publish_after_terminal_record(
11129 module_id,
11130 || {
11131 record_terminal_with_detail(
11132 module_id,
11133 terminal_ring,
11134 spawn_events,
11135 &exit_report,
11136 terminal_disposition(final_state),
11137 detail,
11138 )
11139 },
11140 || {
11141 update_snapshot(snapshot, Some(module_id), |state| {
11142 state.state = final_state;
11143 if final_state == ModuleState::Failed {
11144 state.failed_retry.failed_at = Some(Instant::now());
11145 }
11146 if let Some(enabled) = enabled {
11147 state.enabled = enabled;
11148 }
11149 clear_current_process_facts(state);
11150 state.last_exit = Some(exit_report.clone());
11151 if exit_report.kind == ExitKind::DeliberateSeverance {
11152 state.lifetime_restarts += 1;
11153 }
11154 })
11155 },
11156 )?;
11157 child.drain_stderr(module_id).await;
11158
11159 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11160}
11161
11162#[cfg(unix)]
11182fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
11183 let Some(pid) = child
11184 .id()
11185 .and_then(|pid| i32::try_from(pid).ok())
11186 .and_then(rustix::process::Pid::from_raw)
11187 else {
11188 debug!(
11189 module_id,
11190 "no pid to signal for teardown; falling through to the drain wait"
11191 );
11192 return;
11193 };
11194 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
11195 Ok(()) => debug!(
11196 module_id,
11197 "sent SIGTERM to a module nothing else asked to stop"
11198 ),
11199 Err(err) => debug!(
11200 module_id,
11201 error = %err,
11202 "SIGTERM to module failed; the drain wait and kill still apply"
11203 ),
11204 }
11205}
11206
11207#[cfg(not(unix))]
11215fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11216 debug!(
11217 module_id,
11218 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11219 );
11220}
11221
11222fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11223 match final_state {
11224 ModuleState::Stopped => TerminalDisposition::Stopped,
11225 ModuleState::Disabled => TerminalDisposition::Disabled,
11226 ModuleState::Restarting => TerminalDisposition::Restarting,
11227 ModuleState::Failed => TerminalDisposition::Failed,
11228 ModuleState::Starting
11229 | ModuleState::Running
11230 | ModuleState::Unresponsive
11231 | ModuleState::Draining => {
11232 unreachable!("terminal exits only finish in terminal or restarting states")
11233 }
11234 }
11235}
11236
11237async fn release_dead_registration(
11245 registry: &Registry,
11246 forwarding: Option<&ForwardingTable>,
11247 snapshot: &SharedSnapshot,
11248 module_id: &str,
11249 failure: &FailureContext,
11250) -> Result<(), SuperviseError> {
11251 let result = async {
11252 let registration = registry
11253 .get_module(module_id)
11254 .map_err(SuperviseError::Registry)?;
11255 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11256 Ok(()) => return Ok(()),
11257 Err(SuperviseError::RegistrationStillActive { .. }) => {}
11258 Err(err) => return Err(err),
11259 }
11260 let pid = lock_snapshot(snapshot)?.reaped_pid;
11261 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11262 warn!(
11263 module_id,
11264 pid,
11265 connection_id = registration.connection_id.get(),
11266 "reaped module registration outlived release grace; closing dead connection"
11267 );
11268 forwarding.request_connection_close(
11269 registration.connection_id,
11270 CloseReason::new(
11271 "supervised_process_reaped",
11272 format!("module '{module_id}' pid {pid} exited"),
11273 ),
11274 );
11275 wait_for_slot_registration_release(
11276 registry,
11277 crate::registry::RegistrationSlot::Connection(registration.connection_id),
11278 REGISTRY_RELEASE_TIMEOUT,
11279 )
11280 .await?;
11281 }
11282 wait_for_registration_release(registry, module_id, Duration::ZERO).await
11283 }
11284 .await;
11285 if let Err(err) = &result {
11286 fail_snapshot(snapshot, Some(module_id), None, failure);
11287 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11288 }
11289 result
11290}
11291
11292async fn wait_for_registration_release(
11295 registry: &Registry,
11296 module_id: &str,
11297 wait: Duration,
11298) -> Result<(), SuperviseError> {
11299 wait_for_slot_registration_release(
11300 registry,
11301 crate::registry::RegistrationSlot::Active(module_id),
11302 wait,
11303 )
11304 .await
11305}
11306
11307async fn wait_for_slot_registration_release(
11315 registry: &Registry,
11316 slot: crate::registry::RegistrationSlot<'_>,
11317 wait: Duration,
11318) -> Result<(), SuperviseError> {
11319 let deadline = Instant::now() + wait;
11320 let mut release_events = registration_release_events().subscribe();
11321 let still_active = |registration: &crate::registry::ModuleRegistration| {
11322 SuperviseError::RegistrationStillActive {
11323 module_id: registration.manifest.module_id.clone(),
11324 waited: wait,
11325 }
11326 };
11327 loop {
11328 let _observed_generation = *release_events.borrow_and_update();
11329 let Some(registration) = registry
11330 .registration(slot)
11331 .map_err(SuperviseError::Registry)?
11332 else {
11333 return Ok(());
11334 };
11335
11336 let now = Instant::now();
11337 if now >= deadline {
11338 return Err(still_active(®istration));
11339 }
11340
11341 let remaining = deadline.saturating_duration_since(now);
11342 match timeout(remaining, release_events.changed()).await {
11343 Ok(Ok(())) | Ok(Err(_)) => {}
11344 Err(_) => return Err(still_active(®istration)),
11345 }
11346 }
11347}
11348
11349#[cfg(test)]
11350mod slot_registration_wait_tests {
11351 use super::*;
11352 use crate::registry::{ConnectionId, RegistrationSlot};
11353 use subc_protocol::manifest::ModuleManifest;
11354
11355 #[tokio::test]
11356 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11357 let registry = Arc::new(Registry::default());
11358 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
11359 let runtime = supervisor.runtime_config();
11360 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11361 let spec = ModuleSpec {
11362 module_id: "enable-stale-registration".to_string(),
11363 program: PathBuf::from("/missing/enable-retry-test"),
11364 args: Vec::new(),
11365 env: Vec::new(),
11366 reserved: false,
11367 reserved_prefixes: Vec::new(),
11368 protocol: ModuleProtocol::Subc,
11369 overlap: Default::default(),
11370 };
11371 let connection = ConnectionId::new(90);
11372 registry
11373 .register_with_control_ops(
11374 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11375 1,
11376 connection,
11377 Vec::new(),
11378 )
11379 .unwrap();
11380 let mut child = None;
11381 let err = set_child_enabled(
11382 &spec,
11383 &runtime,
11384 ®istry,
11385 &supervisor.process_liveness,
11386 &snapshot,
11387 &mut child,
11388 true,
11389 &runtime.failure_context(false),
11390 )
11391 .await
11392 .unwrap_err();
11393 assert!(matches!(
11394 err,
11395 SuperviseError::RegistrationStillActive { .. }
11396 ));
11397 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11398 assert!(child.is_none());
11399 registry.deregister_connection(connection).unwrap();
11400 let err = set_child_enabled(
11401 &spec,
11402 &runtime,
11403 ®istry,
11404 &supervisor.process_liveness,
11405 &snapshot,
11406 &mut child,
11407 true,
11408 &runtime.failure_context(false),
11409 )
11410 .await
11411 .unwrap_err();
11412 assert!(
11413 matches!(err, SuperviseError::Spawn { .. }),
11414 "second enable must attempt a spawn: {err}"
11415 );
11416 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11417 }
11418
11419 const INCUMBENT: u64 = 1;
11420 const CANDIDATE: u64 = 2;
11421
11422 fn swapped_registry() -> Arc<Registry> {
11423 let registry = Arc::new(Registry::default());
11424 let manifest = ModuleManifest::builder("m", "0.1.0").build();
11425 registry
11426 .register_with_control_ops(
11427 manifest.clone(),
11428 1,
11429 ConnectionId::new(INCUMBENT),
11430 Vec::new(),
11431 )
11432 .unwrap();
11433 registry
11434 .register_candidate_with_control_ops(
11435 manifest,
11436 1,
11437 ConnectionId::new(CANDIDATE),
11438 Vec::new(),
11439 )
11440 .unwrap();
11441 registry
11442 }
11443
11444 #[tokio::test]
11448 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11449 let registry = swapped_registry();
11450 registry.promote_candidate("m").unwrap().unwrap();
11451
11452 assert!(matches!(
11453 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
11454 Err(SuperviseError::RegistrationStillActive { .. })
11455 ));
11456
11457 assert!(matches!(
11459 wait_for_slot_registration_release(
11460 ®istry,
11461 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11462 Duration::from_millis(50),
11463 )
11464 .await,
11465 Err(SuperviseError::RegistrationStillActive { .. })
11466 ));
11467
11468 let releaser = Arc::clone(®istry);
11469 let release = tokio::spawn(async move {
11470 sleep(Duration::from_millis(20)).await;
11471 releaser
11472 .deregister_connection(ConnectionId::new(INCUMBENT))
11473 .unwrap();
11474 notify_registration_release();
11475 });
11476 wait_for_slot_registration_release(
11477 ®istry,
11478 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11479 Duration::from_secs(5),
11480 )
11481 .await
11482 .expect("the incumbent's own registration is released");
11483 release.await.unwrap();
11484 assert!(registry.get_module("m").unwrap().is_some());
11485 }
11486
11487 #[tokio::test]
11490 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11491 let registry = swapped_registry();
11492 assert!(matches!(
11493 wait_for_slot_registration_release(
11494 ®istry,
11495 RegistrationSlot::Candidate("m"),
11496 Duration::from_millis(50),
11497 )
11498 .await,
11499 Err(SuperviseError::RegistrationStillActive { .. })
11500 ));
11501 registry
11502 .deregister_connection(ConnectionId::new(CANDIDATE))
11503 .unwrap();
11504 wait_for_slot_registration_release(
11505 ®istry,
11506 RegistrationSlot::Candidate("m"),
11507 Duration::from_millis(50),
11508 )
11509 .await
11510 .expect("a candidate slot with no candidate is released");
11511 assert!(registry
11512 .registration(RegistrationSlot::Active("m"))
11513 .unwrap()
11514 .is_some());
11515 }
11516}
11517
11518fn classify_exit(status: &ExitStatus) -> ExitReport {
11519 ExitReport {
11520 kind: if status.success() {
11521 ExitKind::Clean
11522 } else {
11523 ExitKind::Crash
11524 },
11525 code: status.code(),
11526 signal: exit_signal(status),
11527 at_ms: unix_ms_now(),
11528 }
11529}
11530
11531fn wait_error_exit_report() -> ExitReport {
11537 ExitReport {
11538 kind: ExitKind::Crash,
11539 code: None,
11540 signal: None,
11541 at_ms: unix_ms_now(),
11542 }
11543}
11544
11545#[cfg(unix)]
11546fn exit_signal(status: &ExitStatus) -> Option<i32> {
11547 use std::os::unix::process::ExitStatusExt;
11548
11549 status.signal()
11550}
11551
11552#[cfg(not(unix))]
11553fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11554 None
11555}
11556
11557fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11563 update_snapshot(snapshot, Some(module_id), |state| {
11564 state.clear_crash_restarts();
11565 })
11566}
11567
11568fn set_running(
11569 snapshot: &SharedSnapshot,
11570 child: &SupervisedChild,
11571 module_id: &str,
11572 spawn_events: &SpawnEventFeed,
11573) -> Result<(), SuperviseError> {
11574 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11575 module_id: Some(module_id.to_string()),
11576 })?;
11577 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11578 if std::mem::take(&mut state.coalesced_restart_pending) {
11579 let generation = state.spawn_generation;
11580 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11581 }
11582 state.drain_disposition_detail = None;
11583 state.spawn_failure = None;
11584 state.waiting_for_previous_pid = None;
11585 state.waiting_for_previous_detail = None;
11586 state.in_alternate_slot = false;
11589 state.configuration_updated_since_spawn = false;
11590 state.spawned_protocol = Some(child.protocol);
11591 state.state = ModuleState::Running;
11592 state.enabled = true;
11593 state.process_alive = true;
11594 state.pid = child.id();
11595 state.health.status = SupervisorHealthStatus::Unknown;
11598 state.health.consecutive_failures = 0;
11599 state.health.detail = None;
11600 state.health.metrics = None;
11601 state.health.last_probe_ms = None;
11602 state.failed_retry.healthy_since = None;
11603 state.failed_retry.failed_at = None;
11604 state.failed_retry.running_since = Some(child.spawned_at);
11605 #[cfg(windows)]
11606 {
11607 state.pipe_consumed = child
11608 .nonce_delivery
11609 .as_ref()
11610 .map(|delivery| delivery.consumed());
11611 }
11612 #[cfg(target_os = "macos")]
11613 {
11614 state.report_ready = Some(Arc::clone(&child.report_ready));
11615 }
11616 state.spawned_at_ms = Some(child.spawned_at_ms);
11617 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11618 state.spawned_from = Some(child.spawned_from.clone());
11619 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11620 state.spawned_file_identity = child.spawned_file_identity.clone();
11621 state.process_start_time = child.process_start_time;
11622 Ok(())
11623}
11624
11625fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11626 state.failed_retry.healthy_since = None;
11627 state.failed_retry.running_since = None;
11628 state.waiting_for_previous_pid = None;
11629 state.waiting_for_previous_detail = None;
11630 state.process_alive = false;
11631 state.spawned_protocol = None;
11632 state.pid = None;
11633 #[cfg(windows)]
11634 {
11635 state.pipe_consumed = None;
11636 }
11637 #[cfg(target_os = "macos")]
11638 {
11639 state.report_ready = None;
11640 }
11641 state.spawned_at_ms = None;
11642 state.exec_outcome = None;
11643 state.spawned_from = None;
11644 state.launch_spec_digest = None;
11645 state.spawned_file_identity = None;
11646 state.process_start_time = None;
11647 state.deliberate_severance = None;
11648}
11649
11650#[cfg(test)]
11651fn record_deliberate_severance(
11652 snapshot: &SharedSnapshot,
11653 identity: ProcessIdentity,
11654) -> Result<(), SuperviseError> {
11655 update_snapshot(snapshot, None, |state| {
11656 state.deliberate_severance = Some(identity);
11657 })
11658}
11659
11660fn apply_deliberate_severance_marker(
11661 snapshot: &SharedSnapshot,
11662 exited_identity: Option<ProcessIdentity>,
11663 mut exit_report: ExitReport,
11664) -> ExitReport {
11665 let marker = lock_snapshot(snapshot)
11666 .ok()
11667 .and_then(|mut state| state.deliberate_severance.take());
11668 if marker.is_some() && marker == exited_identity {
11669 exit_report.kind = ExitKind::DeliberateSeverance;
11670 }
11671 exit_report
11672}
11673
11674fn classify_reaped_child_exit(
11675 snapshot: &SharedSnapshot,
11676 child: &SupervisedChild,
11677 status: &ExitStatus,
11678) -> ExitReport {
11679 let _ = update_snapshot(snapshot, None, |state| {
11680 state.reaped_pid = Some(child.pid);
11681 state.spawn_failure = child.spawn_failure.clone();
11682 });
11683 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11684}
11685
11686fn fail_snapshot(
11687 snapshot: &SharedSnapshot,
11688 module_id: Option<&str>,
11689 last_exit: Option<ExitReport>,
11690 failure: &FailureContext,
11691) {
11692 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11693 state.state = ModuleState::Failed;
11694 clear_current_process_facts(state);
11695 if failure.may_retry() && state.enabled {
11696 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
11697 } else if failure.shutdown.is_set() {
11698 state.failed_retry.cancel();
11699 }
11700 if let Some(last_exit) = last_exit {
11701 state.last_exit = Some(last_exit);
11702 }
11703 }) {
11704 error!(error = %err, "failed to mark supervisor state failed");
11705 }
11706}
11707
11708fn update_snapshot(
11709 snapshot: &SharedSnapshot,
11710 module_id: Option<&str>,
11711 update: impl FnOnce(&mut SupervisorSnapshot),
11712) -> Result<(), SuperviseError> {
11713 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11714 module_id: module_id.map(ToOwned::to_owned),
11715 })?;
11716 update(&mut state);
11717 Ok(())
11718}
11719
11720const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11721
11722fn lock_snapshot_for_control<'a>(
11723 snapshot: &'a SharedSnapshot,
11724 module_id: &str,
11725 caller: &'static str,
11726) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11727 let started_at = Instant::now();
11728 let guard = lock_snapshot(snapshot)?;
11729 let waited = started_at.elapsed();
11730 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11731 warn!(
11732 module_id = %module_id,
11733 waited_ms = waited.as_millis() as u64,
11734 caller = %caller,
11735 "slow snapshot lock"
11736 );
11737 }
11738 Ok(guard)
11739}
11740
11741fn lock_snapshot(
11742 snapshot: &SharedSnapshot,
11743) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11744 snapshot
11745 .lock()
11746 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11747}
11748
11749#[cfg(test)]
11750mod terminal_history_tests {
11751 use std::{
11752 path::PathBuf,
11753 sync::Arc,
11754 time::{Duration, Instant},
11755 };
11756
11757 use tokio::{sync::mpsc, time::sleep};
11758
11759 use super::{
11760 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11761 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11762 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11763 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11764 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11765 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11766 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11767 SupervisorHealthStatus, SupervisorSnapshot,
11768 };
11769 use super::Instant as ClockInstant;
11774 use crate::{
11775 registry::Registry,
11776 terminal_ring::{TerminalRing, TerminalRingConfig},
11777 };
11778 use std::sync::Mutex;
11779 use subc_control::TerminalDisposition;
11780
11781 pub(super) fn fake_aft_stub_path() -> PathBuf {
11786 let mut path = std::env::current_exe().expect("current_exe available in tests");
11787 path.pop();
11788 path.pop();
11789 path.push(if cfg!(windows) {
11790 "fake-aft-stub.exe"
11791 } else {
11792 "fake-aft-stub"
11793 });
11794 assert!(
11795 path.exists(),
11796 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11797 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11798 path.display()
11799 );
11800 path
11801 }
11802
11803 #[test]
11804 fn reserved_never_spawned_refuses_every_hello() {
11805 let supervisor = SupervisorHandle::default();
11810 supervisor.apply_identity_configuration(&ModuleSpec {
11811 module_id: "never-spawned".to_string(),
11812 program: PathBuf::from("/usr/bin/false"),
11813 args: Vec::new(),
11814 env: Vec::new(),
11815 reserved: true,
11816 reserved_prefixes: Vec::new(),
11817 protocol: ModuleProtocol::Subc,
11818 overlap: Default::default(),
11819 });
11820 assert!(
11821 supervisor
11822 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11823 .is_err(),
11824 "forged nonce must refuse on a reserved never-spawned id"
11825 );
11826 assert!(
11827 supervisor
11828 .reserved_hello_verification("never-spawned", None)
11829 .is_err(),
11830 "absent nonce must refuse on a reserved never-spawned id"
11831 );
11832 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11834 supervisor.apply_identity_configuration(&ModuleSpec {
11835 module_id: "never-spawned".to_string(),
11836 program: PathBuf::from("/usr/bin/false"),
11837 args: Vec::new(),
11838 env: Vec::new(),
11839 reserved: true,
11840 reserved_prefixes: Vec::new(),
11841 protocol: ModuleProtocol::Subc,
11842 overlap: Default::default(),
11843 });
11844 assert!(supervisor
11845 .reserved_hello_verification("never-spawned", Some("minted"))
11846 .is_ok());
11847 assert!(supervisor
11848 .reserved_hello_verification("never-spawned", Some("forged"))
11849 .is_err());
11850 }
11851
11852 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11855 let now = ClockInstant::now();
11856 for _ in 0..count {
11857 state.crash_restarts.push_back(now);
11858 }
11859 }
11860
11861 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11865 let aged = state
11866 .crash_restarts
11867 .front()
11868 .expect("a crash restart must be recorded before it can be aged")
11869 .checked_sub(window + Duration::from_secs(1))
11870 .expect("the test clock is far enough from its origin to age an instant");
11871 state.crash_restarts[0] = aged;
11872 }
11873
11874 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11875 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11876 seed_crash_restarts(&mut state, count);
11877 state
11878 }
11879
11880 #[test]
11881 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11882 let policy = RestartPolicy::new(3, Duration::ZERO);
11883 let now = ClockInstant::now();
11884 assert!(daemon_will_restart(
11885 &mut snapshot_with_restarts(true, 2),
11886 &policy,
11887 now
11888 ));
11889 assert!(!daemon_will_restart(
11890 &mut snapshot_with_restarts(true, 3),
11891 &policy,
11892 now
11893 ));
11894 assert!(!daemon_will_restart(
11895 &mut snapshot_with_restarts(false, 0),
11896 &policy,
11897 now
11898 ));
11899 }
11900
11901 #[test]
11902 fn crash_restart_backoff_escalates_with_in_window_count() {
11903 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11904 .with_max_backoff(Duration::from_secs(30));
11905 let now = ClockInstant::now();
11906 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11907 let schedules = (0..4)
11908 .map(|_| {
11909 state
11910 .next_crash_restart(&policy, now)
11911 .expect("the test policy allows four crash restarts")
11912 })
11913 .collect::<Vec<_>>();
11914
11915 assert_eq!(
11916 schedules
11917 .iter()
11918 .map(|schedule| schedule.restart_in_window)
11919 .collect::<Vec<_>>(),
11920 vec![0, 1, 2, 3]
11921 );
11922 assert_eq!(
11923 schedules
11924 .iter()
11925 .map(|schedule| schedule.delay)
11926 .collect::<Vec<_>>(),
11927 vec![
11928 Duration::from_millis(100),
11929 Duration::from_secs(1),
11930 Duration::from_secs(10),
11931 Duration::from_secs(30),
11932 ]
11933 );
11934 }
11935
11936 #[test]
11937 fn crash_restart_backoff_resets_after_ring_clear() {
11938 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11939 let now = ClockInstant::now();
11940 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11941 assert_eq!(
11942 state.next_crash_restart(&policy, now).unwrap().delay,
11943 Duration::from_millis(100)
11944 );
11945 assert_eq!(
11946 state.next_crash_restart(&policy, now).unwrap().delay,
11947 Duration::from_secs(1)
11948 );
11949
11950 state.clear_crash_restarts();
11951 let schedule = state
11952 .next_crash_restart(&policy, now)
11953 .expect("a cleared ring must allow another restart");
11954 assert_eq!(schedule.restart_in_window, 0);
11955 assert_eq!(schedule.delay, Duration::from_millis(100));
11956 }
11957
11958 #[test]
11959 fn crash_restart_backoff_ignores_aged_restarts() {
11960 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11961 let now = ClockInstant::now();
11962 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11963 state
11964 .next_crash_restart(&policy, now)
11965 .expect("the first restart is allowed");
11966 state
11967 .next_crash_restart(&policy, now)
11968 .expect("the second restart is allowed");
11969 state.crash_restarts[0] = now
11970 .checked_sub(policy.window + Duration::from_secs(1))
11971 .expect("the fake clock can age a restart past the window");
11972
11973 let schedule = state
11974 .next_crash_restart(&policy, now)
11975 .expect("an aged restart must release its slot");
11976 assert_eq!(schedule.restart_in_window, 1);
11977 assert_eq!(schedule.delay, Duration::from_secs(1));
11978 assert_eq!(state.crash_restarts.len(), 2);
11979 }
11980
11981 #[test]
11985 fn a_budget_spent_before_the_window_no_longer_refuses() {
11986 let policy = RestartPolicy::new(3, Duration::ZERO);
11987 let mut state = snapshot_with_restarts(true, 3);
11988 let now = ClockInstant::now();
11989 assert!(!daemon_will_restart(&mut state, &policy, now));
11990
11991 assert!(daemon_will_restart(
11992 &mut state,
11993 &policy,
11994 now + policy.window + Duration::from_secs(1)
11995 ));
11996 assert!(
11997 state.crash_restarts.is_empty(),
11998 "reading the budget must drop the instants that left the window"
11999 );
12000 }
12001
12002 fn module_with_recovery_snapshot(
12003 state: ModuleState,
12004 enabled: bool,
12005 restart_count: u32,
12006 ) -> SupervisedModule {
12007 let registry = Arc::new(Registry::default());
12008 let supervisor =
12009 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
12010 let runtime = supervisor.runtime_config();
12011 let spec = ModuleSpec {
12012 module_id: "recovery-snapshot".to_string(),
12013 program: fake_aft_stub_path(),
12014 args: Vec::new(),
12015 env: Vec::new(),
12016 reserved: false,
12017 reserved_prefixes: Vec::new(),
12018 protocol: ModuleProtocol::Subc,
12019 overlap: Default::default(),
12020 };
12021 let mut snapshot = SupervisorSnapshot::new(state, enabled);
12022 seed_crash_restarts(&mut snapshot, restart_count);
12023 let (commands, _rx) = mpsc::channel(4);
12026 SupervisedModule {
12027 inner: Arc::new(SupervisedModuleInner {
12028 module_id: spec.module_id.clone(),
12029 registry,
12030 snapshot: Arc::new(Mutex::new(snapshot)),
12031 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12032 spec,
12033 health: runtime.health,
12034 })),
12035 stderr_ring: runtime.stderr_ring,
12036 terminal_ring: runtime.terminal_ring,
12037 commands,
12038 monitor: Mutex::new(None),
12039 restart_policy: runtime.restart_policy,
12040 effective_drain_timeout: runtime.effective_drain_timeout,
12041 provenance_probe: supervisor.provenance_probe.clone(),
12042 }),
12043 }
12044 }
12045
12046 #[cfg(target_os = "linux")]
12047 #[tokio::test]
12048 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12049 let supervisor =
12050 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12051 .with_cgroup_placement(None);
12052 let result = supervisor.spawn(ModuleSpec {
12053 module_id: "no-cgroup-placement".to_string(),
12054 program: fake_aft_stub_path(),
12055 args: Vec::new(),
12056 env: Vec::new(),
12057 reserved: false,
12058 reserved_prefixes: Vec::new(),
12059 protocol: ModuleProtocol::Subc,
12060 overlap: Default::default(),
12061 });
12062
12063 assert!(
12064 result.is_ok(),
12065 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12066 );
12067 }
12068
12069 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12070 async fn undecided_snapshot_uses_shared_restart_predicate() {
12071 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12072 .will_recover_after_connection_loss()
12073 .unwrap());
12074 assert!(
12075 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12076 .will_recover_after_connection_loss()
12077 .unwrap()
12078 );
12079 }
12080
12081 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12082 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12083 assert!(
12084 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12085 .will_recover_after_connection_loss()
12086 .unwrap()
12087 );
12088 }
12089
12090 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12091 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12092 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12093 .will_recover_after_connection_loss()
12094 .unwrap());
12095 assert!(
12096 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12097 .will_recover_after_connection_loss()
12098 .unwrap()
12099 );
12100 }
12101
12102 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12103 async fn warming_snapshot_is_limited_to_startup_phases() {
12104 for state in [
12105 ModuleState::Starting,
12106 ModuleState::Running,
12107 ModuleState::Restarting,
12108 ] {
12109 assert!(
12110 module_with_recovery_snapshot(state, true, 0)
12111 .is_warming()
12112 .unwrap(),
12113 "{state:?} should be warming"
12114 );
12115 }
12116 for state in [
12117 ModuleState::Unresponsive,
12118 ModuleState::Draining,
12119 ModuleState::Stopped,
12120 ModuleState::Failed,
12121 ModuleState::Disabled,
12122 ] {
12123 assert!(
12124 !module_with_recovery_snapshot(state, true, 0)
12125 .is_warming()
12126 .unwrap(),
12127 "{state:?} should not be warming"
12128 );
12129 }
12130 }
12131
12132 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12133 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12134 let registry = Arc::new(Registry::default());
12135 let supervisor =
12136 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
12137 let module = supervisor
12138 .spawn(ModuleSpec {
12139 module_id: "terminal-history".to_string(),
12140 program: fake_aft_stub_path(),
12141 args: Vec::new(),
12142 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12143 reserved: false,
12144 reserved_prefixes: Vec::new(),
12145 protocol: ModuleProtocol::Subc,
12146 overlap: Default::default(),
12147 })
12148 .unwrap();
12149
12150 let deadline = Instant::now() + Duration::from_secs(5);
12151 loop {
12152 let history = module.terminal_history();
12153 if history.entries.len() == 2 {
12154 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12155 assert_eq!(history.dropped, 0);
12156 assert_eq!(
12157 history
12158 .entries
12159 .iter()
12160 .map(|entry| entry.exit_code)
12161 .collect::<Vec<_>>(),
12162 vec![Some(23), Some(23)]
12163 );
12164 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
12165 return;
12166 }
12167 assert!(
12168 Instant::now() < deadline,
12169 "module did not retain two terminal exits: {history:?}"
12170 );
12171 sleep(Duration::from_millis(10)).await;
12172 }
12173 }
12174
12175 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12179 async fn disable_during_crash_backoff_cancels_pending_respawn() {
12180 let backoff = Duration::from_secs(2);
12181 let supervisor = Supervisor::new_for_test(
12182 Arc::new(Registry::default()),
12183 RestartPolicy::new(10, backoff),
12184 );
12185 let module = supervisor
12186 .spawn(ModuleSpec {
12187 module_id: "disable-during-backoff".to_string(),
12188 program: fake_aft_stub_path(),
12189 args: Vec::new(),
12190 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12191 reserved: false,
12192 reserved_prefixes: Vec::new(),
12193 protocol: ModuleProtocol::Subc,
12194 overlap: Default::default(),
12195 })
12196 .unwrap();
12197
12198 let deadline = Instant::now() + Duration::from_secs(5);
12200 loop {
12201 if module.status().unwrap().state == ModuleState::Restarting {
12202 break;
12203 }
12204 assert!(
12205 Instant::now() < deadline,
12206 "module never entered the crash backoff"
12207 );
12208 sleep(Duration::from_millis(10)).await;
12209 }
12210
12211 let started = Instant::now();
12212 module.set_enabled(false).await.unwrap();
12213 let waited = started.elapsed();
12214
12215 assert!(
12216 waited < backoff / 2,
12217 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12218 );
12219 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12220
12221 sleep(backoff + Duration::from_millis(500)).await;
12223 let status = module.status().unwrap();
12224 assert_eq!(status.state, ModuleState::Disabled);
12225 assert_eq!(
12226 status.spawn_generation, 1,
12227 "module respawned after the operator disabled it"
12228 );
12229 }
12230
12231 #[cfg(unix)]
12234 fn protocol_none_sigterm_exits_clean_spec(
12235 module_id: &str,
12236 dir: &std::path::Path,
12237 ) -> (ModuleSpec, PathBuf, PathBuf) {
12238 let ready = dir.join("ready");
12239 let marker = dir.join("sigterm");
12240 let spec = ModuleSpec {
12241 module_id: module_id.to_string(),
12242 program: fake_aft_stub_path(),
12243 args: Vec::new(),
12244 env: vec![
12245 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12246 (
12247 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12248 marker.display().to_string(),
12249 ),
12250 (
12251 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12252 ready.display().to_string(),
12253 ),
12254 ],
12255 reserved: false,
12256 reserved_prefixes: Vec::new(),
12257 protocol: ModuleProtocol::None,
12258 overlap: Default::default(),
12259 };
12260 (spec, ready, marker)
12261 }
12262
12263 #[cfg(unix)]
12267 async fn wait_for_file(path: &std::path::Path) {
12268 let deadline = Instant::now() + Duration::from_secs(10);
12269 while !path.exists() {
12270 assert!(
12271 Instant::now() < deadline,
12272 "{} never appeared",
12273 path.display()
12274 );
12275 sleep(Duration::from_millis(10)).await;
12276 }
12277 }
12278
12279 #[cfg(unix)]
12283 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12284 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12285 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12286 let (spec, ready, marker) =
12287 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12288 let supervisor = Supervisor::new_for_test(
12289 Arc::new(Registry::default()),
12290 RestartPolicy::new(3, Duration::ZERO),
12291 );
12292 let module = supervisor.spawn(spec).unwrap();
12293 wait_for_file(&ready).await;
12294 let deadline = Instant::now() + Duration::from_secs(10);
12298 let first_pid = loop {
12299 let status = module.status().unwrap();
12300 if status.state == ModuleState::Running {
12301 if let Some(pid) = status.pid {
12302 break pid;
12303 }
12304 }
12305 assert!(
12306 Instant::now() < deadline,
12307 "a running module must report its pid after exec confirmation: {status:?}"
12308 );
12309 sleep(Duration::from_millis(10)).await;
12310 };
12311
12312 rustix::process::kill_process(
12313 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12314 rustix::process::Signal::TERM,
12315 )
12316 .unwrap();
12317
12318 let deadline = Instant::now() + Duration::from_secs(10);
12319 let respawned = loop {
12320 let status = module.status().unwrap();
12321 if status.state == ModuleState::Running
12322 && status.pid.is_some_and(|pid| pid != first_pid)
12323 {
12324 break status;
12325 }
12326 assert!(
12327 Instant::now() < deadline,
12328 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12329 );
12330 sleep(Duration::from_millis(10)).await;
12331 };
12332 assert_eq!(respawned.spawn_generation, 2);
12333 assert!(
12334 marker.exists(),
12335 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12336 );
12337
12338 let history = module.terminal_history();
12339 assert_eq!(history.entries.len(), 1, "{history:?}");
12340 let entry = &history.entries[0];
12341 assert_eq!(entry.exit_code, Some(0));
12342 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12343 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12344
12345 module.stop().await.unwrap();
12346 }
12347
12348 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12352 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12353 let supervisor = Supervisor::new_for_test(
12354 Arc::new(Registry::default()),
12355 RestartPolicy::new(1, Duration::ZERO),
12356 );
12357 let module = supervisor
12358 .spawn(ModuleSpec {
12359 module_id: "none-clean-exit-budget".to_string(),
12360 program: fake_aft_stub_path(),
12361 args: Vec::new(),
12362 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12363 reserved: false,
12364 reserved_prefixes: Vec::new(),
12365 protocol: ModuleProtocol::None,
12366 overlap: Default::default(),
12367 })
12368 .unwrap();
12369
12370 let deadline = Instant::now() + Duration::from_secs(10);
12373 loop {
12374 let status = module.status().unwrap();
12375 if status.state == ModuleState::Failed {
12376 break;
12377 }
12378 assert!(
12379 Instant::now() < deadline,
12380 "module never exhausted its budget: {status:?} {:?}",
12381 module.terminal_history()
12382 );
12383 sleep(Duration::from_millis(10)).await;
12384 }
12385 let history = module.terminal_history();
12386 assert_eq!(
12387 history
12388 .entries
12389 .iter()
12390 .map(|entry| (entry.exit_code, entry.disposition.clone()))
12391 .collect::<Vec<_>>(),
12392 vec![
12393 (Some(0), TerminalDisposition::Restarting),
12394 (Some(0), TerminalDisposition::Failed),
12395 ]
12396 );
12397 let detail = history.entries[1]
12398 .disposition_detail
12399 .as_deref()
12400 .expect("a budget failure names the budget");
12401 assert!(detail.contains("max_restarts=1"), "{detail}");
12402 assert_eq!(module.status().unwrap().spawn_generation, 2);
12403 }
12404
12405 #[test]
12406 fn restart_budget_failure_is_published_after_its_terminal_record() {
12407 use std::io::{Read, Write};
12408 let supervisor = Supervisor::new_for_test(
12409 Arc::new(Registry::default()),
12410 RestartPolicy::new(0, Duration::ZERO),
12411 );
12412 let runtime = supervisor.runtime_config();
12413 let spec = ModuleSpec {
12414 protocol: ModuleProtocol::None,
12415 ..windowed_crash_spec("budget-publication-order")
12416 };
12417 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12418 ModuleState::Running,
12419 true,
12420 )));
12421 lock_snapshot(&snapshot).unwrap().process_alive = true;
12422 let reaped_snapshot = snapshot.clone();
12423 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12424 let history = runtime.terminal_ring.clone();
12425 let reap = std::thread::spawn(move || {
12426 tokio::runtime::Builder::new_current_thread()
12427 .enable_all()
12428 .build()
12429 .unwrap()
12430 .block_on(on_child_exit(
12431 &spec,
12432 runtime.restart_policy,
12433 &supervisor.registry,
12434 &reaped_snapshot,
12435 &runtime.terminal_ring,
12436 &runtime.spawn_events,
12437 &runtime.child_roster,
12438 ExitReport {
12439 kind: ExitKind::Clean,
12440 code: Some(0),
12441 signal: None,
12442 at_ms: 1,
12443 },
12444 true,
12445 ))
12446 });
12447 let mut reached = [0];
12448 barrier.read_exact(&mut reached).unwrap();
12449 assert_eq!(&reached, b"R");
12450 let (before_state, before_alive) = {
12451 let state = lock_snapshot(&snapshot).unwrap();
12452 (state.state, state.process_alive)
12453 };
12454 let before_history = history.lock().unwrap().snapshot();
12455 barrier.write_all(b"X").unwrap();
12456 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12457 assert!(before_history.entries.is_empty());
12458 assert_ne!(
12459 before_state,
12460 ModuleState::Failed,
12461 "Failed was visible before its terminal record could be written"
12462 );
12463 assert!(before_alive);
12464 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12465 let history = history.lock().unwrap().snapshot();
12466 assert_eq!(history.entries.len(), 1);
12467 assert_eq!(history.entries[0].exit_code, Some(0));
12468 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12469 }
12470
12471 #[test]
12472 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12473 use std::io::{Read, Write};
12474
12475 let supervisor = Supervisor::new_for_test(
12476 Arc::new(Registry::default()),
12477 RestartPolicy::new(3, Duration::ZERO),
12478 );
12479 let mut runtime = supervisor.runtime_config();
12480 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12481 let snapshot = module.inner.snapshot.clone();
12482 lock_snapshot(&snapshot).unwrap().process_alive = true;
12483 runtime.terminal_ring = module.inner.terminal_ring.clone();
12484 let spec = windowed_crash_spec(module.module_id());
12485 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12486 let reaped_snapshot = snapshot.clone();
12487 let reap = std::thread::spawn(move || {
12488 tokio::runtime::Builder::new_current_thread()
12489 .enable_all()
12490 .build()
12491 .unwrap()
12492 .block_on(on_child_exit(
12493 &spec,
12494 runtime.restart_policy,
12495 &supervisor.registry,
12496 &reaped_snapshot,
12497 &runtime.terminal_ring,
12498 &runtime.spawn_events,
12499 &runtime.child_roster,
12500 ExitReport {
12501 kind: ExitKind::Crash,
12502 code: Some(2),
12503 signal: None,
12504 at_ms: 1,
12505 },
12506 true,
12507 ))
12508 });
12509
12510 let mut reached = [0];
12514 barrier.read_exact(&mut reached).unwrap();
12515 assert_eq!(&reached, b"R");
12516 let before_status = module.status().unwrap();
12517 let before_history = module.terminal_history();
12518 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12519 barrier.write_all(b"X").unwrap();
12520 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12521
12522 assert_eq!(before_status.state, ModuleState::Running);
12523 assert!(before_status.process_alive);
12524 assert_eq!(before_status.restart_count, 3);
12525 assert!(before_history.entries.is_empty());
12526 assert!(
12527 before_recovers,
12528 "connection cleanup would report a terminal close for a scheduled crash restart"
12529 );
12530 let restarting = module.status().unwrap();
12531 assert_eq!(restarting.state, ModuleState::Restarting);
12532 assert_eq!(restarting.restart_count, 3);
12533 assert!(module.will_recover_after_connection_loss().unwrap());
12534 let history = module.terminal_history();
12535 assert_eq!(history.entries.len(), 1);
12536 assert_eq!(
12537 history.entries[0].disposition,
12538 TerminalDisposition::Restarting
12539 );
12540
12541 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12544 state.state = ModuleState::Running;
12545 state.respawn_pending = false;
12546 })
12547 .unwrap();
12548 assert!(!module.will_recover_after_connection_loss().unwrap());
12549 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12550 state.enabled = false;
12551 state.respawn_pending = true;
12552 })
12553 .unwrap();
12554 assert!(!module.will_recover_after_connection_loss().unwrap());
12555 }
12556
12557 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12558 use std::net::{TcpListener, TcpStream};
12559 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12560 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12561 let (barrier, _) = listener.accept().unwrap();
12562 for stream in [&writer, &barrier] {
12563 stream
12564 .set_read_timeout(Some(Duration::from_secs(10)))
12565 .unwrap();
12566 stream
12567 .set_write_timeout(Some(Duration::from_secs(10)))
12568 .unwrap();
12569 }
12570 *events.1.lock().unwrap() = Some(writer);
12571 barrier
12572 }
12573
12574 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12575 use std::io::{Read, Write};
12576 let barrier = events.1.lock().unwrap().take();
12577 if let Some(mut barrier) = barrier {
12578 barrier.write_all(b"R").unwrap();
12579 let mut release = [0];
12580 barrier.read_exact(&mut release).unwrap();
12581 assert_eq!(&release, b"X");
12582 }
12583 }
12584
12585 #[test]
12586 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12587 use std::io::{Read, Write};
12588
12589 let supervisor = Supervisor::new_for_test(
12590 Arc::new(Registry::default()),
12591 RestartPolicy::new(0, Duration::ZERO),
12592 );
12593 let mut runtime = supervisor.runtime_config();
12594 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12595 let snapshot = module.inner.snapshot.clone();
12596 runtime.terminal_ring = module.inner.terminal_ring.clone();
12597 let spec = ModuleSpec {
12598 module_id: module.module_id().into(),
12599 program: fake_aft_stub_path(),
12600 protocol: ModuleProtocol::None,
12601 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12602 ..windowed_crash_spec(module.module_id())
12603 };
12604 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12605 let reap = std::thread::spawn(move || {
12606 tokio::runtime::Builder::new_current_thread()
12607 .enable_all()
12608 .build()
12609 .unwrap()
12610 .block_on(async {
12611 let mut child =
12612 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12613 health_restart_child(
12614 &spec,
12615 &runtime,
12616 &supervisor.registry,
12617 &supervisor.process_liveness,
12618 &snapshot,
12619 &mut child,
12620 SupervisorHealthStatus::Failing,
12621 Some("health probe did not answer"),
12622 1,
12623 )
12624 .await
12625 .unwrap();
12626 assert!(child.is_none());
12627 });
12628 });
12629 let mut reached = [0];
12634 barrier.read_exact(&mut reached).unwrap();
12635 assert_eq!(&reached, b"R");
12636 let before_status = module.status().unwrap();
12637 let before_history = module.terminal_history();
12638 barrier.write_all(b"X").unwrap();
12639 reap.join().unwrap();
12640 assert!(before_history.entries.is_empty());
12641 assert_ne!(
12642 before_status.state,
12643 ModuleState::Failed,
12644 "Failed was visible before its terminal record could be written"
12645 );
12646 assert!(
12647 before_status.process_alive,
12648 "process death was visible before its terminal record could be written"
12649 );
12650 let failed = module.status().unwrap();
12651 assert_eq!(failed.state, ModuleState::Failed);
12652 assert!(!failed.process_alive);
12653 let history = module.terminal_history();
12654 assert_eq!(history.entries.len(), 1);
12655 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12656 assert_eq!(
12657 history.entries[0].disposition_detail.as_deref(),
12658 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12659 );
12660 }
12661
12662 #[test]
12663 fn observed_exits_publish_process_death_after_terminal_history() {
12664 use std::io::{Read, Write};
12665 for (kind, enabled, shutdown, wait_error, expected_state) in [
12666 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12667 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12668 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12669 (
12670 ExitKind::DeliberateSeverance,
12671 true,
12672 false,
12673 false,
12674 ModuleState::Restarting,
12675 ),
12676 #[cfg(unix)]
12677 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12678 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12679 ] {
12680 let supervisor = Supervisor::new_for_test(
12681 Arc::new(Registry::default()),
12682 RestartPolicy::new(1, Duration::ZERO),
12683 );
12684 let runtime = supervisor.runtime_config();
12685 #[cfg(unix)]
12686 if shutdown {
12687 runtime.child_roster.close();
12688 }
12689 #[cfg(not(unix))]
12690 let _ = shutdown;
12691 let spec = windowed_crash_spec("exit-publication-order");
12692 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12693 ModuleState::Running,
12694 enabled,
12695 )));
12696 lock_snapshot(&snapshot).unwrap().process_alive = true;
12697 let reaped_snapshot = snapshot.clone();
12698 let history = runtime.terminal_ring.clone();
12699 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12700 let reap = std::thread::spawn(move || {
12701 if wait_error {
12702 super::fail_after_wait_error(
12703 &spec.module_id,
12704 &reaped_snapshot,
12705 &runtime.terminal_ring,
12706 &runtime.spawn_events,
12707 "wait failed".into(),
12708 &runtime.failure_context(true),
12709 );
12710 } else {
12711 tokio::runtime::Builder::new_current_thread()
12712 .enable_all()
12713 .build()
12714 .unwrap()
12715 .block_on(on_child_exit(
12716 &spec,
12717 runtime.restart_policy,
12718 &supervisor.registry,
12719 &reaped_snapshot,
12720 &runtime.terminal_ring,
12721 &runtime.spawn_events,
12722 &runtime.child_roster,
12723 ExitReport {
12724 kind,
12725 code: Some(0),
12726 signal: None,
12727 at_ms: 1,
12728 },
12729 true,
12730 ));
12731 }
12732 });
12733 let mut reached = [0];
12734 barrier.read_exact(&mut reached).unwrap();
12735 assert_eq!(&reached, b"R");
12736 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12737 let before_history = history.lock().unwrap().snapshot();
12738 barrier.write_all(b"X").unwrap();
12739 reap.join().unwrap();
12740 assert!(
12741 before_alive,
12742 "{kind:?}: death was published before recording"
12743 );
12744 assert!(before_history.entries.is_empty());
12745 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12746 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12747 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12748 }
12749 }
12750
12751 #[tokio::test]
12752 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12753 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12754 let path = dir.join("terminals.jsonl");
12755 std::fs::create_dir(&path).unwrap();
12756 let supervisor = Supervisor::new_for_test(
12757 Arc::new(Registry::default()),
12758 RestartPolicy::new(0, Duration::ZERO),
12759 )
12760 .with_terminal_journal(path, "health-budget-journal-failure".into());
12761 let runtime = supervisor.runtime_config();
12762 let spec = ModuleSpec {
12763 program: fake_aft_stub_path(),
12764 protocol: ModuleProtocol::None,
12765 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12766 ..windowed_crash_spec("health-budget-journal-failure")
12767 };
12768 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12769 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12770 health_restart_child(
12771 &spec,
12772 &runtime,
12773 &supervisor.registry,
12774 &supervisor.process_liveness,
12775 &snapshot,
12776 &mut child,
12777 SupervisorHealthStatus::Failing,
12778 None,
12779 1,
12780 )
12781 .await
12782 .unwrap();
12783 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12784 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12785 let history = runtime
12786 .terminal_ring
12787 .lock()
12788 .unwrap()
12789 .durable_history(&spec.module_id);
12790 assert!(history.journal_write_failures > 0);
12791 assert_eq!(history.entries.len(), 1);
12792 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12793 }
12794
12795 #[test]
12796 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12797 let supervisor = Supervisor::new_for_test(
12798 Arc::new(Registry::default()),
12799 RestartPolicy::new(0, Duration::ZERO),
12800 );
12801 let runtime = supervisor.runtime_config();
12802 let spec = ModuleSpec {
12803 program: fake_aft_stub_path(),
12804 protocol: ModuleProtocol::None,
12805 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12806 ..windowed_crash_spec("health-budget-recording-panic")
12807 };
12808 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12809 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12810 tokio::runtime::Builder::new_current_thread()
12811 .enable_all()
12812 .build()
12813 .unwrap()
12814 .block_on(async {
12815 let mut child =
12816 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12817 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12818 health_restart_child(
12819 &spec,
12820 &runtime,
12821 &supervisor.registry,
12822 &supervisor.process_liveness,
12823 &snapshot,
12824 &mut child,
12825 SupervisorHealthStatus::Failing,
12826 None,
12827 1,
12828 )
12829 .await
12830 .unwrap();
12831 });
12832 }));
12833 let panic = result.expect_err("recording must still unwind");
12834 assert_eq!(
12835 panic.downcast_ref::<String>().map(String::as_str),
12836 Some("spawn event sequence exhausted")
12837 );
12838 let state = lock_snapshot(&snapshot).unwrap();
12839 assert_eq!(state.state, ModuleState::Failed);
12840 assert!(!state.process_alive);
12841 assert!(state.last_exit.is_some());
12842 }
12843
12844 #[tokio::test]
12845 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12846 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12847 let path = dir.join("terminals.jsonl");
12848 std::fs::create_dir(&path).unwrap();
12849 let supervisor = Supervisor::new_for_test(
12850 Arc::new(Registry::default()),
12851 RestartPolicy::new(0, Duration::ZERO),
12852 )
12853 .with_terminal_journal(path, "budget-journal-failure".into());
12854 let runtime = supervisor.runtime_config();
12855 let spec = windowed_crash_spec("budget-journal-failure");
12856 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12857 assert!(matches!(
12858 on_child_exit(
12859 &spec,
12860 runtime.restart_policy,
12861 &supervisor.registry,
12862 &snapshot,
12863 &runtime.terminal_ring,
12864 &runtime.spawn_events,
12865 &runtime.child_roster,
12866 crash_exit_report(1),
12867 true,
12868 )
12869 .await,
12870 NextAction::Stop { .. }
12871 ));
12872 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12873 let history = runtime
12874 .terminal_ring
12875 .lock()
12876 .unwrap()
12877 .durable_history(&spec.module_id);
12878 assert!(history.journal_write_failures > 0);
12879 assert_eq!(history.entries.len(), 1);
12880 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12881 }
12882
12883 #[test]
12884 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12885 let supervisor = Supervisor::new_for_test(
12886 Arc::new(Registry::default()),
12887 RestartPolicy::new(0, Duration::ZERO),
12888 );
12889 let runtime = supervisor.runtime_config();
12890 let spec = windowed_crash_spec("budget-recording-panic");
12891 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12892 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12893 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12896 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12897 tokio::runtime::Builder::new_current_thread()
12898 .enable_all()
12899 .build()
12900 .unwrap()
12901 .block_on(on_child_exit(
12902 &spec,
12903 runtime.restart_policy,
12904 &supervisor.registry,
12905 &snapshot,
12906 &runtime.terminal_ring,
12907 &runtime.spawn_events,
12908 &runtime.child_roster,
12909 crash_exit_report(1),
12910 true,
12911 ))
12912 }));
12913 let panic = result.err().expect("recording must still unwind");
12914 assert_eq!(
12915 panic.downcast_ref::<String>().map(String::as_str),
12916 Some("spawn event sequence exhausted")
12917 );
12918 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12919 }
12920
12921 #[cfg(unix)]
12924 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12925 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12926 for disable in [false, true] {
12927 let label = if disable {
12928 "none-requested-disable"
12929 } else {
12930 "none-requested-stop"
12931 };
12932 let dir = cortexkit_test_support::ScratchDir::new(label);
12933 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12934 let supervisor = Supervisor::new_for_test(
12935 Arc::new(Registry::default()),
12936 RestartPolicy::new(3, Duration::ZERO),
12937 );
12938 let module = supervisor.spawn(spec).unwrap();
12939 wait_for_file(&ready).await;
12940
12941 if disable {
12942 module.set_enabled(false).await.unwrap();
12943 } else {
12944 module.stop().await.unwrap();
12945 }
12946 assert!(
12947 marker.exists(),
12948 "{label}: the child must have left through its SIGTERM handler with exit 0"
12949 );
12950
12951 sleep(Duration::from_millis(500)).await;
12954 let status = module.status().unwrap();
12955 let expected = if disable {
12956 ModuleState::Disabled
12957 } else {
12958 ModuleState::Stopped
12959 };
12960 assert_eq!(status.state, expected, "{label}");
12961 assert_eq!(
12962 status.spawn_generation, 1,
12963 "{label}: respawned after a requested stop"
12964 );
12965 let history = module.terminal_history();
12966 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12967 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12968 assert_ne!(
12969 history.entries[0].disposition,
12970 TerminalDisposition::Restarting,
12971 "{label}"
12972 );
12973 }
12974 }
12975
12976 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12979 async fn subc_wire_clean_exit_is_still_a_stop() {
12980 let supervisor = Supervisor::new_for_test(
12981 Arc::new(Registry::default()),
12982 RestartPolicy::new(3, Duration::ZERO),
12983 );
12984 let module = supervisor
12985 .spawn(ModuleSpec {
12986 module_id: "wire-clean-exit".to_string(),
12987 program: fake_aft_stub_path(),
12988 args: Vec::new(),
12989 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12990 reserved: false,
12991 reserved_prefixes: Vec::new(),
12992 protocol: ModuleProtocol::Subc,
12993 overlap: Default::default(),
12994 })
12995 .unwrap();
12996
12997 let deadline = Instant::now() + Duration::from_secs(10);
12998 while module.terminal_history().entries.is_empty() {
12999 assert!(Instant::now() < deadline, "module never exited");
13000 sleep(Duration::from_millis(10)).await;
13001 }
13002 sleep(Duration::from_millis(500)).await;
13004 let status = module.status().unwrap();
13005 assert_eq!(status.state, ModuleState::Stopped);
13006 assert_eq!(status.spawn_generation, 1);
13007 let history = module.terminal_history();
13008 assert_eq!(history.entries.len(), 1, "{history:?}");
13009 assert_eq!(history.entries[0].exit_code, Some(0));
13010 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13011 }
13012
13013 #[cfg(unix)]
13014 #[tokio::test]
13015 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13016 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13017 let record = dir.join("live-children.json");
13018 let supervisor = Supervisor::new_for_test(
13019 Arc::new(Registry::default()),
13020 RestartPolicy::new(0, Duration::ZERO),
13021 );
13022 let mut runtime = supervisor.runtime_config();
13023 runtime.child_roster.record_to(record.clone());
13024 let gate = Arc::new(super::ReloadExitRecordGate::default());
13025 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13026 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13027 let spec = ModuleSpec {
13028 module_id: "reload-exit-roster".into(),
13029 program: fake_aft_stub_path(),
13030 args: Vec::new(),
13031 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13032 reserved: false,
13033 reserved_prefixes: Vec::new(),
13034 protocol: ModuleProtocol::Subc,
13035 overlap: Default::default(),
13036 };
13037 let mut child = None;
13038 let reload = super::finish_reload_child(
13039 &spec,
13040 &runtime,
13041 &supervisor.registry,
13042 &supervisor.process_liveness,
13043 &snapshot,
13044 &mut child,
13045 );
13046 tokio::pin!(reload);
13047 tokio::select! {
13048 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13049 _ = gate.reached.notified() => {}
13050 }
13051 assert!(runtime
13052 .terminal_ring
13053 .lock()
13054 .unwrap()
13055 .snapshot()
13056 .entries
13057 .is_empty());
13058 assert_eq!(
13059 crate::live_children::read_record(&record).unwrap().len(),
13060 1,
13061 "shutdown must still wait for the reaped child until its terminal record exists"
13062 );
13063 runtime.child_roster.close();
13064 gate.resume.notify_one();
13065 assert!(reload.await.is_err());
13066 assert!(crate::live_children::read_record(&record)
13067 .unwrap()
13068 .is_empty());
13069 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13070 assert_eq!(history.entries.len(), 1);
13071 assert_eq!(
13072 history.entries[0].disposition,
13073 TerminalDisposition::DaemonShutdown
13074 );
13075 }
13076
13077 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13081 async fn every_restart_increment_path_advances_lifetime_count() {
13082 let supervisor = Supervisor::new_for_test(
13083 Arc::new(Registry::default()),
13084 RestartPolicy::new(1, Duration::ZERO),
13085 );
13086 let runtime = supervisor.runtime_config();
13087 let spec = ModuleSpec {
13088 module_id: "lifetime-increment-path".to_string(),
13089 program: PathBuf::from("/unused/lifetime-increment-path"),
13090 args: Vec::new(),
13091 env: Vec::new(),
13092 reserved: false,
13093 reserved_prefixes: Vec::new(),
13094 protocol: ModuleProtocol::Subc,
13095 overlap: Default::default(),
13096 };
13097
13098 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13099 assert!(matches!(
13100 on_child_exit(
13101 &spec,
13102 runtime.restart_policy,
13103 &supervisor.registry,
13104 &crash_snapshot,
13105 &runtime.terminal_ring,
13106 &runtime.spawn_events,
13107 &runtime.child_roster,
13108 ExitReport {
13109 kind: ExitKind::Crash,
13110 code: Some(1),
13111 signal: None,
13112 at_ms: 1,
13113 },
13114 true,
13115 )
13116 .await,
13117 NextAction::Restart { schedule: _ }
13118 ));
13119 let (crash_restarts, crash_lifetime) = {
13120 let state = lock_snapshot(&crash_snapshot).unwrap();
13121 (state.crash_restarts.len(), state.lifetime_restarts)
13122 };
13123 assert_eq!(crash_restarts, 1);
13124 assert_eq!(crash_lifetime, 1);
13125
13126 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13127 let mut health_child = None;
13128 assert!(matches!(
13129 health_restart_child(
13130 &spec,
13131 &runtime,
13132 &supervisor.registry,
13133 &supervisor.process_liveness,
13134 &health_snapshot,
13135 &mut health_child,
13136 SupervisorHealthStatus::Failing,
13137 None,
13138 2,
13139 )
13140 .await,
13141 Ok(())
13142 ));
13143 assert!(health_child.is_none());
13144 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13145 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13146 let (health_restarts, health_lifetime) = {
13147 let state = lock_snapshot(&health_snapshot).unwrap();
13148 (state.crash_restarts.len(), state.lifetime_restarts)
13149 };
13150 assert_eq!(health_restarts, 1);
13151 assert_eq!(health_lifetime, 1);
13152
13153 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13154 let mut reload_child = None;
13155 assert!(matches!(
13156 handle_reload_spawn_failure(
13157 &spec,
13158 &runtime,
13159 &supervisor.process_liveness,
13160 &reload_snapshot,
13161 &mut reload_child,
13162 "forced reload spawn failure".to_string(),
13163 )
13164 .await,
13165 Err(SuperviseError::ReloadFailed { .. })
13166 ));
13167 let (reload_restarts, reload_lifetime) = {
13168 let state = lock_snapshot(&reload_snapshot).unwrap();
13169 (state.crash_restarts.len(), state.lifetime_restarts)
13170 };
13171 assert_eq!(reload_restarts, 1);
13172 assert_eq!(reload_lifetime, 1);
13173 }
13174
13175 #[tokio::test]
13176 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
13177 let supervisor = Supervisor::new_for_test(
13178 Arc::new(Registry::default()),
13179 RestartPolicy::new(3, Duration::ZERO),
13180 );
13181 let runtime = supervisor.runtime_config();
13182 let spec = ModuleSpec {
13183 module_id: "deliberately-severed".to_string(),
13184 program: PathBuf::from("/unused/deliberately-severed"),
13185 args: Vec::new(),
13186 env: Vec::new(),
13187 reserved: false,
13188 reserved_prefixes: Vec::new(),
13189 protocol: ModuleProtocol::Subc,
13190 overlap: Default::default(),
13191 };
13192 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13193 let process = ProcessIdentity {
13194 pid: 41,
13195 start_time: 101,
13196 };
13197 record_deliberate_severance(&snapshot, process).unwrap();
13198 let exit_report = apply_deliberate_severance_marker(
13199 &snapshot,
13200 Some(process),
13201 ExitReport {
13202 kind: ExitKind::Crash,
13203 code: Some(1),
13204 signal: None,
13205 at_ms: 1,
13206 },
13207 );
13208 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
13209
13210 assert!(matches!(
13211 on_child_exit(
13212 &spec,
13213 runtime.restart_policy,
13214 &supervisor.registry,
13215 &snapshot,
13216 &runtime.terminal_ring,
13217 &runtime.spawn_events,
13218 &runtime.child_roster,
13219 exit_report,
13220 true,
13221 )
13222 .await,
13223 NextAction::Restart { schedule: _ }
13224 ));
13225 let state = lock_snapshot(&snapshot).unwrap();
13226 assert_eq!(state.lifetime_restarts, 1);
13227 assert_eq!(state.crash_restarts.len(), 0);
13228 }
13229
13230 #[tokio::test]
13231 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13232 let supervisor = Supervisor::new_for_test(
13233 Arc::new(Registry::default()),
13234 RestartPolicy::new(3, Duration::ZERO),
13235 );
13236 let runtime = supervisor.runtime_config();
13237 let spec = ModuleSpec {
13238 module_id: "genuine-crash".to_string(),
13239 program: PathBuf::from("/unused/genuine-crash"),
13240 args: Vec::new(),
13241 env: Vec::new(),
13242 reserved: false,
13243 reserved_prefixes: Vec::new(),
13244 protocol: ModuleProtocol::Subc,
13245 overlap: Default::default(),
13246 };
13247 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13248
13249 assert!(matches!(
13250 on_child_exit(
13251 &spec,
13252 runtime.restart_policy,
13253 &supervisor.registry,
13254 &snapshot,
13255 &runtime.terminal_ring,
13256 &runtime.spawn_events,
13257 &runtime.child_roster,
13258 ExitReport {
13259 kind: ExitKind::Crash,
13260 code: Some(1),
13261 signal: None,
13262 at_ms: 1,
13263 },
13264 true,
13265 )
13266 .await,
13267 NextAction::Restart { schedule: _ }
13268 ));
13269 let state = lock_snapshot(&snapshot).unwrap();
13270 assert_eq!(state.lifetime_restarts, 1);
13271 assert_eq!(state.crash_restarts.len(), 1);
13272 }
13273
13274 fn crash_exit_report(at_ms: u64) -> ExitReport {
13275 ExitReport {
13276 kind: ExitKind::Crash,
13277 code: Some(1),
13278 signal: None,
13279 at_ms,
13280 }
13281 }
13282
13283 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13284 ModuleSpec {
13285 module_id: module_id.to_string(),
13286 program: PathBuf::from("/unused").join(module_id),
13287 args: Vec::new(),
13288 env: Vec::new(),
13289 reserved: false,
13290 reserved_prefixes: Vec::new(),
13291 protocol: ModuleProtocol::Subc,
13292 overlap: Default::default(),
13293 }
13294 }
13295
13296 #[tokio::test]
13302 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13303 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13304 let supervisor = Supervisor::new_for_test(
13305 Arc::new(Registry::default()),
13306 RestartPolicy::new(2, Duration::ZERO),
13307 );
13308 let runtime = supervisor.runtime_config();
13309 let spec = windowed_crash_spec("crash-loop-in-window");
13310 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13311
13312 for attempt in 1..=2 {
13313 assert!(
13314 matches!(
13315 on_child_exit(
13316 &spec,
13317 runtime.restart_policy,
13318 &supervisor.registry,
13319 &snapshot,
13320 &runtime.terminal_ring,
13321 &runtime.spawn_events,
13322 &runtime.child_roster,
13323 crash_exit_report(attempt),
13324 true,
13325 )
13326 .await,
13327 NextAction::Restart { schedule: _ }
13328 ),
13329 "crash {attempt} is inside the budget and must respawn"
13330 );
13331 }
13332
13333 assert!(matches!(
13334 on_child_exit(
13335 &spec,
13336 runtime.restart_policy,
13337 &supervisor.registry,
13338 &snapshot,
13339 &runtime.terminal_ring,
13340 &runtime.spawn_events,
13341 &runtime.child_roster,
13342 crash_exit_report(3),
13343 true,
13344 )
13345 .await,
13346 NextAction::Stop { .. }
13347 ));
13348
13349 {
13350 let state = lock_snapshot(&snapshot).unwrap();
13351 assert_eq!(state.state, ModuleState::Failed);
13352 assert_eq!(state.crash_restarts.len(), 2);
13353 assert_eq!(state.lifetime_restarts, 2);
13354 }
13355
13356 let history = runtime
13357 .terminal_ring
13358 .lock()
13359 .expect("terminal ring is not poisoned")
13360 .snapshot();
13361 let last = history
13362 .entries
13363 .last()
13364 .expect("the refused crash is retained");
13365 assert_eq!(last.disposition, TerminalDisposition::Failed);
13366 assert_eq!(
13367 last.disposition_detail.as_deref(),
13368 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13369 );
13370
13371 let captured = crate::router::test_log::captured_logs(&logs);
13372 assert!(
13373 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13374 "the stop must be logged with its window: {captured}"
13375 );
13376 }
13377
13378 #[tokio::test]
13386 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13387 let supervisor = Supervisor::new_for_test(
13388 Arc::new(Registry::default()),
13389 RestartPolicy::new(2, Duration::ZERO),
13390 );
13391 let runtime = supervisor.runtime_config();
13392 let spec = windowed_crash_spec("crash-across-windows");
13393 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13394
13395 for attempt in 1..=2 {
13396 assert!(matches!(
13397 on_child_exit(
13398 &spec,
13399 runtime.restart_policy,
13400 &supervisor.registry,
13401 &snapshot,
13402 &runtime.terminal_ring,
13403 &runtime.spawn_events,
13404 &runtime.child_roster,
13405 crash_exit_report(attempt),
13406 true,
13407 )
13408 .await,
13409 NextAction::Restart { schedule: _ }
13410 ));
13411 }
13412
13413 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13416 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13417 })
13418 .unwrap();
13419
13420 assert!(
13421 matches!(
13422 on_child_exit(
13423 &spec,
13424 runtime.restart_policy,
13425 &supervisor.registry,
13426 &snapshot,
13427 &runtime.terminal_ring,
13428 &runtime.spawn_events,
13429 &runtime.child_roster,
13430 crash_exit_report(3),
13431 true,
13432 )
13433 .await,
13434 NextAction::Restart { schedule: _ }
13435 ),
13436 "a crash older than the window must not hold a budget slot"
13437 );
13438
13439 let state = lock_snapshot(&snapshot).unwrap();
13440 assert_eq!(state.state, ModuleState::Restarting);
13441 assert_eq!(
13442 state.crash_restarts.len(),
13443 2,
13444 "the aged instant is dropped and the new one takes its place"
13445 );
13446 assert_eq!(
13447 state.lifetime_restarts, 3,
13448 "the ledger counts every restart, including the ones the window forgot"
13449 );
13450 }
13451
13452 #[tokio::test]
13457 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13458 let supervisor = Supervisor::new_for_test(
13459 Arc::new(Registry::default()),
13460 RestartPolicy::new(2, Duration::ZERO),
13461 );
13462 let runtime = supervisor.runtime_config();
13463 let spec = windowed_crash_spec("operator-cleared-budget");
13464 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13465
13466 for attempt in 1..=2 {
13467 assert!(matches!(
13468 on_child_exit(
13469 &spec,
13470 runtime.restart_policy,
13471 &supervisor.registry,
13472 &snapshot,
13473 &runtime.terminal_ring,
13474 &runtime.spawn_events,
13475 &runtime.child_roster,
13476 crash_exit_report(attempt),
13477 true,
13478 )
13479 .await,
13480 NextAction::Restart { schedule: _ }
13481 ));
13482 }
13483
13484 reset_restart_count(&snapshot, &spec.module_id).unwrap();
13485 {
13486 let state = lock_snapshot(&snapshot).unwrap();
13487 assert!(
13488 state.crash_restarts.is_empty(),
13489 "an operator restart returns the full budget"
13490 );
13491 assert_eq!(
13492 state.lifetime_restarts, 2,
13493 "clearing the budget must not unmake the crashes"
13494 );
13495 }
13496
13497 assert!(
13498 matches!(
13499 on_child_exit(
13500 &spec,
13501 runtime.restart_policy,
13502 &supervisor.registry,
13503 &snapshot,
13504 &runtime.terminal_ring,
13505 &runtime.spawn_events,
13506 &runtime.child_roster,
13507 crash_exit_report(3),
13508 true,
13509 )
13510 .await,
13511 NextAction::Restart { schedule: _ }
13512 ),
13513 "the cleared budget must be spendable again"
13514 );
13515 let state = lock_snapshot(&snapshot).unwrap();
13516 assert_eq!(state.crash_restarts.len(), 1);
13517 assert_eq!(state.lifetime_restarts, 3);
13518 }
13519
13520 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13521 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13522 let severed = ProcessIdentity {
13523 pid: 41,
13524 start_time: 101,
13525 };
13526 let successor = ProcessIdentity {
13527 pid: 41,
13528 start_time: 202,
13529 };
13530 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13531 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13532 state.pid = Some(successor.pid);
13533 state.process_start_time = Some(successor.start_time);
13534 })
13535 .unwrap();
13536 assert!(!module.record_deliberate_severance(severed).unwrap());
13537
13538 let exit_report = apply_deliberate_severance_marker(
13539 &module.inner.snapshot,
13540 Some(successor),
13541 ExitReport {
13542 kind: ExitKind::Crash,
13543 code: Some(1),
13544 signal: None,
13545 at_ms: 1,
13546 },
13547 );
13548
13549 assert_eq!(exit_report.kind, ExitKind::Crash);
13550 }
13551
13552 #[tokio::test]
13553 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13554 let registry = Registry::default();
13555 let supervisor = Supervisor::new_for_test(
13556 Arc::new(Registry::default()),
13557 RestartPolicy::new(3, Duration::ZERO),
13558 );
13559 let runtime = supervisor.runtime_config();
13560 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13561 let spec = ModuleSpec {
13562 module_id: "drain-deliberate-severance".to_string(),
13563 program: fake_aft_stub_path(),
13564 args: Vec::new(),
13565 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13566 reserved: false,
13567 reserved_prefixes: Vec::new(),
13568 protocol: ModuleProtocol::Subc,
13569 overlap: Default::default(),
13570 };
13571 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13572 let process = ProcessIdentity {
13573 pid: 41,
13574 start_time: 101,
13575 };
13576 child.process_identity = Some(process);
13577 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13578 state.pid = Some(process.pid);
13579 state.process_start_time = Some(process.start_time);
13580 })
13581 .unwrap();
13582 record_deliberate_severance(&snapshot, process).unwrap();
13583
13584 drain_child_to_state(
13585 &spec.module_id,
13586 spec.protocol,
13587 StopNotice::SentOverConnection,
13590 ®istry,
13591 None,
13592 &snapshot,
13593 &runtime.terminal_ring,
13594 &runtime.spawn_events,
13595 child,
13596 Duration::from_secs(1),
13597 ModuleState::Stopped,
13598 Some(false),
13599 &super::FailureContext::operator(),
13600 )
13601 .await
13602 .unwrap();
13603
13604 let state = lock_snapshot(&snapshot).unwrap();
13605 assert_eq!(
13606 state.last_exit.as_ref().map(|exit| exit.kind),
13607 Some(ExitKind::DeliberateSeverance)
13608 );
13609 assert_eq!(state.lifetime_restarts, 1);
13610 assert_eq!(state.crash_restarts.len(), 0);
13611 drop(state);
13612 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13613 assert_eq!(
13614 history.entries[0].exit_kind,
13615 subc_control::TerminalExitKind::DeliberateSeverance
13616 );
13617 }
13618
13619 #[tokio::test]
13620 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13621 let registry = Registry::default();
13622 let supervisor = Supervisor::new_for_test(
13623 Arc::new(Registry::default()),
13624 RestartPolicy::new(3, Duration::ZERO),
13625 );
13626 let runtime = supervisor.runtime_config();
13627 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13628 let spec = ModuleSpec {
13629 module_id: "ordinary-drain".to_string(),
13630 program: fake_aft_stub_path(),
13631 args: Vec::new(),
13632 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13633 reserved: false,
13634 reserved_prefixes: Vec::new(),
13635 protocol: ModuleProtocol::Subc,
13636 overlap: Default::default(),
13637 };
13638 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13639
13640 drain_child_to_state(
13641 &spec.module_id,
13642 spec.protocol,
13643 StopNotice::SentOverConnection,
13646 ®istry,
13647 None,
13648 &snapshot,
13649 &runtime.terminal_ring,
13650 &runtime.spawn_events,
13651 child,
13652 Duration::from_secs(1),
13653 ModuleState::Stopped,
13654 Some(false),
13655 &super::FailureContext::operator(),
13656 )
13657 .await
13658 .unwrap();
13659
13660 let state = lock_snapshot(&snapshot).unwrap();
13661 assert_eq!(
13662 state.last_exit.as_ref().map(|exit| exit.kind),
13663 Some(ExitKind::Crash)
13664 );
13665 assert_eq!(state.lifetime_restarts, 0);
13666 assert_eq!(state.crash_restarts.len(), 0);
13667 }
13668
13669 #[test]
13670 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13671 assert!(!include_str!("server.rs")
13677 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13678 }
13679
13680 #[test]
13687 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13688 assert!(drained_after_quiescence_wait(&Ok(true)));
13689 assert!(!drained_after_quiescence_wait(&Ok(false)));
13690 assert!(!drained_after_quiescence_wait(&Err(
13691 SuperviseError::StatePoisoned { module_id: None }
13692 )));
13693 }
13694
13695 #[test]
13704 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13705 let ring = Arc::new(Mutex::new(TerminalRing::new(
13706 TerminalRingConfig::default(),
13707 0,
13708 )));
13709 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13710
13711 let snapshot = ring.lock().unwrap().snapshot();
13712 assert_eq!(snapshot.entries.len(), 1);
13713 let entry = &snapshot.entries[0];
13714 assert_eq!(entry.exit_code, None);
13715 assert_eq!(entry.exit_signal, None);
13716 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13717 }
13718
13719 #[test]
13720 fn wait_error_exit_path_preserves_spawn_event_density() {
13721 let feed = super::SpawnEventFeed::default();
13722 feed.configure_incarnation("wait-error-density".to_string());
13723 feed.emit_spawned("wait-error", 41, 1);
13724 let ring = Arc::new(Mutex::new(TerminalRing::new(
13725 TerminalRingConfig::default(),
13726 0,
13727 )));
13728
13729 record_wait_error_terminal("wait-error", &ring, &feed);
13730 feed.emit_spawned("after-wait-error", 42, 2);
13731
13732 let state = feed.0.lock().unwrap();
13733 let sequences = state
13734 .events
13735 .iter()
13736 .map(|event| event.cursor.seq)
13737 .collect::<Vec<_>>();
13738 assert_eq!(sequences, vec![1, 2, 3]);
13739 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13740 assert_eq!(state.events[1].exit_code, None);
13741 assert_eq!(state.events[1].exit_signal, None);
13742 }
13743
13744 #[test]
13748 fn wait_error_exit_report_is_classified_as_a_crash() {
13749 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13750 }
13751}
13752
13753#[cfg(all(test, unix))]
13754mod restart_ordering_tests {
13755 use super::*;
13756 use std::io::Write;
13757 use tokio::io::AsyncReadExt;
13758
13759 #[test]
13760 #[ignore = "child process fixture"]
13761 fn serving_child() {
13762 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13763 return;
13764 };
13765 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13766 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13767 for stream in listener.incoming() {
13768 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13769 }
13770 }
13771
13772 async fn restart_before_publication(requests: usize) {
13773 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13774 let address_file = home.path().join("address");
13775 let spec = ModuleSpec {
13776 module_id: "restart-serving-child".into(),
13777 program: std::env::current_exe().unwrap(),
13778 args: vec![
13779 "--exact".into(),
13780 "supervise::restart_ordering_tests::serving_child".into(),
13781 "--ignored".into(),
13782 "--nocapture".into(),
13783 ],
13784 env: vec![(
13785 "RESTART_TEST_ADDRESS_FILE".into(),
13786 address_file.display().to_string(),
13787 )],
13788 reserved: false,
13789 reserved_prefixes: vec![],
13790 protocol: ModuleProtocol::None,
13791 overlap: Default::default(),
13792 };
13793 let mut supervisor =
13794 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13795 let (spawned_tx, spawned_rx) = oneshot::channel();
13796 let spawned_tx = Mutex::new(Some(spawned_tx));
13797 let (release_tx, release_rx) = std::sync::mpsc::channel();
13798 let release_rx = Mutex::new(release_rx);
13799 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13802 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13803 release_rx
13804 .lock()
13805 .unwrap()
13806 .recv_timeout(Duration::from_secs(20))
13807 .unwrap();
13808 })));
13809 let registered = supervisor
13810 .register_configured_with_health(
13811 spec.clone(),
13812 true,
13813 HealthConfig::default(),
13814 None,
13815 RestartPolicy::new(3, Duration::ZERO),
13816 )
13817 .unwrap();
13818 let module = registered.module.clone();
13819 let startup = tokio::task::spawn_blocking(move || {
13820 supervisor.start_registered_configured(registered, None, true)
13821 });
13822 spawned_rx.await.unwrap();
13823 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13824 let address = loop {
13825 if let Ok(address) = std::fs::read_to_string(&address_file) {
13826 if !address.is_empty() {
13827 break address;
13828 }
13829 }
13830 tokio::time::sleep(Duration::from_millis(1)).await;
13831 };
13832 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13833 let mut reply = String::new();
13834 stream.read_to_string(&mut reply).await.unwrap();
13835 reply.trim().parse::<u32>().unwrap()
13836 })
13837 .await
13838 .expect("child must serve before generation publication");
13839 assert_eq!(
13840 lock_snapshot(&module.inner.snapshot)
13841 .unwrap()
13842 .spawn_generation,
13843 0
13844 );
13845 let mut replies = Vec::new();
13846 for _ in 0..requests {
13847 let mut restart = Box::pin(module.restart(Some(0)));
13848 std::future::poll_fn(|cx| {
13851 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13852 std::task::Poll::Ready(())
13853 })
13854 .await;
13855 replies.push(restart);
13856 }
13857 release_tx.send(()).unwrap();
13858 startup.await.unwrap().unwrap();
13859 for reply in replies {
13860 reply.await.unwrap();
13861 }
13862 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13865 loop {
13866 if lock_snapshot(&module.inner.snapshot)
13867 .unwrap()
13868 .spawn_generation
13869 > 1
13870 {
13871 break;
13872 }
13873 tokio::time::sleep(Duration::from_millis(1)).await;
13874 }
13875 })
13876 .await;
13877 module.update_spec_for_test(spec).await.unwrap();
13881 let (generation, pid) = {
13882 let state = lock_snapshot(&module.inner.snapshot).unwrap();
13883 (state.spawn_generation, state.pid)
13884 };
13885 module.stop().await.unwrap();
13886 replacement.expect("restart must publish a replacement for the already-serving child");
13887 assert_ne!(
13888 pid,
13889 Some(old_pid),
13890 "a serving child predating the restart must be replaced"
13891 );
13892 assert_eq!(generation, 2, "queued restarts must share one replacement");
13893 }
13894
13895 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13896 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13897 restart_before_publication(1).await;
13898 }
13899
13900 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13901 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13902 restart_before_publication(2).await;
13903 }
13904
13905 #[tokio::test]
13906 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13907 let supervisor =
13908 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13909 let registered = supervisor
13910 .register_configured_with_health(
13911 ModuleSpec {
13912 module_id: "queued-before-replacement".into(),
13913 program: "/bin/sleep".into(),
13918 args: vec!["600".into()],
13919 env: vec![],
13920 reserved: false,
13921 reserved_prefixes: vec![],
13922 protocol: ModuleProtocol::None,
13923 overlap: Default::default(),
13924 },
13925 true,
13926 HealthConfig::default(),
13927 None,
13928 RestartPolicy::default(),
13929 )
13930 .unwrap();
13931 let RegisteredConfiguredModule {
13932 mut spec,
13933 mut runtime,
13934 module,
13935 mut commands,
13936 } = registered;
13937 let snapshot = &module.inner.snapshot;
13938 let mut restart = Box::pin(module.restart(Some(0)));
13939 std::future::poll_fn(|cx| {
13940 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13941 std::task::Poll::Ready(())
13942 })
13943 .await;
13944 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13947 let replacement_pid = child.as_ref().unwrap().id();
13948 let command = commands.recv().await.unwrap();
13949 assert!(
13950 handle_supervisor_command(
13951 command,
13952 &mut spec,
13953 &mut runtime,
13954 &supervisor.registry,
13955 &supervisor.process_liveness,
13956 snapshot,
13957 &mut child,
13958 &mut commands,
13959 &mut VecDeque::new(),
13960 )
13961 .await
13962 );
13963 restart.await.unwrap();
13964 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13965 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13966 if let Some(mut child) = child {
13967 child.start_kill().unwrap();
13968 child.wait().await.unwrap();
13969 }
13970 assert_eq!(
13971 kept_pid, replacement_pid,
13972 "a newer replacement must survive"
13973 );
13974 assert!(
13975 !pending,
13976 "a satisfied restart must not schedule another spawn"
13977 );
13978 }
13979}
13980
13981#[cfg(all(test, unix))]
13982mod health_event_tests {
13983 use super::*;
13984 use cortexkit_test_support::ScratchDir;
13985 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13986
13987 struct Actor {
13988 module: SupervisedModule,
13989 registry: Arc<Registry>,
13990 forwarding: Arc<ForwardingTable>,
13991 spec: ModuleSpec,
13992 health: HealthConfig,
13993 rx: mpsc::Receiver<crate::router::OutboundFrame>,
13994 _home: ScratchDir,
13995 }
13996
13997 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13998 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14002 let tick = Instant::now();
14003 loop {
14004 if let Some(value) = observe() {
14005 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14006 return value;
14007 }
14008 assert!(
14009 std::time::Instant::now() < deadline,
14010 "timed out waiting for {reason}"
14011 );
14012 tokio::task::yield_now().await;
14013 }
14014 }
14015
14016 impl Actor {
14017 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14018 let home = ScratchDir::new("health-events");
14019 let registry = Arc::new(Registry::default());
14020 let forwarding = Arc::new(ForwardingTable::default());
14021 let health = HealthConfig {
14022 cadence,
14023 deadline: Duration::from_secs(3600),
14024 ..HealthConfig::default()
14025 };
14026 let supervisor =
14027 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
14028 .with_forwarding(forwarding.clone())
14029 .with_health_config(health.clone());
14030 let spec = ModuleSpec {
14031 module_id: "event-health-child".into(),
14032 program: PathBuf::from("/bin/sleep"),
14040 args: vec!["600".into()],
14043 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14044 .into_iter()
14045 .map(|key| (key.into(), home.path().display().to_string()))
14046 .collect(),
14047 reserved: false,
14048 reserved_prefixes: vec![],
14049 protocol,
14050 overlap: Default::default(),
14051 };
14052 let module = supervisor.spawn(spec.clone()).unwrap();
14053 let (_, rx) = mpsc::channel(8);
14054 let actor = Self {
14055 module,
14056 registry,
14057 forwarding,
14058 spec,
14059 health,
14060 rx,
14061 _home: home,
14062 };
14063 actor
14064 .wait_for_select(|checkpoint| checkpoint.generation > 0)
14065 .await;
14066 actor
14067 }
14068
14069 fn turns(&self) -> u64 {
14070 lock_snapshot(&self.module.inner.snapshot)
14071 .unwrap()
14072 .actor_turns
14073 }
14074
14075 async fn wait_for_select(
14076 &self,
14077 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14078 ) -> ActorSelectCheckpoint {
14079 wait_for("actor select checkpoint", || {
14080 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14081 state.actor_select.filter(|checkpoint| {
14082 checkpoint.generation == state.spawn_generation
14083 && state.state == ModuleState::Running
14084 && state.process_alive
14085 && state.reported_pid().is_some()
14086 && expected(checkpoint)
14087 })
14088 })
14089 .await
14090 }
14091
14092 fn hello(&mut self, connection: u64, health: bool) {
14093 let manifest =
14094 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14095 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14096 .build();
14097 let connection = ConnectionId::new(connection);
14098 let (tx, rx) = mpsc::channel(8);
14099 self.rx = rx;
14100 self.forwarding
14101 .register_module_connection(
14102 connection,
14103 self.spec.module_id.clone(),
14104 subc_protocol::PROTOCOL_VERSION,
14105 Concurrency::ModuleManaged,
14106 FrameSink::new(tx),
14107 )
14108 .unwrap();
14109 self.registry
14110 .register_with_control_ops(
14111 manifest,
14112 subc_protocol::PROTOCOL_VERSION,
14113 connection,
14114 if health {
14115 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
14116 } else {
14117 vec![]
14118 },
14119 )
14120 .unwrap();
14121 }
14122
14123 async fn probe(&mut self) -> crate::router::OutboundFrame {
14124 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
14125 Ok(frame) => Some(frame),
14126 Err(mpsc::error::TryRecvError::Empty) => None,
14127 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
14128 })
14129 .await;
14130 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14131 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14132 frame
14133 }
14134
14135 fn no_probe(&mut self) {
14136 assert!(matches!(
14137 self.rx.try_recv(),
14138 Err(mpsc::error::TryRecvError::Empty)
14139 ));
14140 }
14141
14142 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
14143 self.forwarding
14144 .complete_module_control_rpc(
14145 ConnectionId::new(connection),
14146 frame.header.corr,
14147 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14148 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14149 status: HealthStatus::Ok,
14150 detail: None,
14151 metrics: None,
14152 }),
14153 )
14154 .unwrap();
14155 }
14156 }
14157
14158 #[tokio::test(start_paused = true)]
14159 async fn no_health_children_park_for_sixty_seconds() {
14160 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14161 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14162 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14163 unadvertised.hello(1, false);
14164 for _ in 0..6000 {
14169 tokio::time::advance(Duration::from_millis(10)).await;
14170 tokio::task::yield_now().await;
14171 }
14172 for actor in [&none, &unregistered, &unadvertised] {
14173 assert!(
14174 actor.turns() <= 5,
14175 "no-health actor woke {} times",
14176 actor.turns()
14177 );
14178 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
14179 }
14180 }
14181
14182 #[tokio::test(start_paused = true)]
14183 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
14184 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
14188 let tick = Instant::now();
14189 actor.hello(1, true);
14190 actor.probe().await;
14191 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
14192 }
14193
14194 #[tokio::test(start_paused = true)]
14195 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
14196 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14197 let hello_tick = Instant::now();
14198 actor.hello(1, true);
14199 let armed = actor
14200 .wait_for_select(|checkpoint| {
14201 checkpoint.registered_connection == Some(ConnectionId::new(1))
14202 && checkpoint.next_probe_at.is_some()
14203 })
14204 .await;
14205 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14206 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
14207 let first_deadline = hello_tick + delay;
14208 assert_eq!(armed.next_probe_at, Some(first_deadline));
14209 tokio::time::advance(Duration::from_secs(29)).await;
14210 let turns = actor.turns();
14211 actor
14212 .registry
14213 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
14214 .unwrap();
14215 let updated = actor
14216 .wait_for_select(|checkpoint| checkpoint.turn > turns)
14217 .await;
14218 assert_eq!(
14219 updated.next_probe_at,
14220 Some(first_deadline),
14221 "catalog update must not reset cadence"
14222 );
14223 actor.no_probe();
14224 tokio::time::advance(first_deadline - Instant::now()).await;
14225 let frame = actor.probe().await;
14226 actor.answer(1, frame);
14227 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
14228 let next_deadline = Instant::now() + next;
14229 let rearmed = actor
14230 .wait_for_select(|checkpoint| {
14231 checkpoint
14232 .next_probe_at
14233 .is_some_and(|deadline| deadline > first_deadline)
14234 })
14235 .await;
14236 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14237 tokio::time::advance(next - Duration::from_nanos(1)).await;
14238 actor.no_probe();
14239 tokio::time::advance(Duration::from_nanos(1)).await;
14240 actor.probe().await;
14241 }
14242
14243 #[tokio::test(start_paused = true)]
14244 async fn disconnect_disarms_and_reconnect_rearms_health() {
14245 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14246 actor.hello(1, true);
14247 actor
14248 .wait_for_select(|checkpoint| {
14249 checkpoint.registered_connection == Some(ConnectionId::new(1))
14250 && checkpoint.next_probe_at.is_some()
14251 })
14252 .await;
14253 let turns = actor.turns();
14254 let tick = Instant::now();
14255 actor
14256 .registry
14257 .deregister_connection(ConnectionId::new(1))
14258 .unwrap();
14259 let disarmed = actor
14260 .wait_for_select(|checkpoint| {
14261 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14262 })
14263 .await;
14264 assert_eq!(disarmed.next_probe_at, None);
14265 assert_eq!(disarmed.wake_after, None);
14266 assert_eq!(
14267 Instant::now(),
14268 tick,
14269 "disconnect must disarm in the notification tick"
14270 );
14271 tokio::time::advance(Duration::from_secs(60)).await;
14272 actor.no_probe();
14273 let reconnect_tick = Instant::now();
14274 actor.hello(2, true);
14275 let rearmed = actor
14276 .wait_for_select(|checkpoint| {
14277 checkpoint.registered_connection == Some(ConnectionId::new(2))
14278 && checkpoint.next_probe_at.is_some()
14279 })
14280 .await;
14281 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14282 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14283 tokio::time::advance(delay).await;
14284 actor.probe().await;
14285 }
14286
14287 #[tokio::test(start_paused = true)]
14288 async fn reregistration_without_health_disarms_the_old_schedule() {
14289 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14290 actor.hello(1, true);
14291 actor
14292 .wait_for_select(|checkpoint| {
14293 checkpoint.registered_connection == Some(ConnectionId::new(1))
14294 && checkpoint.next_probe_at.is_some()
14295 })
14296 .await;
14297 actor
14298 .registry
14299 .deregister_connection(ConnectionId::new(1))
14300 .unwrap();
14301 actor.hello(2, false);
14302 let disarmed = actor
14303 .wait_for_select(|checkpoint| {
14304 checkpoint.registered_connection == Some(ConnectionId::new(2))
14305 })
14306 .await;
14307 assert_eq!(disarmed.next_probe_at, None);
14308 assert_eq!(disarmed.wake_after, None);
14309 tokio::time::advance(Duration::from_secs(60)).await;
14310 actor.no_probe();
14311 }
14312
14313 #[tokio::test(start_paused = true)]
14314 async fn restart_waits_for_its_own_hello_before_rearming_health() {
14315 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14316 actor.hello(1, true);
14317 let armed = actor
14318 .wait_for_select(|checkpoint| {
14319 checkpoint.registered_connection == Some(ConnectionId::new(1))
14320 && checkpoint.next_probe_at.is_some()
14321 })
14322 .await;
14323 actor
14327 .registry
14328 .deregister_connection(ConnectionId::new(1))
14329 .unwrap();
14330 actor.module.restart(Some(0)).await.unwrap();
14331 let parked = actor
14332 .wait_for_select(|checkpoint| {
14333 checkpoint.generation > armed.generation
14334 && checkpoint.registered_connection.is_none()
14335 && checkpoint.next_probe_at.is_none()
14336 })
14337 .await;
14338 assert_eq!(parked.wake_after, None);
14339 actor.rx.close();
14340 while actor.rx.try_recv().is_ok() {}
14343 let turns = actor.turns();
14344 tokio::time::advance(Duration::from_secs(60)).await;
14345 let after = actor.turns();
14346 eprintln!(
14347 "restart park turns: before={turns} after={after} delta={}",
14348 after - turns
14349 );
14350 assert_eq!(after, turns, "replacement must park before HELLO");
14351 let hello_tick = Instant::now();
14352 actor.hello(2, true);
14353 let rearmed = actor
14354 .wait_for_select(|checkpoint| {
14355 checkpoint.registered_connection == Some(ConnectionId::new(2))
14356 && checkpoint.next_probe_at.is_some()
14357 })
14358 .await;
14359 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14360 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14361 tokio::time::advance(delay).await;
14362 actor.probe().await;
14363 }
14364
14365 #[tokio::test(start_paused = true)]
14366 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14367 use tokio::{
14368 io::{AsyncReadExt, AsyncWriteExt},
14369 net::TcpListener,
14370 };
14371 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14372 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14373 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14374 let (started_tx, mut started_rx) = mpsc::channel(8);
14375 let peer = tokio::spawn(async move {
14376 loop {
14377 let (mut socket, _) = listener.accept().await.unwrap();
14378 let mut request = [0; 1024];
14379 assert!(socket.read(&mut request).await.unwrap() > 0);
14380 started_tx.send(Instant::now()).await.unwrap();
14381 socket
14382 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14383 .await
14384 .unwrap();
14385 }
14386 });
14387 actor
14388 .module
14389 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14390 .await
14391 .unwrap();
14392 let first_tick = Instant::now();
14393 let first = actor
14394 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
14395 .await;
14396 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
14397 assert!(started_rx.try_recv().is_err());
14398
14399 tokio::time::advance(
14402 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
14403 )
14404 .await;
14405 actor.module.restart(Some(0)).await.unwrap();
14406 let spawn_tick = Instant::now();
14407 let replacement = actor
14408 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
14409 .await;
14410 let deadline = replacement.next_probe_at.unwrap();
14411 assert!(
14412 deadline >= spawn_tick + actor.health.cadence,
14413 "a replacement must not inherit its predecessor's probe deadline"
14414 );
14415 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
14416 assert!(
14417 started_rx.try_recv().is_err(),
14418 "no HTTP probe during startup grace"
14419 );
14420 tokio::time::advance(deadline - Instant::now()).await;
14421 let landed = wait_for("first replacement HTTP probe", || {
14422 started_rx.try_recv().ok()
14423 })
14424 .await;
14425 assert!(landed >= spawn_tick + actor.health.cadence);
14426 wait_for("healthy replacement", || {
14427 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14428 .then_some(())
14429 })
14430 .await;
14431 peer.abort();
14432 }
14433
14434 #[tokio::test(start_paused = true)]
14435 async fn rescan_adds_and_removes_http_health_without_polling() {
14436 use tokio::{
14437 io::{AsyncReadExt, AsyncWriteExt},
14438 net::TcpListener,
14439 };
14440 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14441 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14442 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
14443 actor.health.cadence = Duration::from_millis(1);
14445 let (started_tx, mut started_rx) = mpsc::channel(8);
14446 let peer = tokio::spawn(async move {
14447 loop {
14448 let (mut socket, _) = listener.accept().await.unwrap();
14449 let mut request = [0; 1024];
14450 assert!(socket.read(&mut request).await.unwrap() > 0);
14451 started_tx.send(()).await.unwrap();
14452 socket
14453 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14454 .await
14455 .unwrap();
14456 }
14457 });
14458 let tick = Instant::now();
14459 let turns = actor.turns();
14460 actor
14461 .module
14462 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14463 .await
14464 .unwrap();
14465 let armed = actor
14466 .wait_for_select(|checkpoint| {
14467 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
14468 })
14469 .await;
14470 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
14471 assert_eq!(Instant::now(), tick);
14472 tokio::time::advance(Duration::from_millis(1)).await;
14473 let mut started = false;
14474 wait_for("successful HTTP probe", || {
14475 started |= started_rx.try_recv().is_ok();
14476 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14477 .then_some(())
14478 })
14479 .await;
14480 assert!(started, "added HTTP check must start at its first deadline");
14481 assert_eq!(
14482 actor.module.status().unwrap().health.status,
14483 SupervisorHealthStatus::Ok
14484 );
14485 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
14486 actor.health.http = None;
14487 let turns = actor.turns();
14488 actor
14489 .module
14490 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14491 .await
14492 .unwrap();
14493 let parked = actor
14494 .wait_for_select(|checkpoint| {
14495 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
14496 })
14497 .await;
14498 assert_eq!(parked.wake_after, None);
14499 let turns = actor.turns();
14500 tokio::time::advance(Duration::from_secs(60)).await;
14501 assert_eq!(
14502 actor.turns(),
14503 turns,
14504 "removed HTTP check must park the actor"
14505 );
14506 assert!(
14507 started_rx.try_recv().is_err(),
14508 "removed HTTP check must stay disarmed"
14509 );
14510 assert_eq!(
14511 actor.module.status().unwrap().health,
14512 ModuleHealthStatus::default()
14513 );
14514 peer.abort();
14515 }
14516}
14517
14518#[cfg(test)]
14519mod drain_busy_reprobe_tests {
14520 use super::*;
14524 use std::collections::BTreeMap;
14525 use std::sync::Mutex as StdMutex;
14526 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14527
14528 const MODULE_ID: &str = "drain-reprobe-module";
14529 const CONNECTION: u64 = 7;
14530 const GAUGE: &str = "runs_in_flight";
14531 const CADENCE: Duration = Duration::from_secs(30);
14534 const CEILING: Duration = Duration::from_secs(30);
14535
14536 #[derive(Clone, Copy)]
14538 enum Answer {
14539 BusyUntil { until: Instant, latency: Duration },
14542 OmitsGauge,
14544 Silent,
14546 }
14547
14548 struct Drain {
14549 forwarding: Arc<ForwardingTable>,
14550 runtime: SupervisorRuntimeConfig,
14551 endpoint: crate::ModuleEndpointId,
14552 probes: Arc<StdMutex<Vec<Instant>>>,
14554 responder: tokio::task::JoinHandle<()>,
14555 }
14556
14557 impl Drain {
14558 fn start(answer: Answer) -> Self {
14559 let registry = Arc::new(Registry::default());
14560 let forwarding = Arc::new(ForwardingTable::default());
14561 let health = HealthConfig {
14562 cadence: CADENCE,
14563 ..HealthConfig::default()
14564 };
14565 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14566 .with_forwarding(forwarding.clone())
14567 .with_health_config(health)
14568 .runtime_config();
14569 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14570 let endpoint = forwarding
14571 .register_module_connection(
14572 ConnectionId::new(CONNECTION),
14573 MODULE_ID.to_string(),
14574 subc_protocol::PROTOCOL_VERSION,
14575 Concurrency::ModuleManaged,
14576 FrameSink::new(tx),
14577 )
14578 .unwrap();
14579 let probes = Arc::new(StdMutex::new(Vec::new()));
14580 let responder = tokio::spawn({
14581 let forwarding = forwarding.clone();
14582 let probes = probes.clone();
14583 async move {
14584 while let Some(frame) = rx.recv().await {
14585 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14586 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14587 probes.lock().unwrap().push(Instant::now());
14588 let metrics = match answer {
14589 Answer::BusyUntil { until, latency } => {
14590 sleep(latency).await;
14591 let busy = u64::from(Instant::now() < until);
14592 serde_json::json!({ GAUGE: busy })
14593 }
14594 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14595 Answer::Silent => continue,
14596 };
14597 let _ = forwarding.complete_module_control_rpc(
14600 ConnectionId::new(CONNECTION),
14601 frame.header.corr,
14602 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14603 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14604 status: HealthStatus::Ok,
14605 detail: None,
14606 metrics: Some(metrics),
14607 }),
14608 );
14609 }
14610 }
14611 });
14612 Self {
14613 forwarding,
14614 runtime,
14615 endpoint,
14616 probes,
14617 responder,
14618 }
14619 }
14620
14621 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14624 let (drained, elapsed, _) = self.run_traced(ceiling).await;
14625 (drained, elapsed)
14626 }
14627
14628 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14630 let started = Instant::now();
14631 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14632 let drained = wait_for_forwarding_quiescence(
14633 &self.forwarding,
14634 MODULE_ID,
14635 &self.runtime,
14636 self.endpoint,
14637 started + ceiling,
14638 &[GAUGE.to_string()],
14639 DrainScope::Active,
14640 &mut trace,
14641 )
14642 .await
14643 .unwrap();
14644 (drained, started.elapsed(), trace)
14645 }
14646
14647 fn probes(&self) -> Vec<Instant> {
14648 self.probes.lock().unwrap().clone()
14649 }
14650 }
14651
14652 impl Drop for Drain {
14653 fn drop(&mut self) {
14654 self.responder.abort();
14655 }
14656 }
14657
14658 #[tokio::test(start_paused = true)]
14659 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14660 let drain = Drain::start(Answer::BusyUntil {
14661 until: Instant::now() + Duration::from_secs(1),
14662 latency: Duration::ZERO,
14663 });
14664 let (drained, elapsed) = drain.run(CEILING).await;
14665 assert!(
14666 drained,
14667 "gauges that read zero during the drain must let it finish drained, \
14668 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14669 );
14670 assert!(
14671 elapsed < Duration::from_secs(3),
14672 "drain took {elapsed:?} after the gauges went quiet at 1s"
14673 );
14674 let probes = drain.probes();
14675 assert!(
14676 probes.len() >= 2,
14677 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14678 );
14679 }
14680
14681 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14686 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14687 let drain = Drain::start(Answer::BusyUntil {
14688 until: Instant::now() + Duration::from_millis(600),
14689 latency: Duration::ZERO,
14690 });
14691 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14692 assert!(drained, "the gauge fell at 600ms: {trace}");
14693 assert!(
14694 elapsed < Duration::from_secs(3),
14695 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14696 );
14697 let probes = drain.probes();
14698 assert!(
14699 probes.len() >= 2 && trace.probes as usize == probes.len(),
14700 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14701 );
14702 }
14703
14704 #[tokio::test(start_paused = true)]
14705 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14706 let started = Instant::now();
14707 let drain = Drain::start(Answer::BusyUntil {
14708 until: started + Duration::from_secs(3600),
14709 latency: Duration::ZERO,
14710 });
14711 let (drained, elapsed) = drain.run(CEILING).await;
14712 assert!(!drained, "a module busy throughout must not drain");
14713 assert!(
14714 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14715 "a busy module is held until the ceiling and no longer: {elapsed:?}"
14716 );
14717 let probes = drain.probes();
14720 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14721 assert!(
14722 probes.len() as u128 <= max_probes,
14723 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14724 probes.len()
14725 );
14726 for pair in probes.windows(2) {
14727 assert!(
14728 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14729 "consecutive probes {:?} apart",
14730 pair[1] - pair[0]
14731 );
14732 }
14733 }
14734
14735 #[tokio::test(start_paused = true)]
14736 async fn a_slow_probe_is_never_overlapped_by_the_next() {
14737 let latency = Duration::from_secs(1);
14740 let drain = Drain::start(Answer::BusyUntil {
14741 until: Instant::now() + Duration::from_secs(3600),
14742 latency,
14743 });
14744 let (drained, _) = drain.run(Duration::from_secs(5)).await;
14745 assert!(!drained);
14746 let probes = drain.probes();
14747 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14748 for pair in probes.windows(2) {
14749 assert!(
14750 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14751 "a probe started {:?} after the previous one, before it could finish",
14752 pair[1] - pair[0]
14753 );
14754 }
14755 }
14756
14757 #[tokio::test(start_paused = true)]
14758 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14759 let drain = Drain::start(Answer::Silent);
14760 let ceiling = Duration::from_secs(2);
14761 let (drained, elapsed) = drain.run(ceiling).await;
14762 assert!(!drained, "an unanswered probe counts as busy");
14763 assert!(
14764 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14765 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14766 );
14767 assert_eq!(
14768 drain.probes().len(),
14769 1,
14770 "the unanswered probe holds the only slot until the ceiling"
14771 );
14772 }
14773
14774 #[derive(Clone, Default)]
14776 struct LogCapture {
14777 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14778 }
14779
14780 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14781 fn on_event(
14782 &self,
14783 event: &tracing::Event<'_>,
14784 _context: tracing_subscriber::layer::Context<'_, S>,
14785 ) {
14786 struct Fields(BTreeMap<String, String>);
14787 impl tracing::field::Visit for Fields {
14788 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14789 self.0
14790 .insert(field.name().to_string(), format!("{value:?}"));
14791 }
14792 }
14793 let mut fields = Fields(BTreeMap::new());
14794 event.record(&mut fields);
14795 self.events.lock().unwrap().push(fields.0);
14796 }
14797 }
14798
14799 impl LogCapture {
14800 fn install(&self) -> tracing::subscriber::DefaultGuard {
14801 use tracing_subscriber::layer::SubscriberExt;
14802 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14803 }
14804
14805 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14806 self.events
14807 .lock()
14808 .unwrap()
14809 .iter()
14810 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14811 .cloned()
14812 .collect()
14813 }
14814 }
14815
14816 #[tokio::test(start_paused = true)]
14821 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14822 let capture = LogCapture::default();
14823 let _guard = capture.install();
14824 let drain = Drain::start(Answer::OmitsGauge);
14825 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14826 assert!(!drained, "a gauge never reported cannot read quiet");
14827 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14828
14829 let missing = capture.with_message("lacks a declared busy gauge");
14830 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14831 assert_eq!(
14832 missing[0].get("missing_gauges").map(String::as_str),
14833 Some(GAUGE)
14834 );
14835
14836 warn_drain_timed_out(
14837 MODULE_ID,
14838 Duration::from_secs(2),
14839 RouteCloseReason::Restart,
14840 &Default::default(),
14841 &trace,
14842 );
14843 let timeouts = capture.with_message("route drain timed out");
14844 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14845 let gauges = &timeouts[0]["busy_gauges"];
14846 assert!(
14847 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14848 "the timeout line must name the gauge and why it read busy: {gauges}"
14849 );
14850 assert!(
14851 gauges.ends_with("; no value ever read"),
14852 "no value was ever read for the gauge: {gauges}"
14853 );
14854 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14855 }
14856
14857 #[tokio::test(start_paused = true)]
14861 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14862 let busy = Drain::start(Answer::BusyUntil {
14863 until: Instant::now() + Duration::from_secs(3600),
14864 latency: Duration::ZERO,
14865 });
14866 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14867 let rendered = trace.to_string();
14868 assert!(
14869 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14870 && rendered.contains("; last value non_zero(1) @"),
14871 "{rendered}"
14872 );
14873
14874 let silent = Drain::start(Answer::Silent);
14875 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14876 assert_eq!(
14879 trace.to_string(),
14880 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14881 );
14882 assert_eq!(trace.probes, 1);
14883 }
14884
14885 #[test]
14887 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14888 let gauges = [GAUGE.to_string()];
14889 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14890 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14891 assert_eq!(
14892 busy_gauge_observation(&verdicts),
14893 BusyGaugeObservation::Busy
14894 );
14895 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14896 assert_eq!(
14897 busy_gauge_observation(&zero),
14898 BusyGaugeObservation::Quiescent
14899 );
14900 let none = gauge_verdicts(None, &gauges);
14901 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14902 }
14903}
14904
14905#[cfg(test)]
14906mod health_evidence_tests {
14907 use super::{HealthProbeError, HealthProbeEvidence};
14908 use std::collections::HashSet;
14909
14910 #[test]
14918 fn only_a_dead_lane_is_proof_of_death() {
14919 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14920 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14924 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14925 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14926 }
14927
14928 #[test]
14934 fn every_evidence_class_has_a_distinct_label() {
14935 let labels = [
14936 HealthProbeError::lane_dead("").label(),
14937 HealthProbeError::no_answer("").label(),
14938 HealthProbeError::bad_answer("").label(),
14939 HealthProbeError::misconfigured("").label(),
14940 ];
14941 let unique: HashSet<_> = labels.iter().collect();
14942 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14943 }
14944
14945 #[test]
14951 fn classification_preserves_the_original_message() {
14952 let err = HealthProbeError::no_answer("module did not answer within 5s");
14953 assert_eq!(err.to_string(), "module did not answer within 5s");
14954 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14955 }
14956}
14957
14958#[cfg(test)]
14959mod health_tombstone_tests {
14960 use std::{path::PathBuf, sync::Arc, time::Duration};
14961
14962 use subc_protocol::{
14963 manifest::Concurrency,
14964 session::{HealthStatus, ModuleControlResponse},
14965 };
14966 use tokio::sync::mpsc;
14967
14968 use super::{
14969 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14970 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14971 };
14972 use crate::{
14973 control::ControlHandler,
14974 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14975 registry::{ConnectionId, Registry},
14976 router::FrameSink,
14977 };
14978
14979 struct ProbeHarness {
14980 spec: ModuleSpec,
14981 runtime: SupervisorRuntimeConfig,
14982 forwarding: Arc<ForwardingTable>,
14983 module_connection: ConnectionId,
14984 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14985 handler: ControlHandler,
14986 module: super::SupervisedModule,
14987 }
14988
14989 fn probe_harness() -> ProbeHarness {
14990 let registry = Arc::new(Registry::default());
14991 let forwarding = Arc::new(ForwardingTable::default());
14992 let supervisor_handle = super::SupervisorHandle::new();
14993 let health = HealthConfig {
14994 http: None,
14995 cadence: Duration::from_secs(30),
14996 deadline: Duration::from_secs(5),
14997 failure_threshold: 3,
14998 on_degraded: HealthAction::Report,
14999 on_failing: HealthAction::Report,
15000 critical: false,
15001 };
15002 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15003 .with_forwarding(Arc::clone(&forwarding))
15004 .with_handle(supervisor_handle.clone())
15005 .with_health_config(health);
15006 let spec = ModuleSpec {
15007 module_id: "late-health-module".to_string(),
15008 program: PathBuf::from("disabled-module"),
15009 args: Vec::new(),
15010 env: Vec::new(),
15011 reserved: false,
15012 reserved_prefixes: Vec::new(),
15013 protocol: ModuleProtocol::Subc,
15014 overlap: Default::default(),
15015 };
15016 let module = supervisor
15017 .supervise_configured(spec.clone(), false)
15018 .unwrap();
15019 let runtime = supervisor.runtime_config();
15020 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
15021 .with_supervisor(supervisor_handle);
15022 let module_connection = ConnectionId::new(700);
15023 let (module_tx, module_rx) = mpsc::channel(8);
15024 forwarding
15025 .register_module_connection(
15026 module_connection,
15027 spec.module_id.clone(),
15028 subc_protocol::PROTOCOL_VERSION,
15029 Concurrency::ModuleManaged,
15030 FrameSink::new(module_tx),
15031 )
15032 .unwrap();
15033
15034 ProbeHarness {
15035 spec,
15036 runtime,
15037 forwarding,
15038 module_connection,
15039 module_rx,
15040 handler,
15041 module,
15042 }
15043 }
15044
15045 async fn finish_after(
15046 harness: &mut ProbeHarness,
15047 stall: Duration,
15048 ) -> ModuleControlRpcCompletion {
15049 assert!(stall > harness.runtime.health.deadline);
15050 let deadline = harness.runtime.health.deadline;
15051 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15052 let answer = async {
15053 let frame = harness.module_rx.recv().await.expect("health.check frame");
15054 tokio::time::advance(deadline).await;
15055 tokio::task::yield_now().await;
15056 tokio::time::advance(stall - deadline).await;
15057 harness
15058 .forwarding
15059 .complete_module_control_rpc(
15060 harness.module_connection,
15061 frame.header.corr,
15062 Some("health.check"),
15063 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15064 status: HealthStatus::Ok,
15065 detail: None,
15066 metrics: None,
15067 }),
15068 )
15069 .unwrap()
15070 };
15071 let (probe_result, completion) = tokio::join!(probe, answer);
15072 let err = probe_result.expect_err("probe must miss its deadline");
15073 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15074 completion
15075 }
15076
15077 async fn time_out_without_answer(harness: &mut ProbeHarness) {
15078 let deadline = harness.runtime.health.deadline;
15079 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15080 let exhaust_deadline = async {
15081 let _frame = harness.module_rx.recv().await.expect("health.check frame");
15082 tokio::time::advance(deadline).await;
15083 tokio::task::yield_now().await;
15084 };
15085 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
15086 let err = probe_result.expect_err("probe must miss its deadline");
15087 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15088 }
15089
15090 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
15091 let registry = Arc::clone(&harness.module.inner.registry);
15092 let snapshot = Arc::clone(&harness.module.inner.snapshot);
15093 let process_liveness = super::SupervisorProcessLiveness::default();
15094 let mut child = None;
15095 let cycle = super::run_health_probe_cycle(
15096 &harness.spec,
15097 &harness.runtime,
15098 ®istry,
15099 &process_liveness,
15100 &snapshot,
15101 &mut child,
15102 );
15103 let peer = async {
15104 let frame = harness.module_rx.recv().await.expect("health.check frame");
15105 if answer {
15106 harness
15107 .forwarding
15108 .complete_module_control_rpc(
15109 harness.module_connection,
15110 frame.header.corr,
15111 Some("health.check"),
15112 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15113 status: HealthStatus::Ok,
15114 detail: None,
15115 metrics: Some(serde_json::json!({"ready": true})),
15116 }),
15117 )
15118 .unwrap();
15119 } else {
15120 tokio::time::advance(harness.runtime.health.deadline).await;
15121 tokio::task::yield_now().await;
15122 }
15123 };
15124 tokio::join!(cycle, peer);
15125 }
15126
15127 #[tokio::test(start_paused = true)]
15128 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
15129 let mut harness = probe_harness();
15130 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
15134 monitor.abort();
15135 let _ = monitor.await;
15136 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
15137 state.enabled = true;
15138 state.state = super::ModuleState::Running;
15139 state.process_alive = true;
15140 })
15141 .unwrap();
15142
15143 run_probe_cycle(&mut harness, true).await;
15144 assert_eq!(
15145 harness.module.status().unwrap().health.status,
15146 super::SupervisorHealthStatus::Ok
15147 );
15148
15149 for failures in 1..harness.runtime.health.failure_threshold {
15150 run_probe_cycle(&mut harness, false).await;
15151 let status = harness.module.status().unwrap();
15152 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15153 assert_eq!(status.health.consecutive_failures, failures);
15154 assert!(status.health.last_probe_ms.is_some());
15155 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
15156 assert!(status.health.metrics.is_none());
15157 assert_eq!(status.state, super::ModuleState::Running);
15158 assert!(status.process_alive);
15159 assert_eq!(status.restart_count, 0);
15160 assert_eq!(status.lifetime_restarts, 0);
15161 assert!(status.health.last_action.is_none());
15162 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15163 }
15164
15165 run_probe_cycle(&mut harness, true).await;
15166 let recovered = harness.module.status().unwrap();
15167 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
15168 assert_eq!(recovered.health.consecutive_failures, 0);
15169 assert!(recovered.health.detail.is_none());
15170 assert_eq!(
15171 recovered.health.metrics,
15172 Some(serde_json::json!({"ready": true}))
15173 );
15174 assert_eq!(recovered.lifetime_restarts, 0);
15175
15176 for failures in 1..=harness.runtime.health.failure_threshold {
15177 run_probe_cycle(&mut harness, false).await;
15178 let status = harness.module.status().unwrap();
15179 assert_eq!(status.health.consecutive_failures, failures);
15180 if failures < harness.runtime.health.failure_threshold {
15181 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15182 assert_eq!(status.state, super::ModuleState::Running);
15183 assert_eq!(status.lifetime_restarts, 0);
15184 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15185 } else {
15186 assert_eq!(
15187 status.health.status,
15188 super::SupervisorHealthStatus::Unresponsive
15189 );
15190 assert_eq!(status.state, super::ModuleState::Restarting);
15191 assert_eq!(status.restart_count, 1);
15192 assert_eq!(status.lifetime_restarts, 1);
15193 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
15194 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
15195 }
15196 }
15197 }
15198
15199 #[tokio::test(start_paused = true)]
15200 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
15201 let mut harness = probe_harness();
15202
15203 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
15204 let first_latency = match &first {
15205 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15206 other => panic!("late answer was not retained: {other:?}"),
15207 };
15208 assert!(harness.handler.observe_module_control_completion(first));
15209
15210 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
15211 let second_latency = match &second {
15212 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15213 other => panic!("late answer was not retained: {other:?}"),
15214 };
15215 assert!(harness.handler.observe_module_control_completion(second));
15216
15217 assert_eq!(first_latency, Duration::from_secs(8));
15218 assert_eq!(
15219 second_latency - first_latency,
15220 Duration::from_secs(3),
15221 "latency must grow linearly with the additional stall"
15222 );
15223 let health = harness.module.status().unwrap().health;
15224 assert_eq!(health.late_answer_count, 2);
15225 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
15226 }
15227
15228 #[tokio::test(start_paused = true)]
15236 async fn late_answer_clears_the_consecutive_failure_streak() {
15237 let mut harness = probe_harness();
15238
15239 time_out_without_answer(&mut harness).await;
15241 harness
15242 .module
15243 .record_health_probe_failure_for_test("[no-answer] test miss")
15244 .unwrap();
15245 assert_eq!(
15246 harness.module.status().unwrap().health.consecutive_failures,
15247 1,
15248 "precondition: the miss must be on the streak before the late answer"
15249 );
15250
15251 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15253 assert!(matches!(
15254 late,
15255 ModuleControlRpcCompletion::LateHealthAnswer { .. }
15256 ));
15257 assert!(harness.handler.observe_module_control_completion(late));
15258
15259 let health = harness.module.status().unwrap().health;
15260 assert_eq!(
15261 health.consecutive_failures, 0,
15262 "a late answer is an answer: the streak must reset"
15263 );
15264 assert_eq!(health.late_answer_count, 1);
15265 }
15266
15267 #[tokio::test(start_paused = true)]
15268 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15269 let mut harness = probe_harness();
15270
15271 for _ in 0..20 {
15272 time_out_without_answer(&mut harness).await;
15273 assert_eq!(
15274 harness.forwarding.health_probe_tombstone_count().unwrap(),
15275 1
15276 );
15277 }
15278 }
15279}
15280
15281#[cfg(test)]
15282mod child_env_tests {
15283 #[cfg(windows)]
15284 use super::{
15285 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15286 };
15287 #[cfg(windows)]
15288 use std::sync::{Arc, OnceLock};
15289 #[cfg(windows)]
15290 #[test]
15291 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15292 let spec = spec(Vec::new());
15293 let mut command = Command::new("unused");
15294 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15295 .unwrap()
15296 .unwrap();
15297 let name = command
15298 .as_std()
15299 .get_envs()
15300 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15301 .and_then(|(_, value)| value)
15302 .unwrap();
15303 assert_eq!(name, OsStr::new(handoff.name()));
15304 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15305 assert!(command.as_std().get_envs().any(|(key, value)| key
15306 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15307 && value == Some(OsStr::new("env"))));
15308 assert!(command
15309 .as_std()
15310 .get_envs()
15311 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15312 }
15313
15314 #[cfg(windows)]
15315 #[test]
15316 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15317 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15318 snapshot.process_alive = true;
15319 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15320 assert_eq!(
15321 observed_launch_nonce_source(&snapshot).as_deref(),
15322 Some("env")
15323 );
15324 let consumed = Arc::new(OnceLock::new());
15325 snapshot.pipe_consumed = Some(consumed.clone());
15326 assert_eq!(
15327 observed_launch_nonce_source(&snapshot).as_deref(),
15328 Some("env")
15329 );
15330 consumed.set(()).unwrap();
15331 assert_eq!(
15332 observed_launch_nonce_source(&snapshot).as_deref(),
15333 Some("pipe")
15334 );
15335 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15336 assert_eq!(
15337 observed_launch_nonce_source(&snapshot).as_deref(),
15338 Some("env")
15339 );
15340 clear_current_process_facts(&mut snapshot);
15341 assert_eq!(observed_launch_nonce_source(&snapshot), None);
15342 }
15343 use super::{
15344 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15345 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15346 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15347 };
15348 use std::{ffi::OsStr, path::PathBuf};
15349 use tokio::process::Command;
15350
15351 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15352 ModuleSpec {
15353 module_id: "env-plan".to_string(),
15354 program: PathBuf::from("/nonexistent"),
15355 args: Vec::new(),
15356 env,
15357 reserved: false,
15358 reserved_prefixes: Vec::new(),
15359 protocol: ModuleProtocol::Subc,
15360 overlap: Default::default(),
15361 }
15362 }
15363
15364 #[test]
15365 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15366 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15367 let digest = LaunchSpecDigest::from_spec(&base);
15368 let mut reordered = base.clone();
15369 reordered.env.reverse();
15370 reordered.module_id = "different-id".into();
15371 reordered.reserved = true;
15372 reordered.overlap = Default::default();
15373 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15374 let mut delimiter_collision = base.clone();
15375 delimiter_collision.env[0] = ("A=B".into(), "C".into());
15376 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15377 let mut changed = base.clone();
15378 changed.protocol = ModuleProtocol::None;
15379 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15380 changed = base.clone();
15381 changed.args.push("--flag".into());
15382 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15383 changed = base.clone();
15384 changed.program = PathBuf::from("/other");
15385 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15386 changed = base.clone();
15387 changed.env[1].1 = "other".into();
15388 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15389 assert!(!format!("{digest:?}").contains("B=C"));
15390 assert!(!format!("{digest:?}").contains("[u8"));
15391 }
15392
15393 #[cfg(unix)]
15394 #[test]
15395 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
15396 use std::os::unix::ffi::OsStringExt;
15397 let mut left = spec(Vec::new());
15398 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
15399 let mut right = left.clone();
15400 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
15401 assert_eq!(
15402 left.program.to_string_lossy(),
15403 right.program.to_string_lossy()
15404 );
15405 assert_ne!(
15406 LaunchSpecDigest::from_spec(&left),
15407 LaunchSpecDigest::from_spec(&right)
15408 );
15409 }
15410
15411 #[test]
15425 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
15426 let mut command = Command::new("/nonexistent");
15427 apply_child_env(&mut command, &spec(Vec::new()));
15428 let removed = command
15429 .as_std()
15430 .get_envs()
15431 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
15432 assert!(
15433 removed,
15434 "ambient CK_LOG must be explicitly removed for an unconfigured module"
15435 );
15436
15437 let mut configured = Command::new("/nonexistent");
15438 apply_child_env(
15439 &mut configured,
15440 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
15441 );
15442 let effective = configured
15443 .as_std()
15444 .get_envs()
15445 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
15446 .last()
15447 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15448 assert_eq!(
15449 effective,
15450 Some(Some("debug".to_string())),
15451 "a module's configured CK_LOG must survive the ambient removal"
15452 );
15453 }
15454
15455 #[test]
15464 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
15465 let connection_file = std::path::Path::new("/run/subc-connection.json");
15466 let handle = SupervisorHandle::new();
15467
15468 let mut none_spec = spec(Vec::new());
15469 none_spec.protocol = ModuleProtocol::None;
15470 let mut none = Command::new("/nonexistent");
15471 let none_handoff =
15472 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
15473 .expect("protocol-none spawn args apply");
15474 assert!(
15475 none_handoff.is_none(),
15476 "protocol:none spawn must not receive a nonce descriptor"
15477 );
15478 #[cfg(windows)]
15479 assert!(!none.as_std().get_envs().any(|(key, value)| key
15480 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15481 && value.is_some()));
15482 assert!(
15483 !none.as_std().get_envs().any(|(key, value)| key
15484 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
15485 && value.is_some()),
15486 "protocol:none spawn must not name a nonce descriptor"
15487 );
15488 let none_args: Vec<String> = none
15489 .as_std()
15490 .get_args()
15491 .map(|a| a.to_string_lossy().into_owned())
15492 .collect();
15493 assert!(
15494 !none_args.iter().any(|a| a == SUBC_ARG),
15495 "protocol:none argv must not carry --subc; got {none_args:?}"
15496 );
15497 let none_has_nonce = none
15498 .as_std()
15499 .get_envs()
15500 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
15501 assert!(
15502 !none_has_nonce,
15503 "protocol:none spawn must not receive a launch nonce"
15504 );
15505 let none_has_module_id = none
15506 .as_std()
15507 .get_envs()
15508 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
15509 assert!(
15510 none_has_module_id,
15511 "SUBC_MODULE_ID is inert and stays on every path"
15512 );
15513 assert!(
15514 handle.spawn_nonce(&none_spec.module_id).is_none(),
15515 "no nonce record for a process that will never present one"
15516 );
15517
15518 let wire_spec = spec(Vec::new());
15520 let mut wire = Command::new("/nonexistent");
15521 let wire_handoff =
15522 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15523 .expect("subc-wire spawn args apply");
15524 let wire_fd_env = wire
15525 .as_std()
15526 .get_envs()
15527 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15528 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15529 #[cfg(unix)]
15530 assert_eq!(
15531 wire_fd_env,
15532 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15533 "a subc-wire spawn names the pipe it will receive at descriptor 3"
15534 );
15535 #[cfg(windows)]
15536 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15537 #[cfg(not(any(unix, windows)))]
15538 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15539 let wire_args: Vec<String> = wire
15540 .as_std()
15541 .get_args()
15542 .map(|a| a.to_string_lossy().into_owned())
15543 .collect();
15544 assert_eq!(
15545 wire_args,
15546 vec![
15547 SUBC_ARG.to_string(),
15548 connection_file.to_string_lossy().into_owned()
15549 ],
15550 "a subc-wire spawn still carries --subc <path>"
15551 );
15552 assert_eq!(
15553 wire.as_std()
15554 .get_envs()
15555 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15556 !cfg!(unix),
15557 "only Windows supplies the environment nonce"
15558 );
15559 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15560 }
15561
15562 #[test]
15572 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15573 let role = |command: &Command| {
15574 command
15575 .as_std()
15576 .get_envs()
15577 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15578 .last()
15579 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15580 };
15581 let forged = spec(vec![(
15582 SUBC_SPAWN_ROLE_ENV.to_string(),
15583 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15584 )]);
15585
15586 let mut plain = Command::new("/nonexistent");
15587 apply_child_env(&mut plain, &forged);
15588 apply_spawn_role(&mut plain, SpawnRole::Plain);
15589 assert_eq!(
15590 role(&plain),
15591 Some(None),
15592 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15593 );
15594
15595 let mut candidate = Command::new("/nonexistent");
15596 apply_child_env(&mut candidate, &spec(Vec::new()));
15597 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15598 assert_eq!(
15599 role(&candidate),
15600 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15601 );
15602 }
15603
15604 #[test]
15610 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15611 let mut command = Command::new("/nonexistent");
15612 apply_child_env(
15613 &mut command,
15614 &spec(vec![
15615 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15616 ("KEPT".to_string(), "yes".to_string()),
15617 ]),
15618 );
15619 let keys: Vec<String> = command
15620 .as_std()
15621 .get_envs()
15622 .filter(|(_, value)| value.is_some())
15623 .map(|(key, _)| key.to_string_lossy().into_owned())
15624 .collect();
15625 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15626 assert!(
15627 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15628 "daemon-private capture key leaked to the child: {keys:?}"
15629 );
15630 }
15631}
15632
15633#[cfg(test)]
15634mod jitter_tests {
15635 use super::jittered_health_delay;
15636 use std::{collections::HashSet, time::Duration};
15637
15638 const FLEET: [&str; 14] = [
15647 "aft",
15648 "alfonso-core",
15649 "magic-context",
15650 "broca",
15651 "thalamus",
15652 "quota",
15653 "engram",
15654 "plexus",
15655 "cerebellum",
15656 "astrocyte",
15657 "synapse",
15658 "subc-mcp",
15659 "cortexkit-credentials",
15660 "subc-federation",
15661 ];
15662
15663 #[test]
15671 fn probe_delays_disperse_across_the_fleet() {
15672 let cadence = Duration::from_secs(30);
15673 let delays: HashSet<Duration> = FLEET
15674 .iter()
15675 .map(|id| jittered_health_delay(id, 0, cadence))
15676 .collect();
15677 assert_eq!(
15678 delays.len(),
15679 FLEET.len(),
15680 "every supervised module must land on its own probe offset"
15681 );
15682 }
15683
15684 #[test]
15690 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15691 let cadence = Duration::from_secs(30);
15692 let span = cadence / 10;
15693 for id in FLEET {
15694 for probe_index in 0..8 {
15695 let delay = jittered_health_delay(id, probe_index, cadence);
15696 assert!(
15697 delay >= cadence,
15698 "{id}#{probe_index}: jitter must not shorten the cadence"
15699 );
15700 assert!(
15701 delay < cadence + span,
15702 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15703 );
15704 }
15705 }
15706 }
15707
15708 #[test]
15714 fn a_module_offset_is_stable_across_restarts() {
15715 let cadence = Duration::from_secs(30);
15716 for id in FLEET {
15717 assert_eq!(
15718 jittered_health_delay(id, 0, cadence),
15719 jittered_health_delay(id, 0, cadence),
15720 "{id}: the same module and probe index must produce the same offset"
15721 );
15722 }
15723 }
15724
15725 #[test]
15727 fn zero_cadence_yields_zero_delay() {
15728 assert_eq!(
15729 jittered_health_delay("aft", 0, Duration::ZERO),
15730 Duration::ZERO
15731 );
15732 }
15733}
15734
15735#[cfg(all(test, target_os = "linux"))]
15736mod cgroup_placement_tests {
15737 use super::{
15738 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15739 SuperviseError, SupervisedChild,
15740 };
15741 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15742 use cortexkit_test_support::ScratchDir;
15743 use std::{
15744 fs, io,
15745 path::{Path, PathBuf},
15746 sync::{Arc, Mutex},
15747 };
15748 use tokio::process::Command;
15749
15750 #[tokio::test]
15751 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15752 use super::*;
15753 let dir = ScratchDir::new("unique-spawn-cgroups");
15754 let root = PathBuf::from(format!(
15755 "/sys/fs/cgroup/subc-unique-test-{}-{}",
15756 std::process::id(),
15757 unix_ms_now()
15758 ));
15759 if let Err(error) = fs::create_dir(&root) {
15760 assert!(
15761 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15762 "required cgroup test cannot execute: {error}"
15763 );
15764 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15765 return;
15766 }
15767 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15768 let group_count = || {
15769 fs::read_dir(root.join("subc-modules"))
15770 .unwrap()
15771 .map(|entry| entry.unwrap().file_type().unwrap())
15772 .filter(|kind| kind.is_dir())
15773 .count()
15774 };
15775 let supervisor =
15776 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15777 .with_cgroup_placement(Some(placement.clone()));
15778 let runtime = supervisor.runtime_config();
15779 let mut spec = ModuleSpec {
15780 module_id: "unique-spawn".into(),
15781 program: PathBuf::from("/bin/sleep"),
15782 args: vec!["60".into()],
15783 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15784 .into_iter()
15785 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15786 .collect(),
15787 reserved: false,
15788 reserved_prefixes: vec![],
15789 protocol: ModuleProtocol::None,
15790 overlap: Default::default(),
15791 };
15792 let spawn = |spec: &ModuleSpec| {
15793 spawn_child(
15794 spec,
15795 None,
15796 None,
15797 &runtime.stderr_ring,
15798 None,
15799 &runtime.child_roster,
15800 Some(&placement),
15801 )
15802 .unwrap()
15803 };
15804 let mut live = spawn(&spec);
15805 for _ in 0..3 {
15806 let next = spawn(&spec);
15808 assert_ne!(live.module_id, next.module_id);
15809 live.start_kill().unwrap();
15810 live.wait().await.unwrap();
15811 live = next;
15812 assert!(
15813 live.child.try_wait().unwrap().is_none(),
15814 "retiring the old slot must not kill the replacement"
15815 );
15816 assert_eq!(
15817 group_count(),
15818 1,
15819 "only the live spawn's cgroup should remain"
15820 );
15821 }
15822 supervisor.begin_daemon_shutdown();
15823 let reap = tokio::spawn(async move {
15824 live.wait().await.unwrap();
15825 });
15826 supervisor
15827 .end_children_for_daemon_shutdown(false, std::future::pending())
15828 .await;
15829 reap.await.unwrap();
15830 assert_eq!(group_count(), 0);
15831 spec.program = PathBuf::from("/bin/true");
15833 spec.args.clear();
15834 let fresh_roster = ChildRoster::default();
15835 let mut short = spawn_child(
15836 &spec,
15837 None,
15838 None,
15839 &runtime.stderr_ring,
15840 None,
15841 &fresh_roster,
15842 Some(&placement),
15843 )
15844 .unwrap();
15845 short.wait().await.unwrap();
15846 assert_eq!(group_count(), 0);
15847 spec.module_id = "_".repeat(255);
15848 let mut long_id = spawn_child(
15849 &spec,
15850 None,
15851 None,
15852 &runtime.stderr_ring,
15853 None,
15854 &fresh_roster,
15855 Some(&placement),
15856 )
15857 .unwrap();
15858 long_id.wait().await.unwrap();
15859 assert_eq!(
15860 group_count(),
15861 0,
15862 "valid long module IDs must not exceed cgroup NAME_MAX"
15863 );
15864 fs::remove_dir(root.join("subc-modules")).unwrap();
15865 fs::remove_dir(root).unwrap();
15866 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15867 }
15868
15869 #[test]
15870 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15871 let path = Path::new("/definitely-missing-subc-cgroup");
15872 let mut command = Command::new("true");
15873 let error = apply_cgroup_placement(
15874 &mut command,
15875 &ModuleSpec {
15876 module_id: "broken-cgroup".to_string(),
15877 program: PathBuf::from("true"),
15878 args: Vec::new(),
15879 env: Vec::new(),
15880 reserved: false,
15881 reserved_prefixes: Vec::new(),
15882 protocol: ModuleProtocol::Subc,
15883 overlap: Default::default(),
15884 },
15885 path,
15886 )
15887 .expect_err("a parent cgroup open failure must reject the supervised spawn");
15888 let reason = error.to_string();
15889
15890 assert!(
15891 matches!(error, SuperviseError::Cgroup { .. }),
15892 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15893 );
15894 assert!(
15895 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15896 "parent cgroup open failure must name cgroup.procs: {reason}"
15897 );
15898 }
15899
15900 #[tokio::test]
15901 async fn reaping_a_child_removes_its_empty_module_cgroup() {
15902 let root = ScratchDir::new("supervisor-reap-cgroup");
15903 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15904 let placement = subc_cgroup::prepare_at(&root)
15905 .expect("prepare scratch cgroup root")
15906 .expect("scratch root has a cgroup.procs marker");
15907 let module_id = "reaped-module";
15908 let module = placement
15909 .module_path(module_id)
15910 .expect("create scratch module cgroup");
15911 let child = Command::new("true")
15912 .env("XDG_DATA_HOME", root.path())
15913 .env("XDG_RUNTIME_DIR", root.path())
15914 .env("XDG_CONFIG_HOME", root.path())
15915 .spawn()
15916 .expect("spawn short-lived child");
15917 let pid = child.id().expect("spawned child has pid");
15918 let mut child = SupervisedChild {
15919 child,
15920 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15921 protocol: ModuleProtocol::Subc,
15922 module_id: module_id.to_string(),
15923 cgroup_placement: Some(placement),
15924 stdout_pump: None,
15925 stderr_pump: None,
15926 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15927 spawned_at: tokio::time::Instant::now(),
15928 spawned_at_ms: 0,
15929 spawned_from: PathBuf::from("true"),
15930 launch_spec_digest: LaunchSpecDigest([0; 32]),
15931 spawned_file_identity: None,
15932 process_start_time: None,
15933 process_identity: None,
15934 pid,
15935 roster_guard: None,
15936 #[cfg(target_os = "macos")]
15937 privacy_exec: None,
15938 spawn_failure: None,
15939 };
15940
15941 child.wait().await.expect("reap short-lived child");
15942
15943 assert!(
15944 !module.exists(),
15945 "reaping the supervised child must remove its empty cgroup"
15946 );
15947 }
15948
15949 #[test]
15950 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15951 let root = ScratchDir::new("supervisor-non-empty-cgroup");
15952 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15953 let placement = subc_cgroup::prepare_at(&root)
15954 .expect("prepare scratch cgroup root")
15955 .expect("scratch root has a cgroup.procs marker");
15956 let module = placement
15957 .module_path("surviving-module")
15958 .expect("create scratch module cgroup");
15959 fs::write(module.join("surviving-process"), b"still present")
15960 .expect("make scratch cgroup non-empty");
15961 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15962
15963 remove_module_cgroup(&placement, "surviving-module");
15964
15965 let logs = crate::router::test_log::captured_logs(&logs);
15966 assert!(
15967 module.exists(),
15968 "failed removal must leave the cgroup intact"
15969 );
15970 assert!(
15971 logs.contains("could not remove module cgroup after process exit; continuing teardown")
15972 && logs.contains("surviving-module"),
15973 "best-effort removal must report the failure without returning it: {logs}"
15974 );
15975 }
15976
15977 #[test]
15978 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15979 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15980 let reason = SuperviseError::Spawn {
15981 program: PathBuf::from("/bin/true"),
15982 source: io::Error::from_raw_os_error(13),
15983 cgroup_path: Some(cgroup_path.clone()),
15984 }
15985 .to_string();
15986
15987 assert!(
15988 reason.contains(&cgroup_path.display().to_string()),
15989 "a pre_exec spawn failure must name the cgroup path: {reason}"
15990 );
15991 }
15992}
15993
15994#[cfg(test)]
15995mod spawn_subscriber_lag_tests {
15996 use super::*;
15997
15998 #[tokio::test]
16003 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
16004 let feed = SpawnEventFeed::default();
16005 feed.configure_incarnation("lag-incarnation".to_string());
16006 let (tx, mut rx) = mpsc::channel(1);
16009 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
16010 .expect("subscribe");
16011 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
16012 for index in 0..emitted {
16013 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
16014 tokio::task::yield_now().await;
16017 }
16018 assert_eq!(
16019 feed.subscriber_count(),
16020 0,
16021 "the lagged subscriber must be removed"
16022 );
16023
16024 let mut data = Vec::new();
16025 let mut last = None;
16026 loop {
16027 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16028 .await
16029 .expect("the forwarder must finish once the subscriber is dropped");
16030 let Some(outbound) = next else { break };
16031 let frame = outbound.frame;
16032 if frame.header.ty == FrameType::StreamData {
16033 assert!(last.is_none(), "no data may follow the terminal frame");
16034 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
16035 data.push(event.cursor.seq);
16036 } else {
16037 assert!(last.is_none(), "exactly one terminal frame");
16038 last = Some(frame);
16039 }
16040 }
16041 assert!(!data.is_empty(), "queued frames drain before the terminal");
16042 for pair in data.windows(2) {
16043 assert_eq!(
16044 pair[1],
16045 pair[0] + 1,
16046 "queued frames arrive dense and in order"
16047 );
16048 }
16049 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
16050 assert_eq!(terminal.header.ty, FrameType::Error);
16051 assert_eq!(terminal.header.corr, 7);
16052 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
16053 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
16054 let detail = body.detail.expect("lagged error carries detail");
16055 assert_eq!(
16056 detail["first_undelivered_cursor"]["seq"],
16057 data.last().unwrap() + 1,
16058 "the named cursor is the first event the subscriber did not receive"
16059 );
16060 assert_eq!(
16061 detail["first_undelivered_cursor"]["daemon_incarnation"],
16062 "lag-incarnation"
16063 );
16064 }
16065}
16066
16067#[cfg(test)]
16068mod terminal_history_read_concurrency_tests {
16069 use super::*;
16070 use crate::terminal_journal::read_pause;
16071 use cortexkit_test_support::ScratchDir;
16072 use std::sync::mpsc as std_mpsc;
16073
16074 fn journaled_ring(
16075 journal: &Arc<crate::terminal_journal::TerminalJournal>,
16076 ) -> Arc<Mutex<TerminalRing>> {
16077 Arc::new(Mutex::new(
16078 TerminalRing::new(TerminalRingConfig::default(), 1)
16079 .with_journal(Some(Arc::clone(journal))),
16080 ))
16081 }
16082
16083 fn crash(at_ms: u64) -> ExitReport {
16084 ExitReport {
16085 kind: ExitKind::Crash,
16086 code: Some(1),
16087 signal: None,
16088 at_ms,
16089 }
16090 }
16091
16092 fn record_within(
16095 module_id: &'static str,
16096 ring: &Arc<Mutex<TerminalRing>>,
16097 at_ms: u64,
16098 bound: Duration,
16099 ) -> bool {
16100 let ring = Arc::clone(ring);
16101 let (done, done_rx) = std_mpsc::channel();
16102 std::thread::spawn(move || {
16103 record_terminal(
16104 module_id,
16105 &ring,
16106 &SpawnEventFeed::default(),
16107 &crash(at_ms),
16108 TerminalDisposition::Restarting,
16109 );
16110 let _ = done.send(());
16111 });
16112 done_rx.recv_timeout(bound).is_ok()
16113 }
16114
16115 #[test]
16120 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
16121 let dir = ScratchDir::new("terminal-history-concurrent-read");
16122 let path = dir.join("terminals.jsonl");
16123 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
16124 path.clone(),
16125 "daemon".into(),
16126 ));
16127 let reader_ring = journaled_ring(&journal);
16128 let other_ring = journaled_ring(&journal);
16129 assert!(record_within(
16130 "reader-module",
16131 &reader_ring,
16132 10,
16133 Duration::from_secs(5)
16134 ));
16135
16136 let (started, release) = read_pause::install(&path);
16137 let reading = {
16138 let ring = Arc::clone(&reader_ring);
16139 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
16140 };
16141 started
16142 .recv_timeout(Duration::from_secs(5))
16143 .expect("the history read reached its pause");
16144
16145 let bound = Duration::from_secs(1);
16146 assert!(
16147 record_within("other-module", &other_ring, 20, bound),
16148 "another module's exit waited on a history read (journal writer held)"
16149 );
16150 assert!(
16151 record_within("reader-module", &reader_ring, 30, bound),
16152 "the read module's own exit waited on its history read (ring held)"
16153 );
16154
16155 drop(release);
16156 let paused = reading.join().unwrap();
16157 assert_eq!(
16158 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16159 vec![10],
16160 "an exit recorded after the read began lands in neither half of it"
16161 );
16162 assert_eq!(paused.journal_skipped_lines, 0);
16163 assert_eq!(paused.journal_read_errors, 0);
16164
16165 let after = durable_terminal_history_of(&reader_ring, "reader-module");
16166 assert_eq!(
16167 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16168 vec![10, 30],
16169 "the next read merges ring and journal with no duplicate"
16170 );
16171 assert_eq!(after.journal_skipped_lines, 0);
16172 }
16173}
16174
16175#[cfg(test)]
16180mod stderr_settle_tests {
16181 use std::{
16182 future::Future,
16183 io,
16184 pin::Pin,
16185 sync::{Arc, Mutex},
16186 task::{Context, Poll},
16187 time::Duration,
16188 };
16189
16190 use tokio::{
16191 io::{AsyncRead, ReadBuf},
16192 sync::oneshot,
16193 time::Instant,
16194 };
16195
16196 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
16197 use crate::stderr_tail::{
16198 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
16199 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
16200 };
16201
16202 const BOUND: Duration = Duration::from_millis(250);
16203
16204 struct HeldReader {
16208 before: Option<Vec<u8>>,
16209 gate: Option<oneshot::Receiver<()>>,
16210 after: io::Cursor<Vec<u8>>,
16211 }
16212
16213 impl AsyncRead for HeldReader {
16214 fn poll_read(
16215 mut self: Pin<&mut Self>,
16216 cx: &mut Context<'_>,
16217 buf: &mut ReadBuf<'_>,
16218 ) -> Poll<io::Result<()>> {
16219 if let Some(bytes) = self.before.take() {
16220 buf.put_slice(&bytes);
16221 return Poll::Ready(Ok(()));
16222 }
16223 if let Some(gate) = self.gate.as_mut() {
16224 match Pin::new(gate).poll(cx) {
16225 Poll::Pending => return Poll::Pending,
16226 Poll::Ready(_) => self.gate = None,
16227 }
16228 }
16229 Pin::new(&mut self.after).poll_read(cx, buf)
16230 }
16231 }
16232
16233 struct DiscardSink;
16234
16235 impl OutputSink for DiscardSink {
16236 fn write_line(&mut self, _line: &[u8]) {}
16237 }
16238
16239 fn line(text: &str) -> TailEntry {
16240 TailEntry::Line {
16241 text: text.to_string(),
16242 truncated: false,
16243 at_ms: None,
16244 }
16245 }
16246
16247 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16248 ring.lock().unwrap()
16249 }
16250
16251 fn held_pump(
16255 ring: &Arc<Mutex<StderrRing>>,
16256 before: &str,
16257 after: &str,
16258 ) -> (StderrPump, oneshot::Sender<()>) {
16259 let generation = lock(ring).begin_process();
16260 let (release, gate) = oneshot::channel();
16261 let reader = HeldReader {
16262 before: Some(before.as_bytes().to_vec()),
16263 gate: Some(gate),
16264 after: io::Cursor::new(after.as_bytes().to_vec()),
16265 };
16266 let task = tokio::spawn(pump_stderr_to(
16267 reader,
16268 Arc::clone(ring),
16269 generation,
16270 DiscardSink,
16271 ));
16272 (StderrPump { task, generation }, release)
16273 }
16274
16275 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16276 for _ in 0..1000 {
16277 if done(&lock(ring)) {
16278 return;
16279 }
16280 tokio::time::sleep(Duration::from_millis(1)).await;
16281 }
16282 panic!(
16283 "ring never reached the expected state: {:?}",
16284 lock(ring).snapshot(None, None)
16285 );
16286 }
16287
16288 #[tokio::test(start_paused = true)]
16289 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16290 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16291 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16292
16293 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16294 let before_release = lock(&ring).snapshot(None, None);
16295 assert!(
16296 matches!(before_release.capture, CaptureState::Incomplete { .. }),
16297 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16298 );
16299
16300 let next = lock(&ring).begin_process();
16303 lock(&ring).push_line_from(next, "next process booting");
16304 release.send(()).unwrap();
16305 wait_until(&ring, |ring| {
16306 ring.snapshot(None, None).capture == CaptureState::Captured
16307 })
16308 .await;
16309
16310 assert_eq!(
16311 untimed(lock(&ring).snapshot(None, None).entries),
16312 vec![
16313 line("booting"),
16314 line("config error: missing storage"),
16315 TailEntry::ProcessStart,
16316 line("next process booting"),
16317 ],
16318 "the crash's last line must survive a slow reader and stay in the crashed process's section"
16319 );
16320 }
16321
16322 #[tokio::test(start_paused = true)]
16323 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16324 ) {
16325 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16326 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16329
16330 let started = Instant::now();
16331 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16332 assert_eq!(
16333 started.elapsed(),
16334 BOUND,
16335 "the restart must wait exactly the bound for a pipe that stays open, no longer"
16336 );
16337
16338 let next = lock(&ring).begin_process();
16339 lock(&ring).push_line_from(next, "next process booting");
16340 tokio::time::sleep(Duration::from_secs(60)).await;
16341
16342 let snapshot = lock(&ring).snapshot(None, None);
16343 match &snapshot.capture {
16344 CaptureState::Incomplete { reason } => assert!(
16345 reason.contains("had not reached EOF") && reason.contains("250ms"),
16346 "the reason must say what is missing and after how long: {reason}"
16347 ),
16348 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16349 }
16350 assert_eq!(
16351 untimed(snapshot.entries),
16352 vec![
16353 line("parent exiting"),
16354 TailEntry::ProcessStart,
16355 line("next process booting"),
16356 ]
16357 );
16358 }
16359
16360 #[tokio::test(start_paused = true)]
16361 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16362 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16363 let (pump, release) = held_pump(&ring, "one\n", "two\n");
16364 release.send(()).unwrap();
16365
16366 settle_stderr_pump("clean", &ring, pump, BOUND).await;
16367
16368 let snapshot = lock(&ring).snapshot(None, None);
16369 assert_eq!(snapshot.capture, CaptureState::Captured);
16370 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16371 }
16372
16373 #[tokio::test(start_paused = true)]
16374 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16375 const BURST: usize = 150;
16376 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16377 let path = capture.join("two-pipe.stderr.log");
16378 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16379 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16380 let generation = lock(&ring).begin_process();
16381 let stdout = (0..BURST)
16382 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16383 .collect::<String>();
16384 let stderr = (0..BURST)
16385 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16386 .collect::<String>();
16387 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
16392 let (release, gate) = oneshot::channel();
16393 let reader = HeldReader {
16394 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
16395 gate: Some(gate),
16396 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
16397 };
16398 let (finished, completion) = oneshot::channel();
16399 let stdout_sink = sink.clone();
16400 let pump = tokio::spawn(async move {
16401 pump_stdout_to(reader, stdout_sink).await;
16402 finished.send(()).unwrap();
16403 });
16404 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
16405
16406 let started = Instant::now();
16407 settle_stdout_pump("two-pipe", pump, BOUND).await;
16408 assert_eq!(
16409 started.elapsed(),
16410 BOUND,
16411 "a late reader must not hold up restart"
16412 );
16413 assert_eq!(
16414 std::fs::read_to_string(&path).unwrap().lines().count(),
16415 BURST + 1
16416 );
16417
16418 release
16419 .send(())
16420 .expect("stdout reader must survive the exit drain bound");
16421 tokio::time::timeout(Duration::from_secs(1), completion)
16422 .await
16423 .expect("released stdout reader must reach EOF")
16424 .expect("stdout capture must finish at EOF, not be cancelled");
16425 let contents = std::fs::read_to_string(&path).unwrap();
16426 let mut lines = contents
16427 .lines()
16428 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
16429 .collect::<Vec<_>>();
16430 let mut expected = (0..BURST)
16431 .flat_map(|i| {
16432 [
16433 format!("out-{i:04}-{}", "o".repeat(64)),
16434 format!("err-{i:04}-{}", "e".repeat(64)),
16435 ]
16436 })
16437 .collect::<Vec<_>>();
16438 lines.sort_unstable();
16439 expected.sort_unstable();
16440 assert_eq!(
16441 lines, expected,
16442 "both pipes must retain every complete line exactly once"
16443 );
16444 }
16445}
16446
16447#[cfg(all(test, windows))]
16461mod job_containment_tests {
16462 use super::*;
16463 use cortexkit_test_support::ScratchDir;
16464 use std::{
16465 path::{Path, PathBuf},
16466 sync::{Arc, Mutex},
16467 time::{Duration, Instant},
16468 };
16469
16470 fn stub_path() -> PathBuf {
16476 let mut path = std::env::current_exe().expect("current_exe available in tests");
16477 path.pop();
16478 path.pop();
16479 path.push("fake-aft-stub.exe");
16480 assert!(
16481 path.exists(),
16482 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16483 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16484 path.display()
16485 );
16486 path
16487 }
16488
16489 fn read_grandchild_pid(path: &Path) -> u32 {
16491 let deadline = Instant::now() + Duration::from_secs(10);
16492 loop {
16493 if let Ok(contents) = std::fs::read_to_string(path) {
16494 if let Ok(pid) = contents.trim().parse() {
16495 return pid;
16496 }
16497 }
16498 assert!(
16499 Instant::now() < deadline,
16500 "the stub never recorded a grandchild pid at {}",
16501 path.display()
16502 );
16503 std::thread::sleep(Duration::from_millis(10));
16504 }
16505 }
16506
16507 struct Fixture {
16510 _dir: ScratchDir,
16511 module_id: String,
16512 grandchild: u32,
16513 child: Option<SupervisedChild>,
16514 registry: Arc<Registry>,
16515 snapshot: Arc<Mutex<SupervisorSnapshot>>,
16516 terminal_ring: Arc<Mutex<TerminalRing>>,
16517 spawn_events: SpawnEventFeed,
16518 }
16519
16520 fn fixture(label: &str, module_id: &str) -> Fixture {
16521 let dir = ScratchDir::new(label);
16522 let pid_file = dir.join("grandchild.pid");
16523 let supervisor = Supervisor::new_for_test(
16524 Arc::new(Registry::default()),
16525 RestartPolicy::new(3, Duration::ZERO),
16526 );
16527 let runtime = supervisor.runtime_config();
16528 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16529 let spec = ModuleSpec {
16530 module_id: module_id.to_string(),
16531 program: stub_path(),
16532 args: Vec::new(),
16536 env: vec![
16537 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16538 (
16539 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16540 pid_file.display().to_string(),
16541 ),
16542 ],
16543 reserved: false,
16544 reserved_prefixes: Vec::new(),
16545 protocol: ModuleProtocol::Subc,
16546 overlap: Default::default(),
16547 };
16548 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16549 .expect("spawn the supervised fixture");
16550 let grandchild = read_grandchild_pid(&pid_file);
16551 Fixture {
16552 _dir: dir,
16553 module_id: module_id.to_string(),
16554 grandchild,
16555 child: Some(child),
16556 registry: Arc::new(Registry::default()),
16557 snapshot,
16558 terminal_ring: Arc::clone(&runtime.terminal_ring),
16559 spawn_events: SpawnEventFeed::default(),
16560 }
16561 }
16562
16563 impl Fixture {
16564 async fn drain(&mut self) {
16566 let child = self
16567 .child
16568 .take()
16569 .expect("the fixture child is still present");
16570 drain_child_to_state(
16571 &self.module_id,
16572 ModuleProtocol::Subc,
16573 StopNotice::NotSent,
16576 &self.registry,
16577 None,
16578 &self.snapshot,
16579 &self.terminal_ring,
16580 &self.spawn_events,
16581 child,
16582 Duration::from_millis(500),
16583 ModuleState::Stopped,
16584 Some(false),
16585 &FailureContext::operator(),
16586 )
16587 .await
16588 .expect("drain the supervised fixture");
16589 }
16590 }
16591
16592 #[tokio::test]
16598 async fn teardown_reaps_the_grandchild() {
16599 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16600 let grandchild = fixture.grandchild;
16601
16602 assert!(
16603 subc_jobobject::process_exists(grandchild),
16604 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16605 );
16606
16607 fixture.drain().await;
16608
16609 assert!(
16610 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16611 "grandchild {grandchild} outlived module teardown: the tree was not contained"
16612 );
16613 }
16614
16615 #[test]
16628 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16629 let dir = ScratchDir::new("teardown-uncontained");
16630 let pid_file = dir.join("grandchild.pid");
16631 let mut child = std::process::Command::new(stub_path())
16632 .env("FAKE_AFT_NEVER_CONNECT", "1")
16633 .env(
16634 "FAKE_AFT_GRANDCHILD_PID_FILE",
16635 pid_file.display().to_string(),
16636 )
16637 .stdin(std::process::Stdio::null())
16638 .stdout(std::process::Stdio::null())
16639 .stderr(std::process::Stdio::null())
16640 .spawn()
16641 .expect("spawn the uncontained fixture");
16642 let grandchild = read_grandchild_pid(&pid_file);
16643
16644 child.kill().expect("kill the direct child");
16646 let _ = child.wait();
16647
16648 assert!(
16649 subc_jobobject::process_exists(grandchild),
16650 "grandchild {grandchild} died with the direct child, so this control no longer \
16651 distinguishes contained from uncontained teardown and the regression test is \
16652 passing vacuously"
16653 );
16654
16655 kill_tree(grandchild);
16658 }
16659
16660 #[tokio::test]
16669 async fn dropping_containment_reaps_the_grandchild() {
16670 let mut fixture = fixture("drop-containment", "tree-drop");
16671 let grandchild = fixture.grandchild;
16672
16673 assert!(subc_jobobject::process_exists(grandchild));
16674
16675 fixture.child.as_mut().expect("child present").job = None;
16677
16678 assert!(
16679 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16680 "grandchild {grandchild} survived the containment handle closing, so a daemon \
16681 crash would leave the tree behind"
16682 );
16683 }
16684
16685 fn kill_tree(pid: u32) {
16687 let _ = std::process::Command::new("taskkill.exe")
16688 .args(["/PID", &pid.to_string(), "/T", "/F"])
16689 .stdin(std::process::Stdio::null())
16690 .stdout(std::process::Stdio::null())
16691 .stderr(std::process::Stdio::null())
16692 .status();
16693 assert!(
16694 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16695 "could not clean up grandchild {pid}"
16696 );
16697 }
16698}
16699
16700#[cfg(test)]
16701mod privacy_trampoline_configuration_tests {
16702 #[tokio::test]
16703 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16704 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16705 #[cfg(target_os = "macos")]
16706 {
16707 let supervisor = super::Supervisor::new(
16708 std::sync::Arc::new(crate::Registry::default()),
16709 super::RestartPolicy::default(),
16710 );
16711 let error = supervisor.spawn(spec()).unwrap_err();
16712 assert!(
16713 error
16714 .to_string()
16715 .contains("no privacy trampoline configured"),
16716 "{error}"
16717 );
16718 }
16719 }
16720
16721 #[tokio::test]
16722 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16723 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16724 #[cfg(target_os = "macos")]
16725 {
16726 let supervisor = super::Supervisor::new(
16727 std::sync::Arc::new(crate::Registry::default()),
16728 super::RestartPolicy::default(),
16729 )
16730 .with_privacy_trampoline(std::env::current_exe().unwrap());
16731 let error = supervisor.spawn(spec()).unwrap_err();
16732 assert!(
16733 error
16734 .to_string()
16735 .contains("binary does not implement the privacy trampoline protocol"),
16736 "{error}"
16737 );
16738 }
16739 }
16740
16741 #[cfg(target_os = "macos")]
16742 fn spec() -> super::ModuleSpec {
16743 super::ModuleSpec {
16744 module_id: "privacy-configuration".into(),
16745 program: "/bin/sleep".into(),
16746 args: vec!["30".into()],
16747 env: vec![],
16748 reserved: false,
16749 reserved_prefixes: vec![],
16750 protocol: subc_control::ModuleProtocol::None,
16751 overlap: super::ModuleOverlap::Exclusive,
16752 }
16753 }
16754}
16755
16756#[cfg(test)]
16757mod privacy_exec_boundary_tests {
16758 #[cfg(target_os = "macos")]
16759 use super::*;
16760 #[cfg(target_os = "macos")]
16761 use std::{
16762 io::{Read, Write},
16763 net::{TcpListener, TcpStream},
16764 };
16765
16766 #[cfg(target_os = "macos")]
16769 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16770 if let Some((_, path)) = spec
16771 .env
16772 .iter()
16773 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16774 {
16775 let mut barrier = TcpStream::connect(path).unwrap();
16776 barrier
16777 .set_read_timeout(Some(Duration::from_secs(30)))
16778 .unwrap();
16779 barrier.write_all(&pid.to_ne_bytes()).unwrap();
16780 let mut release = [0];
16781 barrier.read_exact(&mut release).unwrap();
16782 assert_eq!(&release, b"X");
16783 }
16784 }
16785
16786 #[cfg(target_os = "macos")]
16787 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16788 ModuleSpec {
16789 module_id: "privacy-boundary".into(),
16790 program: program.into(),
16791 args: args.iter().map(|arg| (*arg).into()).collect(),
16792 env: vec![],
16793 reserved: false,
16794 reserved_prefixes: vec![],
16795 protocol: ModuleProtocol::None,
16796 overlap: ModuleOverlap::Exclusive,
16797 }
16798 }
16799
16800 #[cfg(target_os = "macos")]
16801 async fn accept(listener: TcpListener) -> TcpStream {
16802 let listener = tokio::net::TcpListener::from_std({
16804 listener.set_nonblocking(true).unwrap();
16805 listener
16806 })
16807 .unwrap();
16808 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16809 .await
16810 .unwrap()
16811 .unwrap();
16812 let stream = stream.into_std().unwrap();
16813 stream.set_nonblocking(false).unwrap();
16814 stream
16815 .set_read_timeout(Some(Duration::from_secs(30)))
16816 .unwrap();
16817 stream
16818 }
16819
16820 #[tokio::test]
16821 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16822 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16823 #[cfg(target_os = "macos")]
16824 {
16825 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16826 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16829 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16830 let record = root.join("live-children.json");
16831 let supervisor =
16832 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16833 .with_live_children_record(&record);
16834 let runtime = supervisor.runtime_config();
16835 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16836 let mut spec = spec("/bin/sleep", &["30"]);
16837 spec.env = vec![
16838 (
16839 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16840 exec_listener.local_addr().unwrap().to_string(),
16841 ),
16842 (
16843 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16844 sample_listener.local_addr().unwrap().to_string(),
16845 ),
16846 ];
16847 let spawn = tokio::task::spawn_blocking(move || {
16848 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16849 });
16850 let mut sample = accept(sample_listener).await;
16851 let mut pid = [0; 4];
16852 sample.read_exact(&mut pid).unwrap();
16853 let pid = u32::from_ne_bytes(pid);
16854 let mut exec = accept(exec_listener).await;
16855 let mut ready = [0];
16856 exec.read_exact(&mut ready).unwrap();
16857 assert_eq!(&ready, b"R");
16858 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16861 assert_eq!(
16862 observe_spawned_image(pid).unwrap().executable,
16863 Some(trampoline)
16864 );
16865 sample.write_all(b"X").unwrap();
16866 let mut child = spawn.await.unwrap();
16867 let early = crate::live_children::read_record(&record).unwrap();
16868 assert_eq!(early.len(), 1);
16869 assert_eq!(early[0].pid, pid);
16870 assert_eq!(
16871 early[0].executable, None,
16872 "unconfirmed trampoline image entered the roster"
16873 );
16874 assert!(child.report_ready.get().is_none());
16875 child.privacy_exec.as_mut().unwrap().deadline =
16878 tokio::time::Instant::now() + Duration::from_secs(30);
16879 exec.write_all(b"X").unwrap();
16880 child.confirm_privacy_exec().await;
16881 assert_eq!(child.spawn_failure, None);
16882 assert!(child.report_ready.get().is_some());
16883 let confirmed = crate::live_children::read_record(&record).unwrap();
16884 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16885 assert_ne!(module, trampoline);
16886 assert_eq!(confirmed[0].executable, Some(module.into()));
16887 child.start_kill().unwrap();
16888 child.wait().await.unwrap();
16889 child.release_roster();
16890 }
16891 }
16892
16893 #[tokio::test]
16894 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16895 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16896 #[cfg(target_os = "macos")]
16897 {
16898 let registry = Arc::new(Registry::default());
16899 let policy = RestartPolicy::new(0, Duration::ZERO);
16900 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
16901 let runtime = supervisor.runtime_config();
16902 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16903 let spec = spec("/bin/sh", &["-c", "exit 121"]);
16904 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16908 let pid = child.pid;
16909 tokio::task::spawn_blocking(move || {
16910 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16911 })
16912 .await
16913 .unwrap()
16914 .unwrap();
16915 child.privacy_exec.as_mut().unwrap().deadline =
16916 tokio::time::Instant::now() + Duration::from_secs(30);
16917 let status = child.wait().await.unwrap();
16918 assert_eq!(status.code(), Some(121));
16919 assert!(child.privacy_exec.is_none());
16920 assert!(
16921 child.report_ready.get().is_none(),
16922 "an exited module must not publish a live pid"
16923 );
16924 let report = classify_reaped_child_exit(&snapshot, &child, &status);
16925 on_child_exit(
16926 &spec,
16927 policy,
16928 ®istry,
16929 &snapshot,
16930 &runtime.terminal_ring,
16931 &runtime.spawn_events,
16932 &runtime.child_roster,
16933 report,
16934 true,
16935 )
16936 .await;
16937 let state = lock_snapshot(&snapshot).unwrap();
16938 assert_eq!(state.state, ModuleState::Failed);
16939 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16940 assert_eq!(state.reported_pid(), None);
16941 drop(state);
16942 let history = runtime.terminal_ring.lock().unwrap().snapshot();
16943 assert_eq!(history.entries.len(), 1);
16944 let terminal = &history.entries[0];
16945 assert_eq!(terminal.exit_code, Some(121));
16946 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16947 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16948 assert_eq!(
16949 terminal.disposition_detail.as_deref(),
16950 Some(policy.budget_exhausted_detail().as_str()),
16951 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16952 );
16953 assert_eq!(child.spawn_failure, None);
16954 child.release_roster();
16955 }
16956 }
16957}
16958
16959#[cfg(all(test, unix))]
16963mod launch_nonce_descriptor_tests {
16964 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16965 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16966 use cortexkit_test_support::ScratchDir;
16967 use std::{
16968 path::PathBuf,
16969 sync::{Arc, Mutex},
16970 time::{Duration, Instant},
16971 };
16972
16973 async fn probe(role: super::SpawnRole) {
16974 let scratch = ScratchDir::new("launch-nonce-descriptor");
16975 let fd_copy = scratch.join("from-descriptor");
16976 let env_copy = scratch.join("environment");
16977 let script = format!(
16978 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16979 fd = fd_copy.display(), env = env_copy.display(),
16980 );
16981 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16982 let spec = ModuleSpec {
16983 module_id: "nonce-descriptor-probe".to_string(),
16984 program: PathBuf::from("/bin/sh"),
16985 args: vec!["-c".to_string(), script],
16986 env: vec![
16987 xdg("XDG_DATA_HOME"),
16988 xdg("XDG_RUNTIME_DIR"),
16989 xdg("XDG_CONFIG_HOME"),
16990 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16991 ],
16992 reserved: true,
16993 reserved_prefixes: Vec::new(),
16994 protocol: ModuleProtocol::Subc,
16995 overlap: Default::default(),
16996 };
16997 let handle = SupervisorHandle::new();
16998 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16999 let roster = ChildRoster::default();
17000 #[cfg(target_os = "macos")]
17001 {
17002 let path = super::test_privacy_trampoline();
17003 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
17004 }
17005 let child = super::spawn_child_in_slot(
17006 &spec,
17007 None,
17008 Some(&handle),
17009 &ring,
17010 None,
17011 &roster,
17012 #[cfg(target_os = "linux")]
17013 None,
17014 role,
17015 matches!(role, super::SpawnRole::SwapCandidate),
17016 )
17017 .expect("spawn probe");
17018 let deadline = Instant::now() + Duration::from_secs(10);
17019 while !(fd_copy.exists() && env_copy.exists()) {
17020 assert!(Instant::now() < deadline, "probe never wrote its copies");
17021 tokio::time::sleep(Duration::from_millis(20)).await;
17022 }
17023 let nonce = std::fs::read_to_string(fd_copy).unwrap();
17024 assert!(!nonce.is_empty());
17025 let environment = std::fs::read_to_string(env_copy).unwrap();
17026 assert!(environment
17027 .lines()
17028 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
17029 let copy = environment
17030 .lines()
17031 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
17032 assert_eq!(
17033 copy, None,
17034 "Unix children must never receive the environment nonce"
17035 );
17036 if matches!(role, super::SpawnRole::Plain) {
17037 assert_eq!(
17038 handle.spawn_nonce(&spec.module_id).as_deref(),
17039 Some(nonce.as_str())
17040 );
17041 }
17042 drop(child);
17043 }
17044
17045 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17046 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
17047 probe(super::SpawnRole::Plain).await;
17048 }
17049
17050 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17051 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
17052 probe(super::SpawnRole::SwapCandidate).await;
17053 }
17054}
17055
17056#[cfg(all(test, target_os = "linux"))]
17057mod cgroup_containment_tests {
17058 use super::*;
17059 use cortexkit_test_support::ScratchDir;
17060
17061 fn running(pid: u32) -> bool {
17062 std::fs::read_to_string(format!("/proc/{pid}/stat"))
17064 .ok()
17065 .and_then(|stat| {
17066 stat.rsplit_once(") ")
17067 .map(|(_, rest)| rest.starts_with('Z'))
17068 })
17069 .is_some_and(|zombie| !zombie)
17070 }
17071
17072 #[tokio::test]
17073 async fn linux_teardown_reaps_the_grandchild() {
17074 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
17075 }
17076
17077 #[tokio::test]
17078 async fn linux_shutdown_straggler_reaps_the_grandchild() {
17079 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
17080 }
17081
17082 async fn teardown_tree(test_name: &str, shutdown: bool) {
17083 let dir = ScratchDir::new(test_name);
17084 let root = PathBuf::from(format!(
17085 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
17086 std::process::id(),
17087 unix_ms_now()
17088 ));
17089 if let Err(error) = std::fs::create_dir(&root) {
17090 assert!(
17091 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17092 "required cgroup test cannot execute: {error}"
17093 );
17094 eprintln!(
17095 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
17096 root.display()
17097 );
17098 return;
17099 }
17100 let placement = subc_cgroup::prepare_at(&root)
17101 .expect("prepare isolated kernel cgroup")
17102 .expect("isolated cgroup is delegated");
17103 let module_id = "tree-teardown";
17104 let module = placement
17105 .module_path(module_id)
17106 .expect("create isolated module cgroup");
17107 if !module.join("cgroup.kill").exists() {
17108 std::fs::remove_dir(&module).unwrap();
17109 std::fs::remove_dir(root.join("subc-modules")).unwrap();
17110 std::fs::remove_dir(&root).unwrap();
17111 assert!(
17112 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17113 "required cgroup.kill interface unavailable"
17114 );
17115 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
17116 return;
17117 }
17118 let supervisor = Supervisor::new_for_test(
17119 Arc::new(Registry::default()),
17120 RestartPolicy::new(3, Duration::ZERO),
17121 )
17122 .with_cgroup_placement(Some(placement));
17123 let mut runtime = supervisor.runtime_config();
17124 runtime.child_roster = runtime
17125 .child_roster
17126 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
17127 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17128 let pid_file = dir.join("grandchild.pid");
17129 let spec = ModuleSpec {
17130 module_id: module_id.to_string(),
17131 program: PathBuf::from("/bin/sh"),
17132 args: vec![
17133 "-c".into(),
17134 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
17135 "fixture".into(),
17136 pid_file.display().to_string(),
17137 ],
17138 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17139 .into_iter()
17140 .map(|key| (key.to_string(), dir.display().to_string()))
17141 .collect(),
17142 reserved: false,
17143 reserved_prefixes: Vec::new(),
17144 protocol: ModuleProtocol::None,
17145 overlap: Default::default(),
17146 };
17147 let child =
17148 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
17149 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17150 let grandchild: u32 = loop {
17151 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
17152 if let Ok(pid) = contents.trim().parse() {
17153 break pid;
17154 }
17155 }
17156 assert!(
17157 tokio::time::Instant::now() < deadline,
17158 "grandchild pid was not recorded"
17159 );
17160 tokio::time::sleep(Duration::from_millis(10)).await;
17161 };
17162 assert!(
17163 running(grandchild),
17164 "grandchild must be alive before teardown"
17165 );
17166 if shutdown {
17167 let mut child = child;
17168 crate::child_roster::end_children_for_daemon_shutdown(
17169 &runtime.child_roster,
17170 false,
17171 std::future::pending(),
17172 )
17173 .await;
17174 child.wait().await.expect("reap shutdown straggler");
17175 } else {
17176 drain_child_to_state(
17177 module_id,
17178 ModuleProtocol::None,
17179 StopNotice::NotSent,
17180 &Registry::default(),
17181 None,
17182 &snapshot,
17183 &runtime.terminal_ring,
17184 &SpawnEventFeed::default(),
17185 child,
17186 Duration::from_millis(100),
17187 ModuleState::Stopped,
17188 Some(false),
17189 &FailureContext::operator(),
17190 )
17191 .await
17192 .expect("real supervisor teardown");
17193 }
17194 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
17195 while running(grandchild) && tokio::time::Instant::now() < deadline {
17196 tokio::time::sleep(Duration::from_millis(10)).await;
17197 }
17198 let survived = running(grandchild);
17199 if survived {
17201 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
17202 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
17203 tokio::time::sleep(Duration::from_millis(100)).await;
17204 }
17205 if module.exists() {
17206 std::fs::remove_dir(&module).expect("remove empty module cgroup");
17207 }
17208 std::fs::remove_dir(root.join("subc-modules")).unwrap();
17209 std::fs::remove_dir(&root).unwrap();
17210 assert!(
17211 !survived,
17212 "grandchild {grandchild} outlived module teardown"
17213 );
17214 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
17215 }
17216}
17217
17218#[cfg(all(test, unix))]
17221mod record_write_stall_tests {
17222 use super::*;
17223 use std::sync::atomic::{AtomicBool, Ordering};
17224 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
17225
17226 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
17229 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
17233
17234 fn sleeping_spec() -> ModuleSpec {
17246 ModuleSpec {
17247 module_id: "record-write-stall".into(),
17248 program: "/bin/sleep".into(),
17249 args: vec!["600".into()],
17250 env: vec![],
17251 reserved: false,
17252 reserved_prefixes: vec![],
17253 protocol: ModuleProtocol::None,
17254 overlap: Default::default(),
17255 }
17256 }
17257
17258 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17259 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17260 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17261 let supervisor =
17262 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17263 .with_live_children_record(home.join("live-children.json"));
17264 let module = supervisor.spawn(sleeping_spec()).unwrap();
17265 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17270 loop {
17271 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17272 .unwrap()
17273 .reported_pid()
17274 {
17275 return pid;
17276 }
17277 tokio::time::sleep(Duration::from_millis(10)).await;
17278 }
17279 })
17280 .await
17281 .expect("the first process was admitted as the module");
17282
17283 let key = vec![0x42; 32];
17286 let daemon_id = [0x24; 16];
17287 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17288 let port = listener.local_addr().unwrap().port();
17289 let server = tokio::spawn(crate::server::serve_listener(
17290 listener,
17291 Arc::new(crate::router::Router::with_default_self_handler()),
17292 crate::server::ServerAuth::with_limits(
17293 key.clone(),
17294 daemon_id,
17295 "test",
17296 crate::server::DEFAULT_AUTH_DEADLINE,
17297 4,
17298 ),
17299 ));
17300
17301 let (held_tx, held_rx) = std::sync::mpsc::channel();
17304 let armed = AtomicBool::new(true);
17305 supervisor
17306 .child_roster
17307 .set_record_write_hook_for_test(move |live| {
17308 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17309 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17310 let _ = held_tx.send(());
17311 std::thread::sleep(HELD_RECORD_WRITE);
17312 }
17313 });
17314
17315 let module_process = std::thread::spawn(move || {
17318 held_rx
17319 .recv_timeout(Duration::from_secs(30))
17320 .expect("the restart reached the replacement's record write");
17321 let conn = ConnectionInfo {
17322 schema: SCHEMA_VERSION,
17323 wire_version: None,
17324 endpoints: vec![Endpoint {
17325 host: "127.0.0.1".to_owned(),
17326 port,
17327 }],
17328 key,
17329 daemon_id,
17330 pid: std::process::id(),
17331 daemon_ver: "test".to_owned(),
17332 };
17333 tokio::runtime::Builder::new_current_thread()
17334 .enable_all()
17335 .build()
17336 .unwrap()
17337 .block_on(async {
17338 let started = Instant::now();
17339 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17340 .await
17341 .unwrap();
17342 let result =
17343 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17344 (result, started.elapsed())
17345 })
17346 });
17347
17348 module.restart(Some(0)).await.unwrap();
17349 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17350 .await
17351 .unwrap();
17352 assert!(
17353 auth.is_ok(),
17354 "the daemon must answer a module's handshake while a restart waits on the \
17355 live-children record write; got {auth:?} after {elapsed:?}"
17356 );
17357
17358 tokio::time::timeout(Duration::from_secs(20), async {
17359 while lock_snapshot(&module.inner.snapshot)
17360 .unwrap()
17361 .reported_pid()
17362 .is_none_or(|pid| pid == old_pid)
17363 {
17364 tokio::time::sleep(Duration::from_millis(10)).await;
17365 }
17366 })
17367 .await
17368 .expect("the restart published its replacement");
17369 module.stop().await.unwrap();
17370 server.abort();
17371 }
17372}