1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136 deadline: tokio::time::Instant,
137 expected: Option<subc_os::FileIdentity>,
138 trampoline: Option<subc_os::FileIdentity>,
139 script: bool,
140 module_id: String,
141 #[cfg(test)]
142 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147 let path = std::env::current_exe()
150 .unwrap()
151 .parent()
152 .unwrap()
153 .parent()
154 .unwrap()
155 .join("privacy-trampoline-fixture");
156 assert!(
160 path.exists(),
161 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162 --bins --features test-support` or `cargo test -p subc-daemon` first",
163 path.display()
164 );
165 path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170 use std::io::Read;
171 let mut probe = std::process::Command::new(path)
172 .args(["__disclaim-exec", "--probe"])
173 .stdin(Stdio::null())
174 .stdout(Stdio::piped())
175 .stderr(Stdio::piped())
176 .spawn()
177 .map_err(|error| {
178 format!(
179 "privacy trampoline probe failed for {}: {error}",
180 path.display()
181 )
182 })?;
183 let deadline = std::time::Instant::now() + Duration::from_secs(5);
184 let status = loop {
185 match probe.try_wait() {
186 Ok(Some(status)) => break status,
187 Ok(None) if std::time::Instant::now() < deadline => {
188 std::thread::sleep(Duration::from_millis(5))
189 }
190 result => {
191 let _ = probe.kill();
192 let _ = probe.wait();
193 return Err(format!(
194 "privacy trampoline probe failed or timed out for {}: {result:?}",
195 path.display()
196 ));
197 }
198 }
199 };
200 let mut answer = String::new();
201 if let Some(stdout) = probe.stdout.take() {
202 let _ = stdout.take(256).read_to_string(&mut answer);
203 }
204 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205 return Ok(());
206 }
207 let mut diagnostic = String::new();
208 if let Some(stderr) = probe.stderr.take() {
209 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210 }
211 let cause = diagnostic
212 .trim()
213 .strip_prefix("ck-subc: own privacy identity refused: ")
214 .unwrap_or("binary does not implement the privacy trampoline protocol");
215 Err(format!(
216 "{cause}: probe of {} exited {status}",
217 path.display()
218 ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223 spec: &ModuleSpec,
224 roster: &ChildRoster,
225) -> Result<
226 (
227 Command,
228 Option<PrivacyExec>,
229 subc_os::privacy_identity::ExecAcknowledgement,
230 ),
231 SuperviseError,
232> {
233 let failure = |cause: String| {
234 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235 SuperviseError::Spawn {
236 program: spec.program.clone(),
237 source: io::Error::other(cause),
238 cgroup_path: None,
239 }
240 };
241 let trampoline = roster.privacy_trampoline().map_err(failure)?;
242 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247 let path = spec
248 .env
249 .iter()
250 .find(|(key, _)| key == "PATH")
251 .map(|(_, value)| std::ffi::OsString::from(value))
252 .or_else(|| std::env::var_os("PATH"))
253 .unwrap_or_else(|| "/usr/bin:/bin".into());
254 std::env::split_paths(&path)
255 .map(|dir| dir.join(&spec.program))
256 .find(|path| path.is_file())
257 .unwrap_or_else(|| spec.program.clone())
258 } else {
259 spec.program.clone()
260 };
261 let expected = subc_os::file_identity(&program);
262 let trampoline_image = subc_os::file_identity(&trampoline);
263 let script = {
264 use std::io::Read;
265 let mut prefix = [0u8; 2];
266 std::fs::File::open(&program)
267 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268 };
269 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270 return Err(failure(
271 "privacy identity module executable is missing or is the trampoline itself".to_string(),
272 ));
273 }
274 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275 .map_err(|error| failure(error.to_string()))?;
276 let reader =
277 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278 let mut command = Command::new(&trampoline);
279 command
280 .arg("__disclaim-exec")
281 .arg(ack.fd().to_string())
282 .arg(&program);
283 ack.install(command.as_std_mut());
284 Ok((
285 command,
286 Some(PrivacyExec {
287 reader,
288 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289 expected,
290 trampoline: trampoline_image,
291 script,
292 module_id: spec.module_id.clone(),
293 #[cfg(test)]
294 confirmation_pause: spec
295 .env
296 .iter()
297 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298 .map(|(_, address)| (address.clone(), None)),
299 }),
300 ack,
301 ))
302}
303
304struct SupervisedChild {
305 child: Child,
306 exec: crate::activity::ExecOwner,
307 #[cfg(windows)]
308 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309 #[cfg(target_os = "macos")]
310 privacy_exec: Option<PrivacyExec>,
311 #[cfg(target_os = "macos")]
318 report_ready: Arc<OnceLock<()>>,
319 spawn_failure: Option<String>,
321 protocol: ModuleProtocol,
326 #[cfg(target_os = "linux")]
332 module_id: String,
333 #[cfg(target_os = "linux")]
334 cgroup_placement: Option<subc_cgroup::Placement>,
335 #[cfg(windows)]
357 job: Option<subc_jobobject::JobObject>,
358 stdout_pump: Option<JoinHandle<()>>,
359 stderr_pump: Option<StderrPump>,
360 stderr_ring: Arc<Mutex<StderrRing>>,
361 spawned_at: Instant,
364 spawned_at_ms: u64,
365 spawned_from: PathBuf,
366 launch_spec_digest: LaunchSpecDigest,
367 spawned_file_identity: Option<SpawnedFileIdentity>,
368 process_start_time: Option<u64>,
369 process_identity: Option<ProcessIdentity>,
370 pid: u32,
371 roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377 fn id(&self) -> Option<u32> {
378 Some(self.pid)
379 }
380
381 fn process_identity(&self) -> Option<ProcessIdentity> {
382 self.process_identity
383 }
384
385 async fn wait(&mut self) -> io::Result<ExitStatus> {
386 #[cfg(target_os = "macos")]
387 self.confirm_privacy_exec().await;
388 let result = self.child.wait().await;
396 #[cfg(target_os = "linux")]
397 if result.is_ok() {
398 if let Some(placement) = self.cgroup_placement.as_ref() {
399 cleanup_reaped_cgroup(placement, &self.module_id).await;
400 self.cgroup_placement = None;
403 }
404 }
405 result
406 }
407
408 #[cfg(target_os = "macos")]
409 async fn confirm_privacy_exec(&mut self) {
410 let Some(pending) = &mut self.privacy_exec else {
411 return;
412 };
413 let result = tokio::time::timeout_at(pending.deadline, async {
414 let mut record = Vec::new();
415 loop {
416 let mut ready = pending.reader.readable().await?;
417 let read = ready.try_io(|reader| {
418 use std::io::Read;
419 let mut reader = reader.get_ref();
420 let mut buffer = [0u8; 256];
421 reader.read(&mut buffer).map(|count| (count, buffer))
422 });
423 match read {
424 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425 Ok(Ok((count, buffer))) => {
426 if record.len() + count > 1024 {
427 return Err(io::Error::other(
428 "privacy exec refusal record is too long",
429 ));
430 }
431 record.extend_from_slice(&buffer[..count]);
432 }
433 Ok(Err(error)) => return Err(error),
434 Err(_) => continue,
435 }
436 }
437 })
438 .await;
439 #[cfg(test)]
440 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441 use tokio::io::{AsyncReadExt, AsyncWriteExt};
442 if stream.is_none() {
443 *stream = Some(
444 tokio::net::TcpStream::connect(address.as_str())
445 .await
446 .unwrap(),
447 );
448 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449 stream
450 .as_mut()
451 .unwrap()
452 .write_all(&self.pid.to_ne_bytes())
453 .await
454 .unwrap();
455 }
456 let mut reply = [0];
457 stream
458 .as_mut()
459 .unwrap()
460 .read_exact(&mut reply)
461 .await
462 .unwrap();
463 pending.confirmation_pause = None;
464 if &reply == b"F" {
465 Ok(Err(io::Error::other("injected late exec refusal")))
466 } else {
467 assert_eq!(&reply, b"X");
468 result
469 }
470 } else {
471 result
472 };
473 let pending = self.privacy_exec.as_ref().expect("pending exec");
476 let cause = match result {
477 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478 Ok(Err(error)) => Some(format!(
479 "privacy identity exec acknowledgement failed: {error}"
480 )),
481 Ok(Ok(record)) if !record.is_empty() => Some(
482 std::str::from_utf8(&record)
483 .ok()
484 .and_then(|record| {
485 record
486 .trim()
487 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488 })
489 .filter(|cause| !cause.is_empty())
490 .unwrap_or("invalid privacy exec refusal record")
491 .to_string(),
492 ),
493 Ok(Ok(_)) => match self.child.try_wait() {
494 Ok(Some(_status)) => None,
498 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499 Ok(None) => {
500 let image = observe_spawned_image(self.pid);
501 if let Some(image) = image.filter(|image| {
502 image.executable.is_some()
503 && image.executable != pending.trampoline
504 && (image.executable == pending.expected || pending.script)
505 }) {
506 if let Err(error) = self
507 .roster_guard
508 .as_ref()
509 .map(|guard| guard.confirm_executable(image))
510 .transpose()
511 {
512 Some(format!(
513 "could not durably record confirmed module image: {error}"
514 ))
515 } else {
516 let _ = self.report_ready.set(());
517 info!(module_id = %pending.module_id, pid = self.pid,
518 "module spawned with own privacy identity (responsibility disclaimed)");
519 None
520 }
521 } else if image.is_none()
522 || image.is_some_and(|image| image.executable.is_none())
523 {
524 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531 Ok(Ok(_status)) => None,
532 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534 }
535 } else {
536 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537 }
538 }
539 },
540 };
541 let pending = self.privacy_exec.take().expect("pending exec");
542 self.exec.0.finish(if cause.is_some() {
545 crate::activity::ExecState::Refused
546 } else {
547 crate::activity::ExecState::CommittedExec
548 });
549 if let Some(cause) = cause {
550 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551 self.spawn_failure = Some(cause);
552 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556 }
557 let _ = self.child.start_kill();
558 }
559 }
560
561 fn release_roster(&mut self) {
565 self.roster_guard = None;
566 }
567
568 fn start_kill(&mut self) -> io::Result<()> {
581 #[cfg(windows)]
582 if let Some(job) = &self.job {
583 if let Err(error) = job.terminate() {
584 debug!(
585 error = %error,
586 "job termination failed; the direct-child kill still owns the outcome"
587 );
588 }
589 }
590 #[cfg(target_os = "linux")]
591 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592 self.child.start_kill()
593 }
594
595 async fn drain_stderr(&mut self, module_id: &str) {
596 if let Some(pump) = self.stdout_pump.take() {
597 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598 }
599
600 let Some(pump) = self.stderr_pump.take() else {
601 return;
602 };
603 settle_stderr_pump(
604 module_id,
605 &self.stderr_ring,
606 pump,
607 STDERR_PUMP_DRAIN_TIMEOUT,
608 )
609 .await;
610 }
611}
612
613async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617 match timeout(bound, &mut pump).await {
618 Ok(Ok(())) => {}
619 Ok(Err(error)) => {
620 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621 }
622 Err(_) => {
623 drop(pump);
624 warn!(
625 module_id,
626 waited = ?bound,
627 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628 );
629 }
630 }
631}
632
633struct StderrPump {
636 task: JoinHandle<()>,
637 generation: u64,
638}
639
640async fn settle_stderr_pump(
646 module_id: &str,
647 ring: &Arc<Mutex<StderrRing>>,
648 pump: StderrPump,
649 bound: Duration,
650) {
651 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652 let StderrPump {
653 mut task,
654 generation,
655 } = pump;
656 lock().retire_pump(generation);
657 match timeout(bound, &mut task).await {
658 Ok(Ok(())) => {}
659 Ok(Err(err)) => {
660 let mut ring = lock();
661 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662 ring.finish_pump(generation);
663 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664 }
665 Err(_) => {
666 drop(task);
668 lock().mark_pump_late(
669 generation,
670 format!(
671 "stderr of the exited process had not reached EOF {bound:?} after it was \
672 retired (a descendant may still hold the pipe open); lines it still \
673 writes are kept in that process's section"
674 ),
675 );
676 warn!(
677 module_id,
678 waited = ?bound,
679 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680 );
681 }
682 }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687 EVENTS.get_or_init(|| {
688 let (sender, _receiver) = watch::channel(0);
689 sender
690 })
691}
692
693pub(crate) fn notify_registration_release() {
694 let events = registration_release_events();
695 let next_generation = (*events.borrow()).wrapping_add(1);
696 events.send_replace(next_generation);
697}
698
699#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702 pub module_id: String,
703 pub program: PathBuf,
704 pub args: Vec<String>,
705 pub env: Vec<(String, String)>,
706 pub reserved: bool,
711 pub reserved_prefixes: Vec<String>,
716 pub protocol: ModuleProtocol,
735 pub overlap: ModuleOverlap,
740}
741
742#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749 f.write_str("LaunchSpecDigest([redacted])")
750 }
751}
752
753impl LaunchSpecDigest {
754 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755 fn field(hash: &mut Sha256, bytes: &[u8]) {
756 hash.update((bytes.len() as u64).to_le_bytes());
757 hash.update(bytes);
758 }
759 let mut hash = Sha256::new();
760 if let Some(program) = spec.program.to_str() {
761 field(&mut hash, b"utf8-program");
762 field(&mut hash, program.as_bytes());
763 } else {
764 #[cfg(unix)]
767 {
768 use std::os::unix::ffi::OsStrExt;
769 field(&mut hash, b"unix-program");
770 field(&mut hash, spec.program.as_os_str().as_bytes());
771 }
772 #[cfg(windows)]
773 {
774 use std::os::windows::ffi::OsStrExt;
775 field(&mut hash, b"windows-program");
776 let bytes = spec
777 .program
778 .as_os_str()
779 .encode_wide()
780 .flat_map(u16::to_le_bytes)
781 .collect::<Vec<_>>();
782 field(&mut hash, &bytes);
783 }
784 #[cfg(not(any(unix, windows)))]
785 field(&mut hash, spec.program.to_string_lossy().as_bytes());
786 }
787 hash.update((spec.args.len() as u64).to_le_bytes());
788 for arg in &spec.args {
789 field(&mut hash, arg.as_bytes());
790 }
791 let mut env = spec.env.iter().collect::<Vec<_>>();
792 env.sort_unstable();
793 hash.update((env.len() as u64).to_le_bytes());
794 for (key, value) in env {
795 field(&mut hash, key.as_bytes());
796 field(&mut hash, value.as_bytes());
797 }
798 field(
799 &mut hash,
800 match spec.protocol {
801 ModuleProtocol::Subc => b"subc",
802 ModuleProtocol::None => b"none",
803 },
804 );
805 Self(hash.finalize().into())
806 }
807}
808
809#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817 #[default]
819 Exclusive,
820 Safe,
832}
833
834impl ModuleOverlap {
835 pub fn as_str(self) -> &'static str {
836 match self {
837 Self::Exclusive => "exclusive",
838 Self::Safe => "safe",
839 }
840 }
841}
842
843pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880 pub max_restarts: u32,
881 pub backoff: Duration,
884 pub max_backoff: Duration,
886 pub window: Duration,
890}
891
892impl RestartPolicy {
893 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897 Self {
898 max_restarts,
899 backoff,
900 max_backoff: DEFAULT_MAX_BACKOFF,
901 window: DEFAULT_RESTART_WINDOW,
902 }
903 }
904
905 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906 self.max_backoff = max_backoff;
907 self
908 }
909
910 pub fn with_window(mut self, window: Duration) -> Self {
911 self.window = window;
912 self
913 }
914
915 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920 if self.backoff.is_zero() || self.max_backoff.is_zero() {
921 return Duration::ZERO;
922 }
923
924 let mut delay = self.backoff;
925 for _ in 0..restart_in_window {
926 if delay >= self.max_backoff {
927 return self.max_backoff;
928 }
929 delay = delay
930 .checked_mul(10)
931 .unwrap_or(self.max_backoff)
932 .min(self.max_backoff);
933 }
934 delay.min(self.max_backoff)
935 }
936
937 fn budget_exhausted_detail(&self) -> String {
942 format!(
943 "crash budget exhausted: max_restarts={} within window_secs={}",
944 self.max_restarts,
945 self.window.as_secs()
946 )
947 }
948}
949
950impl Default for RestartPolicy {
951 fn default() -> Self {
952 Self {
953 max_restarts: DEFAULT_MAX_RESTARTS,
954 backoff: DEFAULT_BACKOFF,
955 max_backoff: DEFAULT_MAX_BACKOFF,
956 window: DEFAULT_RESTART_WINDOW,
957 }
958 }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963 restart_in_window: u32,
964 delay: Duration,
965}
966
967fn daemon_will_restart(
974 state: &mut SupervisorSnapshot,
975 policy: &RestartPolicy,
976 now: Instant,
977) -> bool {
978 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988 Report,
989 Restart,
990 Alert,
991}
992
993impl fmt::Display for HealthAction {
994 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995 f.write_str(match self {
996 Self::Report => "report",
997 Self::Restart => "restart",
998 Self::Alert => "alert",
999 })
1000 }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005 pub http: Option<String>,
1009 pub cadence: Duration,
1010 pub deadline: Duration,
1011 pub failure_threshold: u32,
1012 pub on_degraded: HealthAction,
1013 pub on_failing: HealthAction,
1014 pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018 fn default() -> Self {
1019 Self {
1020 http: None,
1021 cadence: DEFAULT_HEALTH_CADENCE,
1022 deadline: DEFAULT_HEALTH_DEADLINE,
1023 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024 on_degraded: HealthAction::Report,
1025 on_failing: HealthAction::Report,
1026 critical: false,
1027 }
1028 }
1029}
1030
1031#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050 pub status: SupervisorHealthStatus,
1051 pub last_probe_ms: Option<u64>,
1052 pub detail: Option<String>,
1053 pub metrics: Option<Value>,
1054 pub consecutive_failures: u32,
1055 pub late_answer_count: u64,
1058 pub last_late_answer_latency_ms: Option<u64>,
1060 pub last_action: Option<String>,
1061 pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068 fn default() -> Self {
1069 Self {
1070 status: SupervisorHealthStatus::Unknown,
1071 last_probe_ms: None,
1072 detail: None,
1073 metrics: None,
1074 consecutive_failures: 0,
1075 late_answer_count: 0,
1076 last_late_answer_latency_ms: None,
1077 last_action: None,
1078 last_action_ms: None,
1079 }
1080 }
1081}
1082
1083#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086 Starting,
1087 Running,
1088 Unresponsive,
1089 Restarting,
1090 Draining,
1091 Stopped,
1092 Failed,
1093 Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098 f.write_str(match self {
1099 Self::Starting => "starting",
1100 Self::Running => "running",
1101 Self::Unresponsive => "unresponsive",
1102 Self::Restarting => "restarting",
1103 Self::Draining => "draining",
1104 Self::Stopped => "stopped",
1105 Self::Failed => "failed",
1106 Self::Disabled => "disabled",
1107 })
1108 }
1109}
1110
1111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114 Clean,
1115 Crash,
1116 DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120 fn from(kind: ExitKind) -> Self {
1121 match kind {
1122 ExitKind::Clean => Self::Clean,
1123 ExitKind::Crash => Self::Crash,
1124 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125 }
1126 }
1127}
1128
1129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133 pub(crate) pid: u32,
1134 pub(crate) start_time: u64,
1135}
1136
1137#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140 pub kind: ExitKind,
1141 pub code: Option<i32>,
1142 pub signal: Option<i32>,
1143 pub at_ms: u64,
1144}
1145
1146#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150 pub module_id: String,
1151 pub waiting_for_previous_pid: Option<u32>,
1153 pub waiting_for_previous_detail: Option<String>,
1155 pub launch_nonce_source: Option<String>,
1160 pub state: ModuleState,
1161 pub enabled: bool,
1162 pub process_alive: bool,
1163 pub registration_active: bool,
1164 pub protocol: ModuleProtocol,
1169 pub live: bool,
1180 pub restart_count: u32,
1184 pub lifetime_restarts: u32,
1188 pub spawn_generation: u64,
1189 pub max_restarts: u32,
1194 pub restart_window: Duration,
1198 pub drain_timeout: Duration,
1202 pub restart_backoff: Duration,
1203 pub restart_max_backoff: Duration,
1204 pub pid: Option<u32>,
1209 pub spawned_at_ms: Option<u64>,
1210 pub spawned_from: Option<PathBuf>,
1211 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212 pub process_start_time: Option<u64>,
1213 pub last_exit: Option<ExitReport>,
1214 pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220 generation: u64,
1221 turn: u64,
1222 registered_connection: Option<ConnectionId>,
1223 next_probe_at: Option<Instant>,
1224 wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230 #[cfg(windows)]
1231 pipe_consumed: Option<Arc<OnceLock<()>>>,
1232 #[cfg(test)]
1234 actor_turns: u64,
1235 #[cfg(test)]
1238 actor_select: Option<ActorSelectCheckpoint>,
1239 state: ModuleState,
1240 enabled: bool,
1241 process_alive: bool,
1242 spawned_protocol: Option<ModuleProtocol>,
1243 spawn_failure: Option<String>,
1244 waiting_for_previous_pid: Option<u32>,
1245 waiting_for_previous_detail: Option<String>,
1246 crash_restarts: VecDeque<Instant>,
1252 lifetime_restarts: u32,
1253 spawn_generation: u64,
1262 pid: Option<u32>,
1263 #[cfg(target_os = "macos")]
1264 report_ready: Option<Arc<OnceLock<()>>>,
1265 reaped_pid: Option<u32>,
1267 respawn_pending: bool,
1269 coalesced_restart_pending: bool,
1271 spawned_at_ms: Option<u64>,
1272 spawned_from: Option<PathBuf>,
1273 launch_spec_digest: Option<LaunchSpecDigest>,
1274 spawned_file_identity: Option<SpawnedFileIdentity>,
1275 process_start_time: Option<u64>,
1276 deliberate_severance: Option<ProcessIdentity>,
1277 last_exit: Option<ExitReport>,
1278 drain_disposition_detail: Option<String>,
1280 health: ModuleHealthStatus,
1281 in_alternate_slot: bool,
1286 draining_to_replace: bool,
1293 configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302 fn reported_pid(&self) -> Option<u32> {
1307 #[cfg(target_os = "macos")]
1308 if self
1309 .report_ready
1310 .as_ref()
1311 .is_some_and(|ready| ready.get().is_none())
1312 {
1313 return None;
1314 }
1315 self.pid
1316 }
1317
1318 fn starting() -> Self {
1319 Self::new(ModuleState::Starting, true)
1320 }
1321
1322 fn disabled() -> Self {
1323 Self::new(ModuleState::Disabled, false)
1324 }
1325
1326 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330 while let Some(oldest) = self.crash_restarts.front() {
1331 if now.duration_since(*oldest) > window {
1332 self.crash_restarts.pop_front();
1333 } else {
1334 break;
1335 }
1336 }
1337 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338 }
1339
1340 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346 self.crash_restarts.push_back(now);
1347 while self.crash_restarts.len() > policy.max_restarts as usize {
1348 self.crash_restarts.pop_front();
1349 }
1350 self.lifetime_restarts += 1;
1351 }
1352
1353 fn next_crash_restart(
1357 &mut self,
1358 policy: &RestartPolicy,
1359 now: Instant,
1360 ) -> Option<CrashRestartSchedule> {
1361 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362 if restart_in_window >= policy.max_restarts {
1363 return None;
1364 }
1365 self.record_crash_restart(policy, now);
1366 Some(CrashRestartSchedule {
1367 restart_in_window,
1368 delay: policy.delay_for_restart(restart_in_window),
1369 })
1370 }
1371
1372 fn clear_crash_restarts(&mut self) {
1377 self.crash_restarts.clear();
1378 }
1379
1380 fn new(state: ModuleState, enabled: bool) -> Self {
1381 Self {
1382 exec_outcome: None,
1383 #[cfg(test)]
1384 actor_turns: 0,
1385 #[cfg(test)]
1386 actor_select: None,
1387 state,
1388 enabled,
1389 process_alive: false,
1390 #[cfg(windows)]
1391 pipe_consumed: None,
1392 spawned_protocol: None,
1393 spawn_failure: None,
1394 waiting_for_previous_pid: None,
1395 waiting_for_previous_detail: None,
1396 crash_restarts: VecDeque::new(),
1397 lifetime_restarts: 0,
1398 spawn_generation: 0,
1399 pid: None,
1400 #[cfg(target_os = "macos")]
1401 report_ready: None,
1402 reaped_pid: None,
1403 respawn_pending: false,
1404 coalesced_restart_pending: false,
1405 spawned_at_ms: None,
1406 spawned_from: None,
1407 launch_spec_digest: None,
1408 spawned_file_identity: None,
1409 process_start_time: None,
1410 deliberate_severance: None,
1411 last_exit: None,
1412 drain_disposition_detail: None,
1413 health: ModuleHealthStatus::default(),
1414 in_alternate_slot: false,
1415 draining_to_replace: false,
1416 configuration_updated_since_spawn: false,
1417 }
1418 }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427 version: u8,
1428 frames: mpsc::Sender<Frame>,
1429 lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437 daemon_incarnation: String,
1438 seq: u64,
1439 capacity: usize,
1440 live: HashMap<String, LiveSpawn>,
1441 generations: HashMap<String, u64>,
1442 events: VecDeque<SpawnEvent>,
1443 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447 fn default() -> Self {
1448 Self {
1449 daemon_incarnation: "unconfigured".to_string(),
1450 seq: 0,
1451 capacity: SPAWN_EVENT_RING_CAPACITY,
1452 live: HashMap::new(),
1453 generations: HashMap::new(),
1454 events: VecDeque::new(),
1455 subscribers: HashMap::new(),
1456 }
1457 }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462 Arc<Mutex<SpawnEventState>>,
1463 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468 ForeignIncarnation { current: String },
1469 TooOld { oldest: SpawnCursor },
1470 Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474 fn configure_incarnation(&self, daemon_incarnation: String) {
1475 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476 state.daemon_incarnation = daemon_incarnation;
1477 state.seq = 0;
1478 state.live.clear();
1479 state.generations.clear();
1480 state.events.clear();
1481 state.subscribers.clear();
1482 }
1483
1484 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485 SpawnCursor {
1486 daemon_incarnation: state.daemon_incarnation.clone(),
1487 seq: state.seq,
1488 }
1489 }
1490
1491 fn snapshot(&self) -> SpawnSnapshot {
1492 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495 SpawnSnapshot {
1496 cursor: Self::cursor(&state),
1497 ring_bound: state.capacity as u64,
1498 live,
1499 }
1500 }
1501
1502 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504 let generation = state
1505 .generations
1506 .get(module_id)
1507 .copied()
1508 .unwrap_or(0)
1509 .checked_add(1)
1510 .expect("spawn generation exhausted");
1511 state.generations.insert(module_id.to_string(), generation);
1512 let live = LiveSpawn {
1513 module_id: module_id.to_string(),
1514 spawn_generation: generation,
1515 pid,
1516 spawned_at_ms,
1517 };
1518 state.live.insert(module_id.to_string(), live);
1519 Self::emit_locked(
1520 &mut state,
1521 SpawnEventKind::Spawned,
1522 module_id.to_string(),
1523 generation,
1524 pid,
1525 None,
1526 None,
1527 );
1528 generation
1529 }
1530
1531 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533 let Some(live) = state.live.remove(module_id) else {
1534 warn!(
1535 module_id,
1536 "terminal record had no live spawn event identity"
1537 );
1538 return;
1539 };
1540 Self::emit_locked(
1541 &mut state,
1542 SpawnEventKind::Exited,
1543 module_id.to_string(),
1544 live.spawn_generation,
1545 live.pid,
1546 exit_code,
1547 exit_signal,
1548 );
1549 }
1550
1551 fn emit_superseded_exited(
1558 &self,
1559 module_id: &str,
1560 spawn_generation: u64,
1561 pid: u32,
1562 exit_code: Option<i32>,
1563 exit_signal: Option<i32>,
1564 ) {
1565 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566 if state
1567 .live
1568 .get(module_id)
1569 .is_some_and(|live| live.spawn_generation == spawn_generation)
1570 {
1571 state.live.remove(module_id);
1572 }
1573 Self::emit_locked(
1574 &mut state,
1575 SpawnEventKind::Exited,
1576 module_id.to_string(),
1577 spawn_generation,
1578 pid,
1579 exit_code,
1580 exit_signal,
1581 );
1582 }
1583
1584 #[allow(clippy::too_many_arguments)]
1585 fn emit_locked(
1586 state: &mut SpawnEventState,
1587 kind: SpawnEventKind,
1588 module_id: String,
1589 spawn_generation: u64,
1590 pid: u32,
1591 exit_code: Option<i32>,
1592 exit_signal: Option<i32>,
1593 ) {
1594 state.seq = state
1595 .seq
1596 .checked_add(1)
1597 .expect("spawn event sequence exhausted");
1598 let event = SpawnEvent {
1599 cursor: Self::cursor(state),
1600 kind,
1601 module_id,
1602 spawn_generation,
1603 pid,
1604 exit_code,
1605 exit_signal,
1606 };
1607 state.events.push_back(event.clone());
1608 while state.events.len() > state.capacity {
1609 state.events.pop_front();
1610 }
1611 let body = match serde_json::to_vec(&event) {
1612 Ok(body) => body,
1613 Err(error) => {
1614 error!(%error, "failed to serialize supervisor spawn event");
1615 return;
1616 }
1617 };
1618 state.subscribers.retain(|(connection_id, corr), subscriber| {
1619 let frame = Frame::build_with_version(
1620 subscriber.version,
1621 FrameType::StreamData,
1622 control_flags(),
1623 0,
1624 0,
1625 *corr,
1626 body.clone(),
1627 );
1628 match frame {
1629 Ok(frame) => {
1630 if subscriber.frames.try_send(frame).is_ok() {
1631 true
1632 } else {
1633 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634 if let Some(lagged) = subscriber.lagged.take() {
1635 let _ = lagged.send(event.cursor.clone());
1636 }
1637 false
1638 }
1639 }
1640 Err(error) => {
1641 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642 false
1643 }
1644 }
1645 });
1646 }
1647
1648 fn subscribe(
1649 &self,
1650 connection_id: ConnectionId,
1651 corr: u64,
1652 version: u8,
1653 since: Option<SpawnCursor>,
1654 sink: FrameSink,
1655 ) -> Result<(), SpawnSubscribeRefusal> {
1656 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658 {
1659 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660 let replay = if let Some(since) = since {
1661 if since.daemon_incarnation != state.daemon_incarnation {
1662 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663 current: state.daemon_incarnation.clone(),
1664 });
1665 }
1666 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667 if since.seq < oldest.seq.saturating_sub(1) {
1668 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669 }
1670 }
1671 state
1672 .events
1673 .iter()
1674 .filter(|event| event.cursor.seq > since.seq)
1675 .cloned()
1676 .collect::<Vec<_>>()
1677 } else {
1678 Vec::new()
1679 };
1680 for event in replay {
1681 let body = serde_json::to_vec(&event)
1682 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683 let frame = Frame::build_with_version(
1684 version,
1685 FrameType::StreamData,
1686 control_flags(),
1687 0,
1688 0,
1689 corr,
1690 body,
1691 )
1692 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693 frames
1694 .try_send(frame)
1695 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696 }
1697 state.subscribers.insert(
1698 (connection_id, corr),
1699 SpawnSubscriber {
1700 version,
1701 frames,
1702 lagged: Some(lagged),
1703 },
1704 );
1705 }
1706 tokio::spawn(async move {
1717 while let Some(frame) = receiver.recv().await {
1718 if sink.send(frame).await.is_err() {
1719 return;
1720 }
1721 }
1722 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723 return;
1724 };
1725 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726 Ok(frame) => {
1727 let _ = sink.send(frame).await;
1728 }
1729 Err(error) => {
1730 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731 }
1732 }
1733 });
1734 Ok(())
1735 }
1736
1737 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738 let Some(subscriber) = self
1739 .0
1740 .lock()
1741 .unwrap_or_else(|p| p.into_inner())
1742 .subscribers
1743 .remove(&(connection_id, corr))
1744 else {
1745 return false;
1746 };
1747 if let Ok(frame) = Frame::build_with_version(
1748 subscriber.version,
1749 FrameType::StreamEnd,
1750 control_flags(),
1751 0,
1752 0,
1753 corr,
1754 Vec::new(),
1755 ) {
1756 tokio::spawn(async move {
1757 let _ = subscriber.frames.send(frame).await;
1758 });
1759 }
1760 true
1761 }
1762
1763 fn remove_connection(&self, connection_id: ConnectionId) {
1764 self.0
1765 .lock()
1766 .unwrap_or_else(|p| p.into_inner())
1767 .subscribers
1768 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769 }
1770
1771 #[cfg(any(test, feature = "test-support"))]
1772 fn set_capacity(&self, capacity: usize) {
1773 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774 }
1775
1776 #[cfg(any(test, feature = "test-support"))]
1777 fn subscriber_count(&self) -> usize {
1778 self.0
1779 .lock()
1780 .unwrap_or_else(|p| p.into_inner())
1781 .subscribers
1782 .len()
1783 }
1784}
1785
1786fn spawn_subscriber_lagged_frame(
1789 version: u8,
1790 corr: u64,
1791 first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796 .to_string(),
1797 detail: Some(serde_json::json!({
1798 "first_undelivered_cursor": first_undelivered
1799 })),
1800 })
1801 .map_err(|error| error.to_string())?;
1802 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803 .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807 fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809 fn process_replacing(&self, _module_id: &str) -> bool {
1815 false
1816 }
1817}
1818
1819#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826 pub fn new() -> Self {
1827 Self::default()
1828 }
1829
1830 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831 let mut snapshots = self
1832 .snapshots
1833 .lock()
1834 .unwrap_or_else(|poisoned| poisoned.into_inner());
1835 snapshots.insert(module_id, snapshot);
1836 }
1837
1838 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839 let mut snapshots = self
1840 .snapshots
1841 .lock()
1842 .unwrap_or_else(|poisoned| poisoned.into_inner());
1843 let is_current = snapshots
1844 .get(module_id)
1845 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846 .unwrap_or(false);
1847 if is_current {
1848 snapshots.remove(module_id);
1849 }
1850 }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854 fn process_live(&self, module_id: &str) -> Option<bool> {
1855 let snapshot = {
1856 let snapshots = self
1857 .snapshots
1858 .lock()
1859 .unwrap_or_else(|poisoned| poisoned.into_inner());
1860 snapshots.get(module_id).cloned()
1861 }?;
1862 let snapshot = snapshot
1863 .lock()
1864 .unwrap_or_else(|poisoned| poisoned.into_inner());
1865 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866 }
1867
1868 fn process_replacing(&self, module_id: &str) -> bool {
1869 let Some(snapshot) = self
1870 .snapshots
1871 .lock()
1872 .unwrap_or_else(|poisoned| poisoned.into_inner())
1873 .get(module_id)
1874 .cloned()
1875 else {
1876 return false;
1877 };
1878 let snapshot = snapshot
1879 .lock()
1880 .unwrap_or_else(|poisoned| poisoned.into_inner());
1881 snapshot.enabled
1882 && match snapshot.state {
1883 ModuleState::Restarting => true,
1884 ModuleState::Draining => snapshot.draining_to_replace,
1885 ModuleState::Starting
1886 | ModuleState::Running
1887 | ModuleState::Unresponsive
1888 | ModuleState::Stopped
1889 | ModuleState::Failed
1890 | ModuleState::Disabled => false,
1891 }
1892 }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898 reached: tokio::sync::Notify,
1899 resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904 Spawn,
1905 Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910 deadline: Instant,
1911 kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918 start_requester: Option<Principal>,
1919 pending_activity_restart: bool,
1920 provenance_probe: ExecutableIdentityProbe,
1921 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925 restart_policy: RestartPolicy,
1926 drain_timeout: Duration,
1929 effective_drain_timeout: Arc<Mutex<Duration>>,
1932 default_drain_timeout: Duration,
1935 health: HealthConfig,
1936 connection_file_path: Option<PathBuf>,
1937 capture_logs_dir: Option<PathBuf>,
1938 forwarding: Option<Arc<ForwardingTable>>,
1939 supervisor_handle: Option<SupervisorHandle>,
1942 stderr_ring: Arc<Mutex<StderrRing>>,
1949 terminal_ring: Arc<Mutex<TerminalRing>>,
1950 spawn_events: SpawnEventFeed,
1951 child_roster: ChildRoster,
1952 #[cfg(target_os = "linux")]
1953 cgroup_placement: Option<subc_cgroup::Placement>,
1954 #[cfg(test)]
1955 test_seed_stale_facts_before_enable_spawn: bool,
1956 #[cfg(test)]
1957 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962 spec: ModuleSpec,
1963 health: HealthConfig,
1964}
1965
1966#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976 configured_ids: Arc<Mutex<HashSet<String>>>,
1988 spawn_events: SpawnEventFeed,
1989 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024 promotion_observer: PromotionObserverSlot,
2026 operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032pub(crate) trait SwapPromotionObserver: Send + Sync {
2041 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052 f.write_str("PromotionObserverSlot")
2053 }
2054}
2055
2056#[derive(Debug, Clone)]
2058struct OpenSwap {
2059 candidate_nonce: String,
2062 incumbent_nonce: Option<String>,
2067 candidate_admitted: bool,
2071}
2072
2073#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077 NotSwapping,
2080 Candidate,
2082 Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089 Exact {
2090 module_id: String,
2091 },
2092 Prefix {
2093 prefix: String,
2094 owner_module_id: String,
2095 },
2096}
2097
2098impl SupervisorHandle {
2099 pub fn new() -> Self {
2100 Self::default()
2101 }
2102
2103 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104 self.spawn_events.snapshot()
2105 }
2106
2107 pub(crate) fn subscribe_spawns(
2108 &self,
2109 connection_id: ConnectionId,
2110 corr: u64,
2111 version: u8,
2112 since: Option<SpawnCursor>,
2113 sink: FrameSink,
2114 ) -> Result<(), SpawnSubscribeRefusal> {
2115 self.spawn_events
2116 .subscribe(connection_id, corr, version, since, sink)
2117 }
2118
2119 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120 self.spawn_events.cancel(connection_id, corr)
2121 }
2122
2123 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124 self.spawn_events.remove_connection(connection_id);
2125 }
2126
2127 #[cfg(any(test, feature = "test-support"))]
2128 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129 assert!(capacity > 0, "spawn event capacity must be non-zero");
2130 self.spawn_events.set_capacity(capacity);
2131 }
2132
2133 #[cfg(any(test, feature = "test-support"))]
2134 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135 self.spawn_events.subscriber_count()
2136 }
2137
2138 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141 self.spawn_nonces
2142 .lock()
2143 .unwrap_or_else(|poisoned| poisoned.into_inner())
2144 .insert(module_id.to_string(), nonce);
2145 }
2146
2147 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150 self.reserved_nonces
2151 .lock()
2152 .unwrap_or_else(|poisoned| poisoned.into_inner())
2153 .insert(module_id.to_string(), Some(nonce));
2154 }
2155
2156 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158 let mut owners = self
2159 .reserved_prefix_owners
2160 .lock()
2161 .unwrap_or_else(|poisoned| poisoned.into_inner());
2162 owners.retain(|_, owner| owner != owner_module_id);
2163 for prefix in prefixes {
2164 owners.insert(prefix.clone(), owner_module_id.to_string());
2165 }
2166 }
2167
2168 #[cfg(test)]
2170 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171 self.spawn_nonces
2172 .lock()
2173 .unwrap_or_else(|poisoned| poisoned.into_inner())
2174 .get(module_id)
2175 .cloned()
2176 }
2177
2178 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180 let spawn_nonce = self
2181 .spawn_nonces
2182 .lock()
2183 .unwrap_or_else(|poisoned| poisoned.into_inner())
2184 .get(&spec.module_id)
2185 .cloned();
2186 let mut reserved_nonces = self
2187 .reserved_nonces
2188 .lock()
2189 .unwrap_or_else(|poisoned| poisoned.into_inner());
2190 if spec.reserved {
2191 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196 }
2197 drop(reserved_nonces);
2198 self.removal_tombstones
2202 .lock()
2203 .unwrap_or_else(|poisoned| poisoned.into_inner())
2204 .remove(&spec.module_id);
2205 }
2206
2207 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212 self.reserved_hello_rejection(module_id, presented)
2213 .is_none()
2214 }
2215
2216 pub(crate) fn reserved_hello_rejection(
2217 &self,
2218 module_id: &str,
2219 presented: Option<&str>,
2220 ) -> Option<ReservedHelloRejection> {
2221 let nonces = self
2222 .reserved_nonces
2223 .lock()
2224 .unwrap_or_else(|poisoned| poisoned.into_inner());
2225 if let Some(expected) = nonces.get(module_id) {
2226 let authorized = match expected {
2230 Some(expected) => {
2231 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2232 }
2233 None => false,
2234 };
2235 if authorized {
2236 return None;
2237 }
2238 return Some(ReservedHelloRejection::Exact {
2239 module_id: module_id.to_string(),
2240 });
2241 }
2242 drop(nonces);
2243
2244 let matched_prefix = self
2245 .reserved_prefix_owners
2246 .lock()
2247 .unwrap_or_else(|poisoned| poisoned.into_inner())
2248 .iter()
2249 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2250 .max_by_key(|(prefix, _)| prefix.len())
2251 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2252 let (prefix, owner_module_id) = matched_prefix?;
2253
2254 let authorized = presented.is_some_and(|presented| {
2255 self.spawn_nonces
2256 .lock()
2257 .unwrap_or_else(|poisoned| poisoned.into_inner())
2258 .get(&owner_module_id)
2259 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2260 || self.swap_nonce_matches(&owner_module_id, presented)
2263 });
2264 if authorized {
2265 None
2266 } else {
2267 Some(ReservedHelloRejection::Prefix {
2268 prefix,
2269 owner_module_id,
2270 })
2271 }
2272 }
2273
2274 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2279 if presented.is_empty() {
2280 return false;
2281 }
2282 let nonces = self
2283 .spawn_nonces
2284 .lock()
2285 .unwrap_or_else(|poisoned| poisoned.into_inner());
2286 let current = nonces
2287 .get(module_id)
2288 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2289 drop(nonces);
2290 current || self.swap_nonce_matches(module_id, presented)
2295 }
2296
2297 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2299 let swaps = self
2300 .swaps
2301 .lock()
2302 .unwrap_or_else(|poisoned| poisoned.into_inner());
2303 swaps.get(module_id).is_some_and(|swap| {
2304 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2305 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2306 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2307 })
2308 })
2309 }
2310
2311 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2314 let incumbent_nonce = self
2315 .spawn_nonces
2316 .lock()
2317 .unwrap_or_else(|poisoned| poisoned.into_inner())
2318 .get(module_id)
2319 .cloned();
2320 self.swaps
2321 .lock()
2322 .unwrap_or_else(|poisoned| poisoned.into_inner())
2323 .insert(
2324 module_id.to_string(),
2325 OpenSwap {
2326 candidate_nonce,
2327 incumbent_nonce,
2328 candidate_admitted: false,
2329 },
2330 );
2331 }
2332
2333 pub(crate) fn close_swap(&self, module_id: &str) {
2336 self.swaps
2337 .lock()
2338 .unwrap_or_else(|poisoned| poisoned.into_inner())
2339 .remove(module_id);
2340 }
2341
2342 pub(crate) fn set_swap_promotion_observer(
2345 &self,
2346 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2347 ) {
2348 *self
2349 .promotion_observer
2350 .0
2351 .lock()
2352 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2353 }
2354
2355 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2358 let observer = self
2359 .promotion_observer
2360 .0
2361 .lock()
2362 .unwrap_or_else(|poisoned| poisoned.into_inner())
2363 .as_ref()
2364 .and_then(std::sync::Weak::upgrade);
2365 if let Some(observer) = observer {
2366 observer.swap_promoted(registration);
2367 }
2368 }
2369
2370 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2372 self.swaps
2373 .lock()
2374 .unwrap_or_else(|poisoned| poisoned.into_inner())
2375 .contains_key(module_id)
2376 }
2377
2378 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2383 let candidate_nonce = self
2384 .swaps
2385 .lock()
2386 .unwrap_or_else(|poisoned| poisoned.into_inner())
2387 .get(module_id)
2388 .map(|swap| swap.candidate_nonce.clone());
2389 let Some(nonce) = candidate_nonce else {
2390 return;
2391 };
2392 self.set_spawn_nonce(module_id, nonce.clone());
2393 if reserved {
2394 self.set_reserved_nonce(module_id, nonce);
2395 }
2396 }
2397
2398 pub(crate) fn swap_hello_admission(
2413 &self,
2414 module_id: &str,
2415 presented: Option<&str>,
2416 ) -> SwapHelloAdmission {
2417 let swaps = self
2418 .swaps
2419 .lock()
2420 .unwrap_or_else(|poisoned| poisoned.into_inner());
2421 let Some(swap) = swaps.get(module_id) else {
2422 return SwapHelloAdmission::NotSwapping;
2423 };
2424 let Some(presented) = presented else {
2425 return SwapHelloAdmission::Refused;
2426 };
2427 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2428 return if swap.candidate_admitted {
2429 SwapHelloAdmission::Refused
2430 } else {
2431 SwapHelloAdmission::Candidate
2432 };
2433 }
2434 if swap
2435 .incumbent_nonce
2436 .as_deref()
2437 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2438 {
2439 return SwapHelloAdmission::NotSwapping;
2440 }
2441 SwapHelloAdmission::Refused
2442 }
2443
2444 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2447 if let Some(swap) = self
2448 .swaps
2449 .lock()
2450 .unwrap_or_else(|poisoned| poisoned.into_inner())
2451 .get_mut(module_id)
2452 {
2453 swap.candidate_admitted = true;
2454 }
2455 }
2456
2457 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2459 self.spawn_nonces
2460 .lock()
2461 .unwrap_or_else(|poisoned| poisoned.into_inner())
2462 .get(module_id)
2463 .cloned()
2464 }
2465
2466 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2468 self.reserved_nonces
2469 .lock()
2470 .unwrap_or_else(|poisoned| poisoned.into_inner())
2471 .get(module_id)
2472 .cloned()
2473 .flatten()
2474 }
2475
2476 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2477 self.mark_configured(module.module_id());
2481 let mut modules = self
2482 .modules
2483 .lock()
2484 .unwrap_or_else(|poisoned| poisoned.into_inner());
2485 modules.insert(module.module_id().to_string(), module)
2486 }
2487
2488 fn mark_configured(&self, module_id: &str) {
2492 self.configured_ids
2493 .lock()
2494 .unwrap_or_else(|poisoned| poisoned.into_inner())
2495 .insert(module_id.to_string());
2496 }
2497
2498 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2502 let modules = self
2503 .modules
2504 .lock()
2505 .unwrap_or_else(|poisoned| poisoned.into_inner());
2506 if !modules.contains_key(module_id) {
2507 self.configured_ids
2508 .lock()
2509 .unwrap_or_else(|poisoned| poisoned.into_inner())
2510 .remove(module_id);
2511 }
2512 }
2513
2514 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2522 self.configured_ids
2523 .lock()
2524 .unwrap_or_else(|poisoned| poisoned.into_inner())
2525 .contains(module_id)
2526 }
2527
2528 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2529 let modules = self
2530 .modules
2531 .lock()
2532 .unwrap_or_else(|poisoned| poisoned.into_inner());
2533 modules.get(module_id).cloned()
2534 }
2535
2536 pub(crate) fn record_late_health_answer(
2537 &self,
2538 module_id: &str,
2539 latency_ms: u64,
2540 ) -> Result<bool, SuperviseError> {
2541 let Some(module) = self.get(module_id) else {
2542 return Ok(false);
2543 };
2544 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2545 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2546 state.health.last_late_answer_latency_ms = Some(latency_ms);
2547 state.health.consecutive_failures = 0;
2555 })?;
2556 Ok(true)
2557 }
2558
2559 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2565 let Some(module) = self.get(module_id) else {
2566 return Ok(false);
2567 };
2568 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2569 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2570 return Ok(false);
2571 };
2572 drop(snapshot);
2573 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2574 }
2575
2576 pub fn list(&self) -> Vec<SupervisedModule> {
2577 let modules = self
2578 .modules
2579 .lock()
2580 .unwrap_or_else(|poisoned| poisoned.into_inner());
2581 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2582 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2583 modules
2584 }
2585
2586 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2587 self.activity_images
2588 .lock()
2589 .unwrap_or_else(|p| p.into_inner())
2590 .remove(module_id);
2591 self.spawn_nonces
2592 .lock()
2593 .unwrap_or_else(|poisoned| poisoned.into_inner())
2594 .remove(module_id);
2595 self.close_swap(module_id);
2596 let mut reserved_nonces = self
2597 .reserved_nonces
2598 .lock()
2599 .unwrap_or_else(|poisoned| poisoned.into_inner());
2600 if reserved_nonces.contains_key(module_id) {
2601 reserved_nonces.insert(module_id.to_string(), None);
2604 }
2605 drop(reserved_nonces);
2606 self.reserved_prefix_owners
2607 .lock()
2608 .unwrap_or_else(|poisoned| poisoned.into_inner())
2609 .retain(|_, owner| owner != module_id);
2610 let removed = self
2611 .modules
2612 .lock()
2613 .unwrap_or_else(|poisoned| poisoned.into_inner())
2614 .remove(module_id);
2615 self.configured_ids
2616 .lock()
2617 .unwrap_or_else(|poisoned| poisoned.into_inner())
2618 .remove(module_id);
2619 removed
2620 }
2621
2622 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2625 self.removal_tombstones
2626 .lock()
2627 .unwrap_or_else(|poisoned| poisoned.into_inner())
2628 .insert(module_id.to_string(), unix_ms_now());
2629 }
2630
2631 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2633 self.removal_tombstones
2634 .lock()
2635 .unwrap_or_else(|poisoned| poisoned.into_inner())
2636 .get(module_id)
2637 .copied()
2638 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2639 }
2640
2641 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2646 if self.get(module_id).is_some() {
2647 return false;
2648 }
2649 let mut reserved_nonces = self
2650 .reserved_nonces
2651 .lock()
2652 .unwrap_or_else(|poisoned| poisoned.into_inner());
2653 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2654 return false;
2655 }
2656 reserved_nonces.remove(module_id);
2657 true
2658 }
2659
2660 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2661 Arc::clone(&self.operation_lock)
2662 }
2663}
2664
2665#[derive(Debug, Clone)]
2667pub struct Supervisor {
2668 registry: Arc<Registry>,
2669 restart_policy: RestartPolicy,
2670 drain_timeout: Duration,
2671 connection_file_path: Option<PathBuf>,
2672 capture_logs_dir: Option<PathBuf>,
2673 forwarding: Option<Arc<ForwardingTable>>,
2674 process_liveness: Arc<SupervisorProcessLiveness>,
2675 supervisor_handle: Option<SupervisorHandle>,
2676 health: HealthConfig,
2677 daemon_start_clock: crate::clock::StartClock,
2678 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2679 spawn_events: SpawnEventFeed,
2680 provenance_probe: ExecutableIdentityProbe,
2681 child_roster: ChildRoster,
2684 #[cfg(target_os = "linux")]
2685 cgroup_placement: Option<subc_cgroup::Placement>,
2686 #[cfg(test)]
2687 test_after_first_spawn: AfterFirstSpawnHook,
2688}
2689
2690pub(crate) struct RegisteredConfiguredModule {
2694 spec: ModuleSpec,
2695 runtime: SupervisorRuntimeConfig,
2696 module: SupervisedModule,
2697 commands: mpsc::Receiver<SupervisorCommand>,
2698}
2699
2700#[cfg(test)]
2706#[derive(Clone, Default)]
2707struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2708
2709#[cfg(test)]
2710type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2711
2712#[cfg(test)]
2713impl fmt::Debug for AfterFirstSpawnHook {
2714 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2715 f.write_str("AfterFirstSpawnHook")
2716 }
2717}
2718
2719#[cfg(test)]
2720impl AfterFirstSpawnHook {
2721 fn run(&self, module_id: &str) {
2722 if let Some(hook) = &self.0 {
2723 hook(module_id);
2724 }
2725 }
2726}
2727
2728impl Supervisor {
2729 #[cfg(test)]
2730 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2731 let supervisor = Self::new(registry, policy);
2732 #[cfg(target_os = "macos")]
2733 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2734 supervisor
2735 }
2736 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2741 let path = path.into();
2742 #[cfg(target_os = "macos")]
2743 {
2744 let result = probe_privacy_trampoline(&path).map(|()| path);
2745 if let Err(cause) = &result {
2746 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2747 }
2748 self.child_roster.set_privacy_trampoline(result);
2749 }
2750 #[cfg(not(target_os = "macos"))]
2751 let _ = path;
2752 self
2753 }
2754 #[cfg(unix)]
2765 pub(crate) fn begin_daemon_shutdown(&self) {
2766 self.child_roster.close();
2767 if let Some(journal) = &self.terminal_journal {
2768 journal.stamp_shutdown();
2769 }
2770 }
2771
2772 #[cfg(unix)]
2776 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2777 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2778 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2779 let Some(forwarding) = &self.forwarding else {
2780 return Ok(());
2781 };
2782 let modules = forwarding
2783 .begin_daemon_drain()
2784 .map_err(SuperviseError::Forwarding)?;
2785 let deadline_ms =
2786 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2787 let mut notices = tokio::task::JoinSet::new();
2788 let mut drains = Vec::new();
2789 for (module_id, endpoint) in modules {
2790 let Some(target) = forwarding
2791 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2792 .map_err(SuperviseError::Forwarding)?
2793 else {
2794 continue;
2795 };
2796 let routes = forwarding
2797 .endpoint_routes(target.endpoint)
2798 .map_err(SuperviseError::Forwarding)?;
2799 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2805 reason: RouteCloseReason::Restart,
2806 deadline_ms,
2807 })
2808 .expect("module draining serializes");
2809 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2810 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2811 for route in routes {
2812 let client = route.goodbye_target;
2813 if let Some((_, channels)) = clients
2814 .iter_mut()
2815 .find(|(existing, _)| existing.connection_id == client.connection_id)
2816 {
2817 channels.push(client.channel);
2818 } else {
2819 let channel = client.channel;
2820 clients.push((client, vec![channel]));
2821 }
2822 }
2823 for (client, mut channels) in clients {
2824 channels.sort_unstable();
2825 channels.dedup();
2826 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2827 module_id: module_id.clone(),
2828 channels,
2829 reason: RouteCloseReason::Restart,
2830 })
2831 .expect("route closing serializes");
2832 recipients.push((client.sink, client.negotiated_ver, closing));
2833 }
2834 for (sink, version, body) in recipients {
2835 notices.spawn(async move {
2836 let frame = Frame::build_with_version(
2837 version,
2838 FrameType::Push,
2839 control_flags(),
2840 0,
2841 0,
2842 0,
2843 body,
2844 )
2845 .expect("bounded lifecycle notice frame builds");
2846 sink.send_flushed(frame).await
2847 });
2848 }
2849 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2850 drains.push((module_id, target.endpoint, gauges));
2851 }
2852 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2855 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2856 if !matches!(result, Ok(Ok(()))) {
2857 warn!(?result, "daemon shutdown notice delivery failed");
2858 }
2859 }
2860 notices.abort_all();
2861 let deadline = Instant::now() + DRAIN_BUDGET;
2862 let mut waits = tokio::task::JoinSet::new();
2863 for (module_id, endpoint, gauges) in drains {
2864 let forwarding = Arc::clone(forwarding);
2865 let mut runtime = self.runtime_config();
2866 runtime.health.cadence = Duration::from_millis(100);
2867 waits.spawn(async move {
2868 let wait_result = wait_for_forwarding_quiescence(
2869 &forwarding,
2870 &module_id,
2871 &runtime,
2872 endpoint,
2873 deadline,
2874 &gauges,
2875 DrainScope::Endpoint(endpoint),
2876 &mut DrainGaugeTrace::new(&gauges),
2877 )
2878 .await;
2879 forwarding
2883 .record_daemon_drain_result(
2884 endpoint,
2885 drained_after_quiescence_wait(&wait_result),
2886 )
2887 .map_err(SuperviseError::Forwarding)?;
2888 wait_result
2889 });
2890 }
2891 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2892 if !matches!(result, Ok(Ok(true))) {
2893 warn!(?result, "daemon shutdown drain did not reach quiescence");
2894 }
2895 }
2896 Ok(())
2897 }
2898
2899 #[cfg(unix)]
2911 pub(crate) async fn end_children_for_daemon_shutdown(
2912 &self,
2913 already_escalated: bool,
2914 escalate: impl std::future::Future<Output = ()>,
2915 ) {
2916 tokio::pin!(escalate);
2917 let mut escalated = already_escalated;
2918 if let Some(forwarding) = &self.forwarding {
2919 let reason = CloseReason::new(
2920 "daemon_shutdown",
2921 "the daemon is exiting after its shutdown notice and drain",
2922 );
2923 if escalated {
2924 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2927 } else {
2928 tokio::select! {
2929 biased;
2930 _ = escalate.as_mut() => {
2931 info!("second SIGTERM: abandoning module GOODBYE delivery");
2932 escalated = true;
2933 }
2934 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2935 }
2936 }
2937 let closed = forwarding.close_all_connections(&reason);
2938 debug!(closed, "closed established connections for daemon shutdown");
2939 }
2940 let escalated_here = escalated && !already_escalated;
2944 let remaining_escalate = async move {
2945 if escalated_here {
2946 std::future::pending::<()>().await;
2947 } else {
2948 escalate.await;
2949 }
2950 };
2951 crate::child_roster::end_children_for_daemon_shutdown(
2952 &self.child_roster,
2953 escalated,
2954 remaining_escalate,
2955 )
2956 .await;
2957 }
2958
2959 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2960 Self {
2961 registry,
2962 restart_policy,
2963 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2964 connection_file_path: None,
2965 capture_logs_dir: None,
2966 forwarding: None,
2967 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2968 supervisor_handle: None,
2969 health: HealthConfig::default(),
2970 daemon_start_clock: crate::clock::StartClock::capture(),
2971 terminal_journal: None,
2972 spawn_events: SpawnEventFeed::default(),
2973 provenance_probe: ExecutableIdentityProbe::default(),
2974 child_roster: ChildRoster::default(),
2975 #[cfg(target_os = "linux")]
2976 cgroup_placement: None,
2977 #[cfg(test)]
2978 test_after_first_spawn: AfterFirstSpawnHook::default(),
2979 }
2980 }
2981
2982 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2983 self.drain_timeout = drain_timeout;
2984 self
2985 }
2986
2987 pub fn with_process_liveness(
2988 mut self,
2989 process_liveness: Arc<SupervisorProcessLiveness>,
2990 ) -> Self {
2991 self.process_liveness = process_liveness;
2992 self
2993 }
2994
2995 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2996 self.connection_file_path = Some(connection_file_path.into());
2997 self
2998 }
2999
3000 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3002 self.capture_logs_dir = Some(logs_dir.into());
3003 self
3004 }
3005
3006 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3009 self.spawn_events.configure_incarnation(daemon_incarnation);
3013 self
3014 }
3015
3016 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3019 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3020 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3021 path,
3022 daemon_incarnation,
3023 )));
3024 this
3025 }
3026
3027 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3028 self.forwarding = Some(forwarding);
3029 self
3030 }
3031
3032 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3033 self.spawn_events = supervisor_handle.spawn_events.clone();
3034 self.supervisor_handle = Some(supervisor_handle);
3035 self
3036 }
3037
3038 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3039 self.health = health;
3040 self
3041 }
3042
3043 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3047 self.child_roster.record_to(path.into());
3048 self
3049 }
3050
3051 #[cfg(target_os = "linux")]
3052 pub fn with_cgroup_placement(
3053 mut self,
3054 cgroup_placement: Option<subc_cgroup::Placement>,
3055 ) -> Self {
3056 self.cgroup_placement = cgroup_placement;
3057 self
3058 }
3059
3060 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3066 validate_spec(&spec)?;
3067 self.establish_identity(&spec);
3068
3069 let runtime = self.runtime_config();
3070 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3071 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3072 let spawned = spawn_child(
3073 &spec,
3074 runtime.connection_file_path.as_deref(),
3075 self.supervisor_handle.as_ref(),
3076 &runtime.stderr_ring,
3077 runtime.capture_logs_dir.as_deref(),
3078 &runtime.child_roster,
3079 #[cfg(target_os = "linux")]
3080 runtime.cgroup_placement.as_ref(),
3081 );
3082 #[cfg(test)]
3083 self.test_after_first_spawn.run(&spec.module_id);
3084 let child = match spawned {
3085 Ok(child) => child,
3086 Err(err) => {
3087 self.abandon_unrostered(&spec.module_id);
3090 return Err(err);
3091 }
3092 };
3093 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3094 if let Some(spawn) = activity_spawn {
3095 spawn.complete(&snapshot);
3096 }
3097
3098 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3099 }
3100
3101 fn establish_identity(&self, spec: &ModuleSpec) {
3114 if let Some(supervisor_handle) = &self.supervisor_handle {
3115 supervisor_handle.apply_identity_configuration(spec);
3116 supervisor_handle.mark_configured(&spec.module_id);
3117 }
3118 }
3119
3120 fn abandon_unrostered(&self, module_id: &str) {
3123 if let Some(supervisor_handle) = &self.supervisor_handle {
3124 supervisor_handle.unmark_configured_unless_rostered(module_id);
3125 }
3126 }
3127
3128 fn mark_first_process_running(
3131 &self,
3132 spec: &ModuleSpec,
3133 runtime: &SupervisorRuntimeConfig,
3134 snapshot: &SharedSnapshot,
3135 child: &SupervisedChild,
3136 ) -> Result<(), SuperviseError> {
3137 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3138 self.abandon_unrostered(&spec.module_id);
3139 return Err(err);
3140 }
3141 self.process_liveness
3142 .track(spec.module_id.clone(), Arc::clone(snapshot));
3143 Ok(())
3144 }
3145
3146 pub fn supervise_configured(
3152 &self,
3153 spec: ModuleSpec,
3154 enabled: bool,
3155 ) -> Result<SupervisedModule, SuperviseError> {
3156 self.supervise_configured_requested(
3157 spec,
3158 enabled,
3159 self.health.clone(),
3160 None,
3161 self.restart_policy,
3162 None,
3163 )
3164 }
3165
3166 pub fn supervise_configured_with_health(
3172 &self,
3173 spec: ModuleSpec,
3174 enabled: bool,
3175 health: HealthConfig,
3176 drain_timeout_ms: Option<u64>,
3177 restart_policy: RestartPolicy,
3178 ) -> Result<SupervisedModule, SuperviseError> {
3179 self.supervise_configured_requested(
3180 spec,
3181 enabled,
3182 health,
3183 drain_timeout_ms,
3184 restart_policy,
3185 None,
3186 )
3187 }
3188
3189 pub(crate) fn supervise_configured_requested(
3190 &self,
3191 spec: ModuleSpec,
3192 enabled: bool,
3193 health: HealthConfig,
3194 drain_timeout_ms: Option<u64>,
3195 restart_policy: RestartPolicy,
3196 requester: Option<Principal>,
3197 ) -> Result<SupervisedModule, SuperviseError> {
3198 let registered = self.register_configured_with_health(
3199 spec,
3200 enabled,
3201 health,
3202 drain_timeout_ms,
3203 restart_policy,
3204 )?;
3205 self.start_registered_configured(registered, requester)
3206 }
3207
3208 pub(crate) fn register_configured_with_health(
3213 &self,
3214 spec: ModuleSpec,
3215 enabled: bool,
3216 health: HealthConfig,
3217 drain_timeout_ms: Option<u64>,
3218 restart_policy: RestartPolicy,
3219 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3220 validate_spec(&spec)?;
3221 self.establish_identity(&spec);
3222
3223 let mut runtime = self.runtime_config();
3224 runtime.health = health;
3225 runtime.restart_policy = restart_policy;
3226 if let Some(ms) = drain_timeout_ms {
3227 runtime.drain_timeout = Duration::from_millis(ms);
3228 *runtime
3229 .effective_drain_timeout
3230 .lock()
3231 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3232 }
3233 let snapshot = if enabled {
3234 SupervisorSnapshot::starting()
3235 } else {
3236 SupervisorSnapshot::disabled()
3237 };
3238 let snapshot = {
3239 let mut snapshot = snapshot;
3240 if enabled {
3241 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3242 {
3243 snapshot.waiting_for_previous_pid = Some(pid);
3244 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3245 }
3246 }
3247 snapshot
3248 };
3249 let (module, commands) =
3250 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3251 Ok(RegisteredConfiguredModule {
3252 spec,
3253 runtime,
3254 module,
3255 commands,
3256 })
3257 }
3258
3259 pub(crate) fn start_registered_configured(
3262 &self,
3263 registered: RegisteredConfiguredModule,
3264 requester: Option<Principal>,
3265 ) -> Result<SupervisedModule, SuperviseError> {
3266 let RegisteredConfiguredModule {
3267 spec,
3268 mut runtime,
3269 module,
3270 commands,
3271 } = registered;
3272 runtime.start_requester = requester;
3273 let snapshot = &module.inner.snapshot;
3274 let should_spawn = {
3275 let state = lock_snapshot(snapshot)?;
3276 state.enabled && state.waiting_for_previous_pid.is_none()
3277 };
3278 let child = if should_spawn {
3279 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3280 let spawned = spawn_child(
3281 &spec,
3282 runtime.connection_file_path.as_deref(),
3283 self.supervisor_handle.as_ref(),
3284 &runtime.stderr_ring,
3285 runtime.capture_logs_dir.as_deref(),
3286 &runtime.child_roster,
3287 #[cfg(target_os = "linux")]
3288 runtime.cgroup_placement.as_ref(),
3289 );
3290 #[cfg(test)]
3291 self.test_after_first_spawn.run(&spec.module_id);
3292 match spawned {
3293 Ok(child) => {
3294 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3295 if let Some(spawn) = activity_spawn {
3296 spawn.complete(snapshot);
3297 }
3298 Some(child)
3299 }
3300 Err(err) => {
3301 if runtime.health.critical {
3302 error!(
3303 module_id = %spec.module_id,
3304 program = %spec.program.display(),
3305 error = %err,
3306 "critical configured module failed to spawn; marking failed and alerting"
3307 );
3308 } else {
3309 error!(
3310 module_id = %spec.module_id,
3311 program = %spec.program.display(),
3312 error = %err,
3313 "configured module failed to spawn; marking failed and continuing"
3314 );
3315 }
3316 update_snapshot(snapshot, Some(&spec.module_id), |state| {
3317 state.state = ModuleState::Failed;
3318 clear_current_process_facts(state);
3319 })?;
3320 None
3321 }
3322 }
3323 } else {
3324 None
3325 };
3326 self.start_monitor(spec, runtime, &module, child, commands);
3327 Ok(module)
3328 }
3329
3330 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3331 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3332 SupervisorRuntimeConfig {
3333 start_requester: None,
3334 pending_activity_restart: false,
3335 provenance_probe: self.provenance_probe.clone(),
3336 scheduled_respawn: Arc::default(),
3337 deferred_reload_reply: Arc::default(),
3338 restart_policy: self.restart_policy,
3339 drain_timeout: self.drain_timeout,
3340 child_roster: self
3343 .child_roster
3344 .for_module(Arc::clone(&effective_drain_timeout)),
3345 effective_drain_timeout,
3346 default_drain_timeout: self.drain_timeout,
3347 health: self.health.clone(),
3348 connection_file_path: self.connection_file_path.clone(),
3349 capture_logs_dir: self.capture_logs_dir.clone(),
3350 forwarding: self.forwarding.clone(),
3351 supervisor_handle: self.supervisor_handle.clone(),
3352 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3353 terminal_ring: Arc::new(Mutex::new(
3354 TerminalRing::new(
3355 TerminalRingConfig::default(),
3356 self.daemon_start_clock.started_at_ms(),
3357 )
3358 .with_start_clock(self.daemon_start_clock)
3359 .with_journal(self.terminal_journal.clone())
3360 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3361 )),
3362 spawn_events: self.spawn_events.clone(),
3363 #[cfg(target_os = "linux")]
3364 cgroup_placement: self.cgroup_placement.clone(),
3365 #[cfg(test)]
3366 test_seed_stale_facts_before_enable_spawn: false,
3367 #[cfg(test)]
3368 test_reload_exit_record_gate: None,
3369 }
3370 }
3371
3372 fn supervised_module(
3373 &self,
3374 spec: ModuleSpec,
3375 runtime: SupervisorRuntimeConfig,
3376 snapshot: SharedSnapshot,
3377 child: Option<SupervisedChild>,
3378 ) -> SupervisedModule {
3379 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3380 self.start_monitor(spec, runtime, &module, child, commands);
3381 module
3382 }
3383
3384 fn register_module(
3385 &self,
3386 spec: &ModuleSpec,
3387 runtime: &SupervisorRuntimeConfig,
3388 snapshot: SharedSnapshot,
3389 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3390 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3391 spec: spec.clone(),
3392 health: runtime.health.clone(),
3393 }));
3394 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3395 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3396 let restart_policy = runtime.restart_policy;
3400 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3401 let (tx, rx) = mpsc::channel(4);
3402 let module_id = spec.module_id.clone();
3403 let module = SupervisedModule {
3404 inner: Arc::new(SupervisedModuleInner {
3405 module_id: module_id.clone(),
3406 registry: Arc::clone(&self.registry),
3407 snapshot,
3408 configuration,
3409 stderr_ring,
3410 terminal_ring,
3411 commands: tx,
3412 monitor: Mutex::new(None),
3413 restart_policy,
3414 effective_drain_timeout,
3415 provenance_probe: self.provenance_probe.clone(),
3416 }),
3417 };
3418 if let Some(supervisor_handle) = &self.supervisor_handle {
3421 supervisor_handle.insert(module.clone());
3422 }
3423 (module, rx)
3424 }
3425
3426 fn start_monitor(
3427 &self,
3428 spec: ModuleSpec,
3429 runtime: SupervisorRuntimeConfig,
3430 module: &SupervisedModule,
3431 child: Option<SupervisedChild>,
3432 commands: mpsc::Receiver<SupervisorCommand>,
3433 ) {
3434 let monitor = tokio::spawn(supervise_loop(
3435 spec,
3436 runtime,
3437 Arc::clone(&self.registry),
3438 Arc::clone(&self.process_liveness),
3439 Arc::clone(&module.inner.snapshot),
3440 child,
3441 commands,
3442 ));
3443 *module
3444 .inner
3445 .monitor
3446 .lock()
3447 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3448 }
3449}
3450
3451impl Default for Supervisor {
3452 fn default() -> Self {
3453 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3454 }
3455}
3456
3457#[derive(Clone)]
3459pub struct SupervisedModule {
3460 inner: Arc<SupervisedModuleInner>,
3461}
3462
3463struct SupervisedModuleInner {
3464 module_id: String,
3465 registry: Arc<Registry>,
3466 snapshot: SharedSnapshot,
3467 configuration: Arc<Mutex<SupervisedConfiguration>>,
3468 stderr_ring: Arc<Mutex<StderrRing>>,
3469 terminal_ring: Arc<Mutex<TerminalRing>>,
3470 commands: mpsc::Sender<SupervisorCommand>,
3471 monitor: Mutex<Option<JoinHandle<()>>>,
3472 restart_policy: RestartPolicy,
3476 effective_drain_timeout: Arc<Mutex<Duration>>,
3477 provenance_probe: ExecutableIdentityProbe,
3478}
3479
3480impl fmt::Debug for SupervisedModule {
3481 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3482 f.debug_struct("SupervisedModule")
3483 .field("module_id", &self.inner.module_id)
3484 .field("status", &self.status())
3485 .finish_non_exhaustive()
3486 }
3487}
3488
3489impl SupervisedModule {
3490 pub fn module_id(&self) -> &str {
3491 &self.inner.module_id
3492 }
3493
3494 #[cfg(test)]
3498 pub(crate) fn record_health_probe_failure_for_test(
3499 &self,
3500 detail: &str,
3501 ) -> Result<(), SuperviseError> {
3502 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3503 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3504 state.health.detail = Some(detail.to_string());
3505 })
3506 }
3507
3508 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3509 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3510 }
3511
3512 pub(crate) fn registration_end_reason(
3513 &self,
3514 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3515 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3516 Ok(match snapshot.state {
3517 ModuleState::Draining if snapshot.draining_to_replace => {
3518 Some(RegistrationEndReason::SupervisorRestart)
3519 }
3520 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3521 ModuleState::Restarting if snapshot.draining_to_replace => {
3522 Some(RegistrationEndReason::SupervisorRestart)
3523 }
3524 _ => None,
3525 })
3526 }
3527
3528 pub fn stderr_tail(
3535 &self,
3536 max_lines: Option<usize>,
3537 max_bytes: Option<usize>,
3538 ) -> StderrTailSnapshot {
3539 self.inner
3540 .stderr_ring
3541 .lock()
3542 .unwrap_or_else(|poisoned| poisoned.into_inner())
3543 .snapshot(max_lines, max_bytes)
3544 }
3545
3546 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3551 self.inner
3552 .terminal_ring
3553 .lock()
3554 .unwrap_or_else(|poisoned| poisoned.into_inner())
3555 .snapshot()
3556 }
3557
3558 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3563 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3564 }
3565
3566 pub(crate) async fn read_durable_terminal_history(
3571 &self,
3572 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3573 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3574 let module_id = self.inner.module_id.clone();
3575 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3576 .await
3577 }
3578
3579 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3580 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3581 .map(|(status, _)| status)
3582 }
3583
3584 pub(crate) fn record_deliberate_severance(
3585 &self,
3586 identity: ProcessIdentity,
3587 ) -> Result<bool, SuperviseError> {
3588 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3589 if snapshot.pid != Some(identity.pid)
3590 || snapshot.process_start_time != Some(identity.start_time)
3591 {
3592 return Ok(false);
3593 }
3594 snapshot.deliberate_severance = Some(identity);
3595 Ok(true)
3596 }
3597
3598 pub(crate) fn status_for_control(
3603 &self,
3604 caller: &'static str,
3605 ) -> Result<ModuleStatus, SuperviseError> {
3606 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3607 .map(|(status, _)| status)
3608 }
3609
3610 fn status_with_snapshot_lock(
3611 &self,
3612 snapshot: &SharedSnapshot,
3613 caller: Option<&'static str>,
3614 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3615 let mut guard = match caller {
3616 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3617 None => lock_snapshot(snapshot)?,
3618 };
3619 let restart_count =
3622 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3623 let snapshot = guard.clone();
3624 drop(guard);
3625 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3626 SuperviseError::StatePoisoned {
3627 module_id: Some(self.inner.module_id.clone()),
3628 }
3629 })?;
3630 let registration_active = self
3631 .inner
3632 .registry
3633 .get_module(&self.inner.module_id)
3634 .map_err(SuperviseError::Registry)?
3635 .is_some();
3636 let protocol = snapshot
3637 .spawned_protocol
3638 .unwrap_or(self.declared_protocol()?);
3639 let running_process =
3640 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3641 let live = match protocol {
3647 ModuleProtocol::Subc => running_process && registration_active,
3648 ModuleProtocol::None => running_process,
3649 };
3650
3651 Ok((
3652 ModuleStatus {
3653 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3654 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3655 module_id: self.inner.module_id.clone(),
3656 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3657 state: snapshot.state,
3658 enabled: snapshot.enabled,
3659 process_alive: snapshot.process_alive,
3660 registration_active,
3661 protocol,
3662 live,
3663 restart_count,
3664 lifetime_restarts: snapshot.lifetime_restarts,
3665 spawn_generation: snapshot.spawn_generation,
3666 max_restarts: self.inner.restart_policy.max_restarts,
3667 restart_window: self.inner.restart_policy.window,
3668 drain_timeout,
3669 restart_backoff: self.inner.restart_policy.backoff,
3670 restart_max_backoff: self.inner.restart_policy.max_backoff,
3671 pid: snapshot.reported_pid(),
3672 spawned_at_ms: snapshot.spawned_at_ms,
3673 spawned_from: snapshot.spawned_from,
3674 launch_spec_digest: snapshot.launch_spec_digest,
3675 process_start_time: snapshot.process_start_time,
3676 last_exit: snapshot.last_exit,
3677 health: snapshot.health,
3678 },
3679 snapshot.spawned_file_identity.clone(),
3680 ))
3681 }
3682
3683 #[cfg(test)]
3684 pub(crate) fn hold_snapshot_for_test(
3685 &self,
3686 acquired: std::sync::mpsc::Sender<()>,
3687 hold: Duration,
3688 ) -> std::thread::JoinHandle<()> {
3689 let snapshot = Arc::clone(&self.inner.snapshot);
3690 std::thread::spawn(move || {
3691 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3692 acquired
3693 .send(())
3694 .expect("test receiver waits for snapshot lock");
3695 std::thread::sleep(hold);
3696 })
3697 }
3698
3699 pub(crate) async fn status_and_running_image_agreement(
3705 &self,
3706 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3707 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3708 let image = self
3709 .inner
3710 .provenance_probe
3711 .observe(
3712 status.pid,
3713 status.spawned_from.as_deref(),
3714 identity,
3715 status.process_start_time,
3716 )
3717 .await;
3718 Ok((status, image))
3719 }
3720
3721 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3722 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3723 Ok(snapshot) => snapshot.clone(),
3724 Err(_) => {
3725 return subc_control::RunningImageAgreement::Unavailable {
3726 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3727 };
3728 }
3729 };
3730 self.inner
3731 .provenance_probe
3732 .observe(
3733 snapshot.reported_pid(),
3734 snapshot.spawned_from.as_deref(),
3735 snapshot.spawned_file_identity.clone(),
3736 snapshot.process_start_time,
3737 )
3738 .await
3739 }
3740
3741 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3744 #[cfg(windows)]
3745 {
3746 let image = lock_snapshot(&self.inner.snapshot)
3747 .ok()
3748 .and_then(|snapshot| {
3749 snapshot
3750 .spawned_file_identity
3751 .as_ref()
3752 .and_then(|identity| identity.windows_image.clone())
3753 });
3754 match image {
3755 Some(image) => {
3756 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3757 }
3758 None => subc_control::ChildResourceUsage::Unavailable {
3759 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3760 },
3761 }
3762 }
3763 #[cfg(not(windows))]
3764 {
3765 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3766 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3767 Err(_) => {
3768 return subc_control::ChildResourceUsage::Unavailable {
3769 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3770 }
3771 }
3772 };
3773 crate::child_resources::read(pid, start_time)
3774 }
3775 }
3776
3777 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3778 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3779 Ok(match snapshot.state {
3780 ModuleState::Restarting => true,
3781 ModuleState::Failed | ModuleState::Disabled => false,
3782 _ if snapshot.enabled && snapshot.respawn_pending => true,
3783 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3784 })
3785 }
3786
3787 #[cfg(test)]
3788 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3789 self.is_warming_with_snapshot_lock(None)
3790 }
3791
3792 pub(crate) fn is_warming_for_control(
3793 &self,
3794 caller: &'static str,
3795 ) -> Result<bool, SuperviseError> {
3796 self.is_warming_with_snapshot_lock(Some(caller))
3797 }
3798
3799 fn is_warming_with_snapshot_lock(
3800 &self,
3801 caller: Option<&'static str>,
3802 ) -> Result<bool, SuperviseError> {
3803 let snapshot = match caller {
3804 Some(caller) => {
3805 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3806 }
3807 None => lock_snapshot(&self.inner.snapshot)?,
3808 }
3809 .clone();
3810 Ok(matches!(
3811 snapshot.state,
3812 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3813 ))
3814 }
3815
3816 pub async fn drain(&self) -> Result<(), SuperviseError> {
3818 self.stop().await
3819 }
3820
3821 pub(crate) async fn retire_requested(
3822 &self,
3823 requester: Option<Principal>,
3824 ) -> Result<(), SuperviseError> {
3825 match self.state()? {
3826 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3827 ModuleState::Starting
3828 | ModuleState::Running
3829 | ModuleState::Unresponsive
3830 | ModuleState::Restarting
3831 | ModuleState::Draining
3832 | ModuleState::Disabled => {}
3833 }
3834
3835 let (reply_tx, reply_rx) = oneshot::channel();
3836 self.inner
3837 .commands
3838 .send(SupervisorCommand::Retire {
3839 requester,
3840 reply: reply_tx,
3841 })
3842 .await
3843 .map_err(|_| SuperviseError::CommandClosed {
3844 module_id: self.inner.module_id.clone(),
3845 })?;
3846 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3847 module_id: self.inner.module_id.clone(),
3848 })?
3849 }
3850
3851 pub async fn stop(&self) -> Result<(), SuperviseError> {
3852 self.stop_requested(None).await
3853 }
3854
3855 pub(crate) async fn stop_requested(
3856 &self,
3857 requester: Option<Principal>,
3858 ) -> Result<(), SuperviseError> {
3859 match self.state()? {
3860 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3861 ModuleState::Starting
3862 | ModuleState::Running
3863 | ModuleState::Unresponsive
3864 | ModuleState::Restarting
3865 | ModuleState::Draining
3866 | ModuleState::Disabled => {}
3867 }
3868
3869 let (reply_tx, reply_rx) = oneshot::channel();
3870 self.inner
3871 .commands
3872 .send(SupervisorCommand::Drain {
3873 requester,
3874 reply: reply_tx,
3875 })
3876 .await
3877 .map_err(|_| SuperviseError::CommandClosed {
3878 module_id: self.inner.module_id.clone(),
3879 })?;
3880 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3881 module_id: self.inner.module_id.clone(),
3882 })?
3883 }
3884
3885 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3886 self.restart_requested(drain_timeout_ms, None).await
3887 }
3888
3889 pub(crate) async fn restart_requested(
3890 &self,
3891 drain_timeout_ms: Option<u64>,
3892 requester: Option<Principal>,
3893 ) -> Result<(), SuperviseError> {
3894 let queued_at = Instant::now();
3895 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3896 let (reply_tx, reply_rx) = oneshot::channel();
3897 self.inner
3898 .commands
3899 .send(SupervisorCommand::Restart {
3900 requester,
3901 drain_timeout_ms,
3902 received_at_generation,
3903 queued_at,
3904 reply: reply_tx,
3905 })
3906 .await
3907 .map_err(|_| SuperviseError::CommandClosed {
3908 module_id: self.inner.module_id.clone(),
3909 })?;
3910 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3911 module_id: self.inner.module_id.clone(),
3912 })?
3913 }
3914
3915 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3920 self.swap_requested(ready_timeout, None).await
3921 }
3922
3923 pub(crate) async fn swap_requested(
3924 &self,
3925 ready_timeout: Option<Duration>,
3926 requester: Option<Principal>,
3927 ) -> Result<(), SuperviseError> {
3928 let (reply_tx, reply_rx) = oneshot::channel();
3929 self.inner
3930 .commands
3931 .send(SupervisorCommand::Swap {
3932 requester,
3933 ready_timeout,
3934 reply: reply_tx,
3935 })
3936 .await
3937 .map_err(|_| SuperviseError::CommandClosed {
3938 module_id: self.inner.module_id.clone(),
3939 })?;
3940 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3941 module_id: self.inner.module_id.clone(),
3942 })?
3943 }
3944
3945 pub async fn reload(&self) -> Result<(), SuperviseError> {
3946 self.reload_requested(None).await
3947 }
3948
3949 pub(crate) async fn reload_requested(
3950 &self,
3951 requester: Option<Principal>,
3952 ) -> Result<(), SuperviseError> {
3953 let (reply_tx, reply_rx) = oneshot::channel();
3954 self.inner
3955 .commands
3956 .send(SupervisorCommand::Reload {
3957 requester,
3958 reply: reply_tx,
3959 })
3960 .await
3961 .map_err(|_| SuperviseError::CommandClosed {
3962 module_id: self.inner.module_id.clone(),
3963 })?;
3964 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3965 module_id: self.inner.module_id.clone(),
3966 })?
3967 }
3968
3969 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3970 self.set_enabled_requested(enabled, None).await
3971 }
3972
3973 pub(crate) async fn set_enabled_requested(
3974 &self,
3975 enabled: bool,
3976 requester: Option<Principal>,
3977 ) -> Result<bool, SuperviseError> {
3978 let (reply_tx, reply_rx) = oneshot::channel();
3979 self.inner
3980 .commands
3981 .send(SupervisorCommand::SetEnabled {
3982 requester,
3983 enabled,
3984 reply: reply_tx,
3985 })
3986 .await
3987 .map_err(|_| SuperviseError::CommandClosed {
3988 module_id: self.inner.module_id.clone(),
3989 })?;
3990 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3991 module_id: self.inner.module_id.clone(),
3992 })?
3993 }
3994
3995 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3999 let configured = self
4000 .inner
4001 .configuration
4002 .lock()
4003 .map_err(|_| SuperviseError::StatePoisoned {
4004 module_id: Some(self.inner.module_id.clone()),
4005 })?
4006 .spec
4007 .protocol;
4008 let state = lock_snapshot(&self.inner.snapshot)?;
4009 Ok(state.spawned_protocol.unwrap_or(configured))
4010 }
4011
4012 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4013 let configuration =
4014 self.inner
4015 .configuration
4016 .lock()
4017 .map_err(|_| SuperviseError::StatePoisoned {
4018 module_id: Some(self.inner.module_id.clone()),
4019 })?;
4020 Ok((configuration.spec.clone(), configuration.health.clone()))
4021 }
4022
4023 #[cfg(any(test, feature = "test-support"))]
4027 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4028 let (_, health) = self.configuration()?;
4029 let drain_timeout_ms = u64::try_from(
4030 self.inner
4031 .effective_drain_timeout
4032 .lock()
4033 .unwrap_or_else(|poisoned| poisoned.into_inner())
4034 .as_millis(),
4035 )
4036 .ok();
4037 self.update_configuration(spec, health, drain_timeout_ms)
4038 .await
4039 }
4040
4041 pub(crate) async fn update_configuration(
4042 &self,
4043 spec: ModuleSpec,
4044 health: HealthConfig,
4045 drain_timeout_ms: Option<u64>,
4046 ) -> Result<(), SuperviseError> {
4047 if spec.module_id != self.inner.module_id {
4048 return Err(SuperviseError::InvalidSpec {
4049 reason: "a supervised module's module_id cannot be changed".to_string(),
4050 });
4051 }
4052 validate_spec(&spec)?;
4053 let (reply_tx, reply_rx) = oneshot::channel();
4054 self.inner
4055 .commands
4056 .send(SupervisorCommand::UpdateConfiguration {
4057 spec: spec.clone(),
4058 health: health.clone(),
4059 drain_timeout_ms,
4060 reply: reply_tx,
4061 })
4062 .await
4063 .map_err(|_| SuperviseError::CommandClosed {
4064 module_id: self.inner.module_id.clone(),
4065 })?;
4066 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4067 module_id: self.inner.module_id.clone(),
4068 })?;
4069 let mut configuration =
4070 self.inner
4071 .configuration
4072 .lock()
4073 .map_err(|_| SuperviseError::StatePoisoned {
4074 module_id: Some(self.inner.module_id.clone()),
4075 })?;
4076 configuration.spec = spec;
4077 configuration.health = health;
4078 Ok(())
4079 }
4080}
4081
4082impl Drop for SupervisedModuleInner {
4083 fn drop(&mut self) {
4084 let Ok(mut monitor) = self.monitor.lock() else {
4085 return;
4086 };
4087 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4088 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4089 state.state = ModuleState::Stopped;
4090 clear_current_process_facts(state);
4091 });
4092 monitor.abort();
4093 }
4094 let _ = monitor.take();
4095 }
4096}
4097
4098#[derive(Debug)]
4099enum SupervisorCommand {
4100 Drain {
4101 requester: Option<Principal>,
4102 reply: oneshot::Sender<Result<(), SuperviseError>>,
4103 },
4104 Retire {
4105 requester: Option<Principal>,
4106 reply: oneshot::Sender<Result<(), SuperviseError>>,
4107 },
4108 Restart {
4109 requester: Option<Principal>,
4110 drain_timeout_ms: Option<u64>,
4115 received_at_generation: u64,
4119 queued_at: Instant,
4122 reply: oneshot::Sender<Result<(), SuperviseError>>,
4123 },
4124 Reload {
4125 requester: Option<Principal>,
4126 reply: oneshot::Sender<Result<(), SuperviseError>>,
4127 },
4128 SetEnabled {
4129 requester: Option<Principal>,
4130 enabled: bool,
4131 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4132 },
4133 UpdateConfiguration {
4134 spec: ModuleSpec,
4135 health: HealthConfig,
4136 drain_timeout_ms: Option<u64>,
4139 reply: oneshot::Sender<()>,
4140 },
4141 Swap {
4142 requester: Option<Principal>,
4143 ready_timeout: Option<Duration>,
4146 reply: oneshot::Sender<Result<(), SuperviseError>>,
4148 },
4149}
4150
4151#[derive(Debug)]
4152pub enum SuperviseError {
4153 InvalidSpec {
4154 reason: String,
4155 },
4156 Spawn {
4157 program: PathBuf,
4158 source: io::Error,
4159 cgroup_path: Option<PathBuf>,
4160 },
4161 Cgroup {
4162 module_id: String,
4163 source: io::Error,
4164 },
4165 LaunchNonce {
4168 reason: String,
4169 },
4170 Wait {
4171 module_id: String,
4172 source: io::Error,
4173 },
4174 Kill {
4175 module_id: String,
4176 source: io::Error,
4177 },
4178 Forwarding(ForwardingError),
4179 Registry(RegistryError),
4180 ReloadUnavailable {
4181 module_id: String,
4182 reason: String,
4183 },
4184 Disabled {
4189 module_id: String,
4190 },
4191 ReloadFailed {
4192 module_id: String,
4193 reason: String,
4194 },
4195 RegistrationStillActive {
4196 module_id: String,
4197 waited: Duration,
4198 },
4199 StatePoisoned {
4200 module_id: Option<String>,
4201 },
4202 CommandClosed {
4203 module_id: String,
4204 },
4205 SwapInProgress {
4209 module_id: String,
4210 },
4211 SwapRefused {
4213 module_id: String,
4214 reason: SwapRefusal,
4215 },
4216 SwapFailed {
4220 module_id: String,
4221 arm: SwapFailureArm,
4222 detail: String,
4223 candidate_exit: Option<ExitReport>,
4226 },
4227}
4228
4229#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4231pub enum SwapRefusal {
4232 OverlapExclusive,
4234 NotRegistered,
4237 ProtocolNone,
4240 NotConfigured,
4243 AlreadySwapping,
4245}
4246
4247impl SwapRefusal {
4248 pub fn as_str(self) -> &'static str {
4249 match self {
4250 Self::OverlapExclusive => "overlap_exclusive",
4251 Self::NotRegistered => "not_registered",
4252 Self::ProtocolNone => "protocol_none",
4253 Self::NotConfigured => "not_configured",
4254 Self::AlreadySwapping => "already_swapping",
4255 }
4256 }
4257}
4258
4259#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4262pub enum SwapFailureArm {
4263 SpawnFailed,
4265 NeverRegistered,
4267 NeverReady,
4269 CandidateExited,
4271 CandidateUnhealthy,
4273 Interrupted,
4277 CutoverLost,
4282}
4283
4284impl SwapFailureArm {
4285 pub fn as_str(self) -> &'static str {
4286 match self {
4287 Self::SpawnFailed => "spawn_failed",
4288 Self::NeverRegistered => "never_registered",
4289 Self::NeverReady => "never_ready",
4290 Self::CandidateExited => "candidate_exited",
4291 Self::CandidateUnhealthy => "candidate_unhealthy",
4292 Self::Interrupted => "interrupted",
4293 Self::CutoverLost => "cutover_lost",
4294 }
4295 }
4296}
4297
4298impl fmt::Display for SuperviseError {
4299 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4300 match self {
4301 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4302 Self::Spawn {
4303 program,
4304 source,
4305 cgroup_path: Some(cgroup_path),
4306 } => write!(
4307 f,
4308 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4309 cgroup_path.display(),
4310 program.display()
4311 ),
4312 Self::Spawn {
4313 program,
4314 source,
4315 cgroup_path: None,
4316 } => write!(
4317 f,
4318 "failed to spawn module '{}': {source}",
4319 program.display()
4320 ),
4321 Self::Cgroup { module_id, source } => {
4322 write!(
4323 f,
4324 "failed to prepare cgroup for module '{module_id}': {source}"
4325 )
4326 }
4327 Self::LaunchNonce { reason } => {
4328 write!(
4329 f,
4330 "failed to generate reserved-module launch nonce: {reason}"
4331 )
4332 }
4333 Self::Wait { module_id, source } => {
4334 write!(f, "failed to wait for module '{module_id}': {source}")
4335 }
4336 Self::Kill { module_id, source } => {
4337 write!(f, "failed to kill module '{module_id}': {source}")
4338 }
4339 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4340 Self::Registry(err) => write!(f, "registry error: {err}"),
4341 Self::ReloadUnavailable { module_id, reason } => {
4342 write!(f, "reload unavailable for module '{module_id}': {reason}")
4343 }
4344 Self::Disabled { module_id } => {
4345 write!(
4346 f,
4347 "module '{module_id}' is disabled; enable it before restart or reload"
4348 )
4349 }
4350 Self::ReloadFailed { module_id, reason } => {
4351 write!(f, "reload failed for module '{module_id}': {reason}")
4352 }
4353 Self::RegistrationStillActive { module_id, waited } => write!(
4354 f,
4355 "module '{module_id}' registration remained active after waiting {waited:?}"
4356 ),
4357 Self::StatePoisoned { module_id } => match module_id {
4358 Some(module_id) => {
4359 write!(f, "supervisor state for module '{module_id}' was poisoned")
4360 }
4361 None => write!(f, "supervisor state was poisoned"),
4362 },
4363 Self::CommandClosed { module_id } => {
4364 write!(
4365 f,
4366 "supervisor command channel for module '{module_id}' is closed"
4367 )
4368 }
4369 Self::SwapInProgress { module_id } => write!(
4370 f,
4371 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4372 ),
4373 Self::SwapRefused { module_id, reason } => match reason {
4374 SwapRefusal::OverlapExclusive => write!(
4375 f,
4376 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4377 ),
4378 SwapRefusal::NotRegistered => write!(
4379 f,
4380 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4381 ),
4382 SwapRefusal::ProtocolNone => write!(
4383 f,
4384 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4385 ),
4386 SwapRefusal::NotConfigured => write!(
4387 f,
4388 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4389 ),
4390 SwapRefusal::AlreadySwapping => {
4391 write!(f, "module '{module_id}' is already being swapped")
4392 }
4393 },
4394 Self::SwapFailed {
4395 module_id,
4396 arm,
4397 detail,
4398 ..
4399 } => write!(
4400 f,
4401 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4402 arm.as_str()
4403 ),
4404 }
4405 }
4406}
4407
4408impl Error for SuperviseError {
4409 fn source(&self) -> Option<&(dyn Error + 'static)> {
4410 match self {
4411 Self::Spawn { source, .. }
4412 | Self::Cgroup { source, .. }
4413 | Self::Wait { source, .. }
4414 | Self::Kill { source, .. } => Some(source),
4415 Self::Forwarding(err) => Some(err),
4416 Self::Registry(err) => Some(err),
4417 Self::LaunchNonce { .. }
4418 | Self::InvalidSpec { .. }
4419 | Self::ReloadUnavailable { .. }
4420 | Self::Disabled { .. }
4421 | Self::ReloadFailed { .. }
4422 | Self::RegistrationStillActive { .. }
4423 | Self::StatePoisoned { .. }
4424 | Self::CommandClosed { .. }
4425 | Self::SwapInProgress { .. }
4426 | Self::SwapRefused { .. }
4427 | Self::SwapFailed { .. } => None,
4428 }
4429 }
4430}
4431
4432pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4433 if spec.module_id.trim().is_empty() {
4434 return Err(SuperviseError::InvalidSpec {
4435 reason: "module_id must not be empty".to_string(),
4436 });
4437 }
4438
4439 Ok(())
4440}
4441
4442#[derive(Debug, Default)]
4443struct HealthProbeRuntime {
4444 configured_health: Option<HealthConfig>,
4445 registered_connection: Option<crate::ConnectionId>,
4446 advertised: bool,
4447 next_probe_at: Option<Instant>,
4448 probe_index: u64,
4449}
4450
4451fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4452 lock_snapshot(snapshot)
4453 .ok()
4454 .and_then(|state| state.spawned_protocol)
4455 .unwrap_or(spec.protocol)
4456}
4457
4458impl HealthProbeRuntime {
4459 fn refresh_registration(
4460 &mut self,
4461 spec: &ModuleSpec,
4462 runtime: &SupervisorRuntimeConfig,
4463 registry: &Registry,
4464 snapshot: &SharedSnapshot,
4465 ) {
4466 if self.configured_health.as_ref() != Some(&runtime.health) {
4467 self.configured_health = Some(runtime.health.clone());
4468 self.next_probe_at = None;
4469 self.registered_connection = None;
4470 self.probe_index = 0;
4471 }
4472 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4476 self.registered_connection = None;
4477 self.advertised = runtime.health.http.is_some();
4478 if !self.advertised {
4479 self.next_probe_at = None;
4480 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4481 let unknown = ModuleHealthStatus::default();
4482 if state.health != unknown {
4483 state.health = unknown;
4484 }
4485 });
4486 } else if self.next_probe_at.is_none() {
4487 self.next_probe_at = Some(
4488 Instant::now()
4489 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4490 );
4491 }
4492 return;
4493 }
4494
4495 let registration = match registry.get_module(&spec.module_id) {
4496 Ok(registration) => registration,
4497 Err(err) => {
4498 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4499 self.advertised = false;
4500 self.next_probe_at = None;
4501 return;
4502 }
4503 };
4504
4505 let Some(registration) = registration else {
4506 self.registered_connection = None;
4507 self.advertised = false;
4508 self.next_probe_at = None;
4509 return;
4510 };
4511
4512 let advertised = registration
4513 .control_ops
4514 .iter()
4515 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4516 if !advertised {
4517 self.registered_connection = Some(registration.connection_id);
4518 self.advertised = false;
4519 self.next_probe_at = None;
4520 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4521 if state.health.status != SupervisorHealthStatus::Unknown
4522 || state.health.consecutive_failures != 0
4523 || state.health.last_probe_ms.is_some()
4524 || state.health.detail.is_some()
4525 || state.health.metrics.is_some()
4526 {
4527 state.health.status = SupervisorHealthStatus::Unknown;
4528 state.health.consecutive_failures = 0;
4529 state.health.last_probe_ms = None;
4530 state.health.detail = None;
4531 state.health.metrics = None;
4532 }
4533 });
4534 return;
4535 }
4536
4537 let reregistered = self.registered_connection != Some(registration.connection_id);
4538 self.registered_connection = Some(registration.connection_id);
4539 self.advertised = true;
4540 if reregistered || self.next_probe_at.is_none() {
4541 self.probe_index = 0;
4542 self.next_probe_at = Some(
4543 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4544 );
4545 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4546 state.health.status = SupervisorHealthStatus::Unknown;
4547 state.health.consecutive_failures = 0;
4548 state.health.detail = None;
4549 state.health.metrics = None;
4550 });
4551 }
4552 }
4553
4554 fn wake_after(&self) -> Option<Duration> {
4555 if !self.advertised {
4556 return None;
4557 }
4558 self.next_probe_at
4559 .map(|next| next.saturating_duration_since(Instant::now()))
4560 }
4561
4562 fn due(&self) -> bool {
4563 self.advertised
4564 && self
4565 .next_probe_at
4566 .is_some_and(|next| Instant::now() >= next)
4567 }
4568
4569 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4570 self.probe_index = self.probe_index.wrapping_add(1);
4571 self.next_probe_at = Some(
4572 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4573 );
4574 }
4575}
4576
4577#[derive(Debug)]
4612enum HealthProbeEvidence {
4613 LaneDead,
4615 NoAnswer,
4617 BadAnswer,
4619 Misconfigured,
4621}
4622
4623#[derive(Debug)]
4624struct HealthProbeError {
4625 evidence: HealthProbeEvidence,
4626 message: String,
4627}
4628
4629impl HealthProbeError {
4630 fn lane_dead(message: impl Into<String>) -> Self {
4631 Self::with(HealthProbeEvidence::LaneDead, message)
4632 }
4633
4634 fn no_answer(message: impl Into<String>) -> Self {
4635 Self::with(HealthProbeEvidence::NoAnswer, message)
4636 }
4637
4638 fn bad_answer(message: impl Into<String>) -> Self {
4639 Self::with(HealthProbeEvidence::BadAnswer, message)
4640 }
4641
4642 fn misconfigured(message: impl Into<String>) -> Self {
4643 Self::with(HealthProbeEvidence::Misconfigured, message)
4644 }
4645
4646 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4647 Self {
4648 evidence,
4649 message: message.into(),
4650 }
4651 }
4652
4653 #[allow(dead_code)]
4667 fn is_proof_of_death(&self) -> bool {
4668 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4669 }
4670
4671 fn label(&self) -> &'static str {
4679 match self.evidence {
4680 HealthProbeEvidence::LaneDead => "lane-dead",
4681 HealthProbeEvidence::NoAnswer => "no-answer",
4682 HealthProbeEvidence::BadAnswer => "bad-answer",
4683 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4684 }
4685 }
4686}
4687
4688impl fmt::Display for HealthProbeError {
4689 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4690 f.write_str(&self.message)
4691 }
4692}
4693
4694async fn run_health_probe_cycle(
4695 spec: &ModuleSpec,
4696 runtime: &SupervisorRuntimeConfig,
4697 registry: &Registry,
4698 process_liveness: &SupervisorProcessLiveness,
4699 snapshot: &SharedSnapshot,
4700 child: &mut Option<SupervisedChild>,
4701) {
4702 let now_ms = unix_ms_now();
4703 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4704 .then_some(runtime.health.http.as_deref())
4705 .flatten();
4706 let result = match http {
4707 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4708 None => probe_module_health(&spec.module_id, runtime, None).await,
4709 };
4710 match result {
4711 Ok(report) => {
4712 handle_health_report(
4713 spec,
4714 runtime,
4715 registry,
4716 process_liveness,
4717 snapshot,
4718 child,
4719 report,
4720 now_ms,
4721 )
4722 .await;
4723 }
4724 Err(err) => {
4725 if http.is_some() {
4726 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4727 state.health.status = SupervisorHealthStatus::Failing;
4728 });
4729 }
4730 handle_health_probe_failure(
4731 spec,
4732 runtime,
4733 registry,
4734 process_liveness,
4735 snapshot,
4736 child,
4737 err,
4738 now_ms,
4739 )
4740 .await;
4741 }
4742 }
4743}
4744
4745pub(crate) struct HttpProbeTarget<'a> {
4746 address: std::net::SocketAddr,
4747 localhost: bool,
4748 authority: &'a str,
4749 path: String,
4750}
4751
4752pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4755 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4756 return Err("must not contain whitespace, controls, or a fragment".into());
4757 }
4758 let rest = url
4759 .strip_prefix("http://")
4760 .ok_or("must use plain http://")?;
4761 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4762 let (authority, suffix) = rest.split_at(split);
4763 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4764 ("::1", rest)
4765 } else {
4766 let split = authority.find(':').unwrap_or(authority.len());
4767 authority.split_at(split)
4768 };
4769 let ip: std::net::IpAddr = match host {
4770 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4771 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4772 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4773 };
4774 let port = if port.is_empty() {
4775 80
4776 } else {
4777 port.strip_prefix(':')
4778 .and_then(|p| p.parse::<u16>().ok())
4779 .filter(|p| *p > 0)
4780 .ok_or("must have a valid nonzero TCP port")?
4781 };
4782 let path = if suffix.is_empty() {
4783 "/".into()
4784 } else if suffix.starts_with('?') {
4785 format!("/{suffix}")
4786 } else {
4787 suffix.into()
4788 };
4789 Ok(HttpProbeTarget {
4790 address: std::net::SocketAddr::new(ip, port),
4791 localhost: host == "localhost",
4792 authority,
4793 path,
4794 })
4795}
4796
4797async fn probe_http_health(
4798 url: &str,
4799 deadline: Duration,
4800) -> Result<HealthReport, HealthProbeError> {
4801 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4802 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4803 let mut response_status = String::new();
4806 let mut body = Vec::new();
4807 let probe = async {
4808 let connection = match tokio::net::TcpStream::connect(target.address).await {
4811 Err(_) if target.localhost => {
4812 tokio::net::TcpStream::connect((
4813 std::net::Ipv6Addr::LOCALHOST,
4814 target.address.port(),
4815 ))
4816 .await
4817 }
4818 result => result,
4819 };
4820 let mut stream = connection.map_err(|error| {
4821 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4822 })?;
4823 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4824 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4825 let mut reader = BufReader::new(stream);
4826 let mut budget = 16 * 1024;
4827 let status = http_line(&mut reader, &mut budget).await?;
4828 let mut words = status.split_ascii_whitespace();
4829 let version = words.next();
4830 let code = words
4831 .next()
4832 .filter(|word| word.len() == 3)
4833 .and_then(|word| word.parse::<u16>().ok());
4834 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4835 || !code.is_some_and(|code| (100..600).contains(&code))
4836 {
4837 return Err(HealthProbeError::bad_answer(format!(
4838 "invalid HTTP status: {status}"
4839 )));
4840 }
4841 let code = code.expect("validated status code");
4842 response_status = status.clone();
4843 let mut length = None;
4844 let mut chunked = false;
4845 loop {
4846 let line = http_line(&mut reader, &mut budget).await?;
4847 if line.is_empty() {
4848 break;
4849 }
4850 if let Some((name, value)) = line.split_once(':') {
4851 if name.eq_ignore_ascii_case("content-length") {
4852 length = Some(value.trim().parse::<u64>().map_err(|_| {
4853 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4854 })?);
4855 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4856 chunked = value.trim().eq_ignore_ascii_case("chunked");
4857 }
4858 }
4859 }
4860 if chunked {
4861 while body.len() < 200 {
4862 let line = http_line(&mut reader, &mut budget).await?;
4863 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4864 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4865 if size == 0 {
4866 break;
4867 }
4868 let count = size.min((200 - body.len()) as u64) as usize;
4869 let start = body.len();
4870 (&mut reader)
4871 .take(count as u64)
4872 .read_to_end(&mut body)
4873 .await
4874 .map_err(|error| {
4875 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4876 })?;
4877 if body.len() - start != count {
4878 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4879 }
4880 if size > count as u64 || body.len() == 200 {
4881 break;
4882 }
4883 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4884 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4885 }
4886 }
4887 } else {
4888 reader
4889 .take(length.unwrap_or(200).min(200))
4890 .read_to_end(&mut body)
4891 .await
4892 .map_err(|error| {
4893 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4894 })?;
4895 }
4896 if (200..300).contains(&code) {
4897 Ok(HealthReport::ok())
4898 } else {
4899 Err(HealthProbeError::bad_answer(
4900 "HTTP health endpoint returned non-2xx",
4901 ))
4902 }
4903 };
4904 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4905 Err(HealthProbeError::no_answer(format!(
4906 "HTTP probe timed out after {deadline:?}"
4907 )))
4908 });
4909 if let Err(error) = &mut result {
4910 if !response_status.is_empty() {
4911 error.message = format!(
4912 "{}; {response_status}: {}",
4913 error.message,
4914 String::from_utf8_lossy(&body)
4915 );
4916 }
4917 }
4918 result
4919}
4920
4921async fn http_line(
4922 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4923 remaining: &mut usize,
4924) -> Result<String, HealthProbeError> {
4925 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4926 let mut line = Vec::new();
4927 (&mut *reader)
4928 .take(*remaining as u64)
4929 .read_until(b'\n', &mut line)
4930 .await
4931 .map_err(|error| {
4932 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4933 })?;
4934 *remaining -= line.len();
4935 if !line.ends_with(b"\r\n") {
4936 return Err(HealthProbeError::bad_answer(
4937 "HTTP headers are incomplete or exceed 16 KiB",
4938 ));
4939 }
4940 line.truncate(line.len() - 2);
4941 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4942}
4943
4944async fn probe_module_health(
4945 module_id: &str,
4946 runtime: &SupervisorRuntimeConfig,
4947 drain_deadline: Option<Instant>,
4948) -> Result<HealthReport, HealthProbeError> {
4949 let Some(forwarding) = runtime.forwarding.as_ref() else {
4950 return Err(HealthProbeError::misconfigured(
4951 "supervisor was not configured with a forwarding table",
4952 ));
4953 };
4954 let probe_started_at = Instant::now();
4955 let mut deadline = probe_started_at + runtime.health.deadline;
4956 if let Some(drain_deadline) = drain_deadline {
4957 deadline = deadline.min(drain_deadline);
4958 }
4959 let pending = if drain_deadline.is_some() {
4960 forwarding.begin_drain_health_probe_rpc_for(
4961 module_id,
4962 MODULE_CONTROL_OP_HEALTH_CHECK,
4963 probe_started_at,
4964 deadline,
4965 )
4966 } else {
4967 forwarding.begin_health_probe_rpc_for(
4968 module_id,
4969 MODULE_CONTROL_OP_HEALTH_CHECK,
4970 probe_started_at,
4971 deadline,
4972 )
4973 }
4974 .map_err(|err| {
4975 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4978 })?;
4979 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4980}
4981
4982async fn probe_endpoint_health(
4989 endpoint: crate::ModuleEndpointId,
4990 runtime: &SupervisorRuntimeConfig,
4991 deadline_cap: Option<Instant>,
4992) -> Result<HealthReport, HealthProbeError> {
4993 let Some(forwarding) = runtime.forwarding.as_ref() else {
4994 return Err(HealthProbeError::misconfigured(
4995 "supervisor was not configured with a forwarding table",
4996 ));
4997 };
4998 let probe_started_at = Instant::now();
4999 let mut deadline = probe_started_at + runtime.health.deadline;
5000 if let Some(cap) = deadline_cap {
5001 deadline = deadline.min(cap);
5002 }
5003 let pending = forwarding
5004 .begin_endpoint_health_probe_rpc_for(
5005 endpoint,
5006 MODULE_CONTROL_OP_HEALTH_CHECK,
5007 probe_started_at,
5008 deadline,
5009 )
5010 .map_err(|err| {
5011 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5012 })?;
5013 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5014}
5015
5016async fn await_health_probe(
5018 forwarding: &ForwardingTable,
5019 pending: PendingModuleControlRpc,
5020 deadline: Instant,
5021 probe_budget: Duration,
5022) -> Result<HealthReport, HealthProbeError> {
5023 let PendingModuleControlRpc {
5024 endpoint,
5025 module_sink,
5026 negotiated_ver,
5027 corr,
5028 receiver,
5029 } = pending;
5030 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5031 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5032 })?;
5033 let frame = Frame::build_with_version(
5034 negotiated_ver,
5035 FrameType::Request,
5036 control_flags(),
5037 0,
5038 0,
5039 corr,
5040 body,
5041 )
5042 .map_err(|err| {
5043 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5044 })?;
5045
5046 match timeout_at(deadline, module_sink.send(frame)).await {
5052 Ok(Ok(())) => {}
5053 Ok(Err(err)) => {
5054 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5055 return Err(HealthProbeError::lane_dead(format!(
5058 "failed to send health.check: {err}"
5059 )));
5060 }
5061 Err(_elapsed) => {
5062 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5063 return Err(HealthProbeError::no_answer(
5067 "health.check send timed out before enqueue (module egress full)",
5068 ));
5069 }
5070 }
5071
5072 match timeout_at(deadline, receiver).await {
5073 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5077 response.health_report().ok_or_else(|| {
5078 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5079 })
5080 }
5081 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5082 format!("health.check rejected: {}", body.message),
5083 )),
5084 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5085 Err(HealthProbeError::lane_dead(message))
5086 }
5087 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5088 Err(HealthProbeError::bad_answer(message))
5089 }
5090 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5091 Err(HealthProbeError::bad_answer(format!(
5092 "expected module-control op '{expected}', got '{actual}'"
5093 )))
5094 }
5095 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5099 "module answered health.check after its daemon deadline",
5100 )),
5101 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5102 "health.check waiter was canceled before the module responded",
5103 )),
5104 Err(_) => {
5105 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5106 Err(HealthProbeError::no_answer(format!(
5107 "module did not answer health.check within {probe_budget:?}"
5108 )))
5109 }
5110 }
5111}
5112
5113#[allow(clippy::too_many_arguments)]
5114async fn handle_health_report(
5115 spec: &ModuleSpec,
5116 runtime: &SupervisorRuntimeConfig,
5117 registry: &Registry,
5118 process_liveness: &SupervisorProcessLiveness,
5119 snapshot: &SharedSnapshot,
5120 child: &mut Option<SupervisedChild>,
5121 report: HealthReport,
5122 now_ms: u64,
5123) {
5124 let status = supervisor_health_status(report.status);
5125 let detail = report.detail.clone();
5126 let metrics = truncate_health_metrics(report.metrics);
5127 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5128 state.health.status = status;
5129 state.health.last_probe_ms = Some(now_ms);
5130 state.health.detail = detail.clone();
5131 state.health.metrics = metrics.clone();
5132 state.health.consecutive_failures = 0;
5133 });
5134
5135 let action = match report.status {
5136 HealthStatus::Ok => return,
5137 HealthStatus::Degraded => runtime.health.on_degraded,
5138 HealthStatus::Failing => runtime.health.on_failing,
5139 };
5140 apply_l3_health_action(
5141 spec,
5142 runtime,
5143 registry,
5144 process_liveness,
5145 snapshot,
5146 child,
5147 status,
5148 detail.as_deref(),
5149 action,
5150 now_ms,
5151 )
5152 .await;
5153}
5154
5155#[allow(clippy::too_many_arguments)]
5156async fn handle_health_probe_failure(
5157 spec: &ModuleSpec,
5158 runtime: &SupervisorRuntimeConfig,
5159 registry: &Registry,
5160 process_liveness: &SupervisorProcessLiveness,
5161 snapshot: &SharedSnapshot,
5162 child: &mut Option<SupervisedChild>,
5163 err: HealthProbeError,
5164 now_ms: u64,
5165) {
5166 let threshold = runtime.health.failure_threshold.max(1);
5167 let mut failures = 0;
5168 let detail = format!("[{}] {err}", err.label());
5173 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5174 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5177 state.health.status = SupervisorHealthStatus::Unknown;
5178 }
5179 state.health.last_probe_ms = Some(now_ms);
5180 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5181 state.health.detail = Some(detail.clone());
5182 state.health.metrics = None;
5183 failures = state.health.consecutive_failures;
5184 });
5185
5186 if failures < threshold {
5187 warn!(
5188 module_id = %spec.module_id,
5189 consecutive_failures = failures,
5190 threshold,
5191 evidence = err.label(),
5192 detail = %detail,
5193 "health.check probe failed"
5194 );
5195 return;
5196 }
5197
5198 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5199 state.state = ModuleState::Unresponsive;
5200 state.health.status = SupervisorHealthStatus::Unresponsive;
5201 });
5202 if runtime.health.critical {
5206 error!(
5207 module_id = %spec.module_id,
5208 status = "unresponsive",
5209 evidence = err.label(),
5210 detail = %detail,
5211 "critical module health alert"
5212 );
5213 } else {
5214 warn!(
5215 module_id = %spec.module_id,
5216 status = "unresponsive",
5217 evidence = err.label(),
5218 detail = %detail,
5219 "module health threshold breached"
5220 );
5221 }
5222 if let Err(err) = health_restart_child(
5223 spec,
5224 runtime,
5225 registry,
5226 process_liveness,
5227 snapshot,
5228 child,
5229 SupervisorHealthStatus::Unresponsive,
5230 Some(&detail),
5231 now_ms,
5232 )
5233 .await
5234 {
5235 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5236 }
5237}
5238
5239#[allow(clippy::too_many_arguments)]
5240async fn apply_l3_health_action(
5241 spec: &ModuleSpec,
5242 runtime: &SupervisorRuntimeConfig,
5243 registry: &Registry,
5244 process_liveness: &SupervisorProcessLiveness,
5245 snapshot: &SharedSnapshot,
5246 child: &mut Option<SupervisedChild>,
5247 status: SupervisorHealthStatus,
5248 detail: Option<&str>,
5249 action: HealthAction,
5250 now_ms: u64,
5251) {
5252 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5253 match action {
5254 HealthAction::Report => {
5255 info!(
5256 module_id = %spec.module_id,
5257 status = ?status,
5258 detail,
5259 "module reported non-ok health"
5260 );
5261 }
5262 HealthAction::Alert => {
5263 error!(
5264 module_id = %spec.module_id,
5265 status = ?status,
5266 detail,
5267 "module health alert"
5268 );
5269 }
5270 HealthAction::Restart => {
5271 if let Err(err) = health_restart_child(
5272 spec,
5273 runtime,
5274 registry,
5275 process_liveness,
5276 snapshot,
5277 child,
5278 status,
5279 detail,
5280 now_ms,
5281 )
5282 .await
5283 {
5284 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5285 }
5286 }
5287 }
5288}
5289
5290#[allow(clippy::too_many_arguments)]
5291async fn health_restart_child(
5292 spec: &ModuleSpec,
5293 runtime: &SupervisorRuntimeConfig,
5294 registry: &Registry,
5295 process_liveness: &SupervisorProcessLiveness,
5296 snapshot: &SharedSnapshot,
5297 child: &mut Option<SupervisedChild>,
5298 status: SupervisorHealthStatus,
5299 detail: Option<&str>,
5300 now_ms: u64,
5301) -> Result<(), SuperviseError> {
5302 let (enabled, schedule) = {
5303 let mut state = lock_snapshot(snapshot)?;
5304 let enabled = state.enabled;
5305 let schedule = if enabled {
5306 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5307 } else {
5308 None
5309 };
5310 (enabled, schedule)
5311 };
5312
5313 if !enabled {
5314 return Err(SuperviseError::Disabled {
5315 module_id: spec.module_id.clone(),
5316 });
5317 }
5318
5319 if schedule.is_none() {
5320 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5321 error!(
5322 module_id = %spec.module_id,
5323 status = ?status,
5324 detail,
5325 max_restarts = runtime.restart_policy.max_restarts,
5326 window_secs = runtime.restart_policy.window.as_secs(),
5327 reason = %runtime.restart_policy.budget_exhausted_detail(),
5328 "health restart budget exhausted; marking module failed"
5329 );
5330 let stop_notice = begin_forwarding_drain_if_configured(
5331 spec,
5332 runtime,
5333 registry,
5334 snapshot,
5335 Some(true),
5336 RouteCloseReason::Disable,
5337 )
5338 .await?;
5339 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5340 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5341 })?;
5342 drain_optional_child(
5343 &spec.module_id,
5344 spec.protocol,
5345 stop_notice,
5346 registry,
5347 runtime.forwarding.as_deref(),
5348 snapshot,
5349 &runtime.terminal_ring,
5350 &runtime.spawn_events,
5351 child,
5352 runtime.drain_timeout,
5353 ModuleState::Failed,
5354 Some(true),
5355 )
5356 .await?;
5357 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5358 return Ok(());
5359 }
5360
5361 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5362 let mut restart_count = 0;
5363 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5364 restart_count = state.crash_restarts.len();
5365 state.state = ModuleState::Unresponsive;
5366 state.health.status = status;
5367 state.health.last_action = Some(HealthAction::Restart.to_string());
5368 state.health.last_action_ms = Some(now_ms);
5369 })?;
5370 warn!(
5371 module_id = %spec.module_id,
5372 status = ?status,
5373 detail,
5374 restart_count,
5375 restart_in_window = schedule.restart_in_window,
5376 delay_ms = schedule.delay.as_millis() as u64,
5377 "health-triggered module restart"
5378 );
5379
5380 let stop_notice = begin_forwarding_drain_if_configured(
5381 spec,
5382 runtime,
5383 registry,
5384 snapshot,
5385 Some(true),
5386 RouteCloseReason::Restart,
5387 )
5388 .await?;
5389 drain_optional_child(
5390 &spec.module_id,
5391 spec.protocol,
5392 stop_notice,
5393 registry,
5394 runtime.forwarding.as_deref(),
5395 snapshot,
5396 &runtime.terminal_ring,
5397 &runtime.spawn_events,
5398 child,
5399 runtime.drain_timeout,
5400 ModuleState::Restarting,
5401 Some(true),
5402 )
5403 .await?;
5404 schedule_respawn(
5405 runtime,
5406 snapshot,
5407 &spec.module_id,
5408 schedule.delay,
5409 RespawnKind::Spawn,
5410 )
5411}
5412
5413fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5414 if let Some(reply) = runtime
5415 .deferred_reload_reply
5416 .lock()
5417 .unwrap_or_else(|p| p.into_inner())
5418 .take()
5419 {
5420 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5421 module_id: module_id.to_string(),
5422 reason: reason.to_string(),
5423 }));
5424 }
5425}
5426
5427fn schedule_respawn(
5428 runtime: &SupervisorRuntimeConfig,
5429 snapshot: &SharedSnapshot,
5430 module_id: &str,
5431 delay: Duration,
5432 kind: RespawnKind,
5433) -> Result<(), SuperviseError> {
5434 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5435 update_snapshot(snapshot, Some(module_id), |state| {
5436 state.respawn_pending = true
5437 })?;
5438 *runtime
5439 .scheduled_respawn
5440 .lock()
5441 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5442 deadline: Instant::now() + delay,
5443 kind,
5444 });
5445 Ok(())
5446}
5447
5448fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5449 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5450 state.health.last_action = Some(action);
5451 state.health.last_action_ms = Some(now_ms);
5452 });
5453}
5454
5455fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5456 match status {
5457 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5458 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5459 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5460 }
5461}
5462
5463fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5475 let metrics = metrics?;
5476 match serde_json::to_vec(&metrics) {
5477 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5478 "truncated": true,
5479 "original_bytes": encoded.len(),
5480 })),
5481 Ok(_) | Err(_) => Some(metrics),
5482 }
5483}
5484
5485fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5491 if cadence.is_zero() {
5492 return Duration::ZERO;
5493 }
5494 let cadence_ms = cadence.as_millis() as u64;
5495 if cadence_ms == 0 {
5511 return cadence;
5512 }
5513 let jitter_span = (cadence_ms / 10).max(1);
5528 let hash = module_id.as_bytes().iter().fold(
5529 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5530 |acc, byte| {
5531 acc.wrapping_mul(1099511628211)
5532 .wrapping_add(u64::from(*byte))
5533 },
5534 );
5535 cadence + Duration::from_millis(hash % jitter_span)
5536}
5537
5538#[cfg(test)]
5539mod tests {
5540 use super::*;
5541
5542 #[test]
5543 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5544 let handle = SupervisorHandle::new();
5545 let module_id = "readded-tombstone";
5546 handle.record_rescan_removal(module_id);
5547 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5548
5549 handle.apply_identity_configuration(&ModuleSpec {
5550 module_id: module_id.to_string(),
5551 program: PathBuf::from("/test/module"),
5552 args: Vec::new(),
5553 env: Vec::new(),
5554 reserved: false,
5555 reserved_prefixes: Vec::new(),
5556 protocol: ModuleProtocol::Subc,
5557 overlap: Default::default(),
5558 });
5559
5560 assert!(
5561 handle.removal_tombstone_age_ms(module_id).is_none(),
5562 "a re-added module must not retain a stale removal tombstone"
5563 );
5564 }
5565
5566 #[derive(Debug, PartialEq, Eq)]
5569 struct OwnerInSpawnWindow {
5570 module_id: String,
5571 configured: bool,
5572 on_roster: bool,
5573 admission_refusal: Option<&'static str>,
5574 }
5575
5576 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5587 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5588 use crate::scopes::ScopeTable;
5589 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5590
5591 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5592 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5593 module_id: module_id.to_string(),
5594 program,
5595 args: Vec::new(),
5596 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5597 .into_iter()
5598 .map(|key| (key.to_string(), dir.path().display().to_string()))
5599 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5600 .collect(),
5601 reserved: false,
5602 reserved_prefixes: Vec::new(),
5603 protocol: ModuleProtocol::Subc,
5604 overlap: Default::default(),
5605 };
5606 let live = super::terminal_history_tests::fake_aft_stub_path();
5607 let missing = dir.path().join("definitely-missing-module");
5608
5609 let handle = SupervisorHandle::new();
5610 let mut supervisor =
5611 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5612 .with_handle(handle.clone());
5613 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5614 let hook_handle = handle.clone();
5615 let hook_observed = Arc::clone(&observed);
5616 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5617 let configured = hook_handle.is_configured(module_id);
5620 let selector = ScopeSelector::new(
5621 Principal::Reserved {
5622 module_id: module_id.to_string(),
5623 },
5624 "s".to_string(),
5625 Some(1),
5626 );
5627 let carrier = Principal::Reserved {
5628 module_id: "carrier".to_string(),
5629 };
5630 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5631 .admit(&carrier, module_id, &selector, configured)
5632 {
5633 Ok(_) => None,
5634 Err(refusal) => Some(refusal.code),
5635 };
5636 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5637 module_id: module_id.to_string(),
5638 configured,
5639 on_roster: hook_handle.get(module_id).is_some(),
5640 admission_refusal,
5641 });
5642 })));
5643
5644 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5645 let configured = supervisor
5646 .supervise_configured(stub("configured", live.clone()), true)
5647 .unwrap();
5648 let with_health = supervisor
5649 .supervise_configured_with_health(
5650 stub("with-health", live.clone()),
5651 true,
5652 HealthConfig::default(),
5653 None,
5654 RestartPolicy::default(),
5655 )
5656 .unwrap();
5657 let failed = supervisor
5660 .supervise_configured_with_health(
5661 stub("failed-spawn", missing.clone()),
5662 true,
5663 HealthConfig::default(),
5664 None,
5665 RestartPolicy::default(),
5666 )
5667 .unwrap();
5668 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5671
5672 let expected = [
5673 "plain",
5674 "configured",
5675 "with-health",
5676 "failed-spawn",
5677 "spawn-error",
5678 ]
5679 .into_iter()
5680 .map(|module_id| OwnerInSpawnWindow {
5681 module_id: module_id.to_string(),
5682 configured: true,
5683 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5684 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5685 })
5686 .collect::<Vec<_>>();
5687 assert_eq!(*observed.lock().unwrap(), expected);
5688
5689 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5690 assert!(
5691 handle.get(module_id).is_some(),
5692 "{module_id} is on the roster"
5693 );
5694 assert!(
5695 handle.is_configured(module_id),
5696 "{module_id} stays configured"
5697 );
5698 }
5699 assert!(handle.get("spawn-error").is_none());
5700 assert!(
5701 !handle.is_configured("spawn-error"),
5702 "a plain spawn that failed must not leave its module marked configured"
5703 );
5704
5705 handle.retire("failed-spawn");
5707 assert!(!handle.is_configured("failed-spawn"));
5708
5709 for module in [plain, configured, with_health] {
5710 module.stop().await.unwrap();
5711 }
5712 drop(failed);
5713 }
5714
5715 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5716 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5717 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5718 snapshot.process_alive = true;
5719 snapshot.pid = Some(41);
5720 snapshot.spawned_at_ms = Some(42);
5721 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5722 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5723 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5724 device: 43,
5725 inode: 44,
5726 #[cfg(windows)]
5727 windows_image: None,
5728 });
5729 })
5730 .unwrap();
5731 snapshot
5732 }
5733
5734 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5735 let snapshot = lock_snapshot(snapshot).unwrap();
5736 assert!(!snapshot.process_alive);
5737 assert_eq!(snapshot.pid, None);
5738 assert_eq!(snapshot.spawned_at_ms, None);
5739 assert_eq!(snapshot.spawned_from, None);
5740 assert_eq!(snapshot.launch_spec_digest, None);
5741 assert_eq!(snapshot.spawned_file_identity, None);
5742 }
5743
5744 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5745 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5746 let supervisor =
5747 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5748 let mut runtime = supervisor.runtime_config();
5749 runtime.test_seed_stale_facts_before_enable_spawn = true;
5750 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5751 let mut child = None;
5752 let spec = ModuleSpec {
5753 module_id: "failed-enable-clears-facts".to_string(),
5754 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5755 args: Vec::new(),
5756 env: Vec::new(),
5757 reserved: false,
5758 reserved_prefixes: Vec::new(),
5759 protocol: ModuleProtocol::Subc,
5760 overlap: Default::default(),
5761 };
5762
5763 let result = set_child_enabled(
5764 &spec,
5765 &runtime,
5766 &supervisor.registry,
5767 &supervisor.process_liveness,
5768 &snapshot,
5769 &mut child,
5770 true,
5771 )
5772 .await;
5773
5774 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5775 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5776 assert_snapshot_process_facts_cleared(&snapshot);
5777 }
5778
5779 #[tokio::test]
5780 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5781 let supervisor =
5782 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5783 let runtime = supervisor.runtime_config();
5784 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5785 ModuleState::Restarting,
5786 true,
5787 )));
5788 let spec = ModuleSpec {
5789 module_id: "start-stranded-restarting".to_string(),
5790 program: super::terminal_history_tests::fake_aft_stub_path(),
5791 args: Vec::new(),
5792 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5793 reserved: false,
5794 reserved_prefixes: Vec::new(),
5795 protocol: ModuleProtocol::None,
5796 overlap: Default::default(),
5797 };
5798 let mut child = None;
5799 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5800 assert!(!super::set_child_enabled(
5801 &spec,
5802 &runtime,
5803 &Registry::default(),
5804 &supervisor.process_liveness,
5805 &snapshot,
5806 &mut child,
5807 true
5808 )
5809 .await
5810 .unwrap());
5811 assert!(child.is_none());
5812 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5813 assert!(super::set_child_enabled(
5814 &spec,
5815 &runtime,
5816 &Registry::default(),
5817 &supervisor.process_liveness,
5818 &snapshot,
5819 &mut child,
5820 true
5821 )
5822 .await
5823 .unwrap());
5824 assert_eq!(
5825 lock_snapshot(&snapshot).unwrap().state,
5826 ModuleState::Running
5827 );
5828 let mut child = child.unwrap();
5829 child.start_kill().unwrap();
5830 child.wait().await.unwrap();
5831 }
5832
5833 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5834 async fn failed_reload_spawn_clears_current_process_facts() {
5835 let supervisor =
5836 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5837 let mut runtime = supervisor.runtime_config();
5838 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5839 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5840 let mut child = None;
5841 let spec = ModuleSpec {
5842 module_id: "failed-reload-clears-facts".to_string(),
5843 program: PathBuf::from("/unused/failed-reload-module"),
5844 args: Vec::new(),
5845 env: Vec::new(),
5846 reserved: false,
5847 reserved_prefixes: Vec::new(),
5848 protocol: ModuleProtocol::Subc,
5849 overlap: Default::default(),
5850 };
5851
5852 let result = handle_reload_spawn_failure(
5853 &spec,
5854 &runtime,
5855 &supervisor.process_liveness,
5856 &snapshot,
5857 &mut child,
5858 "forced reload spawn failure".to_string(),
5859 )
5860 .await;
5861
5862 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5863 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5864 assert_snapshot_process_facts_cleared(&snapshot);
5865 }
5866
5867 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5868 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5869 let supervisor =
5870 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5871 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5872 let module = supervisor.supervised_module(
5873 ModuleSpec {
5874 module_id: "drop-clears-facts".to_string(),
5875 program: PathBuf::from("/unused/drop-module"),
5876 args: Vec::new(),
5877 env: Vec::new(),
5878 reserved: false,
5879 reserved_prefixes: Vec::new(),
5880 protocol: ModuleProtocol::Subc,
5881 overlap: Default::default(),
5882 },
5883 supervisor.runtime_config(),
5884 Arc::clone(&snapshot),
5885 None,
5886 );
5887 assert!(!module
5888 .inner
5889 .monitor
5890 .lock()
5891 .unwrap()
5892 .as_ref()
5893 .unwrap()
5894 .is_finished());
5895
5896 drop(module);
5897
5898 assert_eq!(
5899 lock_snapshot(&snapshot).unwrap().state,
5900 ModuleState::Stopped
5901 );
5902 assert_snapshot_process_facts_cleared(&snapshot);
5903 }
5904
5905 #[cfg(unix)]
5906 #[tokio::test]
5907 async fn rescan_preserves_running_protocol_until_respawn() {
5908 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5909 let initial = ModuleSpec {
5910 module_id: "rescan-protocol".into(),
5911 program: PathBuf::from("/bin/sleep"),
5912 args: vec!["60".into()],
5913 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5914 .into_iter()
5915 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5916 .collect(),
5917 reserved: false,
5918 reserved_prefixes: vec![],
5919 protocol: ModuleProtocol::None,
5920 overlap: Default::default(),
5921 };
5922 let supervisor =
5923 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5924 let module = supervisor.spawn(initial.clone()).unwrap();
5925 assert!(module.status().unwrap().live);
5926 let mut next = initial;
5927 next.protocol = ModuleProtocol::Subc;
5928 module
5929 .update_configuration(next.clone(), HealthConfig::default(), None)
5930 .await
5931 .unwrap();
5932 assert!(
5933 module.status().unwrap().live,
5934 "rescan must not require HELLO from the old non-wire process"
5935 );
5936 let runtime = supervisor.runtime_config();
5937 let action = on_child_exit(
5938 &next,
5939 RestartPolicy::default(),
5940 &supervisor.registry,
5941 &module.inner.snapshot,
5942 &runtime.terminal_ring,
5943 &runtime.spawn_events,
5944 &runtime.child_roster,
5945 ExitReport {
5946 kind: ExitKind::Clean,
5947 code: Some(0),
5948 signal: None,
5949 at_ms: unix_ms_now(),
5950 },
5951 )
5952 .await;
5953 assert!(
5954 matches!(action, NextAction::Restart { .. }),
5955 "the old non-wire process's clean exit must restart"
5956 );
5957 module.drain().await.unwrap();
5958 }
5959
5960 #[cfg(unix)]
5961 #[tokio::test]
5962 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
5963 use std::os::unix::fs::PermissionsExt;
5964 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
5965 let script = dir.join("module.sh");
5966 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
5967 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
5968 let record_path = dir.join("live-children.json");
5969 let supervisor =
5970 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5971 .with_live_children_record(&record_path);
5972 for (program, args) in [
5973 (PathBuf::from("sleep"), vec!["60".into()]),
5974 (script, vec![]),
5975 ] {
5976 let spec = ModuleSpec {
5977 module_id: "image-identity".into(),
5978 program: program.clone(),
5979 args,
5980 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5981 .into_iter()
5982 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
5983 .collect(),
5984 reserved: false,
5985 reserved_prefixes: vec![],
5986 protocol: ModuleProtocol::None,
5987 overlap: Default::default(),
5988 };
5989 let module = supervisor.spawn(spec).unwrap();
5990 #[cfg(target_os = "macos")]
5991 {
5992 let deadline = Instant::now() + Duration::from_secs(5);
5995 while crate::live_children::read_record(&record_path)
5996 .unwrap()
5997 .iter()
5998 .all(|entry| entry.executable.is_none())
5999 {
6000 assert!(Instant::now() < deadline, "module image was not confirmed");
6001 tokio::time::sleep(Duration::from_millis(5)).await;
6002 }
6003 }
6004 let entry = crate::live_children::read_record(&record_path)
6005 .unwrap()
6006 .pop()
6007 .unwrap();
6008 let observed = subc_os::Process::open(entry.pid)
6009 .unwrap()
6010 .unwrap()
6011 .observe()
6012 .unwrap();
6013 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6014 module.drain().await.unwrap();
6015 assert_eq!(
6016 verdict,
6017 crate::live_children::IdentityVerdict::Matches,
6018 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6019 );
6020 }
6021 }
6022
6023 #[cfg(unix)]
6024 fn http_fixture(
6025 dir: &std::path::Path,
6026 url: &str,
6027 threshold: u32,
6028 ) -> crate::daemon_config::ConfiguredModule {
6029 let path = dir.join("subc.jsonc");
6030 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6031 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6032 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6033 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6034 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6035 }}}).to_string()).unwrap();
6036 crate::daemon_config::load(&path)
6037 .unwrap()
6038 .unwrap()
6039 .modules
6040 .pop()
6041 .unwrap()
6042 }
6043
6044 #[cfg(unix)]
6045 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6046 timeout(Duration::from_secs(5), async {
6047 loop {
6048 if module.status().unwrap().health.status == status {
6049 break;
6050 }
6051 sleep(Duration::from_millis(5)).await;
6052 }
6053 })
6054 .await
6055 .unwrap_or_else(|_| {
6056 panic!(
6057 "expected {status:?}, got {:?}",
6058 module.status().unwrap().health
6059 )
6060 });
6061 }
6062
6063 #[cfg(unix)]
6064 #[tokio::test]
6065 async fn http_health_status_flips_ok_failing_ok() {
6066 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6067 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6068 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6069 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6070 let serving_status = status.clone();
6071 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6072 let server = tokio::spawn(async move {
6073 loop {
6074 let (mut stream, _) = listener.accept().await.unwrap();
6075 let mut request = [0u8; 2048];
6076 let count = stream.read(&mut request).await.unwrap();
6077 assert!(count > 0, "a probe must send an HTTP request");
6078 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6079 let body = if code == 200 {
6080 "ready"
6081 } else {
6082 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6083 };
6084 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6085 let _ = stream.write_all(response.as_bytes()).await;
6086 }
6087 });
6088 let configured = http_fixture(&dir, &url, 1000);
6089 let module =
6090 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6091 .supervise_configured_with_health(
6092 configured.module_spec(),
6093 true,
6094 configured.health,
6095 configured.drain_timeout_ms,
6096 configured.restart,
6097 )
6098 .unwrap();
6099 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6100 status.store(503, std::sync::atomic::Ordering::SeqCst);
6101 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6102 assert!(module
6103 .status()
6104 .unwrap()
6105 .health
6106 .detail
6107 .unwrap()
6108 .contains("scratch failure"));
6109 status.store(200, std::sync::atomic::Ordering::SeqCst);
6110 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6111 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6112 let before = module.status().unwrap();
6113 let (spec, mut health) = module.configuration().unwrap();
6114 health.http = None;
6115 module
6116 .update_configuration(spec.clone(), health.clone(), Some(10))
6117 .await
6118 .unwrap();
6119 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6120 health.http = Some(url);
6121 module
6122 .update_configuration(spec, health, Some(10))
6123 .await
6124 .unwrap();
6125 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6126 assert_eq!(
6127 module.status().unwrap().pid,
6128 before.pid,
6129 "changing a probe must apply live, not restart its process"
6130 );
6131 let (spec, mut health) = module.configuration().unwrap();
6132 health.failure_threshold = 2;
6133 module
6134 .update_configuration(spec, health, Some(10))
6135 .await
6136 .unwrap();
6137 status.store(503, std::sync::atomic::Ordering::SeqCst);
6138 timeout(Duration::from_secs(5), async {
6139 while module.status().unwrap().spawn_generation == before.spawn_generation {
6140 sleep(Duration::from_millis(5)).await;
6141 }
6142 })
6143 .await
6144 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6145 module.drain().await.unwrap();
6146 server.abort();
6147 }
6148
6149 #[cfg(unix)]
6150 #[tokio::test]
6151 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6152 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6153 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6154 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6155 drop(unused);
6156 let configured = http_fixture(&dir, &url, 2);
6157 let module =
6158 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6159 .supervise_configured_with_health(
6160 configured.module_spec(),
6161 true,
6162 configured.health,
6163 configured.drain_timeout_ms,
6164 configured.restart,
6165 )
6166 .unwrap();
6167 let before = module.status().unwrap().spawn_generation;
6168 timeout(Duration::from_secs(5), async {
6169 loop {
6170 let status = module.status().unwrap();
6171 if status.spawn_generation > before {
6172 assert!(status.lifetime_restarts > 0);
6173 break;
6174 }
6175 sleep(Duration::from_millis(5)).await;
6176 }
6177 })
6178 .await
6179 .expect("sustained HTTP refusal must trigger the health restart policy");
6180 module.drain().await.unwrap();
6181 }
6182
6183 #[cfg(unix)]
6184 #[tokio::test]
6185 async fn http_health_timeout_honours_deadline() {
6186 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6187 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6188 let server = tokio::spawn(async move {
6189 let _held = listener.accept().await.unwrap();
6190 std::future::pending::<()>().await;
6191 });
6192 let error = timeout(
6193 Duration::from_secs(1),
6194 probe_http_health(&url, Duration::from_millis(10)),
6195 )
6196 .await
6197 .expect("the probe must enforce its own deadline")
6198 .unwrap_err();
6199 server.abort();
6200 assert!(error.to_string().contains("timed out"));
6201 }
6202
6203 #[cfg(unix)]
6204 #[tokio::test]
6205 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6206 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6207 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6208 let url = format!(
6209 "http://localhost:{}/healthz",
6210 listener.local_addr().unwrap().port()
6211 );
6212 let server = tokio::spawn(async move {
6213 let (mut stream, _) = listener.accept().await.unwrap();
6214 let mut request = [0u8; 2048];
6215 assert!(stream.read(&mut request).await.unwrap() > 0);
6216 stream
6217 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6218 .await
6219 .unwrap();
6220 });
6221 assert_eq!(
6226 probe_http_health(&url, Duration::from_secs(10))
6227 .await
6228 .unwrap()
6229 .status,
6230 HealthStatus::Ok
6231 );
6232 server.await.unwrap();
6233 }
6234
6235 #[cfg(unix)]
6236 #[tokio::test]
6237 async fn http_health_timeout_keeps_partial_status_and_body() {
6238 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6239 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6240 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6241 let server = tokio::spawn(async move {
6242 let (mut stream, _) = listener.accept().await.unwrap();
6243 let mut request = [0u8; 2048];
6244 assert!(stream.read(&mut request).await.unwrap() > 0);
6245 stream
6246 .write_all(
6247 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6248 )
6249 .await
6250 .unwrap();
6251 std::future::pending::<()>().await;
6252 });
6253 let error = probe_http_health(&url, Duration::from_secs(1))
6254 .await
6255 .unwrap_err()
6256 .to_string();
6257 server.abort();
6258 assert!(
6259 error.contains("timed out")
6260 && error.contains("503 Unavailable")
6261 && error.contains("partial diagnostic"),
6262 "{error}"
6263 );
6264 }
6265
6266 #[cfg(unix)]
6267 #[tokio::test]
6268 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6269 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6270 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6271 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6272 let server = tokio::spawn(async move {
6273 let (mut stream, _) = listener.accept().await.unwrap();
6274 let mut request = [0u8; 2048];
6275 assert!(stream.read(&mut request).await.unwrap() > 0);
6276 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6277 let response = format!(
6278 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6279 body.len()
6280 );
6281 stream.write_all(response.as_bytes()).await.unwrap();
6282 });
6283 let error = probe_http_health(&url, Duration::from_secs(1))
6284 .await
6285 .unwrap_err()
6286 .to_string();
6287 server.await.unwrap();
6288 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6289 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6290 assert!(!error.contains("not-in-diagnostic"));
6291 }
6292
6293 #[cfg(unix)]
6294 #[tokio::test]
6295 async fn http_health_real_nats_server_monitoring() {
6296 if std::process::Command::new("nats-server")
6297 .arg("--version")
6298 .env("XDG_DATA_HOME", std::env::temp_dir())
6299 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6300 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6301 .output()
6302 .is_err()
6303 {
6304 eprintln!(
6305 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6306 );
6307 return;
6308 }
6309 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6310 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6311 let port = monitor.local_addr().unwrap().port();
6312 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6313 let client_port = client.local_addr().unwrap().port();
6314 let config = dir.join("server.conf");
6315 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6316 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6317 configured.program = PathBuf::from("nats-server");
6318 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6319 drop(monitor);
6320 drop(client);
6321 let module =
6322 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6323 .supervise_configured_with_health(
6324 configured.module_spec(),
6325 true,
6326 configured.health,
6327 configured.drain_timeout_ms,
6328 configured.restart,
6329 )
6330 .unwrap();
6331 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6332 module.drain().await.unwrap();
6333 }
6334
6335 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6336 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6337 let supervisor =
6338 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6339 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6340 let initial = ModuleSpec {
6341 module_id: "rescan-preserves-spawn-facts".to_string(),
6342 program: PathBuf::from("/spawned/module"),
6343 args: Vec::new(),
6344 env: Vec::new(),
6345 reserved: false,
6346 reserved_prefixes: Vec::new(),
6347 protocol: ModuleProtocol::Subc,
6348 overlap: Default::default(),
6349 };
6350 let module = supervisor.supervised_module(
6351 initial.clone(),
6352 supervisor.runtime_config(),
6353 snapshot,
6354 None,
6355 );
6356 let before = module.status().unwrap();
6357 let mut replacement = initial;
6358 replacement.program = PathBuf::from("/rescanned/replacement-module");
6359
6360 module
6361 .update_configuration(replacement, HealthConfig::default(), None)
6362 .await
6363 .unwrap();
6364
6365 let after = module.status().unwrap();
6366 assert_eq!(after.pid, before.pid);
6367 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6368 assert_eq!(after.spawned_from, before.spawned_from);
6369 drop(module);
6370 }
6371}
6372
6373fn unix_ms_now() -> u64 {
6374 SystemTime::now()
6375 .duration_since(UNIX_EPOCH)
6376 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6377 .unwrap_or(0)
6378}
6379
6380async fn supervise_loop(
6381 mut spec: ModuleSpec,
6382 mut runtime: SupervisorRuntimeConfig,
6383 registry: Arc<Registry>,
6384 process_liveness: Arc<SupervisorProcessLiveness>,
6385 snapshot: SharedSnapshot,
6386 mut child: Option<SupervisedChild>,
6387 mut commands: mpsc::Receiver<SupervisorCommand>,
6388) {
6389 let mut health_probe = HealthProbeRuntime::default();
6390 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6394 Ok(changes) => Some(changes),
6395 Err(err) => {
6396 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6399 None
6400 }
6401 };
6402 let mut pending_respawn: Option<PendingRespawn> = None;
6406 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6409 let mut previous_instance_recheck = tokio::time::interval_at(
6410 Instant::now() + Duration::from_secs(5),
6411 Duration::from_secs(5),
6412 );
6413 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6414 loop {
6415 #[cfg(test)]
6416 {
6417 lock_snapshot(&snapshot).unwrap().actor_select = None;
6418 }
6419 #[cfg(target_os = "macos")]
6420 if let Some(active) = child.as_mut() {
6421 active.confirm_privacy_exec().await;
6422 }
6423 if let Some(scheduled) = runtime
6424 .scheduled_respawn
6425 .lock()
6426 .unwrap_or_else(|p| p.into_inner())
6427 .take()
6428 {
6429 pending_respawn = Some(scheduled);
6430 }
6431 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6432 pending_respawn = None;
6433 cancel_deferred_reload(
6434 &runtime,
6435 &spec.module_id,
6436 "respawn cancelled by a supervisor command",
6437 );
6438 }
6439 if child.is_none()
6440 && pending_respawn.is_none()
6441 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6442 {
6443 cancel_deferred_reload(
6444 &runtime,
6445 &spec.module_id,
6446 "respawn cancelled before a replacement was spawned",
6447 );
6448 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6449 state.respawn_pending = false;
6450 state.coalesced_restart_pending = false;
6451 if matches!(
6452 state.state,
6453 ModuleState::Restarting
6454 | ModuleState::Starting
6455 | ModuleState::Draining
6456 | ModuleState::Unresponsive
6457 ) {
6458 error!(module_id = %spec.module_id, state = ?state.state,
6459 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6460 state.state = ModuleState::Failed;
6461 clear_current_process_facts(state);
6462 }
6463 });
6464 }
6465 if let Some(command) = requeued.pop_front() {
6466 if !handle_supervisor_command(
6467 command,
6468 &mut spec,
6469 &mut runtime,
6470 ®istry,
6471 &process_liveness,
6472 &snapshot,
6473 &mut child,
6474 &mut commands,
6475 &mut requeued,
6476 )
6477 .await
6478 {
6479 return;
6480 }
6481 if child.is_some() || !respawn_still_pending(&snapshot) {
6482 pending_respawn = None;
6483 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6484 state.respawn_pending = false
6485 });
6486 }
6487 continue;
6488 }
6489 if child.is_some() {
6490 #[cfg(test)]
6491 {
6492 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6493 }
6494 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6495 let wake_after = health_probe.wake_after();
6496 let probe_sleep = async move {
6497 match wake_after {
6498 Some(delay) => sleep(delay).await,
6499 None => std::future::pending().await,
6500 }
6501 };
6502 tokio::pin!(probe_sleep);
6503 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6508 #[cfg(test)]
6509 {
6510 let registration_pending = wire_child
6511 && registration_changes
6512 .as_ref()
6513 .is_some_and(|changes| changes.has_changed().unwrap());
6514 let mut state = lock_snapshot(&snapshot).unwrap();
6515 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6516 generation: state.spawn_generation,
6517 turn: state.actor_turns,
6518 registered_connection: health_probe.registered_connection,
6519 next_probe_at: health_probe.next_probe_at,
6520 wake_after,
6521 });
6522 }
6523 let active_child = child.as_mut().expect("child checked above");
6524 tokio::select! {
6525 wait_result = active_child.wait() => {
6526 #[cfg(test)]
6527 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6528 let exit_report = match wait_result {
6537 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6538 Err(err) => {
6539 active_child.drain_stderr(&spec.module_id).await;
6540 fail_after_wait_error(
6541 &spec.module_id,
6542 &snapshot,
6543 &runtime.terminal_ring,
6544 &runtime.spawn_events,
6545 format!("failed to wait for supervised module: {err}"),
6546 );
6547 untrack_if_registration_released(
6548 &process_liveness,
6549 ®istry,
6550 &spec.module_id,
6551 &snapshot,
6552 );
6553 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6554 child = None;
6555 continue;
6556 }
6557 };
6558 active_child.drain_stderr(&spec.module_id).await;
6559
6560 let next = on_child_exit(
6561 &spec,
6562 runtime.restart_policy,
6563 ®istry,
6564 &snapshot,
6565 &runtime.terminal_ring,
6566 &runtime.spawn_events,
6567 &runtime.child_roster,
6568 exit_report,
6569 ).await;
6570 runtime.start_requester = None;
6571 runtime.pending_activity_restart = false;
6572 active_child.release_roster();
6575 match next {
6576 NextAction::Stop { registration_released } => {
6577 if registration_released {
6578 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6579 }
6580 child = None;
6581 }
6582 NextAction::Restart { schedule } => {
6583 let delay = schedule.map_or(
6584 runtime.restart_policy.delay_for_restart(0),
6585 |schedule| schedule.delay,
6586 );
6587 if let Some(schedule) = schedule {
6588 log_crash_respawn(&spec.module_id, schedule);
6589 }
6590 child = None;
6598 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6599 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6600 }
6601 }
6602 }
6603 command = commands.recv() => {
6604 #[cfg(test)]
6605 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6606 let Some(command) = command else {
6607 return;
6608 };
6609 if !handle_supervisor_command(
6610 command,
6611 &mut spec,
6612 &mut runtime,
6613 ®istry,
6614 &process_liveness,
6615 &snapshot,
6616 &mut child,
6617 &mut commands,
6618 &mut requeued,
6619 ).await {
6620 return;
6621 }
6622 }
6623 _ = async {
6624 match registration_changes.as_mut() {
6625 Some(changes) => { let _ = changes.changed().await; }
6626 None => std::future::pending().await,
6627 }
6628 }, if wire_child => {
6629 #[cfg(test)]
6630 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6631 }
6632 _ = &mut probe_sleep => {
6633 #[cfg(test)]
6634 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6635 if health_probe.due() {
6636 run_health_probe_cycle(
6637 &spec,
6638 &runtime,
6639 ®istry,
6640 &process_liveness,
6641 &snapshot,
6642 &mut child,
6643 ).await;
6644 if child.is_none() {
6645 runtime.start_requester = None;
6646 runtime.pending_activity_restart = false;
6647 }
6648 if child.is_some() {
6649 health_probe.schedule_next(&spec, runtime.health.cadence);
6650 }
6651 }
6652 }
6653 }
6654 } else if let Some(pending) = pending_respawn {
6655 tokio::select! {
6656 _ = sleep_until(pending.deadline) => {
6657 pending_respawn = None;
6658 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6659 if !respawn_still_pending(&snapshot) {
6663 continue;
6664 }
6665 if runtime.child_roster.is_closed() {
6670 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6671 state.state = ModuleState::Stopped;
6672 });
6673 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6674 continue;
6675 }
6676 if let Err(err) = release_dead_registration(
6677 ®istry,
6678 runtime.forwarding.as_deref(),
6679 &snapshot,
6680 &spec.module_id,
6681 ).await {
6682 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6683 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6684 continue;
6685 }
6686
6687 if matches!(pending.kind, RespawnKind::Reload) {
6688 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6689 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
6690 if result.is_ok() && runtime.pending_activity_restart {
6691 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6692 }
6693 runtime.pending_activity_restart = false;
6694 if let Some(reply) = reply { let _ = reply.send(result); }
6695 continue;
6696 }
6697 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6698 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6699 Ok(next_child) => {
6700 child = Some(next_child);
6701 if runtime.pending_activity_restart {
6702 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6703 runtime.pending_activity_restart = false;
6704 }
6705 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6706 }
6707 Err(err) => {
6708 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6709 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6710 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6711 }
6712 }
6713 }
6714 command = commands.recv() => {
6715 let Some(command) = command else {
6716 return;
6717 };
6718 if !handle_supervisor_command(
6719 command,
6720 &mut spec,
6721 &mut runtime,
6722 ®istry,
6723 &process_liveness,
6724 &snapshot,
6725 &mut child,
6726 &mut commands,
6727 &mut requeued,
6728 ).await {
6729 return;
6730 }
6731 if child.is_some() || !respawn_still_pending(&snapshot) {
6736 pending_respawn = None;
6737 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6738 }
6739 }
6740 }
6741 } else {
6742 let command = if lock_snapshot(&snapshot)
6743 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6744 {
6745 tokio::select! {
6746 command = commands.recv() => command,
6747 _ = previous_instance_recheck.tick() => {
6748 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6749 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6750 if enabled && !runtime.child_roster.is_closed() {
6751 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6752 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6753 Ok(next_child) => child = Some(next_child),
6754 Err(error) => {
6755 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6756 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6757 }
6758 }
6759 } else {
6760 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6761 state.waiting_for_previous_pid = None;
6762 state.waiting_for_previous_detail = None;
6763 });
6764 }
6765 }
6766 continue;
6767 }
6768 }
6769 } else {
6770 commands.recv().await
6771 };
6772 let Some(command) = command else {
6773 return;
6774 };
6775 if !handle_supervisor_command(
6776 command,
6777 &mut spec,
6778 &mut runtime,
6779 ®istry,
6780 &process_liveness,
6781 &snapshot,
6782 &mut child,
6783 &mut commands,
6784 &mut requeued,
6785 )
6786 .await
6787 {
6788 return;
6789 }
6790 }
6791 }
6792}
6793
6794fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6795 info!(
6796 module_id,
6797 restart_in_window = schedule.restart_in_window,
6798 delay_ms = schedule.delay.as_millis() as u64,
6799 "respawning after crash"
6800 );
6801}
6802
6803fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6809 matches!(
6810 lock_snapshot(snapshot),
6811 Ok(state) if state.enabled && state.state == ModuleState::Restarting
6812 )
6813}
6814
6815enum NextAction {
6816 Stop {
6817 registration_released: bool,
6818 },
6819 Restart {
6820 schedule: Option<CrashRestartSchedule>,
6821 },
6822}
6823
6824#[allow(clippy::too_many_arguments)]
6825async fn handle_supervisor_command(
6826 command: SupervisorCommand,
6827 spec: &mut ModuleSpec,
6828 runtime: &mut SupervisorRuntimeConfig,
6829 registry: &Arc<Registry>,
6830 process_liveness: &SupervisorProcessLiveness,
6831 snapshot: &SharedSnapshot,
6832 child: &mut Option<SupervisedChild>,
6833 commands: &mut mpsc::Receiver<SupervisorCommand>,
6834 requeued: &mut VecDeque<SupervisorCommand>,
6835) -> bool {
6836 let command =
6837 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6838 match command {
6839 SupervisorCommand::Restart { reply, .. }
6840 | SupervisorCommand::Reload { reply, .. }
6841 | SupervisorCommand::Swap { reply, .. } => {
6842 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6843 return true;
6844 }
6845 SupervisorCommand::SetEnabled {
6846 enabled: true,
6847 reply,
6848 ..
6849 } => {
6850 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6851 return true;
6852 }
6853 other => other,
6854 }
6855 } else {
6856 command
6857 };
6858 match command {
6859 SupervisorCommand::Drain { requester, reply } => {
6860 let result = drain_optional_child(
6863 &spec.module_id,
6864 spec.protocol,
6865 StopNotice::NotSent,
6866 registry,
6867 runtime.forwarding.as_deref(),
6868 snapshot,
6869 &runtime.terminal_ring,
6870 &runtime.spawn_events,
6871 child,
6872 runtime.drain_timeout,
6873 ModuleState::Stopped,
6874 None,
6875 )
6876 .await;
6877 let registration_released = result.is_ok();
6878 if registration_released {
6879 runtime.record_activity_transition(
6880 "subc.module_stopped",
6881 &spec.module_id,
6882 requester,
6883 );
6884 }
6885 let _ = reply.send(result);
6886 if registration_released {
6887 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6888 }
6889 false
6890 }
6891 SupervisorCommand::Retire { requester, reply } => {
6892 let result = async {
6893 let stop_notice = begin_forwarding_drain_if_configured(
6894 spec,
6895 runtime,
6896 registry,
6897 snapshot,
6898 None,
6899 RouteCloseReason::Disable,
6900 )
6901 .await?;
6902 drain_optional_child(
6903 &spec.module_id,
6904 spec.protocol,
6905 stop_notice,
6906 registry,
6907 runtime.forwarding.as_deref(),
6908 snapshot,
6909 &runtime.terminal_ring,
6910 &runtime.spawn_events,
6911 child,
6912 runtime.drain_timeout,
6913 ModuleState::Stopped,
6914 None,
6915 )
6916 .await
6917 }
6918 .await;
6919 let registration_released = result.is_ok();
6920 if registration_released {
6921 runtime.record_activity_transition(
6922 "subc.module_stopped",
6923 &spec.module_id,
6924 requester,
6925 );
6926 }
6927 let _ = reply.send(result);
6928 if registration_released {
6929 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6930 }
6931 false
6932 }
6933 SupervisorCommand::Restart {
6934 requester,
6935 drain_timeout_ms,
6936 received_at_generation,
6937 queued_at,
6938 reply,
6939 } => {
6940 info!(
6944 module_id = %spec.module_id,
6945 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6946 "restart command dequeued"
6947 );
6948 let validation = match lock_snapshot(snapshot) {
6960 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
6961 module_id: spec.module_id.clone(),
6962 }),
6963 Ok(_) => Ok(()),
6964 Err(err) => Err(err),
6965 };
6966 let initiated = validation.is_ok();
6967 let _ = reply.send(validation);
6968 let satisfied_by_generation = if initiated
6982 && child
6983 .as_ref()
6984 .is_some_and(|child| child.spawned_at > queued_at)
6985 {
6986 lock_snapshot(snapshot).ok().and_then(|state| {
6987 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
6988 })
6989 } else {
6990 None
6991 };
6992 let satisfied_by_pending = initiated
6993 && child.is_none()
6994 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
6995 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
6999 if pending {
7000 state.coalesced_restart_pending = true;
7001 }
7002 pending
7003 });
7004 if satisfied_by_pending {
7005 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7006 } else if let Some(generation) = satisfied_by_generation {
7007 info!(
7008 module_id = %spec.module_id,
7009 received_at_generation,
7010 "restart already satisfied by generation {generation}; not restarting again"
7011 );
7012 } else if initiated {
7013 let drain_timeout = drain_timeout_ms
7016 .map(Duration::from_millis)
7017 .unwrap_or(runtime.drain_timeout);
7018 if let Err(err) = restart_child(
7019 spec,
7020 runtime,
7021 registry,
7022 process_liveness,
7023 snapshot,
7024 child,
7025 drain_timeout,
7026 )
7027 .await
7028 {
7029 warn!(
7030 module_id = %spec.module_id,
7031 error = %err,
7032 "operator restart failed after initiation ack; module state carries the outcome"
7033 );
7034 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7035 state.state = ModuleState::Failed;
7036 clear_current_process_facts(state);
7037 });
7038 } else {
7039 runtime.start_requester = requester;
7040 runtime.pending_activity_restart = true;
7041 }
7042 }
7043 true
7044 }
7045 SupervisorCommand::Reload { requester, reply } => {
7046 let result =
7047 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7048 if result.is_ok() {
7049 runtime.start_requester = requester;
7050 runtime.pending_activity_restart = true;
7051 }
7052 if result.is_ok()
7053 && runtime
7054 .scheduled_respawn
7055 .lock()
7056 .unwrap_or_else(|p| p.into_inner())
7057 .as_ref()
7058 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7059 {
7060 *runtime
7061 .deferred_reload_reply
7062 .lock()
7063 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7064 } else {
7065 let _ = reply.send(result);
7066 }
7067 true
7068 }
7069 SupervisorCommand::SetEnabled {
7070 enabled,
7071 requester,
7072 reply,
7073 } => {
7074 let previous_requester = runtime.start_requester.clone();
7075 if enabled {
7076 runtime.start_requester = requester.clone();
7077 }
7078 let result = set_child_enabled(
7079 spec,
7080 runtime,
7081 registry,
7082 process_liveness,
7083 snapshot,
7084 child,
7085 enabled,
7086 )
7087 .await;
7088 if matches!(result, Ok(true)) {
7089 runtime.pending_activity_restart = false;
7090 if enabled {
7091 runtime.start_requester = requester;
7092 } else {
7093 runtime.start_requester = None;
7094 runtime.record_activity_transition(
7095 "subc.module_disabled",
7096 &spec.module_id,
7097 requester,
7098 );
7099 }
7100 } else {
7101 runtime.start_requester = previous_requester;
7102 }
7103 let _ = reply.send(result);
7104 true
7105 }
7106 SupervisorCommand::UpdateConfiguration {
7107 spec: next_spec,
7108 health,
7109 drain_timeout_ms,
7110 reply,
7111 } => {
7112 if let Some(handle) = &runtime.supervisor_handle {
7113 handle.apply_identity_configuration(&next_spec);
7114 }
7115 *spec = next_spec;
7116 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7117 state.configuration_updated_since_spawn = true;
7118 });
7119 let health_changed = runtime.health != health;
7120 runtime.health = health;
7121 if health_changed {
7124 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7125 state.health = ModuleHealthStatus::default();
7126 });
7127 }
7128 runtime.drain_timeout = drain_timeout_ms
7129 .map(Duration::from_millis)
7130 .unwrap_or(runtime.default_drain_timeout);
7131 *runtime
7132 .effective_drain_timeout
7133 .lock()
7134 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7135 let _ = reply.send(());
7136 true
7137 }
7138 SupervisorCommand::Swap {
7139 requester,
7140 ready_timeout,
7141 reply,
7142 } => {
7143 let mut swap_runtime = runtime.clone();
7144 swap_runtime.start_requester = requester;
7145 let before = lock_snapshot(snapshot)
7146 .ok()
7147 .map(|state| state.spawn_generation);
7148 let end = swap::run_swap(
7149 spec,
7150 &swap_runtime,
7151 registry,
7152 process_liveness,
7153 snapshot,
7154 child,
7155 commands,
7156 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7157 reply,
7158 )
7159 .await;
7160 if before
7161 != lock_snapshot(snapshot)
7162 .ok()
7163 .map(|state| state.spawn_generation)
7164 {
7165 runtime.start_requester = swap_runtime.start_requester;
7166 }
7167 requeued.extend(end.requeue);
7168 true
7169 }
7170 }
7171}
7172
7173async fn restart_child(
7174 spec: &ModuleSpec,
7175 runtime: &SupervisorRuntimeConfig,
7176 registry: &Registry,
7177 process_liveness: &SupervisorProcessLiveness,
7178 snapshot: &SharedSnapshot,
7179 child: &mut Option<SupervisedChild>,
7180 drain_timeout: Duration,
7181) -> Result<(), SuperviseError> {
7182 if !lock_snapshot(snapshot)?.enabled {
7184 return Err(SuperviseError::Disabled {
7185 module_id: spec.module_id.clone(),
7186 });
7187 }
7188 let stop_notice = begin_forwarding_drain_with_timeout(
7189 spec,
7190 runtime,
7191 registry,
7192 snapshot,
7193 None,
7194 RouteCloseReason::Restart,
7195 drain_timeout,
7196 )
7197 .await?;
7198
7199 if child.is_some() {
7200 drain_optional_child(
7201 &spec.module_id,
7202 spec.protocol,
7203 stop_notice,
7204 registry,
7205 runtime.forwarding.as_deref(),
7206 snapshot,
7207 &runtime.terminal_ring,
7208 &runtime.spawn_events,
7209 child,
7210 drain_timeout,
7211 ModuleState::Restarting,
7212 Some(true),
7213 )
7214 .await?;
7215 } else {
7216 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7217 state.enabled = true;
7218 state.state = ModuleState::Restarting;
7219 clear_current_process_facts(state);
7220 })?;
7221 release_dead_registration(
7222 registry,
7223 runtime.forwarding.as_deref(),
7224 snapshot,
7225 &spec.module_id,
7226 )
7227 .await?;
7228 }
7229
7230 reset_restart_count(snapshot, &spec.module_id)?;
7231 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7232 schedule_respawn(
7233 runtime,
7234 snapshot,
7235 &spec.module_id,
7236 runtime.restart_policy.backoff,
7237 RespawnKind::Spawn,
7238 )
7239}
7240
7241async fn reload_child(
7242 spec: &ModuleSpec,
7243 runtime: &SupervisorRuntimeConfig,
7244 registry: &Registry,
7245 process_liveness: &SupervisorProcessLiveness,
7246 snapshot: &SharedSnapshot,
7247 child: &mut Option<SupervisedChild>,
7248) -> Result<(), SuperviseError> {
7249 if !lock_snapshot(snapshot)?.enabled {
7251 return Err(SuperviseError::Disabled {
7252 module_id: spec.module_id.clone(),
7253 });
7254 }
7255 let stop_notice = begin_forwarding_drain(
7256 spec,
7257 runtime,
7258 registry,
7259 snapshot,
7260 Some(true),
7261 RouteCloseReason::Reload,
7262 )
7263 .await?;
7264
7265 if child.is_some() {
7266 drain_optional_child(
7267 &spec.module_id,
7268 spec.protocol,
7269 stop_notice,
7270 registry,
7271 runtime.forwarding.as_deref(),
7272 snapshot,
7273 &runtime.terminal_ring,
7274 &runtime.spawn_events,
7275 child,
7276 runtime.drain_timeout,
7277 ModuleState::Restarting,
7278 Some(true),
7279 )
7280 .await?;
7281 } else {
7282 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7283 state.enabled = true;
7284 state.state = ModuleState::Restarting;
7285 clear_current_process_facts(state);
7286 })?;
7287 release_dead_registration(
7288 registry,
7289 runtime.forwarding.as_deref(),
7290 snapshot,
7291 &spec.module_id,
7292 )
7293 .await?;
7294 }
7295
7296 reset_restart_count(snapshot, &spec.module_id)?;
7297 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7298 schedule_respawn(
7299 runtime,
7300 snapshot,
7301 &spec.module_id,
7302 runtime.restart_policy.backoff,
7303 RespawnKind::Reload,
7304 )
7305}
7306
7307async fn finish_reload_child(
7308 spec: &ModuleSpec,
7309 runtime: &SupervisorRuntimeConfig,
7310 registry: &Registry,
7311 process_liveness: &SupervisorProcessLiveness,
7312 snapshot: &SharedSnapshot,
7313 child: &mut Option<SupervisedChild>,
7314) -> Result<(), SuperviseError> {
7315 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7316 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7317 Ok(next_child) => next_child,
7318 Err(err) => {
7319 return handle_reload_spawn_failure(
7320 spec,
7321 runtime,
7322 process_liveness,
7323 snapshot,
7324 child,
7325 format!("new child failed to spawn: {err}"),
7326 )
7327 .await;
7328 }
7329 };
7330 *child = Some(next_child);
7331
7332 let wait_outcome = {
7333 let active_child = child.as_mut().expect("new reload child was just stored");
7334 wait_for_registration_after_reload(
7335 registry,
7336 &spec.module_id,
7337 snapshot,
7338 active_child,
7339 REGISTRY_RELEASE_TIMEOUT,
7340 )
7341 .await?
7342 };
7343
7344 match wait_outcome {
7345 RegistrationWaitOutcome::Registered => {
7346 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7347 Ok(())
7348 }
7349 RegistrationWaitOutcome::Exited(exit_report) => {
7350 if let Some(active_child) = child.as_mut() {
7351 active_child.drain_stderr(&spec.module_id).await;
7352 }
7353 let mut exited_child = child.take().expect("exited reload child is still stored");
7356 #[cfg(test)]
7357 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7358 gate.reached.notify_one();
7359 gate.resume.notified().await;
7360 }
7361 let result = handle_reload_child_registration_failure(
7362 spec,
7363 runtime,
7364 registry,
7365 process_liveness,
7366 snapshot,
7367 child,
7368 ReloadRegistrationFailure {
7369 exit_report: registration_failure_exit_report(exit_report),
7370 reason: exited_child
7371 .spawn_failure
7372 .clone()
7373 .unwrap_or_else(|| "new child exited before registering".to_string()),
7374 },
7375 )
7376 .await;
7377 exited_child.release_roster();
7378 result
7379 }
7380 RegistrationWaitOutcome::TimedOut => {
7381 let mut timed_out_child = child
7382 .take()
7383 .expect("timed-out reload child is still running");
7384 timed_out_child
7385 .start_kill()
7386 .map_err(|source| SuperviseError::Kill {
7387 module_id: spec.module_id.clone(),
7388 source,
7389 })?;
7390 let status = timed_out_child
7391 .wait()
7392 .await
7393 .map_err(|source| SuperviseError::Wait {
7394 module_id: spec.module_id.clone(),
7395 source,
7396 })?;
7397 timed_out_child.drain_stderr(&spec.module_id).await;
7398 handle_reload_child_registration_failure(
7399 spec,
7400 runtime,
7401 registry,
7402 process_liveness,
7403 snapshot,
7404 child,
7405 ReloadRegistrationFailure {
7406 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7407 snapshot,
7408 &timed_out_child,
7409 &status,
7410 )),
7411 reason: format!(
7412 "new child did not register within {:?}",
7413 REGISTRY_RELEASE_TIMEOUT
7414 ),
7415 },
7416 )
7417 .await
7418 }
7419 }
7420}
7421
7422async fn set_child_enabled(
7423 spec: &ModuleSpec,
7424 runtime: &SupervisorRuntimeConfig,
7425 registry: &Registry,
7426 process_liveness: &SupervisorProcessLiveness,
7427 snapshot: &SharedSnapshot,
7428 child: &mut Option<SupervisedChild>,
7429 enabled: bool,
7430) -> Result<bool, SuperviseError> {
7431 let (current_enabled, current_state, respawn_pending) = {
7432 let state = lock_snapshot(snapshot)?;
7433 (state.enabled, state.state, state.respawn_pending)
7434 };
7435 let revive_terminal = enabled
7443 && current_enabled
7444 && child.is_none()
7445 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7446 || (current_state == ModuleState::Restarting && !respawn_pending));
7447 if current_enabled == enabled && !revive_terminal {
7448 return Ok(false);
7449 }
7450
7451 if enabled {
7452 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7453 state.enabled = true;
7454 state.state = ModuleState::Starting;
7455 clear_current_process_facts(state);
7456 })?;
7457 #[cfg(test)]
7458 if runtime.test_seed_stale_facts_before_enable_spawn {
7459 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7460 state.process_alive = true;
7461 state.pid = Some(41);
7462 state.spawned_at_ms = Some(42);
7463 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7464 state.spawned_file_identity = Some(SpawnedFileIdentity {
7465 device: 43,
7466 inode: 44,
7467 #[cfg(windows)]
7468 windows_image: None,
7469 });
7470 })?;
7471 }
7472 release_dead_registration(
7473 registry,
7474 runtime.forwarding.as_deref(),
7475 snapshot,
7476 &spec.module_id,
7477 )
7478 .await?;
7479 reset_restart_count(snapshot, &spec.module_id)?;
7480 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7481 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7482 Ok(next_child) => next_child,
7483 Err(err) => {
7484 if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7485 state.state = ModuleState::Failed;
7486 clear_current_process_facts(state);
7487 }) {
7488 error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7489 }
7490 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7491 return Err(err);
7492 }
7493 };
7494 *child = Some(next_child);
7495 debug!(module_id = %spec.module_id, "supervised module enabled");
7496 Ok(true)
7497 } else {
7498 let stop_notice = begin_forwarding_drain_if_configured(
7499 spec,
7500 runtime,
7501 registry,
7502 snapshot,
7503 Some(false),
7504 RouteCloseReason::Disable,
7505 )
7506 .await?;
7507 drain_optional_child(
7508 &spec.module_id,
7509 spec.protocol,
7510 stop_notice,
7511 registry,
7512 runtime.forwarding.as_deref(),
7513 snapshot,
7514 &runtime.terminal_ring,
7515 &runtime.spawn_events,
7516 child,
7517 runtime.drain_timeout,
7518 ModuleState::Disabled,
7519 Some(false),
7520 )
7521 .await?;
7522 debug!(module_id = %spec.module_id, "supervised module disabled");
7523 Ok(true)
7524 }
7525}
7526
7527#[allow(clippy::too_many_arguments)]
7528async fn on_child_exit(
7529 spec: &ModuleSpec,
7530 policy: RestartPolicy,
7531 registry: &Registry,
7532 snapshot: &SharedSnapshot,
7533 terminal_ring: &Arc<Mutex<TerminalRing>>,
7534 spawn_events: &SpawnEventFeed,
7535 roster: &ChildRoster,
7536 exit_report: ExitReport,
7537) -> NextAction {
7538 if roster.is_closed() {
7544 return on_child_exit_during_daemon_shutdown(
7545 spec,
7546 registry,
7547 snapshot,
7548 terminal_ring,
7549 spawn_events,
7550 exit_report,
7551 )
7552 .await;
7553 }
7554 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7569 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7570 match exit_report.kind {
7571 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7572 info!(
7573 module_id = %spec.module_id,
7574 exit_code = ?exit_report.code,
7575 exit_signal = ?exit_report.signal,
7576 "supervised module exited cleanly"
7577 );
7578 if let Err(err) = publish_after_terminal_record(
7579 &spec.module_id,
7580 || {
7581 record_terminal(
7582 &spec.module_id,
7583 terminal_ring,
7584 spawn_events,
7585 &exit_report,
7586 TerminalDisposition::Stopped,
7587 )
7588 },
7589 || {
7590 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7591 state.state = ModuleState::Stopped;
7592 clear_current_process_facts(state);
7593 state.last_exit = Some(exit_report.clone());
7594 })
7595 },
7596 ) {
7597 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7598 }
7599 let registration_released = match wait_for_registration_release(
7600 registry,
7601 &spec.module_id,
7602 REGISTRY_RELEASE_TIMEOUT,
7603 )
7604 .await
7605 {
7606 Ok(()) => true,
7607 Err(err) => {
7608 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7609 false
7610 }
7611 };
7612 NextAction::Stop {
7613 registration_released,
7614 }
7615 }
7616 ExitKind::Clean | ExitKind::Crash => {
7617 if unrequested_clean_exit_of_protocol_none {
7618 warn!(
7619 module_id = %spec.module_id,
7620 exit_code = ?exit_report.code,
7621 exit_signal = ?exit_report.signal,
7622 "protocol-none module exited cleanly without a stop request; handling it as a crash"
7623 );
7624 } else {
7625 warn!(
7626 module_id = %spec.module_id,
7627 exit_code = ?exit_report.code,
7628 exit_signal = ?exit_report.signal,
7629 "supervised module exited abnormally (crash)"
7630 );
7631 }
7632 let mut restart_schedule = None;
7633 let mut disposition = TerminalDisposition::Disabled;
7634 let mut final_state = ModuleState::Disabled;
7635 let mut disposition_detail = lock_snapshot(snapshot)
7639 .ok()
7640 .and_then(|mut state| state.spawn_failure.take());
7641 let now = Instant::now();
7642 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7643 if state.enabled {
7644 if let Some(schedule) = state.next_crash_restart(&policy, now) {
7645 state.respawn_pending = true;
7651 final_state = ModuleState::Restarting;
7652 restart_schedule = Some(schedule);
7653 disposition = TerminalDisposition::Restarting;
7654 } else {
7655 final_state = ModuleState::Failed;
7656 disposition = TerminalDisposition::Failed;
7657 let budget = policy.budget_exhausted_detail();
7658 disposition_detail =
7659 Some(disposition_detail.take().map_or_else(
7660 || budget.clone(),
7661 |cause| format!("{cause}; {budget}"),
7662 ));
7663 }
7664 } else {
7665 disposition = TerminalDisposition::Disabled;
7666 }
7667 }) {
7668 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7669 return NextAction::Stop {
7670 registration_released: false,
7671 };
7672 }
7673 if disposition == TerminalDisposition::Failed {
7674 error!(
7679 module_id = %spec.module_id,
7680 max_restarts = policy.max_restarts,
7681 window_secs = policy.window.as_secs(),
7682 "module stopped: {}",
7683 policy.budget_exhausted_detail()
7684 );
7685 }
7686 if let Err(err) = publish_after_terminal_record(
7687 &spec.module_id,
7688 || {
7689 record_terminal_with_detail(
7690 &spec.module_id,
7691 terminal_ring,
7692 spawn_events,
7693 &exit_report,
7694 disposition,
7695 disposition_detail,
7696 )
7697 },
7698 || {
7699 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7700 state.state = final_state;
7701 clear_current_process_facts(state);
7702 state.last_exit = Some(exit_report.clone());
7703 })
7704 },
7705 ) {
7706 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7707 }
7708
7709 if let Some(schedule) = restart_schedule {
7710 NextAction::Restart {
7711 schedule: Some(schedule),
7712 }
7713 } else {
7714 let registration_released = match wait_for_registration_release(
7715 registry,
7716 &spec.module_id,
7717 REGISTRY_RELEASE_TIMEOUT,
7718 )
7719 .await
7720 {
7721 Ok(()) => true,
7722 Err(err) => {
7723 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7724 false
7725 }
7726 };
7727 NextAction::Stop {
7728 registration_released,
7729 }
7730 }
7731 }
7732 ExitKind::DeliberateSeverance => {
7733 warn!(
7734 module_id = %spec.module_id,
7735 exit_code = ?exit_report.code,
7736 exit_signal = ?exit_report.signal,
7737 "supervised module exited after deliberate connection severance"
7738 );
7739 let should_restart = match lock_snapshot(snapshot) {
7740 Ok(state) => state.enabled,
7741 Err(err) => {
7742 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7743 return NextAction::Stop {
7744 registration_released: false,
7745 };
7746 }
7747 };
7748 let final_state = if should_restart {
7749 ModuleState::Restarting
7750 } else {
7751 ModuleState::Disabled
7752 };
7753 if let Err(err) = publish_after_terminal_record(
7754 &spec.module_id,
7755 || {
7756 record_terminal(
7757 &spec.module_id,
7758 terminal_ring,
7759 spawn_events,
7760 &exit_report,
7761 terminal_disposition(final_state),
7762 )
7763 },
7764 || {
7765 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7766 state.state = final_state;
7767 clear_current_process_facts(state);
7768 state.last_exit = Some(exit_report.clone());
7769 state.lifetime_restarts += 1;
7770 })
7771 },
7772 ) {
7773 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7774 return NextAction::Stop {
7775 registration_released: false,
7776 };
7777 }
7778
7779 if should_restart {
7780 NextAction::Restart { schedule: None }
7781 } else {
7782 let registration_released = match wait_for_registration_release(
7783 registry,
7784 &spec.module_id,
7785 REGISTRY_RELEASE_TIMEOUT,
7786 )
7787 .await
7788 {
7789 Ok(()) => true,
7790 Err(err) => {
7791 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7792 false
7793 }
7794 };
7795 NextAction::Stop {
7796 registration_released,
7797 }
7798 }
7799 }
7800 }
7801}
7802
7803async fn on_child_exit_during_daemon_shutdown(
7804 spec: &ModuleSpec,
7805 registry: &Registry,
7806 snapshot: &SharedSnapshot,
7807 terminal_ring: &Arc<Mutex<TerminalRing>>,
7808 spawn_events: &SpawnEventFeed,
7809 exit_report: ExitReport,
7810) -> NextAction {
7811 info!(
7812 module_id = %spec.module_id,
7813 exit_code = ?exit_report.code,
7814 exit_signal = ?exit_report.signal,
7815 exit_kind = ?exit_report.kind,
7816 "supervised module exited during daemon shutdown; not restarting it"
7817 );
7818 if let Err(err) = publish_after_terminal_record(
7819 &spec.module_id,
7820 || {
7821 record_terminal(
7822 &spec.module_id,
7823 terminal_ring,
7824 spawn_events,
7825 &exit_report,
7826 TerminalDisposition::DaemonShutdown,
7827 )
7828 },
7829 || {
7830 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7831 state.state = ModuleState::Stopped;
7832 clear_current_process_facts(state);
7833 state.last_exit = Some(exit_report.clone());
7834 })
7835 },
7836 ) {
7837 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7838 }
7839 let registration_released =
7840 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7841 .await
7842 .is_ok();
7843 NextAction::Stop {
7844 registration_released,
7845 }
7846}
7847
7848fn publish_after_terminal_record<T>(
7853 module_id: &str,
7854 record: impl FnOnce(),
7855 publish: impl FnOnce() -> T,
7856) -> T {
7857 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7858 let published = publish();
7859 if let Err(panic) = recorded {
7860 error!(
7861 module_id,
7862 "terminal recording panicked; published final state before resuming unwind"
7863 );
7864 std::panic::resume_unwind(panic);
7865 }
7866 published
7867}
7868
7869fn fail_after_wait_error(
7870 module_id: &str,
7871 snapshot: &SharedSnapshot,
7872 terminal_ring: &Arc<Mutex<TerminalRing>>,
7873 spawn_events: &SpawnEventFeed,
7874 detail: String,
7875) {
7876 let report = wait_error_exit_report();
7877 publish_after_terminal_record(
7878 module_id,
7879 || {
7880 record_terminal_with_detail(
7881 module_id,
7882 terminal_ring,
7883 spawn_events,
7884 &report,
7885 TerminalDisposition::Failed,
7886 Some(detail),
7887 )
7888 },
7889 || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7890 );
7891}
7892
7893#[cfg(test)]
7894fn record_wait_error_terminal(
7895 module_id: &str,
7896 terminal_ring: &Arc<Mutex<TerminalRing>>,
7897 spawn_events: &SpawnEventFeed,
7898) {
7899 record_terminal(
7900 module_id,
7901 terminal_ring,
7902 spawn_events,
7903 &wait_error_exit_report(),
7904 TerminalDisposition::Failed,
7905 );
7906}
7907
7908fn record_terminal(
7909 module_id: &str,
7910 terminal_ring: &Arc<Mutex<TerminalRing>>,
7911 spawn_events: &SpawnEventFeed,
7912 exit_report: &ExitReport,
7913 disposition: TerminalDisposition,
7914) {
7915 record_terminal_with_detail(
7916 module_id,
7917 terminal_ring,
7918 spawn_events,
7919 exit_report,
7920 disposition,
7921 None,
7922 );
7923}
7924
7925fn durable_terminal_history_of(
7929 terminal_ring: &Mutex<TerminalRing>,
7930 module_id: &str,
7931) -> subc_control::TerminalHistory {
7932 let read = terminal_ring
7933 .lock()
7934 .unwrap_or_else(|p| p.into_inner())
7935 .capture_durable_history();
7936 read.read(module_id)
7937}
7938
7939fn record_terminal_with_detail(
7940 module_id: &str,
7941 terminal_ring: &Arc<Mutex<TerminalRing>>,
7942 spawn_events: &SpawnEventFeed,
7943 exit_report: &ExitReport,
7944 disposition: TerminalDisposition,
7945 disposition_detail: Option<String>,
7946) {
7947 #[cfg(test)]
7948 terminal_history_tests::before_terminal_record(spawn_events);
7949 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7950 let record = TerminalRecord {
7951 exit_code: exit_report.code,
7952 exit_signal: exit_report.signal,
7953 at_ms: exit_report.at_ms,
7954 disposition,
7955 exit_kind: exit_report.kind.into(),
7956 disposition_detail,
7957 };
7958 terminal_ring
7959 .lock()
7960 .unwrap_or_else(|poisoned| poisoned.into_inner())
7961 .record_exit(module_id, record);
7962}
7963
7964fn untrack_if_registration_released(
7965 process_liveness: &SupervisorProcessLiveness,
7966 registry: &Registry,
7967 module_id: &str,
7968 snapshot: &SharedSnapshot,
7969) {
7970 match registry.get_module(module_id) {
7971 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
7972 Ok(Some(_)) => {}
7973 Err(err) => {
7974 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
7975 }
7976 }
7977}
7978
7979#[cfg(test)]
7993fn apply_wire_spawn_args(
7994 command: &mut Command,
7995 spec: &ModuleSpec,
7996 connection_file_path: Option<&std::path::Path>,
7997 handle: Option<&SupervisorHandle>,
7998) -> Result<Option<NonceHandoff>, SuperviseError> {
7999 apply_wire_spawn_args_for_role(
8000 command,
8001 spec,
8002 connection_file_path,
8003 handle,
8004 SpawnRole::Plain,
8005 )
8006}
8007
8008#[cfg(unix)]
8013type NonceHandoff = subc_os::LaunchNonceHandoff;
8014#[cfg(windows)]
8015type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8016#[cfg(not(any(unix, windows)))]
8017type NonceHandoff = std::convert::Infallible;
8018
8019fn apply_wire_spawn_args_for_role(
8040 command: &mut Command,
8041 spec: &ModuleSpec,
8042 connection_file_path: Option<&std::path::Path>,
8043 handle: Option<&SupervisorHandle>,
8044 role: SpawnRole,
8045) -> Result<Option<NonceHandoff>, SuperviseError> {
8046 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8047 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8052 #[cfg(windows)]
8053 {
8054 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8055 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8056 }
8057 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8059 if spec.protocol == ModuleProtocol::None {
8060 return Ok(None);
8061 }
8062 if let Some(connection_file_path) = connection_file_path {
8063 command.arg(SUBC_ARG).arg(connection_file_path);
8064 }
8065
8066 let nonce = generate_launch_nonce()?;
8070 if let Some(handle) = handle {
8071 match role {
8072 SpawnRole::Plain => {
8073 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8074 if spec.reserved {
8075 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8076 }
8077 }
8078 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8079 }
8080 }
8081 #[cfg(unix)]
8082 let handoff = {
8083 let handoff =
8084 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8085 program: spec.program.clone(),
8086 source,
8087 cgroup_path: None,
8088 })?;
8089 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8090 Some(handoff)
8091 };
8092 #[cfg(windows)]
8093 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8094 #[cfg(not(any(unix, windows)))]
8095 let handoff = None;
8096 #[cfg(not(unix))]
8100 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8101 Ok(handoff)
8102}
8103
8104fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8105 command.env_remove(CK_LOG_ENV);
8106 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8113 for (key, value) in &spec.env {
8114 if matches!(
8118 key.as_str(),
8119 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8120 ) || key == SUBC_SPAWN_ROLE_ENV
8121 {
8122 continue;
8123 }
8124 command.env(key, value);
8125 }
8126}
8127
8128#[cfg(windows)]
8129fn prepare_windows_nonce_pipe(
8130 command: &mut Command,
8131 spec: &ModuleSpec,
8132 nonce: &str,
8133) -> Option<NonceHandoff> {
8134 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8135 Ok(handoff) => {
8136 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8137 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8142 Some(handoff)
8143 }
8144 Err(error) => {
8145 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8146 None
8147 }
8148 }
8149}
8150
8151#[cfg(windows)]
8152fn start_windows_nonce_pipe(
8153 handoff: Option<NonceHandoff>,
8154 pid: u32,
8155 spec: &ModuleSpec,
8156) -> Option<subc_os::LaunchNoncePipeDelivery> {
8157 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8161 Ok(delivery) => Some(delivery),
8162 Err(error) => {
8163 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8164 None
8165 }
8166 })
8167}
8168
8169fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8170 #[cfg(windows)]
8171 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8172 return Some(
8173 if snapshot
8174 .pipe_consumed
8175 .as_ref()
8176 .is_some_and(|consumed| consumed.get().is_some())
8177 {
8178 "pipe"
8179 } else {
8180 "env"
8181 }
8182 .to_owned(),
8183 );
8184 }
8185 let _ = snapshot;
8186 None
8187}
8188
8189#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8192enum SpawnRole {
8193 Plain,
8194 SwapCandidate,
8195}
8196
8197fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8200 if role == SpawnRole::SwapCandidate {
8201 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8202 }
8203}
8204
8205fn previous_instance_error(
8206 spec: &ModuleSpec,
8207 pid: u32,
8208 reason: crate::live_children::RetainedReason,
8209) -> SuperviseError {
8210 SuperviseError::Spawn {
8211 program: spec.program.clone(),
8212 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8213 cgroup_path: None,
8214 }
8215}
8216
8217fn spawn_child(
8218 spec: &ModuleSpec,
8219 connection_file_path: Option<&std::path::Path>,
8220 handle: Option<&SupervisorHandle>,
8221 ring: &Arc<Mutex<StderrRing>>,
8222 capture_logs_dir: Option<&std::path::Path>,
8223 roster: &ChildRoster,
8224 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8225) -> Result<SupervisedChild, SuperviseError> {
8226 spawn_child_in_slot(
8227 spec,
8228 connection_file_path,
8229 handle,
8230 ring,
8231 capture_logs_dir,
8232 roster,
8233 #[cfg(target_os = "linux")]
8234 cgroup_placement,
8235 SpawnRole::Plain,
8236 false,
8237 )
8238}
8239
8240#[allow(clippy::too_many_arguments)]
8253fn spawn_child_in_slot(
8254 spec: &ModuleSpec,
8255 connection_file_path: Option<&std::path::Path>,
8256 handle: Option<&SupervisorHandle>,
8257 ring: &Arc<Mutex<StderrRing>>,
8258 capture_logs_dir: Option<&std::path::Path>,
8259 roster: &ChildRoster,
8260 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8261 role: SpawnRole,
8262 alternate_slot: bool,
8263) -> Result<SupervisedChild, SuperviseError> {
8264 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8265 return Err(previous_instance_error(spec, pid, reason));
8266 }
8267 if roster.is_closed() {
8268 return Err(SuperviseError::Spawn {
8269 program: spec.program.clone(),
8270 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8271 cgroup_path: None,
8272 });
8273 }
8274 #[cfg(target_os = "linux")]
8275 let cgroup_name = {
8276 if cgroup_placement.is_none() {
8281 swap::cgroup_name(&spec.module_id, alternate_slot)
8282 } else {
8283 let nonce = generate_launch_nonce()?;
8284 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8285 let mut end = spec.module_id.len().min(64);
8288 while !spec.module_id.is_char_boundary(end) {
8289 end -= 1;
8290 }
8291 format!(
8292 "{}_{suffix}",
8293 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8294 )
8295 }
8296 };
8297 #[cfg(not(target_os = "linux"))]
8298 let _ = alternate_slot;
8299 #[cfg(target_os = "macos")]
8300 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8301 #[cfg(windows)]
8302 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8303 program: spec.program.clone(),
8304 source,
8305 cgroup_path: None,
8306 })?;
8307 #[cfg(windows)]
8308 let mut command = Command::new(image_capture.path());
8309 #[cfg(not(any(target_os = "macos", windows)))]
8310 let mut command = Command::new(&spec.program);
8311 command.args(&spec.args);
8312 apply_child_env(&mut command, spec);
8342 apply_spawn_role(&mut command, role);
8343 let nonce_handoff =
8344 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8345
8346 #[cfg(target_os = "linux")]
8347 let cgroup_path = cgroup_placement
8348 .map(|placement| placement.module_path(&cgroup_name))
8349 .transpose()
8350 .map_err(|source| SuperviseError::Cgroup {
8351 module_id: spec.module_id.clone(),
8352 source,
8353 })?;
8354 #[cfg(not(target_os = "linux"))]
8355 let cgroup_path: Option<PathBuf> = None;
8356 #[cfg(target_os = "linux")]
8357 if let Some(path) = &cgroup_path {
8358 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8359 if let Some(placement) = cgroup_placement {
8360 remove_module_cgroup(placement, &cgroup_name);
8361 }
8362 return Err(error);
8363 }
8364 }
8365
8366 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8367 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8368 match ChildOutputSink::open(&path, capture_retention(spec)) {
8369 Ok(sink) => sink,
8370 Err(error) => {
8371 warn!(
8372 module_id = %spec.module_id,
8373 path = %path.display(),
8374 error = %error,
8375 "could not open child output capture file; forwarding to stderr"
8376 );
8377 ChildOutputSink::Stderr
8378 }
8379 }
8380 } else {
8381 ChildOutputSink::Stderr
8382 };
8383
8384 command.stdout(Stdio::piped());
8385 command.stderr(Stdio::piped());
8386 command.kill_on_drop(true);
8387 #[cfg(unix)]
8404 command.process_group(0);
8405 command.stdin(Stdio::null());
8406 #[cfg(unix)]
8410 if let Some(handoff) = nonce_handoff {
8411 handoff.install_last(command.as_std_mut());
8412 }
8413 #[cfg(not(any(unix, windows)))]
8414 let _ = nonce_handoff;
8415
8416 #[cfg(windows)]
8421 {
8422 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8423 program: spec.program.clone(),
8424 source,
8425 cgroup_path: None,
8426 })?;
8427 subc_jobobject::suspend_on_create_async(&mut command);
8428 }
8429 let spawned_at = Instant::now();
8433 let mut child = match command.spawn() {
8434 Ok(child) => child,
8435 Err(source) => {
8436 #[cfg(target_os = "linux")]
8437 if let Some(placement) = cgroup_placement {
8438 remove_module_cgroup(placement, &cgroup_name);
8439 }
8440 return Err(SuperviseError::Spawn {
8441 program: spec.program.clone(),
8442 source,
8443 cgroup_path,
8444 });
8445 }
8446 };
8447 #[cfg(test)]
8448 spawn_record_tests::after_create(&child, spec);
8449 #[cfg(target_os = "macos")]
8454 drop(exec_ack);
8455
8456 #[cfg(windows)]
8461 let windows_image = match image_capture.bind_tokio(&child) {
8462 Ok(image) => Arc::new(image),
8463 Err(source) => {
8464 kill_and_reap_rejected_child(&mut child);
8465 return Err(SuperviseError::Spawn {
8466 program: spec.program.clone(),
8467 source,
8468 cgroup_path: None,
8469 });
8470 }
8471 };
8472 #[cfg(windows)]
8475 let job = contain_spawned_child(&mut child, spec)?;
8476 let spawned_at_ms = unix_ms_now();
8477 #[cfg(not(windows))]
8478 let spawned_from = spec.program.clone();
8479 #[cfg(windows)]
8480 let spawned_from = windows_image.path().to_path_buf();
8481 #[cfg(not(windows))]
8482 let spawned_file_identity = spawned_file_identity(&spawned_from);
8483 #[cfg(windows)]
8484 let spawned_file_identity = Some(SpawnedFileIdentity {
8485 device: 0,
8486 inode: 0,
8487 windows_image: Some(Arc::clone(&windows_image)),
8488 });
8489 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8490 program: spec.program.clone(),
8491 source: io::Error::other("spawned child exposed no live pid"),
8492 cgroup_path: cgroup_path.clone(),
8493 })?;
8494 #[cfg(windows)]
8495 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8496 #[cfg(not(windows))]
8497 let process_start_time = crate::provenance::process_start_time(pid);
8498 #[cfg(windows)]
8499 let process_start_time = Some(windows_image.start_time());
8500 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8501 #[cfg(all(test, target_os = "macos"))]
8502 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8503 #[cfg(not(windows))]
8508 let recorded_image = observe_spawned_image(pid);
8509 #[cfg(target_os = "macos")]
8513 let recorded_image = if privacy_exec.is_some() {
8514 None
8515 } else {
8516 recorded_image
8517 };
8518 #[cfg(target_os = "linux")]
8519 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8520 #[cfg(not(target_os = "linux"))]
8521 let recorded_cgroup_name = None;
8522 let roster_guard = roster.admit(
8523 spec.module_id.clone(),
8524 pid,
8525 spec.protocol,
8526 process_start_time,
8527 crate::child_roster::RecordedIdentity {
8528 #[cfg(not(windows))]
8529 start_time: recorded_image.map(|image| image.start_time),
8530 #[cfg(windows)]
8531 start_time: process_start_time,
8532 #[cfg(not(windows))]
8533 executable: recorded_image
8534 .and_then(|image| image.executable)
8535 .map(crate::live_children::ExecutableIdentity::from),
8536 #[cfg(windows)]
8537 executable: Some(crate::live_children::ExecutableIdentity::from(
8538 windows_image.identity(),
8539 )),
8540 cgroup_name: recorded_cgroup_name,
8541 #[cfg(target_os = "linux")]
8542 cgroup_placement: cgroup_placement.cloned(),
8543 },
8544 );
8545 let roster_guard = match roster_guard {
8546 Ok(guard) => guard,
8547 Err(source) => {
8548 #[cfg(windows)]
8549 if let Some(job) = &job {
8550 let _ = job.terminate();
8551 }
8552 #[cfg(target_os = "linux")]
8553 kill_module_cgroup(cgroup_placement, &cgroup_name);
8554 kill_and_reap_rejected_child(&mut child);
8555 #[cfg(target_os = "linux")]
8556 if let Some(placement) = cgroup_placement {
8557 remove_module_cgroup(placement, &cgroup_name);
8558 }
8559 return Err(SuperviseError::Spawn {
8560 program: spec.program.clone(),
8561 source: io::Error::other(format!(
8562 "could not durably record module child: {source}"
8563 )),
8564 cgroup_path,
8565 });
8566 }
8567 };
8568 if roster.is_closed() {
8577 #[cfg(target_os = "linux")]
8579 kill_module_cgroup(cgroup_placement, &cgroup_name);
8580 if let Err(error) = child.start_kill() {
8581 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8582 }
8583 #[cfg(target_os = "linux")]
8584 if let Some(placement) = cgroup_placement {
8585 while matches!(child.try_wait(), Ok(None)) {
8589 std::thread::yield_now();
8590 }
8591 if matches!(
8592 subc_cgroup::kill_module(Some(placement), &cgroup_name),
8593 subc_cgroup::KillOutcome::Killed
8594 ) {
8595 if let Ok(path) = placement.module_path(&cgroup_name) {
8596 while std::fs::read_to_string(path.join("cgroup.events"))
8597 .ok()
8598 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8599 {
8600 std::thread::yield_now();
8601 }
8602 }
8603 }
8604 remove_module_cgroup(placement, &cgroup_name);
8605 }
8606 drop(roster_guard);
8607 return Err(SuperviseError::Spawn {
8608 program: spec.program.clone(),
8609 source: io::Error::other(
8610 "the daemon began shutting down while this process was starting; ended it",
8611 ),
8612 cgroup_path,
8613 });
8614 }
8615
8616 #[cfg(windows)]
8617 if let Err(error) = resume_suspended_child(pid, spec) {
8618 kill_and_reap_rejected_child(&mut child);
8619 return Err(error);
8620 }
8621
8622 let stdout_pump = match child.stdout.take() {
8623 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8624 None => {
8625 warn!(
8626 module_id = %spec.module_id,
8627 "spawned child exposed no stdout pipe; file capture will be incomplete"
8628 );
8629 None
8630 }
8631 };
8632 let stderr_pump = match child.stderr.take() {
8633 Some(stderr) => {
8634 let generation = ring
8635 .lock()
8636 .unwrap_or_else(|poisoned| poisoned.into_inner())
8637 .begin_process();
8638 Some(StderrPump {
8639 task: tokio::spawn(pump_stderr_to(
8640 stderr,
8641 Arc::clone(ring),
8642 generation,
8643 output_sink,
8644 )),
8645 generation,
8646 })
8647 }
8648 None => {
8649 ring.lock()
8653 .unwrap_or_else(|poisoned| poisoned.into_inner())
8654 .mark_not_captured("stderr pipe was not available on spawn");
8655 warn!(
8656 module_id = %spec.module_id,
8657 "spawned child exposed no stderr pipe; tail will be unavailable"
8658 );
8659 None
8660 }
8661 };
8662
8663 let exec = Arc::new(crate::activity::ExecOutcome::new());
8664 #[cfg(target_os = "macos")]
8665 let pending_exec = privacy_exec.is_some();
8666 #[cfg(not(target_os = "macos"))]
8667 let pending_exec = false;
8668 if !pending_exec {
8669 exec.finish(crate::activity::ExecState::CommittedExec);
8670 }
8671 Ok(SupervisedChild {
8672 child,
8673 exec: crate::activity::ExecOwner(exec),
8674 #[cfg(windows)]
8675 nonce_delivery,
8676 protocol: spec.protocol,
8677 #[cfg(target_os = "linux")]
8678 module_id: cgroup_name,
8679 #[cfg(target_os = "linux")]
8680 cgroup_placement: cgroup_placement.cloned(),
8681 #[cfg(windows)]
8682 job,
8683 stdout_pump,
8684 stderr_pump,
8685 stderr_ring: Arc::clone(ring),
8686 spawned_at,
8687 spawned_at_ms,
8688 spawned_from,
8689 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8690 spawned_file_identity,
8691 process_start_time,
8692 process_identity,
8693 pid,
8694 roster_guard: Some(roster_guard),
8695 #[cfg(target_os = "macos")]
8696 privacy_exec,
8697 #[cfg(target_os = "macos")]
8698 report_ready: Arc::new(OnceLock::new()),
8699 spawn_failure: None,
8700 })
8701}
8702
8703#[cfg(test)]
8704mod spawn_record_tests {
8705 use super::*;
8706
8707 thread_local! {
8708 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8709 }
8710
8711 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8712 LAST_PID.set(child.id());
8713 #[cfg(windows)]
8714 if let Some((_, address)) = _spec
8715 .env
8716 .iter()
8717 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8718 {
8719 use std::io::{Read, Write};
8720 let mut stream = std::net::TcpStream::connect(address).unwrap();
8721 stream
8722 .write_all(&child.id().unwrap().to_ne_bytes())
8723 .unwrap();
8724 let mut release = [0];
8725 stream.read_exact(&mut release).unwrap();
8726 }
8727 }
8728
8729 #[cfg(windows)]
8730 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8731 if spec
8732 .env
8733 .iter()
8734 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8735 {
8736 Err(io::Error::other("injected module job assignment failure"))
8737 } else {
8738 Ok(())
8739 }
8740 }
8741
8742 fn spec() -> ModuleSpec {
8743 ModuleSpec {
8744 module_id: "spawn-record-worker".into(),
8745 #[cfg(unix)]
8746 program: "/bin/sleep".into(),
8747 #[cfg(windows)]
8748 program: std::env::current_exe().unwrap(),
8749 #[cfg(unix)]
8750 args: vec!["60".into()],
8751 #[cfg(windows)]
8752 args: vec![
8753 "--exact".into(),
8754 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8755 "--nocapture".into(),
8756 ],
8757 env: vec![],
8758 reserved: false,
8759 reserved_prefixes: vec![],
8760 protocol: ModuleProtocol::None,
8761 overlap: Default::default(),
8762 }
8763 }
8764
8765 #[tokio::test]
8766 async fn record_sync_failure_fails_spawn_and_reaps_child() {
8767 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8768 let supervisor =
8769 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8770 .with_live_children_record(home.join("live-children.json"));
8771 crate::live_children::fail_next_file_sync();
8772 let result = supervisor.spawn(spec());
8773 let failed = result.is_err();
8774 if let Ok(module) = result {
8775 module.stop().await.unwrap();
8776 }
8777 assert!(failed, "record failure must fail spawn");
8778 let pid = LAST_PID.get().expect("the real OS child was created");
8779 assert!(
8780 subc_os::Process::open(pid).unwrap().is_none(),
8781 "child {pid} survived rejected admission or was not reaped"
8782 );
8783 }
8784
8785 #[cfg(target_os = "macos")]
8786 #[test]
8787 #[ignore = "manual on-device latency measurement"]
8788 fn durable_record_write_latency_50() {
8789 let home = cortexkit_test_support::ScratchDir::new("record-latency");
8790 let path = home.join("live-children.json");
8791 let children = vec![crate::live_children::LiveChild {
8792 module_id: "latency-worker".into(),
8793 pid: std::process::id(),
8794 protocol: ModuleProtocol::None,
8795 start_time: Some(1),
8796 executable: None,
8797 cgroup_name: None,
8798 }];
8799 let mut samples = Vec::new();
8800 for _ in 0..50 {
8801 let start = std::time::Instant::now();
8802 crate::live_children::write_record(&path, &children).unwrap();
8803 samples.push(start.elapsed());
8804 }
8805 samples.sort();
8806 eprintln!(
8807 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8808 (samples[24] + samples[25]) / 2,
8809 samples[47]
8810 );
8811 }
8812
8813 #[cfg(windows)]
8814 #[test]
8815 fn windows_store_touch_fixture() {
8816 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8817 std::fs::write(path, b"module ran").unwrap();
8818 }
8819 }
8820
8821 #[cfg(windows)]
8822 #[tokio::test]
8823 async fn windows_daemon_fixture() {
8824 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8825 return;
8826 };
8827 let mut spec = spec();
8828 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8829 let supervisor =
8830 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8831 let _module = supervisor.spawn(spec).unwrap();
8832 }
8833
8834 #[cfg(windows)]
8835 #[test]
8836 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8837 use std::io::Read;
8838 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8839 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8842 command
8843 .args([
8844 "--exact",
8845 "supervise::spawn_record_tests::windows_daemon_fixture",
8846 "--nocapture",
8847 ])
8848 .env(
8849 "SUBC_TEST_DAEMON_BARRIER",
8850 listener.local_addr().unwrap().to_string(),
8851 );
8852 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8853 listener.set_nonblocking(true).unwrap();
8854 let deadline = std::time::Instant::now() + Duration::from_secs(30);
8855 let mut stream = loop {
8856 if let Ok((stream, _)) = listener.accept() {
8857 break stream;
8858 }
8859 assert!(
8860 daemon.child.try_wait().unwrap().is_none(),
8861 "daemon failed before rendezvous"
8862 );
8863 assert!(
8864 std::time::Instant::now() < deadline,
8865 "daemon did not reach CreateProcess failpoint"
8866 );
8867 std::thread::yield_now();
8868 };
8869 stream
8870 .set_read_timeout(Some(Duration::from_secs(30)))
8871 .unwrap();
8872 let mut pid = [0; 4];
8873 stream.read_exact(&mut pid).unwrap();
8874 let pid = u32::from_ne_bytes(pid);
8875 assert!(subc_jobobject::process_exists(pid));
8876 daemon.child.kill().unwrap();
8879 daemon.child.wait().unwrap();
8880 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8881 if !exited {
8882 daemon.job.terminate().unwrap();
8883 }
8884 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8885 }
8886
8887 #[cfg(windows)]
8888 #[tokio::test]
8889 async fn windows_containment_failure_never_resumes() {
8890 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8891 let sentinel = home.join("store-touched");
8892 let mut spec = spec();
8893 spec.env
8894 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8895 spec.env.push((
8896 "SUBC_TEST_STORE_TOUCH".into(),
8897 sentinel.display().to_string(),
8898 ));
8899 let supervisor =
8900 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8901 let result = supervisor.spawn(spec);
8902 if let Ok(module) = result {
8905 let _ = tokio::time::timeout(Duration::from_secs(10), async {
8906 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8907 tokio::task::yield_now().await;
8908 }
8909 })
8910 .await;
8911 let _ = module.stop().await;
8912 panic!(
8913 "assignment failure resumed a module; store touched={}",
8914 sentinel.exists()
8915 );
8916 }
8917 assert!(
8918 !sentinel.exists(),
8919 "refused module executed store-touch sentinel"
8920 );
8921 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8922 }
8923
8924 #[cfg(windows)]
8925 #[tokio::test]
8926 async fn windows_nested_daemon_fixture() {
8927 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8928 return;
8929 };
8930 let mut spec = spec();
8931 spec.env.push((
8932 "SUBC_TEST_STORE_TOUCH".into(),
8933 sentinel.to_str().unwrap().into(),
8934 ));
8935 let supervisor =
8936 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8937 let module = supervisor
8938 .spawn(spec)
8939 .expect("per-module job must nest below daemon and host jobs");
8940 tokio::time::timeout(Duration::from_secs(10), async {
8941 while !PathBuf::from(&sentinel).exists() {
8942 tokio::task::yield_now().await;
8943 }
8944 })
8945 .await
8946 .unwrap();
8947 module.stop().await.unwrap();
8948 }
8949
8950 #[cfg(windows)]
8951 #[test]
8952 fn windows_module_job_nests_under_host_job() {
8953 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
8954 let sentinel = home.join("nested-store-touched");
8955 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8956 command
8957 .args([
8958 "--exact",
8959 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
8960 "--nocapture",
8961 ])
8962 .env("SUBC_TEST_NESTED_STORE", &sentinel);
8963 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8964 assert!(daemon.child.wait().unwrap().success());
8965 assert!(sentinel.exists(), "nested protocol:none module never ran");
8966 }
8967}
8968
8969#[cfg(target_os = "linux")]
8970pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
8971 use subc_cgroup::KillOutcome;
8972 match subc_cgroup::kill_module(placement, module_id) {
8973 KillOutcome::Killed => {}
8974 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
8975 debug!(
8976 module_id,
8977 "cgroup tree kill unavailable; using direct-child kill"
8978 );
8979 }
8980 KillOutcome::IoError { path, error } => {
8981 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
8982 }
8983 }
8984}
8985
8986#[cfg(windows)]
8994fn contain_spawned_child(
8995 child: &mut Child,
8996 spec: &ModuleSpec,
8997) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
8998 let assignment = || {
8999 let job = subc_jobobject::JobObject::new()?;
9000 #[cfg(test)]
9001 spawn_record_tests::assignment_failpoint(spec)?;
9002 job.assign(child)?;
9003 Ok::<_, io::Error>(job)
9004 };
9005 match assignment() {
9006 Ok(job) => Ok(Some(job)),
9007 Err(source) => {
9008 kill_and_reap_rejected_child(child);
9009 Err(SuperviseError::Spawn {
9010 program: spec.program.clone(),
9011 source: io::Error::other(format!(
9012 "Windows module job containment failed: {source}"
9013 )),
9014 cgroup_path: None,
9015 })
9016 }
9017 }
9018}
9019
9020fn kill_and_reap_rejected_child(child: &mut Child) {
9023 #[cfg(unix)]
9024 if let Some(pid) = child
9025 .id()
9026 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9027 {
9028 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9029 }
9030 let _ = child.start_kill();
9031 while matches!(child.try_wait(), Ok(None)) {
9032 std::thread::yield_now();
9033 }
9034}
9035
9036#[cfg(windows)]
9041fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9042 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9043 let _ = std::process::Command::new("taskkill.exe")
9047 .args(["/PID", &pid.to_string(), "/T", "/F"])
9048 .stdin(Stdio::null())
9049 .stdout(Stdio::null())
9050 .stderr(Stdio::null())
9051 .status();
9052 return Err(SuperviseError::Spawn {
9053 program: spec.program.clone(),
9054 source,
9055 cgroup_path: None,
9056 });
9057 }
9058 Ok(())
9059}
9060
9061#[cfg(target_os = "linux")]
9062fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9063 match placement.remove_module(module_id) {
9064 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9065 Err(error) => warn!(
9066 module_id,
9067 error = %error,
9068 "could not remove module cgroup after process exit; continuing teardown"
9069 ),
9070 }
9071}
9072
9073#[cfg(target_os = "linux")]
9074async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9075 if matches!(
9079 subc_cgroup::kill_module(Some(placement), module_id),
9080 subc_cgroup::KillOutcome::Killed
9081 ) {
9082 if let Ok(path) = placement.module_path(module_id) {
9083 while std::fs::read_to_string(path.join("cgroup.events"))
9084 .ok()
9085 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9086 {
9087 sleep(Duration::from_millis(1)).await;
9088 }
9089 }
9090 }
9091 remove_module_cgroup(placement, module_id);
9092}
9093
9094#[cfg(target_os = "linux")]
9095fn apply_cgroup_placement(
9096 command: &mut Command,
9097 spec: &ModuleSpec,
9098 path: &std::path::Path,
9099) -> Result<(), SuperviseError> {
9100 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9101 module_id: spec.module_id.clone(),
9102 source,
9103 })
9104}
9105
9106fn capture_retention(spec: &ModuleSpec) -> Retention {
9107 let defaults = Retention::default();
9108 let value = |name: &str| {
9109 spec.env
9110 .iter()
9111 .rev()
9112 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9113 };
9114 Retention {
9115 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9116 .and_then(|value| value.parse().ok())
9117 .unwrap_or(defaults.max_file_mb),
9118 keep: value(CAPTURE_KEEP_ENV)
9119 .and_then(|value| value.parse().ok())
9120 .unwrap_or(defaults.keep),
9121 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9122 .and_then(|value| value.parse().ok())
9123 .unwrap_or(defaults.max_age_days),
9124 }
9125}
9126
9127fn generate_launch_nonce() -> Result<String, SuperviseError> {
9130 let mut bytes = [0u8; 32];
9131 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9132 reason: source.to_string(),
9133 })?;
9134 let mut hex = String::with_capacity(64);
9135 for b in bytes {
9136 use std::fmt::Write;
9137 let _ = write!(hex, "{b:02x}");
9138 }
9139 Ok(hex)
9140}
9141
9142fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9145 if a.len() != b.len() {
9146 return false;
9147 }
9148 let mut diff = 0u8;
9149 for (x, y) in a.iter().zip(b.iter()) {
9150 diff |= x ^ y;
9151 }
9152 diff == 0
9153}
9154
9155#[cfg(not(windows))]
9159fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9160 let process = subc_os::Process::open(pid).ok().flatten()?;
9161 #[cfg(target_os = "linux")]
9162 return sample_after_linux_exec(
9163 || {
9164 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9165 let committed = linux_exec_committed(&stat);
9166 #[cfg(test)]
9167 spawn_image_tests::report_exec_state(committed);
9168 committed
9169 },
9170 || process.observe(),
9171 );
9172 #[cfg(not(target_os = "linux"))]
9173 process.observe()
9174}
9175
9176#[cfg(target_os = "linux")]
9180fn linux_exec_committed(stat: &str) -> Option<bool> {
9181 const PF_FORKNOEXEC: u64 = 0x40;
9182 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9183 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9184}
9185
9186#[cfg(target_os = "linux")]
9187fn sample_after_linux_exec(
9188 mut exec_committed: impl FnMut() -> Option<bool>,
9189 observe: impl FnOnce() -> Option<subc_os::Observation>,
9190) -> Option<subc_os::Observation> {
9191 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9192 while !exec_committed()? {
9193 if std::time::Instant::now() >= deadline {
9197 return None;
9198 }
9199 std::thread::sleep(Duration::from_micros(50));
9200 }
9201 observe()
9202}
9203
9204#[cfg(all(test, target_os = "linux"))]
9205mod spawn_image_tests {
9206 use super::*;
9207 use std::cell::{Cell, RefCell};
9208
9209 enum ExecEvent {
9210 Uncommitted,
9211 Image(Option<subc_os::Observation>),
9212 }
9213
9214 thread_local! {
9215 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9216 }
9217
9218 pub(super) fn report_exec_state(committed: Option<bool>) {
9219 if committed == Some(false) {
9220 EXEC_EVENTS.with(|events| {
9221 if let Some(events) = events.borrow_mut().take() {
9222 let _ = events.send(ExecEvent::Uncommitted);
9223 }
9224 });
9225 }
9226 }
9227
9228 #[test]
9229 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9230 use std::io::{Read, Write};
9231 use std::os::fd::AsRawFd;
9232
9233 let descriptor = std::fs::File::open("/dev/null").unwrap();
9234 let mut command = std::process::Command::new("/bin/sleep");
9235 command.arg("30");
9236 let (mut ready, mut release) =
9237 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9238 .unwrap();
9239 let spawn = std::thread::spawn(move || command.spawn());
9240 let mut message = [0u8; 8];
9241 ready.read_exact(&mut message).unwrap();
9242 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9243 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9244 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9245 let (events, received) = std::sync::mpsc::channel();
9246 let observer = std::thread::spawn(move || {
9247 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9248 let image = observe_spawned_image(pid);
9249 let _ = events.send(ExecEvent::Image(image));
9250 });
9251 let first = received.recv_timeout(Duration::from_secs(5));
9255 release.write_all(&[1]).unwrap();
9256 let mut child = spawn.join().unwrap().unwrap();
9257 let after = match first {
9258 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9259 Ok(ExecEvent::Image(image)) => image,
9260 _ => None,
9261 },
9262 Ok(ExecEvent::Image(image)) => image,
9263 Err(_) => None,
9264 };
9265 observer.join().unwrap();
9266 child.kill().unwrap();
9267 child.wait().unwrap();
9268
9269 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9270 assert_eq!(
9271 inherited.unwrap().executable,
9272 subc_os::file_identity(&std::env::current_exe().unwrap())
9273 );
9274 assert_eq!(
9275 after.unwrap().executable,
9276 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9277 );
9278 }
9279
9280 #[test]
9281 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9282 let committed = Cell::new(false);
9283 let inherited = subc_os::Observation {
9284 start_time: 23,
9285 executable: Some(subc_os::FileIdentity {
9286 device: 29,
9287 inode: 31,
9288 }),
9289 };
9290 let module = subc_os::Observation {
9291 start_time: 23,
9292 executable: Some(subc_os::FileIdentity {
9293 device: 29,
9294 inode: 37,
9295 }),
9296 };
9297 let polls = Cell::new(0);
9298 let image = sample_after_linux_exec(
9299 || {
9300 polls.set(polls.get() + 1);
9301 committed.set(polls.get() == 2);
9304 Some(committed.get())
9305 },
9306 || Some(if committed.get() { module } else { inherited }),
9307 );
9308 assert_eq!(image, Some(module));
9309 assert_eq!(polls.get(), 2);
9310 }
9311
9312 #[test]
9313 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9314 assert_eq!(
9315 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9316 Some(false)
9317 );
9318 assert_eq!(
9319 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9320 Some(true)
9321 );
9322 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9323 assert_eq!(linux_exec_committed("17 (a) R"), None);
9324 }
9325
9326 #[test]
9327 fn unreadable_exec_state_never_samples_an_inherited_image() {
9328 assert_eq!(
9329 sample_after_linux_exec(
9330 || None,
9331 || panic!("must not sample before exec confirmation")
9332 ),
9333 None
9334 );
9335 }
9336
9337 #[test]
9338 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9339 assert_eq!(
9340 sample_after_linux_exec(
9341 || Some(false),
9342 || panic!("must not sample after an exec timeout")
9343 ),
9344 None
9345 );
9346 }
9347}
9348
9349#[cfg(windows)]
9352fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9353 let mut program = spec.program.clone();
9354 if program.is_absolute() || program.components().count() > 1 {
9355 match subc_os::ExecutableCapture::open(&program) {
9358 Ok(capture) => return Ok(capture),
9359 Err(error)
9360 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9361 Err(error) => return Err(error),
9362 }
9363 program.set_extension("exe");
9364 return subc_os::ExecutableCapture::open(&program);
9365 }
9366 if program.extension().is_none() {
9367 program.set_extension("exe");
9368 }
9369 let custom_path = spec
9370 .env
9371 .iter()
9372 .rev()
9373 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9374 let mut directories = Vec::new();
9375 if let Some(parent) = std::env::current_exe()?.parent() {
9378 directories.push(parent.to_path_buf());
9379 }
9380 directories.push(std::env::current_dir()?);
9381 if let Some(windows) = std::env::var_os("SystemRoot") {
9382 let windows = PathBuf::from(windows);
9383 directories.push(windows.join("System32"));
9384 directories.push(windows.clone());
9385 }
9386 let path = custom_path
9387 .map(|(_, value)| std::ffi::OsString::from(value))
9388 .or_else(|| std::env::var_os("PATH"));
9389 if let Some(path) = path {
9390 directories.extend(std::env::split_paths(&path));
9391 }
9392 for directory in directories {
9393 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9394 Ok(capture) => return Ok(capture),
9395 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9396 Err(error) => return Err(error),
9397 }
9398 }
9399 Err(io::Error::new(
9400 io::ErrorKind::NotFound,
9401 "resolved executable not found",
9402 ))
9403}
9404
9405fn spawn_and_mark_running(
9406 spec: &ModuleSpec,
9407 runtime: &SupervisorRuntimeConfig,
9408 snapshot: &SharedSnapshot,
9409) -> Result<SupervisedChild, SuperviseError> {
9410 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9411 let child = spawn_child(
9412 spec,
9413 runtime.connection_file_path.as_deref(),
9414 runtime.supervisor_handle.as_ref(),
9415 &runtime.stderr_ring,
9416 runtime.capture_logs_dir.as_deref(),
9417 &runtime.child_roster,
9418 #[cfg(target_os = "linux")]
9419 runtime.cgroup_placement.as_ref(),
9420 )?;
9421 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9422 if let Some(spawn) = activity_spawn {
9423 spawn.complete(snapshot);
9424 }
9425 Ok(child)
9426}
9427
9428enum RegistrationWaitOutcome {
9429 Registered,
9430 Exited(ExitReport),
9431 TimedOut,
9432}
9433
9434struct ReloadRegistrationFailure {
9435 exit_report: ExitReport,
9436 reason: String,
9437}
9438
9439#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9440enum BusyGaugeObservation {
9441 Quiescent,
9442 Busy,
9443 Omitted,
9444}
9445
9446#[derive(Debug, Clone, PartialEq, Eq)]
9448enum GaugeVerdict {
9449 Zero,
9451 NonZero(u64),
9453 NotACount(String),
9457 Missing,
9459 ProbeFailed(&'static str),
9462}
9463
9464impl GaugeVerdict {
9465 fn reads_quiet(&self) -> bool {
9466 matches!(self, Self::Zero)
9467 }
9468}
9469
9470impl fmt::Display for GaugeVerdict {
9471 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9472 match self {
9473 Self::Zero => f.write_str("zero"),
9474 Self::NonZero(value) => write!(f, "non_zero({value})"),
9475 Self::NotACount(value) => write!(f, "not_a_count({value})"),
9476 Self::Missing => f.write_str("missing_from_health_body"),
9477 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9478 }
9479 }
9480}
9481
9482const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9485
9486fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9488 let metrics = metrics.and_then(Value::as_object);
9489 gauges
9490 .iter()
9491 .map(
9492 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9493 None => GaugeVerdict::Missing,
9494 Some(value) => match value.as_u64() {
9495 Some(0) => GaugeVerdict::Zero,
9496 Some(count) => GaugeVerdict::NonZero(count),
9497 None => {
9498 let mut text = value.to_string();
9499 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9500 let mut end = GAUGE_VALUE_TEXT_LIMIT;
9501 while !text.is_char_boundary(end) {
9502 end -= 1;
9503 }
9504 text.truncate(end);
9505 text.push_str("...");
9506 }
9507 GaugeVerdict::NotACount(text)
9508 }
9509 },
9510 },
9511 )
9512 .collect()
9513}
9514
9515fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9518 if verdicts
9519 .iter()
9520 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9521 {
9522 BusyGaugeObservation::Omitted
9523 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9524 BusyGaugeObservation::Quiescent
9525 } else {
9526 BusyGaugeObservation::Busy
9527 }
9528}
9529
9530#[derive(Debug, Clone, Default, PartialEq, Eq)]
9535struct GaugeTrace {
9536 latest: Option<(GaugeVerdict, Duration)>,
9537 last_value: Option<(GaugeVerdict, Duration)>,
9538}
9539
9540#[derive(Debug, Clone, Default, PartialEq, Eq)]
9546struct DrainGaugeTrace {
9547 probes: u32,
9549 gauges: Vec<(String, GaugeTrace)>,
9551}
9552
9553impl DrainGaugeTrace {
9554 fn new(gauges: &[String]) -> Self {
9555 Self {
9556 probes: 0,
9557 gauges: gauges
9558 .iter()
9559 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9560 .collect(),
9561 }
9562 }
9563
9564 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9566 self.probes = self.probes.saturating_add(1);
9567 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
9568 trace.latest = Some((verdict.clone(), at));
9569 if matches!(
9570 verdict,
9571 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
9572 ) {
9573 trace.last_value = Some((verdict.clone(), at));
9574 }
9575 }
9576 }
9577}
9578
9579impl fmt::Display for DrainGaugeTrace {
9584 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9585 if self.gauges.is_empty() {
9586 return f.write_str("none declared");
9587 }
9588 for (index, (name, trace)) in self.gauges.iter().enumerate() {
9589 if index > 0 {
9590 f.write_str(" | ")?;
9591 }
9592 write!(f, "{name}: ")?;
9593 match &trace.latest {
9594 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
9595 None => f.write_str("never probed")?,
9596 }
9597 match &trace.last_value {
9598 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
9599 None => f.write_str("; no value ever read")?,
9600 }
9601 }
9602 Ok(())
9603 }
9604}
9605
9606fn declared_busy_gauges(
9607 registry: &Registry,
9608 module_id: &str,
9609) -> Result<Vec<String>, SuperviseError> {
9610 busy_gauges_of(
9611 registry
9612 .get_module(module_id)
9613 .map_err(SuperviseError::Registry)?,
9614 )
9615}
9616
9617fn declared_busy_gauges_for_connection(
9621 registry: &Registry,
9622 connection_id: ConnectionId,
9623) -> Result<Vec<String>, SuperviseError> {
9624 busy_gauges_of(
9625 registry
9626 .get_module_by_connection(connection_id)
9627 .map_err(SuperviseError::Registry)?,
9628 )
9629}
9630
9631fn busy_gauges_of(
9632 registration: Option<crate::registry::ModuleRegistration>,
9633) -> Result<Vec<String>, SuperviseError> {
9634 let Some(registration) = registration else {
9635 return Ok(Vec::new());
9636 };
9637 let Some(self_signals) = registration.manifest.self_signals else {
9638 return Ok(Vec::new());
9639 };
9640
9641 let mut gauges = Vec::new();
9642 for declaration in self_signals {
9643 if declaration.kind != SelfSignalKind::Busy {
9644 continue;
9645 }
9646 match declaration.anchored_to {
9647 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9648 gauges.extend(declared)
9649 }
9650 _ => {
9651 gauges.push(String::new());
9654 }
9655 }
9656 }
9657 Ok(gauges)
9658}
9659
9660#[allow(clippy::too_many_arguments)]
9669async fn wait_for_forwarding_quiescence(
9670 forwarding: &ForwardingTable,
9671 module_id: &str,
9672 runtime: &SupervisorRuntimeConfig,
9673 endpoint: crate::ModuleEndpointId,
9674 deadline: Instant,
9675 busy_gauges: &[String],
9676 scope: DrainScope,
9677 trace: &mut DrainGaugeTrace,
9678) -> Result<bool, SuperviseError> {
9679 let wait_started = Instant::now();
9680 let mut gauges_quiescent = busy_gauges.is_empty();
9681 let mut next_probe_at = wait_started;
9682 let mut omission_reported = false;
9683
9684 loop {
9685 let now = Instant::now();
9686 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9687 let report = match scope {
9688 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9689 DrainScope::Endpoint(endpoint) => {
9690 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9691 }
9692 };
9693 let verdicts = match &report {
9694 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
9695 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
9696 };
9697 trace.record(&verdicts, wait_started.elapsed());
9698 gauges_quiescent = match report {
9699 Ok(_) => match busy_gauge_observation(&verdicts) {
9700 BusyGaugeObservation::Quiescent => true,
9701 BusyGaugeObservation::Busy => false,
9702 BusyGaugeObservation::Omitted => {
9703 if !omission_reported {
9706 forwarding
9707 .counters()
9708 .increment_drains_with_undeclared_gauge();
9709 let missing = busy_gauges
9710 .iter()
9711 .zip(&verdicts)
9712 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
9713 .map(|(gauge, _)| gauge.as_str())
9714 .collect::<Vec<_>>()
9715 .join(",");
9716 warn!(
9717 module_id,
9718 missing_gauges = %missing,
9719 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
9720 );
9721 omission_reported = true;
9722 }
9723 false
9724 }
9725 },
9726 Err(err) => {
9727 warn!(
9728 module_id,
9729 error = %err,
9730 "drain health.check did not produce declared busy gauges; treating module as busy"
9731 );
9732 false
9733 }
9734 };
9735 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9738 }
9739
9740 let in_flight = forwarding
9741 .endpoint_in_flight_count(endpoint)
9742 .map_err(SuperviseError::Forwarding)?;
9743 if in_flight == 0 && gauges_quiescent {
9744 return Ok(true);
9745 }
9746
9747 let now = Instant::now();
9748 if now >= deadline {
9749 return Ok(false);
9750 }
9751 let mut wait = deadline
9752 .saturating_duration_since(now)
9753 .min(REGISTRY_RELEASE_POLL);
9754 if !busy_gauges.is_empty() {
9755 wait = wait.min(next_probe_at.saturating_duration_since(now));
9756 }
9757 sleep(wait).await;
9758 }
9759}
9760
9761fn drain_busy_reprobe_interval() -> Duration {
9765 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9766}
9767
9768fn warn_drain_timed_out(
9783 module_id: &str,
9784 drain_timeout: Duration,
9785 reason: RouteCloseReason,
9786 holdouts: &crate::forwarding::DrainHoldouts,
9787 gauges: &DrainGaugeTrace,
9788) {
9789 warn!(
9790 module_id = %module_id,
9791 waited = ?drain_timeout,
9792 ?reason,
9793 held_requests = holdouts.requests,
9794 held_routes = holdouts.routes,
9795 total_routes = holdouts.total_routes,
9796 top_connections = ?holdouts.top_connections,
9797 held = %holdouts
9800 .held
9801 .iter()
9802 .map(|(channel, corr)| format!("{channel}:{corr}"))
9803 .collect::<Vec<_>>()
9804 .join(","),
9805 gauge_probes = gauges.probes,
9806 busy_gauges = %gauges,
9807 "route drain timed out before request quiescence; forcing teardown"
9808 );
9809}
9810
9811fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9819 match wait_result {
9820 Ok(drained) => *drained,
9821 Err(_) => false,
9822 }
9823}
9824
9825fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9826 for released in released_routes {
9827 let frame = match Frame::build_with_version(
9828 released.negotiated_ver,
9829 FrameType::Goodbye,
9830 control_flags(),
9831 released.channel,
9832 released.epoch,
9833 0,
9834 Vec::new(),
9835 ) {
9836 Ok(frame) => frame,
9837 Err(err) => {
9838 warn!(
9839 route_channel = released.channel,
9840 error = %err,
9841 "failed to build supervisor drain route GOODBYE frame"
9842 );
9843 continue;
9844 }
9845 };
9846 if !released.close_on_delivery_failure() {
9847 crate::forwarding::send_module_route_goodbye(
9848 &forwarding.counters(),
9849 &released.sink,
9850 frame,
9851 released.module_id.as_deref(),
9852 "supervisor drain",
9853 );
9854 continue;
9855 }
9856 if let Err(err) = released.sink.try_send(frame) {
9857 warn!(
9858 target_connection_id = released.connection_id.get(),
9859 route_channel = released.channel,
9860 error = %err,
9861 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9862 );
9863 let _ = forwarding.escalate_client_delivery_failure(
9864 released.connection_id,
9865 released.channel,
9866 released.epoch,
9867 CloseReason::new(
9868 "route_goodbye_delivery_failed",
9869 format!(
9870 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9871 released.channel
9872 ),
9873 ),
9874 crate::forwarding::UndeliveredFrame {
9875 module_id: released.module_id.as_deref(),
9876 sink: &released.sink,
9877 },
9878 );
9879 }
9880 }
9881}
9882
9883fn send_module_draining(
9884 module_id: &str,
9885 reason: RouteCloseReason,
9886 deadline_ms: u64,
9887 target: &ModuleDrainTarget,
9888) {
9889 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9890 reason,
9891 deadline_ms,
9892 }) {
9893 Ok(body) => body,
9894 Err(err) => {
9895 warn!(
9896 module_id,
9897 error = %err,
9898 "failed to encode module draining command"
9899 );
9900 return;
9901 }
9902 };
9903 let frame = match Frame::build_with_version(
9904 target.negotiated_ver,
9905 FrameType::Push,
9906 control_flags(),
9907 0,
9908 0,
9909 0,
9910 body,
9911 ) {
9912 Ok(frame) => frame,
9913 Err(err) => {
9914 warn!(
9915 module_id,
9916 error = %err,
9917 "failed to build module draining command frame"
9918 );
9919 return;
9920 }
9921 };
9922 if let Err(err) = target.sink.try_send(frame) {
9923 warn!(
9924 module_id,
9925 target_connection_id = target.endpoint.connection_id.get(),
9926 error = %err,
9927 "module draining command was not delivered to peer"
9928 );
9929 }
9930}
9931
9932fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9934 match Frame::build_with_version(
9935 negotiated_ver,
9936 FrameType::Goodbye,
9937 control_flags(),
9938 0,
9939 0,
9940 0,
9941 Vec::new(),
9942 ) {
9943 Ok(frame) => Some(frame),
9944 Err(err) => {
9945 warn!(
9946 module_id,
9947 error = %err,
9948 "failed to build module GOODBYE frame"
9949 );
9950 None
9951 }
9952 }
9953}
9954
9955#[cfg(unix)]
9969async fn send_module_goodbyes_for_daemon_shutdown(
9970 forwarding: &Arc<ForwardingTable>,
9971 reason: &CloseReason,
9972 wait_for_flush: bool,
9973) {
9974 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
9975 let targets = match forwarding.module_connections() {
9976 Ok(targets) => targets,
9977 Err(err) => {
9978 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
9979 return;
9980 }
9981 };
9982 let deadline = Instant::now() + GOODBYE_BUDGET;
9983 let mut sends = tokio::task::JoinSet::new();
9984 for target in targets {
9985 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
9986 continue;
9987 };
9988 if !wait_for_flush {
9989 if let Err(err) = target.sink.try_send(frame) {
9990 debug!(
9991 module_id = %target.module_id,
9992 error = %err,
9993 "shutdown module GOODBYE was not queued"
9994 );
9995 }
9996 continue;
9997 }
9998 let forwarding = Arc::clone(forwarding);
9999 let reason = reason.clone();
10000 sends.spawn(async move {
10001 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10002 Ok(Ok(())) => {}
10003 Ok(Err(err)) => debug!(
10004 module_id = %target.module_id,
10005 error = %err,
10006 "module connection closed before its shutdown GOODBYE was written"
10007 ),
10008 Err(_) => warn!(
10009 module_id = %target.module_id,
10010 budget = ?GOODBYE_BUDGET,
10011 "shutdown module GOODBYE was not written within its budget; closing anyway"
10012 ),
10013 }
10014 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10015 });
10016 }
10017 while sends.join_next().await.is_some() {}
10019}
10020
10021fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10022 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10023 return;
10024 };
10025 if let Err(err) = target.sink.try_send(frame) {
10026 warn!(
10027 module_id,
10028 target_connection_id = target.endpoint.connection_id.get(),
10029 error = %err,
10030 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10031 );
10032 forwarding.request_connection_close(
10033 target.endpoint.connection_id,
10034 CloseReason::new(
10035 "module_goodbye_delivery_failed",
10036 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10037 ),
10038 );
10039 }
10040}
10041
10042#[derive(Clone, Copy)]
10043struct ForwardingDrainContext<'a> {
10044 spec: &'a ModuleSpec,
10045 runtime: &'a SupervisorRuntimeConfig,
10046 registry: &'a Registry,
10047 scope: DrainScope,
10048}
10049
10050#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10052enum DrainScope {
10053 Active,
10056 Endpoint(crate::ModuleEndpointId),
10061}
10062
10063#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10071enum StopNotice {
10072 SentOverConnection,
10075 NoConnection,
10079 NotSent,
10083}
10084
10085async fn begin_forwarding_drain(
10086 spec: &ModuleSpec,
10087 runtime: &SupervisorRuntimeConfig,
10088 registry: &Registry,
10089 snapshot: &SharedSnapshot,
10090 enabled: Option<bool>,
10091 reason: RouteCloseReason,
10092) -> Result<StopNotice, SuperviseError> {
10093 let Some(forwarding) = runtime.forwarding.as_ref() else {
10094 return Err(SuperviseError::ReloadUnavailable {
10095 module_id: spec.module_id.clone(),
10096 reason: "supervisor was not configured with a forwarding table".to_string(),
10097 });
10098 };
10099
10100 begin_forwarding_drain_with(
10101 forwarding,
10102 ForwardingDrainContext {
10103 spec,
10104 runtime,
10105 registry,
10106 scope: DrainScope::Active,
10107 },
10108 snapshot,
10109 enabled,
10110 reason,
10111 runtime.drain_timeout,
10112 )
10113 .await
10114}
10115
10116async fn begin_forwarding_drain_if_configured(
10117 spec: &ModuleSpec,
10118 runtime: &SupervisorRuntimeConfig,
10119 registry: &Registry,
10120 snapshot: &SharedSnapshot,
10121 enabled: Option<bool>,
10122 reason: RouteCloseReason,
10123) -> Result<StopNotice, SuperviseError> {
10124 begin_forwarding_drain_with_timeout(
10125 spec,
10126 runtime,
10127 registry,
10128 snapshot,
10129 enabled,
10130 reason,
10131 runtime.drain_timeout,
10132 )
10133 .await
10134}
10135
10136async fn begin_forwarding_drain_with_timeout(
10140 spec: &ModuleSpec,
10141 runtime: &SupervisorRuntimeConfig,
10142 registry: &Registry,
10143 snapshot: &SharedSnapshot,
10144 enabled: Option<bool>,
10145 reason: RouteCloseReason,
10146 drain_timeout: Duration,
10147) -> Result<StopNotice, SuperviseError> {
10148 let Some(forwarding) = runtime.forwarding.as_ref() else {
10149 return Ok(StopNotice::NotSent);
10150 };
10151
10152 begin_forwarding_drain_with(
10153 forwarding,
10154 ForwardingDrainContext {
10155 spec,
10156 runtime,
10157 registry,
10158 scope: DrainScope::Active,
10159 },
10160 snapshot,
10161 enabled,
10162 reason,
10163 drain_timeout,
10164 )
10165 .await
10166}
10167
10168async fn begin_forwarding_drain_with(
10169 forwarding: &ForwardingTable,
10170 context: ForwardingDrainContext<'_>,
10171 snapshot: &SharedSnapshot,
10172 enabled: Option<bool>,
10173 reason: RouteCloseReason,
10174 drain_timeout: Duration,
10175) -> Result<StopNotice, SuperviseError> {
10176 let ForwardingDrainContext {
10177 spec,
10178 runtime,
10179 registry,
10180 scope,
10181 } = context;
10182 debug_assert_ne!(reason, RouteCloseReason::Crash);
10183 let terminal = matches!(reason, RouteCloseReason::Disable);
10184 let drain_started_at = Instant::now();
10185 let drain_deadline = drain_started_at + drain_timeout;
10186 let deadline_ms =
10187 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10188 let busy_gauges = match scope {
10189 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10190 DrainScope::Endpoint(endpoint) => {
10191 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10192 }
10193 };
10194
10195 let gate_started = Instant::now();
10198 let drain_target = match scope {
10199 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10200 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10201 }
10202 .map_err(SuperviseError::Forwarding)?;
10203 info!(
10208 module_id = %spec.module_id,
10209 ?reason,
10210 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10211 connected = drain_target.is_some(),
10212 "module drain began; route admission closed"
10213 );
10214 if scope == DrainScope::Active {
10215 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10216 state.state = ModuleState::Draining;
10217 state.draining_to_replace =
10218 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10219 if let Some(enabled) = enabled {
10220 state.enabled = enabled;
10221 }
10222 })?;
10223 }
10224
10225 let Some(target) = drain_target.as_ref() else {
10226 return Ok(StopNotice::NoConnection);
10230 };
10231 {
10232 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10233 #[cfg(feature = "test-support")]
10234 forwarding.wait_before_route_closing_for_test().await;
10235 let routes = forwarding
10236 .endpoint_routes(target.endpoint)
10237 .map_err(SuperviseError::Forwarding)?;
10238 let routes_notified = routes.len();
10239 crate::control::send_route_control_pushes(
10240 forwarding,
10241 routes.clone(),
10242 ClientControlPush::RouteClosing {
10243 module_id: spec.module_id.clone(),
10244 channels: Vec::new(),
10245 reason,
10246 },
10247 );
10248 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10249
10250 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10256 let wait_result = wait_for_forwarding_quiescence(
10257 forwarding,
10258 &spec.module_id,
10259 runtime,
10260 target.endpoint,
10261 drain_deadline,
10262 &busy_gauges,
10263 scope,
10264 &mut gauge_trace,
10265 )
10266 .await;
10267 let drained = drained_after_quiescence_wait(&wait_result);
10268 if let Err(err) = &wait_result {
10269 error!(
10270 module_id = %spec.module_id,
10271 ?reason,
10272 error = %err,
10273 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10274 );
10275 } else if !drained {
10276 let holdouts = forwarding
10278 .endpoint_drain_holdouts(target.endpoint)
10279 .unwrap_or_default();
10280 warn_drain_timed_out(
10281 &spec.module_id,
10282 drain_timeout,
10283 reason,
10284 &holdouts,
10285 &gauge_trace,
10286 );
10287 }
10288 crate::control::send_route_control_pushes(
10289 forwarding,
10290 routes,
10291 ClientControlPush::RouteClosed {
10292 module_id: spec.module_id.clone(),
10293 channels: Vec::new(),
10294 reason,
10295 drained,
10296 abandoned: target.abandoned_bindings.len() as u32,
10297 excluded_subscriptions: target.excluded_subscriptions,
10298 terminal: Some(terminal),
10299 },
10300 );
10301 wait_result?;
10302
10303 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10309 Ok(routes) => routes,
10310 Err(err) => {
10311 warn!(
10312 module_id = %spec.module_id,
10313 ?reason,
10314 error = %err,
10315 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10316 );
10317 send_module_goodbye(&spec.module_id, forwarding, target);
10318 return Err(SuperviseError::Forwarding(err));
10319 }
10320 };
10321 let route_goodbye_count = released_routes.len();
10322 send_route_goodbyes(forwarding, released_routes);
10323 send_module_goodbye(&spec.module_id, forwarding, target);
10324
10325 info!(
10331 module_id = %spec.module_id,
10332 ?reason,
10333 routes_notified,
10334 route_goodbyes = route_goodbye_count,
10335 abandoned_reservations = target.abandoned_bindings.len(),
10336 excluded_subscriptions = target.excluded_subscriptions,
10337 drained,
10338 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10339 );
10340 }
10341
10342 Ok(StopNotice::SentOverConnection)
10343}
10344
10345async fn wait_for_registration_after_reload(
10348 registry: &Registry,
10349 module_id: &str,
10350 snapshot: &SharedSnapshot,
10351 child: &mut SupervisedChild,
10352 wait: Duration,
10353) -> Result<RegistrationWaitOutcome, SuperviseError> {
10354 wait_for_slot_registration(
10355 registry,
10356 crate::registry::RegistrationSlot::Active(module_id),
10357 module_id,
10358 snapshot,
10359 child,
10360 wait,
10361 )
10362 .await
10363}
10364
10365async fn wait_for_slot_registration(
10373 registry: &Registry,
10374 slot: crate::registry::RegistrationSlot<'_>,
10375 module_id: &str,
10376 snapshot: &SharedSnapshot,
10377 child: &mut SupervisedChild,
10378 wait: Duration,
10379) -> Result<RegistrationWaitOutcome, SuperviseError> {
10380 let deadline = Instant::now() + wait;
10381 loop {
10382 if registry
10383 .registration(slot)
10384 .map_err(SuperviseError::Registry)?
10385 .is_some()
10386 {
10387 return Ok(RegistrationWaitOutcome::Registered);
10388 }
10389
10390 let now = Instant::now();
10391 if now >= deadline {
10392 return Ok(RegistrationWaitOutcome::TimedOut);
10393 }
10394 let remaining = deadline.saturating_duration_since(now);
10395 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10396
10397 tokio::select! {
10398 wait_result = child.wait() => {
10399 let status = wait_result.map_err(|source| SuperviseError::Wait {
10400 module_id: module_id.to_string(),
10401 source,
10402 })?;
10403 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10404 snapshot,
10405 child,
10406 &status,
10407 )));
10408 }
10409 _ = sleep(poll) => {}
10410 }
10411 }
10412}
10413
10414fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10415 if exit_report.kind != ExitKind::DeliberateSeverance {
10418 exit_report.kind = ExitKind::Crash;
10419 }
10420 exit_report
10421}
10422
10423async fn handle_reload_child_registration_failure(
10424 spec: &ModuleSpec,
10425 runtime: &SupervisorRuntimeConfig,
10426 registry: &Registry,
10427 process_liveness: &SupervisorProcessLiveness,
10428 snapshot: &SharedSnapshot,
10429 _child: &mut Option<SupervisedChild>,
10430 failure: ReloadRegistrationFailure,
10431) -> Result<(), SuperviseError> {
10432 let ReloadRegistrationFailure {
10433 exit_report,
10434 reason,
10435 } = failure;
10436 match on_child_exit(
10437 spec,
10438 runtime.restart_policy,
10439 registry,
10440 snapshot,
10441 &runtime.terminal_ring,
10442 &runtime.spawn_events,
10443 &runtime.child_roster,
10444 exit_report,
10445 )
10446 .await
10447 {
10448 NextAction::Stop {
10449 registration_released,
10450 } => {
10451 if registration_released {
10452 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10453 }
10454 }
10455 NextAction::Restart { schedule } => {
10456 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10457 schedule.delay
10458 });
10459 if let Some(schedule) = schedule {
10460 log_crash_respawn(&spec.module_id, schedule);
10461 }
10462 schedule_respawn(
10463 runtime,
10464 snapshot,
10465 &spec.module_id,
10466 delay,
10467 RespawnKind::Spawn,
10468 )?;
10469 }
10470 }
10471 Err(SuperviseError::ReloadFailed {
10472 module_id: spec.module_id.clone(),
10473 reason,
10474 })
10475}
10476
10477async fn handle_reload_spawn_failure(
10478 spec: &ModuleSpec,
10479 runtime: &SupervisorRuntimeConfig,
10480 process_liveness: &SupervisorProcessLiveness,
10481 snapshot: &SharedSnapshot,
10482 _child: &mut Option<SupervisedChild>,
10483 reason: String,
10484) -> Result<(), SuperviseError> {
10485 let now = Instant::now();
10486 let mut schedule = None;
10487 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10488 clear_current_process_facts(state);
10489 if state.enabled {
10490 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10491 state.state = if schedule.is_some() {
10492 ModuleState::Restarting
10493 } else {
10494 ModuleState::Failed
10495 };
10496 } else {
10497 state.state = ModuleState::Disabled;
10498 }
10499 })?;
10500 if let Some(schedule) = schedule {
10501 schedule_respawn(
10502 runtime,
10503 snapshot,
10504 &spec.module_id,
10505 schedule.delay,
10506 RespawnKind::Spawn,
10507 )?;
10508 } else {
10509 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10510 }
10511 Err(SuperviseError::ReloadFailed {
10512 module_id: spec.module_id.clone(),
10513 reason,
10514 })
10515}
10516
10517fn control_flags() -> Flags {
10518 Flags::new(false, Priority::Passive, false)
10519}
10520
10521#[allow(clippy::too_many_arguments)]
10522async fn drain_optional_child(
10523 module_id: &str,
10524 protocol: ModuleProtocol,
10525 stop_notice: StopNotice,
10526 registry: &Registry,
10527 forwarding: Option<&ForwardingTable>,
10528 snapshot: &SharedSnapshot,
10529 terminal_ring: &Arc<Mutex<TerminalRing>>,
10530 spawn_events: &SpawnEventFeed,
10531 child: &mut Option<SupervisedChild>,
10532 drain_timeout: Duration,
10533 final_state: ModuleState,
10534 enabled: Option<bool>,
10535) -> Result<(), SuperviseError> {
10536 if let Some(child) = child.take() {
10537 drain_child_to_state(
10538 module_id,
10539 protocol,
10540 stop_notice,
10541 registry,
10542 forwarding,
10543 snapshot,
10544 terminal_ring,
10545 spawn_events,
10546 child,
10547 drain_timeout,
10548 final_state,
10549 enabled,
10550 )
10551 .await
10552 } else {
10553 update_snapshot(snapshot, Some(module_id), |state| {
10554 state.state = final_state;
10555 if let Some(enabled) = enabled {
10556 state.enabled = enabled;
10557 }
10558 clear_current_process_facts(state);
10559 })?;
10560 release_dead_registration(registry, forwarding, snapshot, module_id).await
10561 }
10562}
10563
10564#[allow(clippy::too_many_arguments)]
10565async fn drain_child_to_state(
10566 module_id: &str,
10567 _protocol: ModuleProtocol,
10568 stop_notice: StopNotice,
10569 registry: &Registry,
10570 forwarding: Option<&ForwardingTable>,
10571 snapshot: &SharedSnapshot,
10572 terminal_ring: &Arc<Mutex<TerminalRing>>,
10573 spawn_events: &SpawnEventFeed,
10574 mut child: SupervisedChild,
10575 drain_timeout: Duration,
10576 final_state: ModuleState,
10577 enabled: Option<bool>,
10578) -> Result<(), SuperviseError> {
10579 let protocol = child.protocol;
10580 update_snapshot(snapshot, Some(module_id), |state| {
10581 state.state = ModuleState::Draining;
10582 state.draining_to_replace = final_state == ModuleState::Restarting;
10583 if let Some(enabled) = enabled {
10584 state.enabled = enabled;
10585 }
10586 })?;
10587
10588 if stop_notice != StopNotice::SentOverConnection {
10599 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10600 info!(
10601 module_id,
10602 pid = child.pid,
10603 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10604 "module has no connection yet; requesting stop by signal"
10605 );
10606 }
10607 request_graceful_stop(module_id, &child);
10608 }
10609
10610 let exit_report = match timeout(drain_timeout, child.wait()).await {
10611 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10612 Ok(Err(source)) => {
10613 fail_after_wait_error(
10614 module_id,
10615 snapshot,
10616 terminal_ring,
10617 spawn_events,
10618 format!("failed to wait during drain: {source}"),
10619 );
10620 return Err(SuperviseError::Wait {
10621 module_id: module_id.to_string(),
10622 source,
10623 });
10624 }
10625 Err(_) => {
10626 warn!(
10639 module_id,
10640 pid = child.pid,
10641 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10642 reason = ?final_state,
10643 ?stop_notice,
10644 "drain budget expired before the module exited; killing it"
10645 );
10646 child.start_kill().map_err(|source| {
10647 fail_after_wait_error(
10648 module_id,
10649 snapshot,
10650 terminal_ring,
10651 spawn_events,
10652 format!("failed to kill during drain: {source}"),
10653 );
10654 SuperviseError::Kill {
10655 module_id: module_id.to_string(),
10656 source,
10657 }
10658 })?;
10659 let status = child.wait().await.map_err(|source| {
10660 fail_after_wait_error(
10661 module_id,
10662 snapshot,
10663 terminal_ring,
10664 spawn_events,
10665 format!("failed to wait after drain kill: {source}"),
10666 );
10667 SuperviseError::Wait {
10668 module_id: module_id.to_string(),
10669 source,
10670 }
10671 })?;
10672 classify_reaped_child_exit(snapshot, &child, &status)
10673 }
10674 };
10675
10676 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10677 publish_after_terminal_record(
10678 module_id,
10679 || {
10680 record_terminal_with_detail(
10681 module_id,
10682 terminal_ring,
10683 spawn_events,
10684 &exit_report,
10685 terminal_disposition(final_state),
10686 detail,
10687 )
10688 },
10689 || {
10690 update_snapshot(snapshot, Some(module_id), |state| {
10691 state.state = final_state;
10692 if let Some(enabled) = enabled {
10693 state.enabled = enabled;
10694 }
10695 clear_current_process_facts(state);
10696 state.last_exit = Some(exit_report.clone());
10697 if exit_report.kind == ExitKind::DeliberateSeverance {
10698 state.lifetime_restarts += 1;
10699 }
10700 })
10701 },
10702 )?;
10703 child.drain_stderr(module_id).await;
10704
10705 release_dead_registration(registry, forwarding, snapshot, module_id).await
10706}
10707
10708#[cfg(unix)]
10728fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10729 let Some(pid) = child
10730 .id()
10731 .and_then(|pid| i32::try_from(pid).ok())
10732 .and_then(rustix::process::Pid::from_raw)
10733 else {
10734 debug!(
10735 module_id,
10736 "no pid to signal for teardown; falling through to the drain wait"
10737 );
10738 return;
10739 };
10740 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10741 Ok(()) => debug!(
10742 module_id,
10743 "sent SIGTERM to a module nothing else asked to stop"
10744 ),
10745 Err(err) => debug!(
10746 module_id,
10747 error = %err,
10748 "SIGTERM to module failed; the drain wait and kill still apply"
10749 ),
10750 }
10751}
10752
10753#[cfg(not(unix))]
10761fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10762 debug!(
10763 module_id,
10764 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10765 );
10766}
10767
10768fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10769 match final_state {
10770 ModuleState::Stopped => TerminalDisposition::Stopped,
10771 ModuleState::Disabled => TerminalDisposition::Disabled,
10772 ModuleState::Restarting => TerminalDisposition::Restarting,
10773 ModuleState::Failed => TerminalDisposition::Failed,
10774 ModuleState::Starting
10775 | ModuleState::Running
10776 | ModuleState::Unresponsive
10777 | ModuleState::Draining => {
10778 unreachable!("terminal exits only finish in terminal or restarting states")
10779 }
10780 }
10781}
10782
10783async fn release_dead_registration(
10791 registry: &Registry,
10792 forwarding: Option<&ForwardingTable>,
10793 snapshot: &SharedSnapshot,
10794 module_id: &str,
10795) -> Result<(), SuperviseError> {
10796 let result = async {
10797 let registration = registry
10798 .get_module(module_id)
10799 .map_err(SuperviseError::Registry)?;
10800 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10801 Ok(()) => return Ok(()),
10802 Err(SuperviseError::RegistrationStillActive { .. }) => {}
10803 Err(err) => return Err(err),
10804 }
10805 let pid = lock_snapshot(snapshot)?.reaped_pid;
10806 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10807 warn!(
10808 module_id,
10809 pid,
10810 connection_id = registration.connection_id.get(),
10811 "reaped module registration outlived release grace; closing dead connection"
10812 );
10813 forwarding.request_connection_close(
10814 registration.connection_id,
10815 CloseReason::new(
10816 "supervised_process_reaped",
10817 format!("module '{module_id}' pid {pid} exited"),
10818 ),
10819 );
10820 wait_for_slot_registration_release(
10821 registry,
10822 crate::registry::RegistrationSlot::Connection(registration.connection_id),
10823 REGISTRY_RELEASE_TIMEOUT,
10824 )
10825 .await?;
10826 }
10827 wait_for_registration_release(registry, module_id, Duration::ZERO).await
10828 }
10829 .await;
10830 if let Err(err) = &result {
10831 fail_snapshot(snapshot, Some(module_id), None);
10832 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10833 }
10834 result
10835}
10836
10837async fn wait_for_registration_release(
10840 registry: &Registry,
10841 module_id: &str,
10842 wait: Duration,
10843) -> Result<(), SuperviseError> {
10844 wait_for_slot_registration_release(
10845 registry,
10846 crate::registry::RegistrationSlot::Active(module_id),
10847 wait,
10848 )
10849 .await
10850}
10851
10852async fn wait_for_slot_registration_release(
10860 registry: &Registry,
10861 slot: crate::registry::RegistrationSlot<'_>,
10862 wait: Duration,
10863) -> Result<(), SuperviseError> {
10864 let deadline = Instant::now() + wait;
10865 let mut release_events = registration_release_events().subscribe();
10866 let still_active = |registration: &crate::registry::ModuleRegistration| {
10867 SuperviseError::RegistrationStillActive {
10868 module_id: registration.manifest.module_id.clone(),
10869 waited: wait,
10870 }
10871 };
10872 loop {
10873 let _observed_generation = *release_events.borrow_and_update();
10874 let Some(registration) = registry
10875 .registration(slot)
10876 .map_err(SuperviseError::Registry)?
10877 else {
10878 return Ok(());
10879 };
10880
10881 let now = Instant::now();
10882 if now >= deadline {
10883 return Err(still_active(®istration));
10884 }
10885
10886 let remaining = deadline.saturating_duration_since(now);
10887 match timeout(remaining, release_events.changed()).await {
10888 Ok(Ok(())) | Ok(Err(_)) => {}
10889 Err(_) => return Err(still_active(®istration)),
10890 }
10891 }
10892}
10893
10894#[cfg(test)]
10895mod slot_registration_wait_tests {
10896 use super::*;
10897 use crate::registry::{ConnectionId, RegistrationSlot};
10898 use subc_protocol::manifest::ModuleManifest;
10899
10900 #[tokio::test]
10901 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10902 let registry = Arc::new(Registry::default());
10903 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
10904 let runtime = supervisor.runtime_config();
10905 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10906 let spec = ModuleSpec {
10907 module_id: "enable-stale-registration".to_string(),
10908 program: PathBuf::from("/missing/enable-retry-test"),
10909 args: Vec::new(),
10910 env: Vec::new(),
10911 reserved: false,
10912 reserved_prefixes: Vec::new(),
10913 protocol: ModuleProtocol::Subc,
10914 overlap: Default::default(),
10915 };
10916 let connection = ConnectionId::new(90);
10917 registry
10918 .register_with_control_ops(
10919 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10920 1,
10921 connection,
10922 Vec::new(),
10923 )
10924 .unwrap();
10925 let mut child = None;
10926 let err = set_child_enabled(
10927 &spec,
10928 &runtime,
10929 ®istry,
10930 &supervisor.process_liveness,
10931 &snapshot,
10932 &mut child,
10933 true,
10934 )
10935 .await
10936 .unwrap_err();
10937 assert!(matches!(
10938 err,
10939 SuperviseError::RegistrationStillActive { .. }
10940 ));
10941 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10942 assert!(child.is_none());
10943 registry.deregister_connection(connection).unwrap();
10944 let err = set_child_enabled(
10945 &spec,
10946 &runtime,
10947 ®istry,
10948 &supervisor.process_liveness,
10949 &snapshot,
10950 &mut child,
10951 true,
10952 )
10953 .await
10954 .unwrap_err();
10955 assert!(
10956 matches!(err, SuperviseError::Spawn { .. }),
10957 "second enable must attempt a spawn: {err}"
10958 );
10959 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10960 }
10961
10962 const INCUMBENT: u64 = 1;
10963 const CANDIDATE: u64 = 2;
10964
10965 fn swapped_registry() -> Arc<Registry> {
10966 let registry = Arc::new(Registry::default());
10967 let manifest = ModuleManifest::builder("m", "0.1.0").build();
10968 registry
10969 .register_with_control_ops(
10970 manifest.clone(),
10971 1,
10972 ConnectionId::new(INCUMBENT),
10973 Vec::new(),
10974 )
10975 .unwrap();
10976 registry
10977 .register_candidate_with_control_ops(
10978 manifest,
10979 1,
10980 ConnectionId::new(CANDIDATE),
10981 Vec::new(),
10982 )
10983 .unwrap();
10984 registry
10985 }
10986
10987 #[tokio::test]
10991 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
10992 let registry = swapped_registry();
10993 registry.promote_candidate("m").unwrap().unwrap();
10994
10995 assert!(matches!(
10996 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
10997 Err(SuperviseError::RegistrationStillActive { .. })
10998 ));
10999
11000 assert!(matches!(
11002 wait_for_slot_registration_release(
11003 ®istry,
11004 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11005 Duration::from_millis(50),
11006 )
11007 .await,
11008 Err(SuperviseError::RegistrationStillActive { .. })
11009 ));
11010
11011 let releaser = Arc::clone(®istry);
11012 let release = tokio::spawn(async move {
11013 sleep(Duration::from_millis(20)).await;
11014 releaser
11015 .deregister_connection(ConnectionId::new(INCUMBENT))
11016 .unwrap();
11017 notify_registration_release();
11018 });
11019 wait_for_slot_registration_release(
11020 ®istry,
11021 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11022 Duration::from_secs(5),
11023 )
11024 .await
11025 .expect("the incumbent's own registration is released");
11026 release.await.unwrap();
11027 assert!(registry.get_module("m").unwrap().is_some());
11028 }
11029
11030 #[tokio::test]
11033 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11034 let registry = swapped_registry();
11035 assert!(matches!(
11036 wait_for_slot_registration_release(
11037 ®istry,
11038 RegistrationSlot::Candidate("m"),
11039 Duration::from_millis(50),
11040 )
11041 .await,
11042 Err(SuperviseError::RegistrationStillActive { .. })
11043 ));
11044 registry
11045 .deregister_connection(ConnectionId::new(CANDIDATE))
11046 .unwrap();
11047 wait_for_slot_registration_release(
11048 ®istry,
11049 RegistrationSlot::Candidate("m"),
11050 Duration::from_millis(50),
11051 )
11052 .await
11053 .expect("a candidate slot with no candidate is released");
11054 assert!(registry
11055 .registration(RegistrationSlot::Active("m"))
11056 .unwrap()
11057 .is_some());
11058 }
11059}
11060
11061fn classify_exit(status: &ExitStatus) -> ExitReport {
11062 ExitReport {
11063 kind: if status.success() {
11064 ExitKind::Clean
11065 } else {
11066 ExitKind::Crash
11067 },
11068 code: status.code(),
11069 signal: exit_signal(status),
11070 at_ms: unix_ms_now(),
11071 }
11072}
11073
11074fn wait_error_exit_report() -> ExitReport {
11080 ExitReport {
11081 kind: ExitKind::Crash,
11082 code: None,
11083 signal: None,
11084 at_ms: unix_ms_now(),
11085 }
11086}
11087
11088#[cfg(unix)]
11089fn exit_signal(status: &ExitStatus) -> Option<i32> {
11090 use std::os::unix::process::ExitStatusExt;
11091
11092 status.signal()
11093}
11094
11095#[cfg(not(unix))]
11096fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11097 None
11098}
11099
11100fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11106 update_snapshot(snapshot, Some(module_id), |state| {
11107 state.clear_crash_restarts();
11108 })
11109}
11110
11111fn set_running(
11112 snapshot: &SharedSnapshot,
11113 child: &SupervisedChild,
11114 module_id: &str,
11115 spawn_events: &SpawnEventFeed,
11116) -> Result<(), SuperviseError> {
11117 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11118 module_id: Some(module_id.to_string()),
11119 })?;
11120 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11121 if std::mem::take(&mut state.coalesced_restart_pending) {
11122 let generation = state.spawn_generation;
11123 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11124 }
11125 state.drain_disposition_detail = None;
11126 state.spawn_failure = None;
11127 state.waiting_for_previous_pid = None;
11128 state.waiting_for_previous_detail = None;
11129 state.in_alternate_slot = false;
11132 state.configuration_updated_since_spawn = false;
11133 state.spawned_protocol = Some(child.protocol);
11134 state.state = ModuleState::Running;
11135 state.enabled = true;
11136 state.process_alive = true;
11137 state.pid = child.id();
11138 #[cfg(windows)]
11139 {
11140 state.pipe_consumed = child
11141 .nonce_delivery
11142 .as_ref()
11143 .map(|delivery| delivery.consumed());
11144 }
11145 #[cfg(target_os = "macos")]
11146 {
11147 state.report_ready = Some(Arc::clone(&child.report_ready));
11148 }
11149 state.spawned_at_ms = Some(child.spawned_at_ms);
11150 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11151 state.spawned_from = Some(child.spawned_from.clone());
11152 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11153 state.spawned_file_identity = child.spawned_file_identity.clone();
11154 state.process_start_time = child.process_start_time;
11155 Ok(())
11156}
11157
11158fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11159 state.waiting_for_previous_pid = None;
11160 state.waiting_for_previous_detail = None;
11161 state.process_alive = false;
11162 state.spawned_protocol = None;
11163 state.pid = None;
11164 #[cfg(windows)]
11165 {
11166 state.pipe_consumed = None;
11167 }
11168 #[cfg(target_os = "macos")]
11169 {
11170 state.report_ready = None;
11171 }
11172 state.spawned_at_ms = None;
11173 state.exec_outcome = None;
11174 state.spawned_from = None;
11175 state.launch_spec_digest = None;
11176 state.spawned_file_identity = None;
11177 state.process_start_time = None;
11178 state.deliberate_severance = None;
11179}
11180
11181#[cfg(test)]
11182fn record_deliberate_severance(
11183 snapshot: &SharedSnapshot,
11184 identity: ProcessIdentity,
11185) -> Result<(), SuperviseError> {
11186 update_snapshot(snapshot, None, |state| {
11187 state.deliberate_severance = Some(identity);
11188 })
11189}
11190
11191fn apply_deliberate_severance_marker(
11192 snapshot: &SharedSnapshot,
11193 exited_identity: Option<ProcessIdentity>,
11194 mut exit_report: ExitReport,
11195) -> ExitReport {
11196 let marker = lock_snapshot(snapshot)
11197 .ok()
11198 .and_then(|mut state| state.deliberate_severance.take());
11199 if marker.is_some() && marker == exited_identity {
11200 exit_report.kind = ExitKind::DeliberateSeverance;
11201 }
11202 exit_report
11203}
11204
11205fn classify_reaped_child_exit(
11206 snapshot: &SharedSnapshot,
11207 child: &SupervisedChild,
11208 status: &ExitStatus,
11209) -> ExitReport {
11210 let _ = update_snapshot(snapshot, None, |state| {
11211 state.reaped_pid = Some(child.pid);
11212 state.spawn_failure = child.spawn_failure.clone();
11213 });
11214 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11215}
11216
11217fn fail_snapshot(
11218 snapshot: &SharedSnapshot,
11219 module_id: Option<&str>,
11220 last_exit: Option<ExitReport>,
11221) {
11222 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11223 state.state = ModuleState::Failed;
11224 clear_current_process_facts(state);
11225 if let Some(last_exit) = last_exit {
11226 state.last_exit = Some(last_exit);
11227 }
11228 }) {
11229 error!(error = %err, "failed to mark supervisor state failed");
11230 }
11231}
11232
11233fn update_snapshot(
11234 snapshot: &SharedSnapshot,
11235 module_id: Option<&str>,
11236 update: impl FnOnce(&mut SupervisorSnapshot),
11237) -> Result<(), SuperviseError> {
11238 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11239 module_id: module_id.map(ToOwned::to_owned),
11240 })?;
11241 update(&mut state);
11242 Ok(())
11243}
11244
11245const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11246
11247fn lock_snapshot_for_control<'a>(
11248 snapshot: &'a SharedSnapshot,
11249 module_id: &str,
11250 caller: &'static str,
11251) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11252 let started_at = Instant::now();
11253 let guard = lock_snapshot(snapshot)?;
11254 let waited = started_at.elapsed();
11255 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11256 warn!(
11257 module_id = %module_id,
11258 waited_ms = waited.as_millis() as u64,
11259 caller = %caller,
11260 "slow snapshot lock"
11261 );
11262 }
11263 Ok(guard)
11264}
11265
11266fn lock_snapshot(
11267 snapshot: &SharedSnapshot,
11268) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11269 snapshot
11270 .lock()
11271 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11272}
11273
11274#[cfg(test)]
11275mod terminal_history_tests {
11276 use std::{
11277 path::PathBuf,
11278 sync::Arc,
11279 time::{Duration, Instant},
11280 };
11281
11282 use tokio::{sync::mpsc, time::sleep};
11283
11284 use super::{
11285 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11286 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11287 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11288 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11289 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11290 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11291 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11292 SupervisorHealthStatus, SupervisorSnapshot,
11293 };
11294 use super::Instant as ClockInstant;
11299 use crate::{
11300 registry::Registry,
11301 terminal_ring::{TerminalRing, TerminalRingConfig},
11302 };
11303 use std::sync::Mutex;
11304 use subc_control::TerminalDisposition;
11305
11306 pub(super) fn fake_aft_stub_path() -> PathBuf {
11311 let mut path = std::env::current_exe().expect("current_exe available in tests");
11312 path.pop();
11313 path.pop();
11314 path.push(if cfg!(windows) {
11315 "fake-aft-stub.exe"
11316 } else {
11317 "fake-aft-stub"
11318 });
11319 assert!(
11320 path.exists(),
11321 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11322 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11323 path.display()
11324 );
11325 path
11326 }
11327
11328 #[test]
11329 fn reserved_never_spawned_refuses_every_hello() {
11330 let supervisor = SupervisorHandle::default();
11335 supervisor.apply_identity_configuration(&ModuleSpec {
11336 module_id: "never-spawned".to_string(),
11337 program: PathBuf::from("/usr/bin/false"),
11338 args: Vec::new(),
11339 env: Vec::new(),
11340 reserved: true,
11341 reserved_prefixes: Vec::new(),
11342 protocol: ModuleProtocol::Subc,
11343 overlap: Default::default(),
11344 });
11345 assert!(
11346 supervisor
11347 .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11348 .is_some(),
11349 "forged nonce must refuse on a reserved never-spawned id"
11350 );
11351 assert!(
11352 supervisor
11353 .reserved_hello_rejection("never-spawned", None)
11354 .is_some(),
11355 "absent nonce must refuse on a reserved never-spawned id"
11356 );
11357 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11359 supervisor.apply_identity_configuration(&ModuleSpec {
11360 module_id: "never-spawned".to_string(),
11361 program: PathBuf::from("/usr/bin/false"),
11362 args: Vec::new(),
11363 env: Vec::new(),
11364 reserved: true,
11365 reserved_prefixes: Vec::new(),
11366 protocol: ModuleProtocol::Subc,
11367 overlap: Default::default(),
11368 });
11369 assert!(supervisor
11370 .reserved_hello_rejection("never-spawned", Some("minted"))
11371 .is_none());
11372 assert!(supervisor
11373 .reserved_hello_rejection("never-spawned", Some("forged"))
11374 .is_some());
11375 }
11376
11377 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11380 let now = ClockInstant::now();
11381 for _ in 0..count {
11382 state.crash_restarts.push_back(now);
11383 }
11384 }
11385
11386 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11390 let aged = state
11391 .crash_restarts
11392 .front()
11393 .expect("a crash restart must be recorded before it can be aged")
11394 .checked_sub(window + Duration::from_secs(1))
11395 .expect("the test clock is far enough from its origin to age an instant");
11396 state.crash_restarts[0] = aged;
11397 }
11398
11399 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11400 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11401 seed_crash_restarts(&mut state, count);
11402 state
11403 }
11404
11405 #[test]
11406 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11407 let policy = RestartPolicy::new(3, Duration::ZERO);
11408 let now = ClockInstant::now();
11409 assert!(daemon_will_restart(
11410 &mut snapshot_with_restarts(true, 2),
11411 &policy,
11412 now
11413 ));
11414 assert!(!daemon_will_restart(
11415 &mut snapshot_with_restarts(true, 3),
11416 &policy,
11417 now
11418 ));
11419 assert!(!daemon_will_restart(
11420 &mut snapshot_with_restarts(false, 0),
11421 &policy,
11422 now
11423 ));
11424 }
11425
11426 #[test]
11427 fn crash_restart_backoff_escalates_with_in_window_count() {
11428 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11429 .with_max_backoff(Duration::from_secs(30));
11430 let now = ClockInstant::now();
11431 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11432 let schedules = (0..4)
11433 .map(|_| {
11434 state
11435 .next_crash_restart(&policy, now)
11436 .expect("the test policy allows four crash restarts")
11437 })
11438 .collect::<Vec<_>>();
11439
11440 assert_eq!(
11441 schedules
11442 .iter()
11443 .map(|schedule| schedule.restart_in_window)
11444 .collect::<Vec<_>>(),
11445 vec![0, 1, 2, 3]
11446 );
11447 assert_eq!(
11448 schedules
11449 .iter()
11450 .map(|schedule| schedule.delay)
11451 .collect::<Vec<_>>(),
11452 vec![
11453 Duration::from_millis(100),
11454 Duration::from_secs(1),
11455 Duration::from_secs(10),
11456 Duration::from_secs(30),
11457 ]
11458 );
11459 }
11460
11461 #[test]
11462 fn crash_restart_backoff_resets_after_ring_clear() {
11463 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11464 let now = ClockInstant::now();
11465 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11466 assert_eq!(
11467 state.next_crash_restart(&policy, now).unwrap().delay,
11468 Duration::from_millis(100)
11469 );
11470 assert_eq!(
11471 state.next_crash_restart(&policy, now).unwrap().delay,
11472 Duration::from_secs(1)
11473 );
11474
11475 state.clear_crash_restarts();
11476 let schedule = state
11477 .next_crash_restart(&policy, now)
11478 .expect("a cleared ring must allow another restart");
11479 assert_eq!(schedule.restart_in_window, 0);
11480 assert_eq!(schedule.delay, Duration::from_millis(100));
11481 }
11482
11483 #[test]
11484 fn crash_restart_backoff_ignores_aged_restarts() {
11485 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11486 let now = ClockInstant::now();
11487 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11488 state
11489 .next_crash_restart(&policy, now)
11490 .expect("the first restart is allowed");
11491 state
11492 .next_crash_restart(&policy, now)
11493 .expect("the second restart is allowed");
11494 state.crash_restarts[0] = now
11495 .checked_sub(policy.window + Duration::from_secs(1))
11496 .expect("the fake clock can age a restart past the window");
11497
11498 let schedule = state
11499 .next_crash_restart(&policy, now)
11500 .expect("an aged restart must release its slot");
11501 assert_eq!(schedule.restart_in_window, 1);
11502 assert_eq!(schedule.delay, Duration::from_secs(1));
11503 assert_eq!(state.crash_restarts.len(), 2);
11504 }
11505
11506 #[test]
11510 fn a_budget_spent_before_the_window_no_longer_refuses() {
11511 let policy = RestartPolicy::new(3, Duration::ZERO);
11512 let mut state = snapshot_with_restarts(true, 3);
11513 let now = ClockInstant::now();
11514 assert!(!daemon_will_restart(&mut state, &policy, now));
11515
11516 assert!(daemon_will_restart(
11517 &mut state,
11518 &policy,
11519 now + policy.window + Duration::from_secs(1)
11520 ));
11521 assert!(
11522 state.crash_restarts.is_empty(),
11523 "reading the budget must drop the instants that left the window"
11524 );
11525 }
11526
11527 fn module_with_recovery_snapshot(
11528 state: ModuleState,
11529 enabled: bool,
11530 restart_count: u32,
11531 ) -> SupervisedModule {
11532 let registry = Arc::new(Registry::default());
11533 let supervisor =
11534 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
11535 let runtime = supervisor.runtime_config();
11536 let spec = ModuleSpec {
11537 module_id: "recovery-snapshot".to_string(),
11538 program: fake_aft_stub_path(),
11539 args: Vec::new(),
11540 env: Vec::new(),
11541 reserved: false,
11542 reserved_prefixes: Vec::new(),
11543 protocol: ModuleProtocol::Subc,
11544 overlap: Default::default(),
11545 };
11546 let mut snapshot = SupervisorSnapshot::new(state, enabled);
11547 seed_crash_restarts(&mut snapshot, restart_count);
11548 let (commands, _rx) = mpsc::channel(4);
11551 SupervisedModule {
11552 inner: Arc::new(SupervisedModuleInner {
11553 module_id: spec.module_id.clone(),
11554 registry,
11555 snapshot: Arc::new(Mutex::new(snapshot)),
11556 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11557 spec,
11558 health: runtime.health,
11559 })),
11560 stderr_ring: runtime.stderr_ring,
11561 terminal_ring: runtime.terminal_ring,
11562 commands,
11563 monitor: Mutex::new(None),
11564 restart_policy: runtime.restart_policy,
11565 effective_drain_timeout: runtime.effective_drain_timeout,
11566 provenance_probe: supervisor.provenance_probe.clone(),
11567 }),
11568 }
11569 }
11570
11571 #[cfg(target_os = "linux")]
11572 #[tokio::test]
11573 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11574 let supervisor =
11575 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11576 .with_cgroup_placement(None);
11577 let result = supervisor.spawn(ModuleSpec {
11578 module_id: "no-cgroup-placement".to_string(),
11579 program: fake_aft_stub_path(),
11580 args: Vec::new(),
11581 env: Vec::new(),
11582 reserved: false,
11583 reserved_prefixes: Vec::new(),
11584 protocol: ModuleProtocol::Subc,
11585 overlap: Default::default(),
11586 });
11587
11588 assert!(
11589 result.is_ok(),
11590 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11591 );
11592 }
11593
11594 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11595 async fn undecided_snapshot_uses_shared_restart_predicate() {
11596 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11597 .will_recover_after_connection_loss()
11598 .unwrap());
11599 assert!(
11600 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11601 .will_recover_after_connection_loss()
11602 .unwrap()
11603 );
11604 }
11605
11606 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11607 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11608 assert!(
11609 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11610 .will_recover_after_connection_loss()
11611 .unwrap()
11612 );
11613 }
11614
11615 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11616 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11617 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11618 .will_recover_after_connection_loss()
11619 .unwrap());
11620 assert!(
11621 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11622 .will_recover_after_connection_loss()
11623 .unwrap()
11624 );
11625 }
11626
11627 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11628 async fn warming_snapshot_is_limited_to_startup_phases() {
11629 for state in [
11630 ModuleState::Starting,
11631 ModuleState::Running,
11632 ModuleState::Restarting,
11633 ] {
11634 assert!(
11635 module_with_recovery_snapshot(state, true, 0)
11636 .is_warming()
11637 .unwrap(),
11638 "{state:?} should be warming"
11639 );
11640 }
11641 for state in [
11642 ModuleState::Unresponsive,
11643 ModuleState::Draining,
11644 ModuleState::Stopped,
11645 ModuleState::Failed,
11646 ModuleState::Disabled,
11647 ] {
11648 assert!(
11649 !module_with_recovery_snapshot(state, true, 0)
11650 .is_warming()
11651 .unwrap(),
11652 "{state:?} should not be warming"
11653 );
11654 }
11655 }
11656
11657 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11658 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11659 let registry = Arc::new(Registry::default());
11660 let supervisor =
11661 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
11662 let module = supervisor
11663 .spawn(ModuleSpec {
11664 module_id: "terminal-history".to_string(),
11665 program: fake_aft_stub_path(),
11666 args: Vec::new(),
11667 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11668 reserved: false,
11669 reserved_prefixes: Vec::new(),
11670 protocol: ModuleProtocol::Subc,
11671 overlap: Default::default(),
11672 })
11673 .unwrap();
11674
11675 let deadline = Instant::now() + Duration::from_secs(5);
11676 loop {
11677 let history = module.terminal_history();
11678 if history.entries.len() == 2 {
11679 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11680 assert_eq!(history.dropped, 0);
11681 assert_eq!(
11682 history
11683 .entries
11684 .iter()
11685 .map(|entry| entry.exit_code)
11686 .collect::<Vec<_>>(),
11687 vec![Some(23), Some(23)]
11688 );
11689 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11690 return;
11691 }
11692 assert!(
11693 Instant::now() < deadline,
11694 "module did not retain two terminal exits: {history:?}"
11695 );
11696 sleep(Duration::from_millis(10)).await;
11697 }
11698 }
11699
11700 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11704 async fn disable_during_crash_backoff_cancels_pending_respawn() {
11705 let backoff = Duration::from_secs(2);
11706 let supervisor = Supervisor::new_for_test(
11707 Arc::new(Registry::default()),
11708 RestartPolicy::new(10, backoff),
11709 );
11710 let module = supervisor
11711 .spawn(ModuleSpec {
11712 module_id: "disable-during-backoff".to_string(),
11713 program: fake_aft_stub_path(),
11714 args: Vec::new(),
11715 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11716 reserved: false,
11717 reserved_prefixes: Vec::new(),
11718 protocol: ModuleProtocol::Subc,
11719 overlap: Default::default(),
11720 })
11721 .unwrap();
11722
11723 let deadline = Instant::now() + Duration::from_secs(5);
11725 loop {
11726 if module.status().unwrap().state == ModuleState::Restarting {
11727 break;
11728 }
11729 assert!(
11730 Instant::now() < deadline,
11731 "module never entered the crash backoff"
11732 );
11733 sleep(Duration::from_millis(10)).await;
11734 }
11735
11736 let started = Instant::now();
11737 module.set_enabled(false).await.unwrap();
11738 let waited = started.elapsed();
11739
11740 assert!(
11741 waited < backoff / 2,
11742 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11743 );
11744 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11745
11746 sleep(backoff + Duration::from_millis(500)).await;
11748 let status = module.status().unwrap();
11749 assert_eq!(status.state, ModuleState::Disabled);
11750 assert_eq!(
11751 status.spawn_generation, 1,
11752 "module respawned after the operator disabled it"
11753 );
11754 }
11755
11756 #[cfg(unix)]
11759 fn protocol_none_sigterm_exits_clean_spec(
11760 module_id: &str,
11761 dir: &std::path::Path,
11762 ) -> (ModuleSpec, PathBuf, PathBuf) {
11763 let ready = dir.join("ready");
11764 let marker = dir.join("sigterm");
11765 let spec = ModuleSpec {
11766 module_id: module_id.to_string(),
11767 program: fake_aft_stub_path(),
11768 args: Vec::new(),
11769 env: vec![
11770 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11771 (
11772 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11773 marker.display().to_string(),
11774 ),
11775 (
11776 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11777 ready.display().to_string(),
11778 ),
11779 ],
11780 reserved: false,
11781 reserved_prefixes: Vec::new(),
11782 protocol: ModuleProtocol::None,
11783 overlap: Default::default(),
11784 };
11785 (spec, ready, marker)
11786 }
11787
11788 #[cfg(unix)]
11792 async fn wait_for_file(path: &std::path::Path) {
11793 let deadline = Instant::now() + Duration::from_secs(10);
11794 while !path.exists() {
11795 assert!(
11796 Instant::now() < deadline,
11797 "{} never appeared",
11798 path.display()
11799 );
11800 sleep(Duration::from_millis(10)).await;
11801 }
11802 }
11803
11804 #[cfg(unix)]
11808 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11809 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11810 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11811 let (spec, ready, marker) =
11812 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11813 let supervisor = Supervisor::new_for_test(
11814 Arc::new(Registry::default()),
11815 RestartPolicy::new(3, Duration::ZERO),
11816 );
11817 let module = supervisor.spawn(spec).unwrap();
11818 wait_for_file(&ready).await;
11819 let deadline = Instant::now() + Duration::from_secs(10);
11823 let first_pid = loop {
11824 let status = module.status().unwrap();
11825 if status.state == ModuleState::Running {
11826 if let Some(pid) = status.pid {
11827 break pid;
11828 }
11829 }
11830 assert!(
11831 Instant::now() < deadline,
11832 "a running module must report its pid after exec confirmation: {status:?}"
11833 );
11834 sleep(Duration::from_millis(10)).await;
11835 };
11836
11837 rustix::process::kill_process(
11838 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11839 rustix::process::Signal::TERM,
11840 )
11841 .unwrap();
11842
11843 let deadline = Instant::now() + Duration::from_secs(10);
11844 let respawned = loop {
11845 let status = module.status().unwrap();
11846 if status.state == ModuleState::Running
11847 && status.pid.is_some_and(|pid| pid != first_pid)
11848 {
11849 break status;
11850 }
11851 assert!(
11852 Instant::now() < deadline,
11853 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11854 );
11855 sleep(Duration::from_millis(10)).await;
11856 };
11857 assert_eq!(respawned.spawn_generation, 2);
11858 assert!(
11859 marker.exists(),
11860 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11861 );
11862
11863 let history = module.terminal_history();
11864 assert_eq!(history.entries.len(), 1, "{history:?}");
11865 let entry = &history.entries[0];
11866 assert_eq!(entry.exit_code, Some(0));
11867 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11868 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11869
11870 module.stop().await.unwrap();
11871 }
11872
11873 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11877 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11878 let supervisor = Supervisor::new_for_test(
11879 Arc::new(Registry::default()),
11880 RestartPolicy::new(1, Duration::ZERO),
11881 );
11882 let module = supervisor
11883 .spawn(ModuleSpec {
11884 module_id: "none-clean-exit-budget".to_string(),
11885 program: fake_aft_stub_path(),
11886 args: Vec::new(),
11887 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11888 reserved: false,
11889 reserved_prefixes: Vec::new(),
11890 protocol: ModuleProtocol::None,
11891 overlap: Default::default(),
11892 })
11893 .unwrap();
11894
11895 let deadline = Instant::now() + Duration::from_secs(10);
11898 loop {
11899 let status = module.status().unwrap();
11900 if status.state == ModuleState::Failed {
11901 break;
11902 }
11903 assert!(
11904 Instant::now() < deadline,
11905 "module never exhausted its budget: {status:?} {:?}",
11906 module.terminal_history()
11907 );
11908 sleep(Duration::from_millis(10)).await;
11909 }
11910 let history = module.terminal_history();
11911 assert_eq!(
11912 history
11913 .entries
11914 .iter()
11915 .map(|entry| (entry.exit_code, entry.disposition.clone()))
11916 .collect::<Vec<_>>(),
11917 vec![
11918 (Some(0), TerminalDisposition::Restarting),
11919 (Some(0), TerminalDisposition::Failed),
11920 ]
11921 );
11922 let detail = history.entries[1]
11923 .disposition_detail
11924 .as_deref()
11925 .expect("a budget failure names the budget");
11926 assert!(detail.contains("max_restarts=1"), "{detail}");
11927 assert_eq!(module.status().unwrap().spawn_generation, 2);
11928 }
11929
11930 #[test]
11931 fn restart_budget_failure_is_published_after_its_terminal_record() {
11932 use std::io::{Read, Write};
11933 let supervisor = Supervisor::new_for_test(
11934 Arc::new(Registry::default()),
11935 RestartPolicy::new(0, Duration::ZERO),
11936 );
11937 let runtime = supervisor.runtime_config();
11938 let spec = ModuleSpec {
11939 protocol: ModuleProtocol::None,
11940 ..windowed_crash_spec("budget-publication-order")
11941 };
11942 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11943 ModuleState::Running,
11944 true,
11945 )));
11946 lock_snapshot(&snapshot).unwrap().process_alive = true;
11947 let reaped_snapshot = snapshot.clone();
11948 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11949 let history = runtime.terminal_ring.clone();
11950 let reap = std::thread::spawn(move || {
11951 tokio::runtime::Builder::new_current_thread()
11952 .enable_all()
11953 .build()
11954 .unwrap()
11955 .block_on(on_child_exit(
11956 &spec,
11957 runtime.restart_policy,
11958 &supervisor.registry,
11959 &reaped_snapshot,
11960 &runtime.terminal_ring,
11961 &runtime.spawn_events,
11962 &runtime.child_roster,
11963 ExitReport {
11964 kind: ExitKind::Clean,
11965 code: Some(0),
11966 signal: None,
11967 at_ms: 1,
11968 },
11969 ))
11970 });
11971 let mut reached = [0];
11972 barrier.read_exact(&mut reached).unwrap();
11973 assert_eq!(&reached, b"R");
11974 let (before_state, before_alive) = {
11975 let state = lock_snapshot(&snapshot).unwrap();
11976 (state.state, state.process_alive)
11977 };
11978 let before_history = history.lock().unwrap().snapshot();
11979 barrier.write_all(b"X").unwrap();
11980 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
11981 assert!(before_history.entries.is_empty());
11982 assert_ne!(
11983 before_state,
11984 ModuleState::Failed,
11985 "Failed was visible before its terminal record could be written"
11986 );
11987 assert!(before_alive);
11988 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11989 let history = history.lock().unwrap().snapshot();
11990 assert_eq!(history.entries.len(), 1);
11991 assert_eq!(history.entries[0].exit_code, Some(0));
11992 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11993 }
11994
11995 #[test]
11996 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
11997 use std::io::{Read, Write};
11998
11999 let supervisor = Supervisor::new_for_test(
12000 Arc::new(Registry::default()),
12001 RestartPolicy::new(3, Duration::ZERO),
12002 );
12003 let mut runtime = supervisor.runtime_config();
12004 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12005 let snapshot = module.inner.snapshot.clone();
12006 lock_snapshot(&snapshot).unwrap().process_alive = true;
12007 runtime.terminal_ring = module.inner.terminal_ring.clone();
12008 let spec = windowed_crash_spec(module.module_id());
12009 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12010 let reaped_snapshot = snapshot.clone();
12011 let reap = std::thread::spawn(move || {
12012 tokio::runtime::Builder::new_current_thread()
12013 .enable_all()
12014 .build()
12015 .unwrap()
12016 .block_on(on_child_exit(
12017 &spec,
12018 runtime.restart_policy,
12019 &supervisor.registry,
12020 &reaped_snapshot,
12021 &runtime.terminal_ring,
12022 &runtime.spawn_events,
12023 &runtime.child_roster,
12024 ExitReport {
12025 kind: ExitKind::Crash,
12026 code: Some(2),
12027 signal: None,
12028 at_ms: 1,
12029 },
12030 ))
12031 });
12032
12033 let mut reached = [0];
12037 barrier.read_exact(&mut reached).unwrap();
12038 assert_eq!(&reached, b"R");
12039 let before_status = module.status().unwrap();
12040 let before_history = module.terminal_history();
12041 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12042 barrier.write_all(b"X").unwrap();
12043 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12044
12045 assert_eq!(before_status.state, ModuleState::Running);
12046 assert!(before_status.process_alive);
12047 assert_eq!(before_status.restart_count, 3);
12048 assert!(before_history.entries.is_empty());
12049 assert!(
12050 before_recovers,
12051 "connection cleanup would report a terminal close for a scheduled crash restart"
12052 );
12053 let restarting = module.status().unwrap();
12054 assert_eq!(restarting.state, ModuleState::Restarting);
12055 assert_eq!(restarting.restart_count, 3);
12056 assert!(module.will_recover_after_connection_loss().unwrap());
12057 let history = module.terminal_history();
12058 assert_eq!(history.entries.len(), 1);
12059 assert_eq!(
12060 history.entries[0].disposition,
12061 TerminalDisposition::Restarting
12062 );
12063
12064 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12067 state.state = ModuleState::Running;
12068 state.respawn_pending = false;
12069 })
12070 .unwrap();
12071 assert!(!module.will_recover_after_connection_loss().unwrap());
12072 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12073 state.enabled = false;
12074 state.respawn_pending = true;
12075 })
12076 .unwrap();
12077 assert!(!module.will_recover_after_connection_loss().unwrap());
12078 }
12079
12080 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12081 use std::net::{TcpListener, TcpStream};
12082 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12083 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12084 let (barrier, _) = listener.accept().unwrap();
12085 for stream in [&writer, &barrier] {
12086 stream
12087 .set_read_timeout(Some(Duration::from_secs(10)))
12088 .unwrap();
12089 stream
12090 .set_write_timeout(Some(Duration::from_secs(10)))
12091 .unwrap();
12092 }
12093 *events.1.lock().unwrap() = Some(writer);
12094 barrier
12095 }
12096
12097 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12098 use std::io::{Read, Write};
12099 let barrier = events.1.lock().unwrap().take();
12100 if let Some(mut barrier) = barrier {
12101 barrier.write_all(b"R").unwrap();
12102 let mut release = [0];
12103 barrier.read_exact(&mut release).unwrap();
12104 assert_eq!(&release, b"X");
12105 }
12106 }
12107
12108 #[test]
12109 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12110 use std::io::{Read, Write};
12111
12112 let supervisor = Supervisor::new_for_test(
12113 Arc::new(Registry::default()),
12114 RestartPolicy::new(0, Duration::ZERO),
12115 );
12116 let mut runtime = supervisor.runtime_config();
12117 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12118 let snapshot = module.inner.snapshot.clone();
12119 runtime.terminal_ring = module.inner.terminal_ring.clone();
12120 let spec = ModuleSpec {
12121 module_id: module.module_id().into(),
12122 program: fake_aft_stub_path(),
12123 protocol: ModuleProtocol::None,
12124 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12125 ..windowed_crash_spec(module.module_id())
12126 };
12127 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12128 let reap = std::thread::spawn(move || {
12129 tokio::runtime::Builder::new_current_thread()
12130 .enable_all()
12131 .build()
12132 .unwrap()
12133 .block_on(async {
12134 let mut child =
12135 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12136 health_restart_child(
12137 &spec,
12138 &runtime,
12139 &supervisor.registry,
12140 &supervisor.process_liveness,
12141 &snapshot,
12142 &mut child,
12143 SupervisorHealthStatus::Failing,
12144 Some("health probe did not answer"),
12145 1,
12146 )
12147 .await
12148 .unwrap();
12149 assert!(child.is_none());
12150 });
12151 });
12152 let mut reached = [0];
12157 barrier.read_exact(&mut reached).unwrap();
12158 assert_eq!(&reached, b"R");
12159 let before_status = module.status().unwrap();
12160 let before_history = module.terminal_history();
12161 barrier.write_all(b"X").unwrap();
12162 reap.join().unwrap();
12163 assert!(before_history.entries.is_empty());
12164 assert_ne!(
12165 before_status.state,
12166 ModuleState::Failed,
12167 "Failed was visible before its terminal record could be written"
12168 );
12169 assert!(
12170 before_status.process_alive,
12171 "process death was visible before its terminal record could be written"
12172 );
12173 let failed = module.status().unwrap();
12174 assert_eq!(failed.state, ModuleState::Failed);
12175 assert!(!failed.process_alive);
12176 let history = module.terminal_history();
12177 assert_eq!(history.entries.len(), 1);
12178 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12179 assert_eq!(
12180 history.entries[0].disposition_detail.as_deref(),
12181 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12182 );
12183 }
12184
12185 #[test]
12186 fn observed_exits_publish_process_death_after_terminal_history() {
12187 use std::io::{Read, Write};
12188 for (kind, enabled, shutdown, wait_error, expected_state) in [
12189 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12190 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12191 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12192 (
12193 ExitKind::DeliberateSeverance,
12194 true,
12195 false,
12196 false,
12197 ModuleState::Restarting,
12198 ),
12199 #[cfg(unix)]
12200 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12201 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12202 ] {
12203 let supervisor = Supervisor::new_for_test(
12204 Arc::new(Registry::default()),
12205 RestartPolicy::new(1, Duration::ZERO),
12206 );
12207 let runtime = supervisor.runtime_config();
12208 #[cfg(unix)]
12209 if shutdown {
12210 runtime.child_roster.close();
12211 }
12212 #[cfg(not(unix))]
12213 let _ = shutdown;
12214 let spec = windowed_crash_spec("exit-publication-order");
12215 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12216 ModuleState::Running,
12217 enabled,
12218 )));
12219 lock_snapshot(&snapshot).unwrap().process_alive = true;
12220 let reaped_snapshot = snapshot.clone();
12221 let history = runtime.terminal_ring.clone();
12222 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12223 let reap = std::thread::spawn(move || {
12224 if wait_error {
12225 super::fail_after_wait_error(
12226 &spec.module_id,
12227 &reaped_snapshot,
12228 &runtime.terminal_ring,
12229 &runtime.spawn_events,
12230 "wait failed".into(),
12231 );
12232 } else {
12233 tokio::runtime::Builder::new_current_thread()
12234 .enable_all()
12235 .build()
12236 .unwrap()
12237 .block_on(on_child_exit(
12238 &spec,
12239 runtime.restart_policy,
12240 &supervisor.registry,
12241 &reaped_snapshot,
12242 &runtime.terminal_ring,
12243 &runtime.spawn_events,
12244 &runtime.child_roster,
12245 ExitReport {
12246 kind,
12247 code: Some(0),
12248 signal: None,
12249 at_ms: 1,
12250 },
12251 ));
12252 }
12253 });
12254 let mut reached = [0];
12255 barrier.read_exact(&mut reached).unwrap();
12256 assert_eq!(&reached, b"R");
12257 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12258 let before_history = history.lock().unwrap().snapshot();
12259 barrier.write_all(b"X").unwrap();
12260 reap.join().unwrap();
12261 assert!(
12262 before_alive,
12263 "{kind:?}: death was published before recording"
12264 );
12265 assert!(before_history.entries.is_empty());
12266 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12267 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12268 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12269 }
12270 }
12271
12272 #[tokio::test]
12273 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12274 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12275 let path = dir.join("terminals.jsonl");
12276 std::fs::create_dir(&path).unwrap();
12277 let supervisor = Supervisor::new_for_test(
12278 Arc::new(Registry::default()),
12279 RestartPolicy::new(0, Duration::ZERO),
12280 )
12281 .with_terminal_journal(path, "health-budget-journal-failure".into());
12282 let runtime = supervisor.runtime_config();
12283 let spec = ModuleSpec {
12284 program: fake_aft_stub_path(),
12285 protocol: ModuleProtocol::None,
12286 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12287 ..windowed_crash_spec("health-budget-journal-failure")
12288 };
12289 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12290 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12291 health_restart_child(
12292 &spec,
12293 &runtime,
12294 &supervisor.registry,
12295 &supervisor.process_liveness,
12296 &snapshot,
12297 &mut child,
12298 SupervisorHealthStatus::Failing,
12299 None,
12300 1,
12301 )
12302 .await
12303 .unwrap();
12304 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12305 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12306 let history = runtime
12307 .terminal_ring
12308 .lock()
12309 .unwrap()
12310 .durable_history(&spec.module_id);
12311 assert!(history.journal_write_failures > 0);
12312 assert_eq!(history.entries.len(), 1);
12313 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12314 }
12315
12316 #[test]
12317 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12318 let supervisor = Supervisor::new_for_test(
12319 Arc::new(Registry::default()),
12320 RestartPolicy::new(0, Duration::ZERO),
12321 );
12322 let runtime = supervisor.runtime_config();
12323 let spec = ModuleSpec {
12324 program: fake_aft_stub_path(),
12325 protocol: ModuleProtocol::None,
12326 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12327 ..windowed_crash_spec("health-budget-recording-panic")
12328 };
12329 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12330 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12331 tokio::runtime::Builder::new_current_thread()
12332 .enable_all()
12333 .build()
12334 .unwrap()
12335 .block_on(async {
12336 let mut child =
12337 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12338 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12339 health_restart_child(
12340 &spec,
12341 &runtime,
12342 &supervisor.registry,
12343 &supervisor.process_liveness,
12344 &snapshot,
12345 &mut child,
12346 SupervisorHealthStatus::Failing,
12347 None,
12348 1,
12349 )
12350 .await
12351 .unwrap();
12352 });
12353 }));
12354 let panic = result.expect_err("recording must still unwind");
12355 assert_eq!(
12356 panic.downcast_ref::<String>().map(String::as_str),
12357 Some("spawn event sequence exhausted")
12358 );
12359 let state = lock_snapshot(&snapshot).unwrap();
12360 assert_eq!(state.state, ModuleState::Failed);
12361 assert!(!state.process_alive);
12362 assert!(state.last_exit.is_some());
12363 }
12364
12365 #[tokio::test]
12366 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12367 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12368 let path = dir.join("terminals.jsonl");
12369 std::fs::create_dir(&path).unwrap();
12370 let supervisor = Supervisor::new_for_test(
12371 Arc::new(Registry::default()),
12372 RestartPolicy::new(0, Duration::ZERO),
12373 )
12374 .with_terminal_journal(path, "budget-journal-failure".into());
12375 let runtime = supervisor.runtime_config();
12376 let spec = windowed_crash_spec("budget-journal-failure");
12377 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12378 assert!(matches!(
12379 on_child_exit(
12380 &spec,
12381 runtime.restart_policy,
12382 &supervisor.registry,
12383 &snapshot,
12384 &runtime.terminal_ring,
12385 &runtime.spawn_events,
12386 &runtime.child_roster,
12387 crash_exit_report(1),
12388 )
12389 .await,
12390 NextAction::Stop { .. }
12391 ));
12392 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12393 let history = runtime
12394 .terminal_ring
12395 .lock()
12396 .unwrap()
12397 .durable_history(&spec.module_id);
12398 assert!(history.journal_write_failures > 0);
12399 assert_eq!(history.entries.len(), 1);
12400 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12401 }
12402
12403 #[test]
12404 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12405 let supervisor = Supervisor::new_for_test(
12406 Arc::new(Registry::default()),
12407 RestartPolicy::new(0, Duration::ZERO),
12408 );
12409 let runtime = supervisor.runtime_config();
12410 let spec = windowed_crash_spec("budget-recording-panic");
12411 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12412 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12413 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12416 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12417 tokio::runtime::Builder::new_current_thread()
12418 .enable_all()
12419 .build()
12420 .unwrap()
12421 .block_on(on_child_exit(
12422 &spec,
12423 runtime.restart_policy,
12424 &supervisor.registry,
12425 &snapshot,
12426 &runtime.terminal_ring,
12427 &runtime.spawn_events,
12428 &runtime.child_roster,
12429 crash_exit_report(1),
12430 ))
12431 }));
12432 let panic = result.err().expect("recording must still unwind");
12433 assert_eq!(
12434 panic.downcast_ref::<String>().map(String::as_str),
12435 Some("spawn event sequence exhausted")
12436 );
12437 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12438 }
12439
12440 #[cfg(unix)]
12443 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12444 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12445 for disable in [false, true] {
12446 let label = if disable {
12447 "none-requested-disable"
12448 } else {
12449 "none-requested-stop"
12450 };
12451 let dir = cortexkit_test_support::ScratchDir::new(label);
12452 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12453 let supervisor = Supervisor::new_for_test(
12454 Arc::new(Registry::default()),
12455 RestartPolicy::new(3, Duration::ZERO),
12456 );
12457 let module = supervisor.spawn(spec).unwrap();
12458 wait_for_file(&ready).await;
12459
12460 if disable {
12461 module.set_enabled(false).await.unwrap();
12462 } else {
12463 module.stop().await.unwrap();
12464 }
12465 assert!(
12466 marker.exists(),
12467 "{label}: the child must have left through its SIGTERM handler with exit 0"
12468 );
12469
12470 sleep(Duration::from_millis(500)).await;
12473 let status = module.status().unwrap();
12474 let expected = if disable {
12475 ModuleState::Disabled
12476 } else {
12477 ModuleState::Stopped
12478 };
12479 assert_eq!(status.state, expected, "{label}");
12480 assert_eq!(
12481 status.spawn_generation, 1,
12482 "{label}: respawned after a requested stop"
12483 );
12484 let history = module.terminal_history();
12485 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12486 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12487 assert_ne!(
12488 history.entries[0].disposition,
12489 TerminalDisposition::Restarting,
12490 "{label}"
12491 );
12492 }
12493 }
12494
12495 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12498 async fn subc_wire_clean_exit_is_still_a_stop() {
12499 let supervisor = Supervisor::new_for_test(
12500 Arc::new(Registry::default()),
12501 RestartPolicy::new(3, Duration::ZERO),
12502 );
12503 let module = supervisor
12504 .spawn(ModuleSpec {
12505 module_id: "wire-clean-exit".to_string(),
12506 program: fake_aft_stub_path(),
12507 args: Vec::new(),
12508 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12509 reserved: false,
12510 reserved_prefixes: Vec::new(),
12511 protocol: ModuleProtocol::Subc,
12512 overlap: Default::default(),
12513 })
12514 .unwrap();
12515
12516 let deadline = Instant::now() + Duration::from_secs(10);
12517 while module.terminal_history().entries.is_empty() {
12518 assert!(Instant::now() < deadline, "module never exited");
12519 sleep(Duration::from_millis(10)).await;
12520 }
12521 sleep(Duration::from_millis(500)).await;
12523 let status = module.status().unwrap();
12524 assert_eq!(status.state, ModuleState::Stopped);
12525 assert_eq!(status.spawn_generation, 1);
12526 let history = module.terminal_history();
12527 assert_eq!(history.entries.len(), 1, "{history:?}");
12528 assert_eq!(history.entries[0].exit_code, Some(0));
12529 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12530 }
12531
12532 #[cfg(unix)]
12533 #[tokio::test]
12534 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12535 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12536 let record = dir.join("live-children.json");
12537 let supervisor = Supervisor::new_for_test(
12538 Arc::new(Registry::default()),
12539 RestartPolicy::new(0, Duration::ZERO),
12540 );
12541 let mut runtime = supervisor.runtime_config();
12542 runtime.child_roster.record_to(record.clone());
12543 let gate = Arc::new(super::ReloadExitRecordGate::default());
12544 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12545 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12546 let spec = ModuleSpec {
12547 module_id: "reload-exit-roster".into(),
12548 program: fake_aft_stub_path(),
12549 args: Vec::new(),
12550 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12551 reserved: false,
12552 reserved_prefixes: Vec::new(),
12553 protocol: ModuleProtocol::Subc,
12554 overlap: Default::default(),
12555 };
12556 let mut child = None;
12557 let reload = super::finish_reload_child(
12558 &spec,
12559 &runtime,
12560 &supervisor.registry,
12561 &supervisor.process_liveness,
12562 &snapshot,
12563 &mut child,
12564 );
12565 tokio::pin!(reload);
12566 tokio::select! {
12567 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12568 _ = gate.reached.notified() => {}
12569 }
12570 assert!(runtime
12571 .terminal_ring
12572 .lock()
12573 .unwrap()
12574 .snapshot()
12575 .entries
12576 .is_empty());
12577 assert_eq!(
12578 crate::live_children::read_record(&record).unwrap().len(),
12579 1,
12580 "shutdown must still wait for the reaped child until its terminal record exists"
12581 );
12582 runtime.child_roster.close();
12583 gate.resume.notify_one();
12584 assert!(reload.await.is_err());
12585 assert!(crate::live_children::read_record(&record)
12586 .unwrap()
12587 .is_empty());
12588 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12589 assert_eq!(history.entries.len(), 1);
12590 assert_eq!(
12591 history.entries[0].disposition,
12592 TerminalDisposition::DaemonShutdown
12593 );
12594 }
12595
12596 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12600 async fn every_restart_increment_path_advances_lifetime_count() {
12601 let supervisor = Supervisor::new_for_test(
12602 Arc::new(Registry::default()),
12603 RestartPolicy::new(1, Duration::ZERO),
12604 );
12605 let runtime = supervisor.runtime_config();
12606 let spec = ModuleSpec {
12607 module_id: "lifetime-increment-path".to_string(),
12608 program: PathBuf::from("/unused/lifetime-increment-path"),
12609 args: Vec::new(),
12610 env: Vec::new(),
12611 reserved: false,
12612 reserved_prefixes: Vec::new(),
12613 protocol: ModuleProtocol::Subc,
12614 overlap: Default::default(),
12615 };
12616
12617 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12618 assert!(matches!(
12619 on_child_exit(
12620 &spec,
12621 runtime.restart_policy,
12622 &supervisor.registry,
12623 &crash_snapshot,
12624 &runtime.terminal_ring,
12625 &runtime.spawn_events,
12626 &runtime.child_roster,
12627 ExitReport {
12628 kind: ExitKind::Crash,
12629 code: Some(1),
12630 signal: None,
12631 at_ms: 1,
12632 },
12633 )
12634 .await,
12635 NextAction::Restart { schedule: _ }
12636 ));
12637 let (crash_restarts, crash_lifetime) = {
12638 let state = lock_snapshot(&crash_snapshot).unwrap();
12639 (state.crash_restarts.len(), state.lifetime_restarts)
12640 };
12641 assert_eq!(crash_restarts, 1);
12642 assert_eq!(crash_lifetime, 1);
12643
12644 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12645 let mut health_child = None;
12646 assert!(matches!(
12647 health_restart_child(
12648 &spec,
12649 &runtime,
12650 &supervisor.registry,
12651 &supervisor.process_liveness,
12652 &health_snapshot,
12653 &mut health_child,
12654 SupervisorHealthStatus::Failing,
12655 None,
12656 2,
12657 )
12658 .await,
12659 Ok(())
12660 ));
12661 assert!(health_child.is_none());
12662 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12663 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12664 let (health_restarts, health_lifetime) = {
12665 let state = lock_snapshot(&health_snapshot).unwrap();
12666 (state.crash_restarts.len(), state.lifetime_restarts)
12667 };
12668 assert_eq!(health_restarts, 1);
12669 assert_eq!(health_lifetime, 1);
12670
12671 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12672 let mut reload_child = None;
12673 assert!(matches!(
12674 handle_reload_spawn_failure(
12675 &spec,
12676 &runtime,
12677 &supervisor.process_liveness,
12678 &reload_snapshot,
12679 &mut reload_child,
12680 "forced reload spawn failure".to_string(),
12681 )
12682 .await,
12683 Err(SuperviseError::ReloadFailed { .. })
12684 ));
12685 let (reload_restarts, reload_lifetime) = {
12686 let state = lock_snapshot(&reload_snapshot).unwrap();
12687 (state.crash_restarts.len(), state.lifetime_restarts)
12688 };
12689 assert_eq!(reload_restarts, 1);
12690 assert_eq!(reload_lifetime, 1);
12691 }
12692
12693 #[tokio::test]
12694 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12695 let supervisor = Supervisor::new_for_test(
12696 Arc::new(Registry::default()),
12697 RestartPolicy::new(3, Duration::ZERO),
12698 );
12699 let runtime = supervisor.runtime_config();
12700 let spec = ModuleSpec {
12701 module_id: "deliberately-severed".to_string(),
12702 program: PathBuf::from("/unused/deliberately-severed"),
12703 args: Vec::new(),
12704 env: Vec::new(),
12705 reserved: false,
12706 reserved_prefixes: Vec::new(),
12707 protocol: ModuleProtocol::Subc,
12708 overlap: Default::default(),
12709 };
12710 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12711 let process = ProcessIdentity {
12712 pid: 41,
12713 start_time: 101,
12714 };
12715 record_deliberate_severance(&snapshot, process).unwrap();
12716 let exit_report = apply_deliberate_severance_marker(
12717 &snapshot,
12718 Some(process),
12719 ExitReport {
12720 kind: ExitKind::Crash,
12721 code: Some(1),
12722 signal: None,
12723 at_ms: 1,
12724 },
12725 );
12726 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12727
12728 assert!(matches!(
12729 on_child_exit(
12730 &spec,
12731 runtime.restart_policy,
12732 &supervisor.registry,
12733 &snapshot,
12734 &runtime.terminal_ring,
12735 &runtime.spawn_events,
12736 &runtime.child_roster,
12737 exit_report,
12738 )
12739 .await,
12740 NextAction::Restart { schedule: _ }
12741 ));
12742 let state = lock_snapshot(&snapshot).unwrap();
12743 assert_eq!(state.lifetime_restarts, 1);
12744 assert_eq!(state.crash_restarts.len(), 0);
12745 }
12746
12747 #[tokio::test]
12748 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12749 let supervisor = Supervisor::new_for_test(
12750 Arc::new(Registry::default()),
12751 RestartPolicy::new(3, Duration::ZERO),
12752 );
12753 let runtime = supervisor.runtime_config();
12754 let spec = ModuleSpec {
12755 module_id: "genuine-crash".to_string(),
12756 program: PathBuf::from("/unused/genuine-crash"),
12757 args: Vec::new(),
12758 env: Vec::new(),
12759 reserved: false,
12760 reserved_prefixes: Vec::new(),
12761 protocol: ModuleProtocol::Subc,
12762 overlap: Default::default(),
12763 };
12764 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12765
12766 assert!(matches!(
12767 on_child_exit(
12768 &spec,
12769 runtime.restart_policy,
12770 &supervisor.registry,
12771 &snapshot,
12772 &runtime.terminal_ring,
12773 &runtime.spawn_events,
12774 &runtime.child_roster,
12775 ExitReport {
12776 kind: ExitKind::Crash,
12777 code: Some(1),
12778 signal: None,
12779 at_ms: 1,
12780 },
12781 )
12782 .await,
12783 NextAction::Restart { schedule: _ }
12784 ));
12785 let state = lock_snapshot(&snapshot).unwrap();
12786 assert_eq!(state.lifetime_restarts, 1);
12787 assert_eq!(state.crash_restarts.len(), 1);
12788 }
12789
12790 fn crash_exit_report(at_ms: u64) -> ExitReport {
12791 ExitReport {
12792 kind: ExitKind::Crash,
12793 code: Some(1),
12794 signal: None,
12795 at_ms,
12796 }
12797 }
12798
12799 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12800 ModuleSpec {
12801 module_id: module_id.to_string(),
12802 program: PathBuf::from("/unused").join(module_id),
12803 args: Vec::new(),
12804 env: Vec::new(),
12805 reserved: false,
12806 reserved_prefixes: Vec::new(),
12807 protocol: ModuleProtocol::Subc,
12808 overlap: Default::default(),
12809 }
12810 }
12811
12812 #[tokio::test]
12818 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12819 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12820 let supervisor = Supervisor::new_for_test(
12821 Arc::new(Registry::default()),
12822 RestartPolicy::new(2, Duration::ZERO),
12823 );
12824 let runtime = supervisor.runtime_config();
12825 let spec = windowed_crash_spec("crash-loop-in-window");
12826 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12827
12828 for attempt in 1..=2 {
12829 assert!(
12830 matches!(
12831 on_child_exit(
12832 &spec,
12833 runtime.restart_policy,
12834 &supervisor.registry,
12835 &snapshot,
12836 &runtime.terminal_ring,
12837 &runtime.spawn_events,
12838 &runtime.child_roster,
12839 crash_exit_report(attempt),
12840 )
12841 .await,
12842 NextAction::Restart { schedule: _ }
12843 ),
12844 "crash {attempt} is inside the budget and must respawn"
12845 );
12846 }
12847
12848 assert!(matches!(
12849 on_child_exit(
12850 &spec,
12851 runtime.restart_policy,
12852 &supervisor.registry,
12853 &snapshot,
12854 &runtime.terminal_ring,
12855 &runtime.spawn_events,
12856 &runtime.child_roster,
12857 crash_exit_report(3),
12858 )
12859 .await,
12860 NextAction::Stop { .. }
12861 ));
12862
12863 {
12864 let state = lock_snapshot(&snapshot).unwrap();
12865 assert_eq!(state.state, ModuleState::Failed);
12866 assert_eq!(state.crash_restarts.len(), 2);
12867 assert_eq!(state.lifetime_restarts, 2);
12868 }
12869
12870 let history = runtime
12871 .terminal_ring
12872 .lock()
12873 .expect("terminal ring is not poisoned")
12874 .snapshot();
12875 let last = history
12876 .entries
12877 .last()
12878 .expect("the refused crash is retained");
12879 assert_eq!(last.disposition, TerminalDisposition::Failed);
12880 assert_eq!(
12881 last.disposition_detail.as_deref(),
12882 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12883 );
12884
12885 let captured = crate::router::test_log::captured_logs(&logs);
12886 assert!(
12887 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12888 "the stop must be logged with its window: {captured}"
12889 );
12890 }
12891
12892 #[tokio::test]
12900 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12901 let supervisor = Supervisor::new_for_test(
12902 Arc::new(Registry::default()),
12903 RestartPolicy::new(2, Duration::ZERO),
12904 );
12905 let runtime = supervisor.runtime_config();
12906 let spec = windowed_crash_spec("crash-across-windows");
12907 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12908
12909 for attempt in 1..=2 {
12910 assert!(matches!(
12911 on_child_exit(
12912 &spec,
12913 runtime.restart_policy,
12914 &supervisor.registry,
12915 &snapshot,
12916 &runtime.terminal_ring,
12917 &runtime.spawn_events,
12918 &runtime.child_roster,
12919 crash_exit_report(attempt),
12920 )
12921 .await,
12922 NextAction::Restart { schedule: _ }
12923 ));
12924 }
12925
12926 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12929 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12930 })
12931 .unwrap();
12932
12933 assert!(
12934 matches!(
12935 on_child_exit(
12936 &spec,
12937 runtime.restart_policy,
12938 &supervisor.registry,
12939 &snapshot,
12940 &runtime.terminal_ring,
12941 &runtime.spawn_events,
12942 &runtime.child_roster,
12943 crash_exit_report(3),
12944 )
12945 .await,
12946 NextAction::Restart { schedule: _ }
12947 ),
12948 "a crash older than the window must not hold a budget slot"
12949 );
12950
12951 let state = lock_snapshot(&snapshot).unwrap();
12952 assert_eq!(state.state, ModuleState::Restarting);
12953 assert_eq!(
12954 state.crash_restarts.len(),
12955 2,
12956 "the aged instant is dropped and the new one takes its place"
12957 );
12958 assert_eq!(
12959 state.lifetime_restarts, 3,
12960 "the ledger counts every restart, including the ones the window forgot"
12961 );
12962 }
12963
12964 #[tokio::test]
12969 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
12970 let supervisor = Supervisor::new_for_test(
12971 Arc::new(Registry::default()),
12972 RestartPolicy::new(2, Duration::ZERO),
12973 );
12974 let runtime = supervisor.runtime_config();
12975 let spec = windowed_crash_spec("operator-cleared-budget");
12976 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12977
12978 for attempt in 1..=2 {
12979 assert!(matches!(
12980 on_child_exit(
12981 &spec,
12982 runtime.restart_policy,
12983 &supervisor.registry,
12984 &snapshot,
12985 &runtime.terminal_ring,
12986 &runtime.spawn_events,
12987 &runtime.child_roster,
12988 crash_exit_report(attempt),
12989 )
12990 .await,
12991 NextAction::Restart { schedule: _ }
12992 ));
12993 }
12994
12995 reset_restart_count(&snapshot, &spec.module_id).unwrap();
12996 {
12997 let state = lock_snapshot(&snapshot).unwrap();
12998 assert!(
12999 state.crash_restarts.is_empty(),
13000 "an operator restart returns the full budget"
13001 );
13002 assert_eq!(
13003 state.lifetime_restarts, 2,
13004 "clearing the budget must not unmake the crashes"
13005 );
13006 }
13007
13008 assert!(
13009 matches!(
13010 on_child_exit(
13011 &spec,
13012 runtime.restart_policy,
13013 &supervisor.registry,
13014 &snapshot,
13015 &runtime.terminal_ring,
13016 &runtime.spawn_events,
13017 &runtime.child_roster,
13018 crash_exit_report(3),
13019 )
13020 .await,
13021 NextAction::Restart { schedule: _ }
13022 ),
13023 "the cleared budget must be spendable again"
13024 );
13025 let state = lock_snapshot(&snapshot).unwrap();
13026 assert_eq!(state.crash_restarts.len(), 1);
13027 assert_eq!(state.lifetime_restarts, 3);
13028 }
13029
13030 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13031 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13032 let severed = ProcessIdentity {
13033 pid: 41,
13034 start_time: 101,
13035 };
13036 let successor = ProcessIdentity {
13037 pid: 41,
13038 start_time: 202,
13039 };
13040 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13041 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13042 state.pid = Some(successor.pid);
13043 state.process_start_time = Some(successor.start_time);
13044 })
13045 .unwrap();
13046 assert!(!module.record_deliberate_severance(severed).unwrap());
13047
13048 let exit_report = apply_deliberate_severance_marker(
13049 &module.inner.snapshot,
13050 Some(successor),
13051 ExitReport {
13052 kind: ExitKind::Crash,
13053 code: Some(1),
13054 signal: None,
13055 at_ms: 1,
13056 },
13057 );
13058
13059 assert_eq!(exit_report.kind, ExitKind::Crash);
13060 }
13061
13062 #[tokio::test]
13063 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13064 let registry = Registry::default();
13065 let supervisor = Supervisor::new_for_test(
13066 Arc::new(Registry::default()),
13067 RestartPolicy::new(3, Duration::ZERO),
13068 );
13069 let runtime = supervisor.runtime_config();
13070 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13071 let spec = ModuleSpec {
13072 module_id: "drain-deliberate-severance".to_string(),
13073 program: fake_aft_stub_path(),
13074 args: Vec::new(),
13075 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13076 reserved: false,
13077 reserved_prefixes: Vec::new(),
13078 protocol: ModuleProtocol::Subc,
13079 overlap: Default::default(),
13080 };
13081 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13082 let process = ProcessIdentity {
13083 pid: 41,
13084 start_time: 101,
13085 };
13086 child.process_identity = Some(process);
13087 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13088 state.pid = Some(process.pid);
13089 state.process_start_time = Some(process.start_time);
13090 })
13091 .unwrap();
13092 record_deliberate_severance(&snapshot, process).unwrap();
13093
13094 drain_child_to_state(
13095 &spec.module_id,
13096 spec.protocol,
13097 StopNotice::SentOverConnection,
13100 ®istry,
13101 None,
13102 &snapshot,
13103 &runtime.terminal_ring,
13104 &runtime.spawn_events,
13105 child,
13106 Duration::from_secs(1),
13107 ModuleState::Stopped,
13108 Some(false),
13109 )
13110 .await
13111 .unwrap();
13112
13113 let state = lock_snapshot(&snapshot).unwrap();
13114 assert_eq!(
13115 state.last_exit.as_ref().map(|exit| exit.kind),
13116 Some(ExitKind::DeliberateSeverance)
13117 );
13118 assert_eq!(state.lifetime_restarts, 1);
13119 assert_eq!(state.crash_restarts.len(), 0);
13120 drop(state);
13121 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13122 assert_eq!(
13123 history.entries[0].exit_kind,
13124 subc_control::TerminalExitKind::DeliberateSeverance
13125 );
13126 }
13127
13128 #[tokio::test]
13129 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13130 let registry = Registry::default();
13131 let supervisor = Supervisor::new_for_test(
13132 Arc::new(Registry::default()),
13133 RestartPolicy::new(3, Duration::ZERO),
13134 );
13135 let runtime = supervisor.runtime_config();
13136 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13137 let spec = ModuleSpec {
13138 module_id: "ordinary-drain".to_string(),
13139 program: fake_aft_stub_path(),
13140 args: Vec::new(),
13141 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13142 reserved: false,
13143 reserved_prefixes: Vec::new(),
13144 protocol: ModuleProtocol::Subc,
13145 overlap: Default::default(),
13146 };
13147 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13148
13149 drain_child_to_state(
13150 &spec.module_id,
13151 spec.protocol,
13152 StopNotice::SentOverConnection,
13155 ®istry,
13156 None,
13157 &snapshot,
13158 &runtime.terminal_ring,
13159 &runtime.spawn_events,
13160 child,
13161 Duration::from_secs(1),
13162 ModuleState::Stopped,
13163 Some(false),
13164 )
13165 .await
13166 .unwrap();
13167
13168 let state = lock_snapshot(&snapshot).unwrap();
13169 assert_eq!(
13170 state.last_exit.as_ref().map(|exit| exit.kind),
13171 Some(ExitKind::Crash)
13172 );
13173 assert_eq!(state.lifetime_restarts, 0);
13174 assert_eq!(state.crash_restarts.len(), 0);
13175 }
13176
13177 #[test]
13178 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13179 assert!(!include_str!("server.rs")
13185 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13186 }
13187
13188 #[test]
13195 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13196 assert!(drained_after_quiescence_wait(&Ok(true)));
13197 assert!(!drained_after_quiescence_wait(&Ok(false)));
13198 assert!(!drained_after_quiescence_wait(&Err(
13199 SuperviseError::StatePoisoned { module_id: None }
13200 )));
13201 }
13202
13203 #[test]
13212 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13213 let ring = Arc::new(Mutex::new(TerminalRing::new(
13214 TerminalRingConfig::default(),
13215 0,
13216 )));
13217 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13218
13219 let snapshot = ring.lock().unwrap().snapshot();
13220 assert_eq!(snapshot.entries.len(), 1);
13221 let entry = &snapshot.entries[0];
13222 assert_eq!(entry.exit_code, None);
13223 assert_eq!(entry.exit_signal, None);
13224 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13225 }
13226
13227 #[test]
13228 fn wait_error_exit_path_preserves_spawn_event_density() {
13229 let feed = super::SpawnEventFeed::default();
13230 feed.configure_incarnation("wait-error-density".to_string());
13231 feed.emit_spawned("wait-error", 41, 1);
13232 let ring = Arc::new(Mutex::new(TerminalRing::new(
13233 TerminalRingConfig::default(),
13234 0,
13235 )));
13236
13237 record_wait_error_terminal("wait-error", &ring, &feed);
13238 feed.emit_spawned("after-wait-error", 42, 2);
13239
13240 let state = feed.0.lock().unwrap();
13241 let sequences = state
13242 .events
13243 .iter()
13244 .map(|event| event.cursor.seq)
13245 .collect::<Vec<_>>();
13246 assert_eq!(sequences, vec![1, 2, 3]);
13247 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13248 assert_eq!(state.events[1].exit_code, None);
13249 assert_eq!(state.events[1].exit_signal, None);
13250 }
13251
13252 #[test]
13256 fn wait_error_exit_report_is_classified_as_a_crash() {
13257 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13258 }
13259}
13260
13261#[cfg(all(test, unix))]
13262mod restart_ordering_tests {
13263 use super::*;
13264 use std::io::Write;
13265 use tokio::io::AsyncReadExt;
13266
13267 #[test]
13268 #[ignore = "child process fixture"]
13269 fn serving_child() {
13270 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13271 return;
13272 };
13273 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13274 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13275 for stream in listener.incoming() {
13276 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13277 }
13278 }
13279
13280 async fn restart_before_publication(requests: usize) {
13281 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13282 let address_file = home.path().join("address");
13283 let spec = ModuleSpec {
13284 module_id: "restart-serving-child".into(),
13285 program: std::env::current_exe().unwrap(),
13286 args: vec![
13287 "--exact".into(),
13288 "supervise::restart_ordering_tests::serving_child".into(),
13289 "--ignored".into(),
13290 "--nocapture".into(),
13291 ],
13292 env: vec![(
13293 "RESTART_TEST_ADDRESS_FILE".into(),
13294 address_file.display().to_string(),
13295 )],
13296 reserved: false,
13297 reserved_prefixes: vec![],
13298 protocol: ModuleProtocol::None,
13299 overlap: Default::default(),
13300 };
13301 let mut supervisor =
13302 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13303 let (spawned_tx, spawned_rx) = oneshot::channel();
13304 let spawned_tx = Mutex::new(Some(spawned_tx));
13305 let (release_tx, release_rx) = std::sync::mpsc::channel();
13306 let release_rx = Mutex::new(release_rx);
13307 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13310 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13311 release_rx
13312 .lock()
13313 .unwrap()
13314 .recv_timeout(Duration::from_secs(20))
13315 .unwrap();
13316 })));
13317 let registered = supervisor
13318 .register_configured_with_health(
13319 spec.clone(),
13320 true,
13321 HealthConfig::default(),
13322 None,
13323 RestartPolicy::new(3, Duration::ZERO),
13324 )
13325 .unwrap();
13326 let module = registered.module.clone();
13327 let startup = tokio::task::spawn_blocking(move || {
13328 supervisor.start_registered_configured(registered, None)
13329 });
13330 spawned_rx.await.unwrap();
13331 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13332 let address = loop {
13333 if let Ok(address) = std::fs::read_to_string(&address_file) {
13334 if !address.is_empty() {
13335 break address;
13336 }
13337 }
13338 tokio::time::sleep(Duration::from_millis(1)).await;
13339 };
13340 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13341 let mut reply = String::new();
13342 stream.read_to_string(&mut reply).await.unwrap();
13343 reply.trim().parse::<u32>().unwrap()
13344 })
13345 .await
13346 .expect("child must serve before generation publication");
13347 assert_eq!(
13348 lock_snapshot(&module.inner.snapshot)
13349 .unwrap()
13350 .spawn_generation,
13351 0
13352 );
13353 let mut replies = Vec::new();
13354 for _ in 0..requests {
13355 let mut restart = Box::pin(module.restart(Some(0)));
13356 std::future::poll_fn(|cx| {
13359 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13360 std::task::Poll::Ready(())
13361 })
13362 .await;
13363 replies.push(restart);
13364 }
13365 release_tx.send(()).unwrap();
13366 startup.await.unwrap().unwrap();
13367 for reply in replies {
13368 reply.await.unwrap();
13369 }
13370 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13373 loop {
13374 if lock_snapshot(&module.inner.snapshot)
13375 .unwrap()
13376 .spawn_generation
13377 > 1
13378 {
13379 break;
13380 }
13381 tokio::time::sleep(Duration::from_millis(1)).await;
13382 }
13383 })
13384 .await;
13385 module.update_spec_for_test(spec).await.unwrap();
13389 let (generation, pid) = {
13390 let state = lock_snapshot(&module.inner.snapshot).unwrap();
13391 (state.spawn_generation, state.pid)
13392 };
13393 module.stop().await.unwrap();
13394 replacement.expect("restart must publish a replacement for the already-serving child");
13395 assert_ne!(
13396 pid,
13397 Some(old_pid),
13398 "a serving child predating the restart must be replaced"
13399 );
13400 assert_eq!(generation, 2, "queued restarts must share one replacement");
13401 }
13402
13403 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13404 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13405 restart_before_publication(1).await;
13406 }
13407
13408 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13409 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13410 restart_before_publication(2).await;
13411 }
13412
13413 #[tokio::test]
13414 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13415 let supervisor =
13416 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13417 let registered = supervisor
13418 .register_configured_with_health(
13419 ModuleSpec {
13420 module_id: "queued-before-replacement".into(),
13421 program: "/bin/sh".into(),
13422 args: vec!["-c".into(), "exec sleep 600".into()],
13423 env: vec![],
13424 reserved: false,
13425 reserved_prefixes: vec![],
13426 protocol: ModuleProtocol::None,
13427 overlap: Default::default(),
13428 },
13429 true,
13430 HealthConfig::default(),
13431 None,
13432 RestartPolicy::default(),
13433 )
13434 .unwrap();
13435 let RegisteredConfiguredModule {
13436 mut spec,
13437 mut runtime,
13438 module,
13439 mut commands,
13440 } = registered;
13441 let snapshot = &module.inner.snapshot;
13442 let mut restart = Box::pin(module.restart(Some(0)));
13443 std::future::poll_fn(|cx| {
13444 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13445 std::task::Poll::Ready(())
13446 })
13447 .await;
13448 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13451 let replacement_pid = child.as_ref().unwrap().id();
13452 let command = commands.recv().await.unwrap();
13453 assert!(
13454 handle_supervisor_command(
13455 command,
13456 &mut spec,
13457 &mut runtime,
13458 &supervisor.registry,
13459 &supervisor.process_liveness,
13460 snapshot,
13461 &mut child,
13462 &mut commands,
13463 &mut VecDeque::new(),
13464 )
13465 .await
13466 );
13467 restart.await.unwrap();
13468 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13469 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13470 if let Some(mut child) = child {
13471 child.start_kill().unwrap();
13472 child.wait().await.unwrap();
13473 }
13474 assert_eq!(
13475 kept_pid, replacement_pid,
13476 "a newer replacement must survive"
13477 );
13478 assert!(
13479 !pending,
13480 "a satisfied restart must not schedule another spawn"
13481 );
13482 }
13483}
13484
13485#[cfg(all(test, unix))]
13486mod health_event_tests {
13487 use super::*;
13488 use cortexkit_test_support::ScratchDir;
13489 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13490
13491 struct Actor {
13492 module: SupervisedModule,
13493 registry: Arc<Registry>,
13494 forwarding: Arc<ForwardingTable>,
13495 spec: ModuleSpec,
13496 health: HealthConfig,
13497 rx: mpsc::Receiver<crate::router::OutboundFrame>,
13498 _home: ScratchDir,
13499 }
13500
13501 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13502 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13506 let tick = Instant::now();
13507 loop {
13508 if let Some(value) = observe() {
13509 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13510 return value;
13511 }
13512 assert!(
13513 std::time::Instant::now() < deadline,
13514 "timed out waiting for {reason}"
13515 );
13516 tokio::task::yield_now().await;
13517 }
13518 }
13519
13520 impl Actor {
13521 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13522 let home = ScratchDir::new("health-events");
13523 let registry = Arc::new(Registry::default());
13524 let forwarding = Arc::new(ForwardingTable::default());
13525 let health = HealthConfig {
13526 cadence,
13527 deadline: Duration::from_secs(3600),
13528 ..HealthConfig::default()
13529 };
13530 let supervisor =
13531 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13532 .with_forwarding(forwarding.clone())
13533 .with_health_config(health.clone());
13534 let spec = ModuleSpec {
13535 module_id: "event-health-child".into(),
13536 program: PathBuf::from("/bin/sh"),
13537 args: vec!["-c".into(), "exec sleep 600".into()],
13540 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13541 .into_iter()
13542 .map(|key| (key.into(), home.path().display().to_string()))
13543 .collect(),
13544 reserved: false,
13545 reserved_prefixes: vec![],
13546 protocol,
13547 overlap: Default::default(),
13548 };
13549 let module = supervisor.spawn(spec.clone()).unwrap();
13550 let (_, rx) = mpsc::channel(8);
13551 let actor = Self {
13552 module,
13553 registry,
13554 forwarding,
13555 spec,
13556 health,
13557 rx,
13558 _home: home,
13559 };
13560 actor
13561 .wait_for_select(|checkpoint| checkpoint.generation > 0)
13562 .await;
13563 actor
13564 }
13565
13566 fn turns(&self) -> u64 {
13567 lock_snapshot(&self.module.inner.snapshot)
13568 .unwrap()
13569 .actor_turns
13570 }
13571
13572 async fn wait_for_select(
13573 &self,
13574 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13575 ) -> ActorSelectCheckpoint {
13576 wait_for("actor select checkpoint", || {
13577 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13578 state.actor_select.filter(|checkpoint| {
13579 checkpoint.generation == state.spawn_generation
13580 && state.state == ModuleState::Running
13581 && state.process_alive
13582 && state.reported_pid().is_some()
13583 && expected(checkpoint)
13584 })
13585 })
13586 .await
13587 }
13588
13589 fn hello(&mut self, connection: u64, health: bool) {
13590 let manifest =
13591 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13592 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13593 .build();
13594 let connection = ConnectionId::new(connection);
13595 let (tx, rx) = mpsc::channel(8);
13596 self.rx = rx;
13597 self.forwarding
13598 .register_module_connection(
13599 connection,
13600 self.spec.module_id.clone(),
13601 subc_protocol::PROTOCOL_VERSION,
13602 Concurrency::ModuleManaged,
13603 FrameSink::new(tx),
13604 )
13605 .unwrap();
13606 self.registry
13607 .register_with_control_ops(
13608 manifest,
13609 subc_protocol::PROTOCOL_VERSION,
13610 connection,
13611 if health {
13612 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13613 } else {
13614 vec![]
13615 },
13616 )
13617 .unwrap();
13618 }
13619
13620 async fn probe(&mut self) -> crate::router::OutboundFrame {
13621 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13622 Ok(frame) => Some(frame),
13623 Err(mpsc::error::TryRecvError::Empty) => None,
13624 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13625 })
13626 .await;
13627 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13628 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13629 frame
13630 }
13631
13632 fn no_probe(&mut self) {
13633 assert!(matches!(
13634 self.rx.try_recv(),
13635 Err(mpsc::error::TryRecvError::Empty)
13636 ));
13637 }
13638
13639 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13640 self.forwarding
13641 .complete_module_control_rpc(
13642 ConnectionId::new(connection),
13643 frame.header.corr,
13644 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13645 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13646 status: HealthStatus::Ok,
13647 detail: None,
13648 metrics: None,
13649 }),
13650 )
13651 .unwrap();
13652 }
13653 }
13654
13655 #[tokio::test(start_paused = true)]
13656 async fn no_health_children_park_for_sixty_seconds() {
13657 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13658 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13659 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13660 unadvertised.hello(1, false);
13661 for _ in 0..6000 {
13666 tokio::time::advance(Duration::from_millis(10)).await;
13667 tokio::task::yield_now().await;
13668 }
13669 for actor in [&none, &unregistered, &unadvertised] {
13670 assert!(
13671 actor.turns() <= 5,
13672 "no-health actor woke {} times",
13673 actor.turns()
13674 );
13675 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13676 }
13677 }
13678
13679 #[tokio::test(start_paused = true)]
13680 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13681 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13685 let tick = Instant::now();
13686 actor.hello(1, true);
13687 actor.probe().await;
13688 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13689 }
13690
13691 #[tokio::test(start_paused = true)]
13692 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13693 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13694 let hello_tick = Instant::now();
13695 actor.hello(1, true);
13696 let armed = actor
13697 .wait_for_select(|checkpoint| {
13698 checkpoint.registered_connection == Some(ConnectionId::new(1))
13699 && checkpoint.next_probe_at.is_some()
13700 })
13701 .await;
13702 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13703 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13704 let first_deadline = hello_tick + delay;
13705 assert_eq!(armed.next_probe_at, Some(first_deadline));
13706 tokio::time::advance(Duration::from_secs(29)).await;
13707 let turns = actor.turns();
13708 actor
13709 .registry
13710 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13711 .unwrap();
13712 let updated = actor
13713 .wait_for_select(|checkpoint| checkpoint.turn > turns)
13714 .await;
13715 assert_eq!(
13716 updated.next_probe_at,
13717 Some(first_deadline),
13718 "catalog update must not reset cadence"
13719 );
13720 actor.no_probe();
13721 tokio::time::advance(first_deadline - Instant::now()).await;
13722 let frame = actor.probe().await;
13723 actor.answer(1, frame);
13724 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13725 let next_deadline = Instant::now() + next;
13726 let rearmed = actor
13727 .wait_for_select(|checkpoint| {
13728 checkpoint
13729 .next_probe_at
13730 .is_some_and(|deadline| deadline > first_deadline)
13731 })
13732 .await;
13733 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13734 tokio::time::advance(next - Duration::from_nanos(1)).await;
13735 actor.no_probe();
13736 tokio::time::advance(Duration::from_nanos(1)).await;
13737 actor.probe().await;
13738 }
13739
13740 #[tokio::test(start_paused = true)]
13741 async fn disconnect_disarms_and_reconnect_rearms_health() {
13742 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13743 actor.hello(1, true);
13744 actor
13745 .wait_for_select(|checkpoint| {
13746 checkpoint.registered_connection == Some(ConnectionId::new(1))
13747 && checkpoint.next_probe_at.is_some()
13748 })
13749 .await;
13750 let turns = actor.turns();
13751 let tick = Instant::now();
13752 actor
13753 .registry
13754 .deregister_connection(ConnectionId::new(1))
13755 .unwrap();
13756 let disarmed = actor
13757 .wait_for_select(|checkpoint| {
13758 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13759 })
13760 .await;
13761 assert_eq!(disarmed.next_probe_at, None);
13762 assert_eq!(disarmed.wake_after, None);
13763 assert_eq!(
13764 Instant::now(),
13765 tick,
13766 "disconnect must disarm in the notification tick"
13767 );
13768 tokio::time::advance(Duration::from_secs(60)).await;
13769 actor.no_probe();
13770 let reconnect_tick = Instant::now();
13771 actor.hello(2, true);
13772 let rearmed = actor
13773 .wait_for_select(|checkpoint| {
13774 checkpoint.registered_connection == Some(ConnectionId::new(2))
13775 && checkpoint.next_probe_at.is_some()
13776 })
13777 .await;
13778 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13779 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13780 tokio::time::advance(delay).await;
13781 actor.probe().await;
13782 }
13783
13784 #[tokio::test(start_paused = true)]
13785 async fn reregistration_without_health_disarms_the_old_schedule() {
13786 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13787 actor.hello(1, true);
13788 actor
13789 .wait_for_select(|checkpoint| {
13790 checkpoint.registered_connection == Some(ConnectionId::new(1))
13791 && checkpoint.next_probe_at.is_some()
13792 })
13793 .await;
13794 actor
13795 .registry
13796 .deregister_connection(ConnectionId::new(1))
13797 .unwrap();
13798 actor.hello(2, false);
13799 let disarmed = actor
13800 .wait_for_select(|checkpoint| {
13801 checkpoint.registered_connection == Some(ConnectionId::new(2))
13802 })
13803 .await;
13804 assert_eq!(disarmed.next_probe_at, None);
13805 assert_eq!(disarmed.wake_after, None);
13806 tokio::time::advance(Duration::from_secs(60)).await;
13807 actor.no_probe();
13808 }
13809
13810 #[tokio::test(start_paused = true)]
13811 async fn restart_waits_for_its_own_hello_before_rearming_health() {
13812 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13813 actor.hello(1, true);
13814 let armed = actor
13815 .wait_for_select(|checkpoint| {
13816 checkpoint.registered_connection == Some(ConnectionId::new(1))
13817 && checkpoint.next_probe_at.is_some()
13818 })
13819 .await;
13820 actor
13824 .registry
13825 .deregister_connection(ConnectionId::new(1))
13826 .unwrap();
13827 actor.module.restart(Some(0)).await.unwrap();
13828 let parked = actor
13829 .wait_for_select(|checkpoint| {
13830 checkpoint.generation > armed.generation
13831 && checkpoint.registered_connection.is_none()
13832 && checkpoint.next_probe_at.is_none()
13833 })
13834 .await;
13835 assert_eq!(parked.wake_after, None);
13836 actor.rx.close();
13837 while actor.rx.try_recv().is_ok() {}
13840 let turns = actor.turns();
13841 tokio::time::advance(Duration::from_secs(60)).await;
13842 let after = actor.turns();
13843 eprintln!(
13844 "restart park turns: before={turns} after={after} delta={}",
13845 after - turns
13846 );
13847 assert_eq!(after, turns, "replacement must park before HELLO");
13848 let hello_tick = Instant::now();
13849 actor.hello(2, true);
13850 let rearmed = actor
13851 .wait_for_select(|checkpoint| {
13852 checkpoint.registered_connection == Some(ConnectionId::new(2))
13853 && checkpoint.next_probe_at.is_some()
13854 })
13855 .await;
13856 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13857 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13858 tokio::time::advance(delay).await;
13859 actor.probe().await;
13860 }
13861
13862 #[tokio::test(start_paused = true)]
13863 async fn rescan_adds_and_removes_http_health_without_polling() {
13864 use tokio::{
13865 io::{AsyncReadExt, AsyncWriteExt},
13866 net::TcpListener,
13867 };
13868 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13869 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13870 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13871 actor.health.cadence = Duration::from_millis(1);
13873 let (started_tx, mut started_rx) = mpsc::channel(8);
13874 let peer = tokio::spawn(async move {
13875 loop {
13876 let (mut socket, _) = listener.accept().await.unwrap();
13877 let mut request = [0; 1024];
13878 assert!(socket.read(&mut request).await.unwrap() > 0);
13879 started_tx.send(()).await.unwrap();
13880 socket
13881 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13882 .await
13883 .unwrap();
13884 }
13885 });
13886 let tick = Instant::now();
13887 let turns = actor.turns();
13888 actor
13889 .module
13890 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13891 .await
13892 .unwrap();
13893 let armed = actor
13894 .wait_for_select(|checkpoint| {
13895 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13896 })
13897 .await;
13898 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13899 assert_eq!(Instant::now(), tick);
13900 tokio::time::advance(Duration::from_millis(1)).await;
13901 let mut started = false;
13902 wait_for("successful HTTP probe", || {
13903 started |= started_rx.try_recv().is_ok();
13904 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13905 .then_some(())
13906 })
13907 .await;
13908 assert!(started, "added HTTP check must start at its first deadline");
13909 assert_eq!(
13910 actor.module.status().unwrap().health.status,
13911 SupervisorHealthStatus::Ok
13912 );
13913 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13914 actor.health.http = None;
13915 let turns = actor.turns();
13916 actor
13917 .module
13918 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13919 .await
13920 .unwrap();
13921 let parked = actor
13922 .wait_for_select(|checkpoint| {
13923 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13924 })
13925 .await;
13926 assert_eq!(parked.wake_after, None);
13927 let turns = actor.turns();
13928 tokio::time::advance(Duration::from_secs(60)).await;
13929 assert_eq!(
13930 actor.turns(),
13931 turns,
13932 "removed HTTP check must park the actor"
13933 );
13934 assert!(
13935 started_rx.try_recv().is_err(),
13936 "removed HTTP check must stay disarmed"
13937 );
13938 assert_eq!(
13939 actor.module.status().unwrap().health,
13940 ModuleHealthStatus::default()
13941 );
13942 peer.abort();
13943 }
13944}
13945
13946#[cfg(test)]
13947mod drain_busy_reprobe_tests {
13948 use super::*;
13952 use std::collections::BTreeMap;
13953 use std::sync::Mutex as StdMutex;
13954 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13955
13956 const MODULE_ID: &str = "drain-reprobe-module";
13957 const CONNECTION: u64 = 7;
13958 const GAUGE: &str = "runs_in_flight";
13959 const CADENCE: Duration = Duration::from_secs(30);
13962 const CEILING: Duration = Duration::from_secs(30);
13963
13964 #[derive(Clone, Copy)]
13966 enum Answer {
13967 BusyUntil { until: Instant, latency: Duration },
13970 OmitsGauge,
13972 Silent,
13974 }
13975
13976 struct Drain {
13977 forwarding: Arc<ForwardingTable>,
13978 runtime: SupervisorRuntimeConfig,
13979 endpoint: crate::ModuleEndpointId,
13980 probes: Arc<StdMutex<Vec<Instant>>>,
13982 responder: tokio::task::JoinHandle<()>,
13983 }
13984
13985 impl Drain {
13986 fn start(answer: Answer) -> Self {
13987 let registry = Arc::new(Registry::default());
13988 let forwarding = Arc::new(ForwardingTable::default());
13989 let health = HealthConfig {
13990 cadence: CADENCE,
13991 ..HealthConfig::default()
13992 };
13993 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
13994 .with_forwarding(forwarding.clone())
13995 .with_health_config(health)
13996 .runtime_config();
13997 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
13998 let endpoint = forwarding
13999 .register_module_connection(
14000 ConnectionId::new(CONNECTION),
14001 MODULE_ID.to_string(),
14002 subc_protocol::PROTOCOL_VERSION,
14003 Concurrency::ModuleManaged,
14004 FrameSink::new(tx),
14005 )
14006 .unwrap();
14007 let probes = Arc::new(StdMutex::new(Vec::new()));
14008 let responder = tokio::spawn({
14009 let forwarding = forwarding.clone();
14010 let probes = probes.clone();
14011 async move {
14012 while let Some(frame) = rx.recv().await {
14013 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14014 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14015 probes.lock().unwrap().push(Instant::now());
14016 let metrics = match answer {
14017 Answer::BusyUntil { until, latency } => {
14018 sleep(latency).await;
14019 let busy = u64::from(Instant::now() < until);
14020 serde_json::json!({ GAUGE: busy })
14021 }
14022 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14023 Answer::Silent => continue,
14024 };
14025 let _ = forwarding.complete_module_control_rpc(
14028 ConnectionId::new(CONNECTION),
14029 frame.header.corr,
14030 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14031 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14032 status: HealthStatus::Ok,
14033 detail: None,
14034 metrics: Some(metrics),
14035 }),
14036 );
14037 }
14038 }
14039 });
14040 Self {
14041 forwarding,
14042 runtime,
14043 endpoint,
14044 probes,
14045 responder,
14046 }
14047 }
14048
14049 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14052 let (drained, elapsed, _) = self.run_traced(ceiling).await;
14053 (drained, elapsed)
14054 }
14055
14056 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14058 let started = Instant::now();
14059 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14060 let drained = wait_for_forwarding_quiescence(
14061 &self.forwarding,
14062 MODULE_ID,
14063 &self.runtime,
14064 self.endpoint,
14065 started + ceiling,
14066 &[GAUGE.to_string()],
14067 DrainScope::Active,
14068 &mut trace,
14069 )
14070 .await
14071 .unwrap();
14072 (drained, started.elapsed(), trace)
14073 }
14074
14075 fn probes(&self) -> Vec<Instant> {
14076 self.probes.lock().unwrap().clone()
14077 }
14078 }
14079
14080 impl Drop for Drain {
14081 fn drop(&mut self) {
14082 self.responder.abort();
14083 }
14084 }
14085
14086 #[tokio::test(start_paused = true)]
14087 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14088 let drain = Drain::start(Answer::BusyUntil {
14089 until: Instant::now() + Duration::from_secs(1),
14090 latency: Duration::ZERO,
14091 });
14092 let (drained, elapsed) = drain.run(CEILING).await;
14093 assert!(
14094 drained,
14095 "gauges that read zero during the drain must let it finish drained, \
14096 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14097 );
14098 assert!(
14099 elapsed < Duration::from_secs(3),
14100 "drain took {elapsed:?} after the gauges went quiet at 1s"
14101 );
14102 let probes = drain.probes();
14103 assert!(
14104 probes.len() >= 2,
14105 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14106 );
14107 }
14108
14109 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14114 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14115 let drain = Drain::start(Answer::BusyUntil {
14116 until: Instant::now() + Duration::from_millis(600),
14117 latency: Duration::ZERO,
14118 });
14119 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14120 assert!(drained, "the gauge fell at 600ms: {trace}");
14121 assert!(
14122 elapsed < Duration::from_secs(3),
14123 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14124 );
14125 let probes = drain.probes();
14126 assert!(
14127 probes.len() >= 2 && trace.probes as usize == probes.len(),
14128 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14129 );
14130 }
14131
14132 #[tokio::test(start_paused = true)]
14133 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14134 let started = Instant::now();
14135 let drain = Drain::start(Answer::BusyUntil {
14136 until: started + Duration::from_secs(3600),
14137 latency: Duration::ZERO,
14138 });
14139 let (drained, elapsed) = drain.run(CEILING).await;
14140 assert!(!drained, "a module busy throughout must not drain");
14141 assert!(
14142 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14143 "a busy module is held until the ceiling and no longer: {elapsed:?}"
14144 );
14145 let probes = drain.probes();
14148 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14149 assert!(
14150 probes.len() as u128 <= max_probes,
14151 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14152 probes.len()
14153 );
14154 for pair in probes.windows(2) {
14155 assert!(
14156 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14157 "consecutive probes {:?} apart",
14158 pair[1] - pair[0]
14159 );
14160 }
14161 }
14162
14163 #[tokio::test(start_paused = true)]
14164 async fn a_slow_probe_is_never_overlapped_by_the_next() {
14165 let latency = Duration::from_secs(1);
14168 let drain = Drain::start(Answer::BusyUntil {
14169 until: Instant::now() + Duration::from_secs(3600),
14170 latency,
14171 });
14172 let (drained, _) = drain.run(Duration::from_secs(5)).await;
14173 assert!(!drained);
14174 let probes = drain.probes();
14175 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14176 for pair in probes.windows(2) {
14177 assert!(
14178 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14179 "a probe started {:?} after the previous one, before it could finish",
14180 pair[1] - pair[0]
14181 );
14182 }
14183 }
14184
14185 #[tokio::test(start_paused = true)]
14186 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14187 let drain = Drain::start(Answer::Silent);
14188 let ceiling = Duration::from_secs(2);
14189 let (drained, elapsed) = drain.run(ceiling).await;
14190 assert!(!drained, "an unanswered probe counts as busy");
14191 assert!(
14192 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14193 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14194 );
14195 assert_eq!(
14196 drain.probes().len(),
14197 1,
14198 "the unanswered probe holds the only slot until the ceiling"
14199 );
14200 }
14201
14202 #[derive(Clone, Default)]
14204 struct LogCapture {
14205 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14206 }
14207
14208 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14209 fn on_event(
14210 &self,
14211 event: &tracing::Event<'_>,
14212 _context: tracing_subscriber::layer::Context<'_, S>,
14213 ) {
14214 struct Fields(BTreeMap<String, String>);
14215 impl tracing::field::Visit for Fields {
14216 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14217 self.0
14218 .insert(field.name().to_string(), format!("{value:?}"));
14219 }
14220 }
14221 let mut fields = Fields(BTreeMap::new());
14222 event.record(&mut fields);
14223 self.events.lock().unwrap().push(fields.0);
14224 }
14225 }
14226
14227 impl LogCapture {
14228 fn install(&self) -> tracing::subscriber::DefaultGuard {
14229 use tracing_subscriber::layer::SubscriberExt;
14230 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14231 }
14232
14233 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14234 self.events
14235 .lock()
14236 .unwrap()
14237 .iter()
14238 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14239 .cloned()
14240 .collect()
14241 }
14242 }
14243
14244 #[tokio::test(start_paused = true)]
14249 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14250 let capture = LogCapture::default();
14251 let _guard = capture.install();
14252 let drain = Drain::start(Answer::OmitsGauge);
14253 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14254 assert!(!drained, "a gauge never reported cannot read quiet");
14255 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14256
14257 let missing = capture.with_message("lacks a declared busy gauge");
14258 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14259 assert_eq!(
14260 missing[0].get("missing_gauges").map(String::as_str),
14261 Some(GAUGE)
14262 );
14263
14264 warn_drain_timed_out(
14265 MODULE_ID,
14266 Duration::from_secs(2),
14267 RouteCloseReason::Restart,
14268 &Default::default(),
14269 &trace,
14270 );
14271 let timeouts = capture.with_message("route drain timed out");
14272 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14273 let gauges = &timeouts[0]["busy_gauges"];
14274 assert!(
14275 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14276 "the timeout line must name the gauge and why it read busy: {gauges}"
14277 );
14278 assert!(
14279 gauges.ends_with("; no value ever read"),
14280 "no value was ever read for the gauge: {gauges}"
14281 );
14282 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14283 }
14284
14285 #[tokio::test(start_paused = true)]
14289 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14290 let busy = Drain::start(Answer::BusyUntil {
14291 until: Instant::now() + Duration::from_secs(3600),
14292 latency: Duration::ZERO,
14293 });
14294 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14295 let rendered = trace.to_string();
14296 assert!(
14297 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14298 && rendered.contains("; last value non_zero(1) @"),
14299 "{rendered}"
14300 );
14301
14302 let silent = Drain::start(Answer::Silent);
14303 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14304 assert_eq!(
14307 trace.to_string(),
14308 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14309 );
14310 assert_eq!(trace.probes, 1);
14311 }
14312
14313 #[test]
14315 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14316 let gauges = [GAUGE.to_string()];
14317 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14318 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14319 assert_eq!(
14320 busy_gauge_observation(&verdicts),
14321 BusyGaugeObservation::Busy
14322 );
14323 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14324 assert_eq!(
14325 busy_gauge_observation(&zero),
14326 BusyGaugeObservation::Quiescent
14327 );
14328 let none = gauge_verdicts(None, &gauges);
14329 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14330 }
14331}
14332
14333#[cfg(test)]
14334mod health_evidence_tests {
14335 use super::{HealthProbeError, HealthProbeEvidence};
14336 use std::collections::HashSet;
14337
14338 #[test]
14346 fn only_a_dead_lane_is_proof_of_death() {
14347 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14348 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14352 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14353 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14354 }
14355
14356 #[test]
14362 fn every_evidence_class_has_a_distinct_label() {
14363 let labels = [
14364 HealthProbeError::lane_dead("").label(),
14365 HealthProbeError::no_answer("").label(),
14366 HealthProbeError::bad_answer("").label(),
14367 HealthProbeError::misconfigured("").label(),
14368 ];
14369 let unique: HashSet<_> = labels.iter().collect();
14370 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14371 }
14372
14373 #[test]
14379 fn classification_preserves_the_original_message() {
14380 let err = HealthProbeError::no_answer("module did not answer within 5s");
14381 assert_eq!(err.to_string(), "module did not answer within 5s");
14382 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14383 }
14384}
14385
14386#[cfg(test)]
14387mod health_tombstone_tests {
14388 use std::{path::PathBuf, sync::Arc, time::Duration};
14389
14390 use subc_protocol::{
14391 manifest::Concurrency,
14392 session::{HealthStatus, ModuleControlResponse},
14393 };
14394 use tokio::sync::mpsc;
14395
14396 use super::{
14397 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14398 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14399 };
14400 use crate::{
14401 control::ControlHandler,
14402 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14403 registry::{ConnectionId, Registry},
14404 router::FrameSink,
14405 };
14406
14407 struct ProbeHarness {
14408 spec: ModuleSpec,
14409 runtime: SupervisorRuntimeConfig,
14410 forwarding: Arc<ForwardingTable>,
14411 module_connection: ConnectionId,
14412 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14413 handler: ControlHandler,
14414 module: super::SupervisedModule,
14415 }
14416
14417 fn probe_harness() -> ProbeHarness {
14418 let registry = Arc::new(Registry::default());
14419 let forwarding = Arc::new(ForwardingTable::default());
14420 let supervisor_handle = super::SupervisorHandle::new();
14421 let health = HealthConfig {
14422 http: None,
14423 cadence: Duration::from_secs(30),
14424 deadline: Duration::from_secs(5),
14425 failure_threshold: 3,
14426 on_degraded: HealthAction::Report,
14427 on_failing: HealthAction::Report,
14428 critical: false,
14429 };
14430 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14431 .with_forwarding(Arc::clone(&forwarding))
14432 .with_handle(supervisor_handle.clone())
14433 .with_health_config(health);
14434 let spec = ModuleSpec {
14435 module_id: "late-health-module".to_string(),
14436 program: PathBuf::from("disabled-module"),
14437 args: Vec::new(),
14438 env: Vec::new(),
14439 reserved: false,
14440 reserved_prefixes: Vec::new(),
14441 protocol: ModuleProtocol::Subc,
14442 overlap: Default::default(),
14443 };
14444 let module = supervisor
14445 .supervise_configured(spec.clone(), false)
14446 .unwrap();
14447 let runtime = supervisor.runtime_config();
14448 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14449 .with_supervisor(supervisor_handle);
14450 let module_connection = ConnectionId::new(700);
14451 let (module_tx, module_rx) = mpsc::channel(8);
14452 forwarding
14453 .register_module_connection(
14454 module_connection,
14455 spec.module_id.clone(),
14456 subc_protocol::PROTOCOL_VERSION,
14457 Concurrency::ModuleManaged,
14458 FrameSink::new(module_tx),
14459 )
14460 .unwrap();
14461
14462 ProbeHarness {
14463 spec,
14464 runtime,
14465 forwarding,
14466 module_connection,
14467 module_rx,
14468 handler,
14469 module,
14470 }
14471 }
14472
14473 async fn finish_after(
14474 harness: &mut ProbeHarness,
14475 stall: Duration,
14476 ) -> ModuleControlRpcCompletion {
14477 assert!(stall > harness.runtime.health.deadline);
14478 let deadline = harness.runtime.health.deadline;
14479 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14480 let answer = async {
14481 let frame = harness.module_rx.recv().await.expect("health.check frame");
14482 tokio::time::advance(deadline).await;
14483 tokio::task::yield_now().await;
14484 tokio::time::advance(stall - deadline).await;
14485 harness
14486 .forwarding
14487 .complete_module_control_rpc(
14488 harness.module_connection,
14489 frame.header.corr,
14490 Some("health.check"),
14491 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14492 status: HealthStatus::Ok,
14493 detail: None,
14494 metrics: None,
14495 }),
14496 )
14497 .unwrap()
14498 };
14499 let (probe_result, completion) = tokio::join!(probe, answer);
14500 let err = probe_result.expect_err("probe must miss its deadline");
14501 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14502 completion
14503 }
14504
14505 async fn time_out_without_answer(harness: &mut ProbeHarness) {
14506 let deadline = harness.runtime.health.deadline;
14507 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14508 let exhaust_deadline = async {
14509 let _frame = harness.module_rx.recv().await.expect("health.check frame");
14510 tokio::time::advance(deadline).await;
14511 tokio::task::yield_now().await;
14512 };
14513 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14514 let err = probe_result.expect_err("probe must miss its deadline");
14515 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14516 }
14517
14518 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14519 let registry = Arc::clone(&harness.module.inner.registry);
14520 let snapshot = Arc::clone(&harness.module.inner.snapshot);
14521 let process_liveness = super::SupervisorProcessLiveness::default();
14522 let mut child = None;
14523 let cycle = super::run_health_probe_cycle(
14524 &harness.spec,
14525 &harness.runtime,
14526 ®istry,
14527 &process_liveness,
14528 &snapshot,
14529 &mut child,
14530 );
14531 let peer = async {
14532 let frame = harness.module_rx.recv().await.expect("health.check frame");
14533 if answer {
14534 harness
14535 .forwarding
14536 .complete_module_control_rpc(
14537 harness.module_connection,
14538 frame.header.corr,
14539 Some("health.check"),
14540 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14541 status: HealthStatus::Ok,
14542 detail: None,
14543 metrics: Some(serde_json::json!({"ready": true})),
14544 }),
14545 )
14546 .unwrap();
14547 } else {
14548 tokio::time::advance(harness.runtime.health.deadline).await;
14549 tokio::task::yield_now().await;
14550 }
14551 };
14552 tokio::join!(cycle, peer);
14553 }
14554
14555 #[tokio::test(start_paused = true)]
14556 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14557 let mut harness = probe_harness();
14558 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14562 monitor.abort();
14563 let _ = monitor.await;
14564 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14565 state.enabled = true;
14566 state.state = super::ModuleState::Running;
14567 state.process_alive = true;
14568 })
14569 .unwrap();
14570
14571 run_probe_cycle(&mut harness, true).await;
14572 assert_eq!(
14573 harness.module.status().unwrap().health.status,
14574 super::SupervisorHealthStatus::Ok
14575 );
14576
14577 for failures in 1..harness.runtime.health.failure_threshold {
14578 run_probe_cycle(&mut harness, false).await;
14579 let status = harness.module.status().unwrap();
14580 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14581 assert_eq!(status.health.consecutive_failures, failures);
14582 assert!(status.health.last_probe_ms.is_some());
14583 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14584 assert!(status.health.metrics.is_none());
14585 assert_eq!(status.state, super::ModuleState::Running);
14586 assert!(status.process_alive);
14587 assert_eq!(status.restart_count, 0);
14588 assert_eq!(status.lifetime_restarts, 0);
14589 assert!(status.health.last_action.is_none());
14590 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14591 }
14592
14593 run_probe_cycle(&mut harness, true).await;
14594 let recovered = harness.module.status().unwrap();
14595 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14596 assert_eq!(recovered.health.consecutive_failures, 0);
14597 assert!(recovered.health.detail.is_none());
14598 assert_eq!(
14599 recovered.health.metrics,
14600 Some(serde_json::json!({"ready": true}))
14601 );
14602 assert_eq!(recovered.lifetime_restarts, 0);
14603
14604 for failures in 1..=harness.runtime.health.failure_threshold {
14605 run_probe_cycle(&mut harness, false).await;
14606 let status = harness.module.status().unwrap();
14607 assert_eq!(status.health.consecutive_failures, failures);
14608 if failures < harness.runtime.health.failure_threshold {
14609 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14610 assert_eq!(status.state, super::ModuleState::Running);
14611 assert_eq!(status.lifetime_restarts, 0);
14612 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14613 } else {
14614 assert_eq!(
14615 status.health.status,
14616 super::SupervisorHealthStatus::Unresponsive
14617 );
14618 assert_eq!(status.state, super::ModuleState::Restarting);
14619 assert_eq!(status.restart_count, 1);
14620 assert_eq!(status.lifetime_restarts, 1);
14621 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14622 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14623 }
14624 }
14625 }
14626
14627 #[tokio::test(start_paused = true)]
14628 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14629 let mut harness = probe_harness();
14630
14631 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14632 let first_latency = match &first {
14633 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14634 other => panic!("late answer was not retained: {other:?}"),
14635 };
14636 assert!(harness.handler.observe_module_control_completion(first));
14637
14638 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14639 let second_latency = match &second {
14640 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14641 other => panic!("late answer was not retained: {other:?}"),
14642 };
14643 assert!(harness.handler.observe_module_control_completion(second));
14644
14645 assert_eq!(first_latency, Duration::from_secs(8));
14646 assert_eq!(
14647 second_latency - first_latency,
14648 Duration::from_secs(3),
14649 "latency must grow linearly with the additional stall"
14650 );
14651 let health = harness.module.status().unwrap().health;
14652 assert_eq!(health.late_answer_count, 2);
14653 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14654 }
14655
14656 #[tokio::test(start_paused = true)]
14664 async fn late_answer_clears_the_consecutive_failure_streak() {
14665 let mut harness = probe_harness();
14666
14667 time_out_without_answer(&mut harness).await;
14669 harness
14670 .module
14671 .record_health_probe_failure_for_test("[no-answer] test miss")
14672 .unwrap();
14673 assert_eq!(
14674 harness.module.status().unwrap().health.consecutive_failures,
14675 1,
14676 "precondition: the miss must be on the streak before the late answer"
14677 );
14678
14679 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14681 assert!(matches!(
14682 late,
14683 ModuleControlRpcCompletion::LateHealthAnswer { .. }
14684 ));
14685 assert!(harness.handler.observe_module_control_completion(late));
14686
14687 let health = harness.module.status().unwrap().health;
14688 assert_eq!(
14689 health.consecutive_failures, 0,
14690 "a late answer is an answer: the streak must reset"
14691 );
14692 assert_eq!(health.late_answer_count, 1);
14693 }
14694
14695 #[tokio::test(start_paused = true)]
14696 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14697 let mut harness = probe_harness();
14698
14699 for _ in 0..20 {
14700 time_out_without_answer(&mut harness).await;
14701 assert_eq!(
14702 harness.forwarding.health_probe_tombstone_count().unwrap(),
14703 1
14704 );
14705 }
14706 }
14707}
14708
14709#[cfg(test)]
14710mod child_env_tests {
14711 #[cfg(windows)]
14712 use super::{
14713 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14714 };
14715 #[cfg(windows)]
14716 use std::sync::{Arc, OnceLock};
14717 #[cfg(windows)]
14718 #[test]
14719 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14720 let spec = spec(Vec::new());
14721 let mut command = Command::new("unused");
14722 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14723 .unwrap()
14724 .unwrap();
14725 let name = command
14726 .as_std()
14727 .get_envs()
14728 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14729 .and_then(|(_, value)| value)
14730 .unwrap();
14731 assert_eq!(name, OsStr::new(handoff.name()));
14732 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14733 assert!(command.as_std().get_envs().any(|(key, value)| key
14734 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14735 && value == Some(OsStr::new("env"))));
14736 assert!(command
14737 .as_std()
14738 .get_envs()
14739 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14740 }
14741
14742 #[cfg(windows)]
14743 #[test]
14744 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14745 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14746 snapshot.process_alive = true;
14747 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14748 assert_eq!(
14749 observed_launch_nonce_source(&snapshot).as_deref(),
14750 Some("env")
14751 );
14752 let consumed = Arc::new(OnceLock::new());
14753 snapshot.pipe_consumed = Some(consumed.clone());
14754 assert_eq!(
14755 observed_launch_nonce_source(&snapshot).as_deref(),
14756 Some("env")
14757 );
14758 consumed.set(()).unwrap();
14759 assert_eq!(
14760 observed_launch_nonce_source(&snapshot).as_deref(),
14761 Some("pipe")
14762 );
14763 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14764 assert_eq!(
14765 observed_launch_nonce_source(&snapshot).as_deref(),
14766 Some("env")
14767 );
14768 clear_current_process_facts(&mut snapshot);
14769 assert_eq!(observed_launch_nonce_source(&snapshot), None);
14770 }
14771 use super::{
14772 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
14773 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
14774 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14775 };
14776 use std::{ffi::OsStr, path::PathBuf};
14777 use tokio::process::Command;
14778
14779 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14780 ModuleSpec {
14781 module_id: "env-plan".to_string(),
14782 program: PathBuf::from("/nonexistent"),
14783 args: Vec::new(),
14784 env,
14785 reserved: false,
14786 reserved_prefixes: Vec::new(),
14787 protocol: ModuleProtocol::Subc,
14788 overlap: Default::default(),
14789 }
14790 }
14791
14792 #[test]
14793 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
14794 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
14795 let digest = LaunchSpecDigest::from_spec(&base);
14796 let mut reordered = base.clone();
14797 reordered.env.reverse();
14798 reordered.module_id = "different-id".into();
14799 reordered.reserved = true;
14800 reordered.overlap = Default::default();
14801 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
14802 let mut delimiter_collision = base.clone();
14803 delimiter_collision.env[0] = ("A=B".into(), "C".into());
14804 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
14805 let mut changed = base.clone();
14806 changed.protocol = ModuleProtocol::None;
14807 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14808 changed = base.clone();
14809 changed.args.push("--flag".into());
14810 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14811 changed = base.clone();
14812 changed.program = PathBuf::from("/other");
14813 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14814 changed = base.clone();
14815 changed.env[1].1 = "other".into();
14816 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14817 assert!(!format!("{digest:?}").contains("B=C"));
14818 assert!(!format!("{digest:?}").contains("[u8"));
14819 }
14820
14821 #[cfg(unix)]
14822 #[test]
14823 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
14824 use std::os::unix::ffi::OsStringExt;
14825 let mut left = spec(Vec::new());
14826 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
14827 let mut right = left.clone();
14828 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
14829 assert_eq!(
14830 left.program.to_string_lossy(),
14831 right.program.to_string_lossy()
14832 );
14833 assert_ne!(
14834 LaunchSpecDigest::from_spec(&left),
14835 LaunchSpecDigest::from_spec(&right)
14836 );
14837 }
14838
14839 #[test]
14853 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14854 let mut command = Command::new("/nonexistent");
14855 apply_child_env(&mut command, &spec(Vec::new()));
14856 let removed = command
14857 .as_std()
14858 .get_envs()
14859 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14860 assert!(
14861 removed,
14862 "ambient CK_LOG must be explicitly removed for an unconfigured module"
14863 );
14864
14865 let mut configured = Command::new("/nonexistent");
14866 apply_child_env(
14867 &mut configured,
14868 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14869 );
14870 let effective = configured
14871 .as_std()
14872 .get_envs()
14873 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14874 .last()
14875 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14876 assert_eq!(
14877 effective,
14878 Some(Some("debug".to_string())),
14879 "a module's configured CK_LOG must survive the ambient removal"
14880 );
14881 }
14882
14883 #[test]
14892 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14893 let connection_file = std::path::Path::new("/run/subc-connection.json");
14894 let handle = SupervisorHandle::new();
14895
14896 let mut none_spec = spec(Vec::new());
14897 none_spec.protocol = ModuleProtocol::None;
14898 let mut none = Command::new("/nonexistent");
14899 let none_handoff =
14900 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14901 .expect("protocol-none spawn args apply");
14902 assert!(
14903 none_handoff.is_none(),
14904 "protocol:none spawn must not receive a nonce descriptor"
14905 );
14906 #[cfg(windows)]
14907 assert!(!none.as_std().get_envs().any(|(key, value)| key
14908 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14909 && value.is_some()));
14910 assert!(
14911 !none.as_std().get_envs().any(|(key, value)| key
14912 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14913 && value.is_some()),
14914 "protocol:none spawn must not name a nonce descriptor"
14915 );
14916 let none_args: Vec<String> = none
14917 .as_std()
14918 .get_args()
14919 .map(|a| a.to_string_lossy().into_owned())
14920 .collect();
14921 assert!(
14922 !none_args.iter().any(|a| a == SUBC_ARG),
14923 "protocol:none argv must not carry --subc; got {none_args:?}"
14924 );
14925 let none_has_nonce = none
14926 .as_std()
14927 .get_envs()
14928 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
14929 assert!(
14930 !none_has_nonce,
14931 "protocol:none spawn must not receive a launch nonce"
14932 );
14933 let none_has_module_id = none
14934 .as_std()
14935 .get_envs()
14936 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
14937 assert!(
14938 none_has_module_id,
14939 "SUBC_MODULE_ID is inert and stays on every path"
14940 );
14941 assert!(
14942 handle.spawn_nonce(&none_spec.module_id).is_none(),
14943 "no nonce record for a process that will never present one"
14944 );
14945
14946 let wire_spec = spec(Vec::new());
14948 let mut wire = Command::new("/nonexistent");
14949 let wire_handoff =
14950 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
14951 .expect("subc-wire spawn args apply");
14952 let wire_fd_env = wire
14953 .as_std()
14954 .get_envs()
14955 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
14956 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14957 #[cfg(unix)]
14958 assert_eq!(
14959 wire_fd_env,
14960 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
14961 "a subc-wire spawn names the pipe it will receive at descriptor 3"
14962 );
14963 #[cfg(windows)]
14964 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
14965 #[cfg(not(any(unix, windows)))]
14966 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
14967 let wire_args: Vec<String> = wire
14968 .as_std()
14969 .get_args()
14970 .map(|a| a.to_string_lossy().into_owned())
14971 .collect();
14972 assert_eq!(
14973 wire_args,
14974 vec![
14975 SUBC_ARG.to_string(),
14976 connection_file.to_string_lossy().into_owned()
14977 ],
14978 "a subc-wire spawn still carries --subc <path>"
14979 );
14980 assert_eq!(
14981 wire.as_std()
14982 .get_envs()
14983 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
14984 !cfg!(unix),
14985 "only Windows supplies the environment nonce"
14986 );
14987 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
14988 }
14989
14990 #[test]
15000 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15001 let role = |command: &Command| {
15002 command
15003 .as_std()
15004 .get_envs()
15005 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15006 .last()
15007 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15008 };
15009 let forged = spec(vec![(
15010 SUBC_SPAWN_ROLE_ENV.to_string(),
15011 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15012 )]);
15013
15014 let mut plain = Command::new("/nonexistent");
15015 apply_child_env(&mut plain, &forged);
15016 apply_spawn_role(&mut plain, SpawnRole::Plain);
15017 assert_eq!(
15018 role(&plain),
15019 Some(None),
15020 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15021 );
15022
15023 let mut candidate = Command::new("/nonexistent");
15024 apply_child_env(&mut candidate, &spec(Vec::new()));
15025 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15026 assert_eq!(
15027 role(&candidate),
15028 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15029 );
15030 }
15031
15032 #[test]
15038 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15039 let mut command = Command::new("/nonexistent");
15040 apply_child_env(
15041 &mut command,
15042 &spec(vec![
15043 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15044 ("KEPT".to_string(), "yes".to_string()),
15045 ]),
15046 );
15047 let keys: Vec<String> = command
15048 .as_std()
15049 .get_envs()
15050 .filter(|(_, value)| value.is_some())
15051 .map(|(key, _)| key.to_string_lossy().into_owned())
15052 .collect();
15053 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15054 assert!(
15055 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15056 "daemon-private capture key leaked to the child: {keys:?}"
15057 );
15058 }
15059}
15060
15061#[cfg(test)]
15062mod jitter_tests {
15063 use super::jittered_health_delay;
15064 use std::{collections::HashSet, time::Duration};
15065
15066 const FLEET: [&str; 14] = [
15075 "aft",
15076 "alfonso-core",
15077 "magic-context",
15078 "broca",
15079 "thalamus",
15080 "quota",
15081 "engram",
15082 "plexus",
15083 "cerebellum",
15084 "astrocyte",
15085 "synapse",
15086 "subc-mcp",
15087 "cortexkit-credentials",
15088 "subc-federation",
15089 ];
15090
15091 #[test]
15099 fn probe_delays_disperse_across_the_fleet() {
15100 let cadence = Duration::from_secs(30);
15101 let delays: HashSet<Duration> = FLEET
15102 .iter()
15103 .map(|id| jittered_health_delay(id, 0, cadence))
15104 .collect();
15105 assert_eq!(
15106 delays.len(),
15107 FLEET.len(),
15108 "every supervised module must land on its own probe offset"
15109 );
15110 }
15111
15112 #[test]
15118 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15119 let cadence = Duration::from_secs(30);
15120 let span = cadence / 10;
15121 for id in FLEET {
15122 for probe_index in 0..8 {
15123 let delay = jittered_health_delay(id, probe_index, cadence);
15124 assert!(
15125 delay >= cadence,
15126 "{id}#{probe_index}: jitter must not shorten the cadence"
15127 );
15128 assert!(
15129 delay < cadence + span,
15130 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15131 );
15132 }
15133 }
15134 }
15135
15136 #[test]
15142 fn a_module_offset_is_stable_across_restarts() {
15143 let cadence = Duration::from_secs(30);
15144 for id in FLEET {
15145 assert_eq!(
15146 jittered_health_delay(id, 0, cadence),
15147 jittered_health_delay(id, 0, cadence),
15148 "{id}: the same module and probe index must produce the same offset"
15149 );
15150 }
15151 }
15152
15153 #[test]
15155 fn zero_cadence_yields_zero_delay() {
15156 assert_eq!(
15157 jittered_health_delay("aft", 0, Duration::ZERO),
15158 Duration::ZERO
15159 );
15160 }
15161}
15162
15163#[cfg(all(test, target_os = "linux"))]
15164mod cgroup_placement_tests {
15165 use super::{
15166 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15167 SuperviseError, SupervisedChild,
15168 };
15169 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15170 use cortexkit_test_support::ScratchDir;
15171 use std::{
15172 fs, io,
15173 path::{Path, PathBuf},
15174 sync::{Arc, Mutex},
15175 };
15176 use tokio::process::Command;
15177
15178 #[tokio::test]
15179 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15180 use super::*;
15181 let dir = ScratchDir::new("unique-spawn-cgroups");
15182 let root = PathBuf::from(format!(
15183 "/sys/fs/cgroup/subc-unique-test-{}-{}",
15184 std::process::id(),
15185 unix_ms_now()
15186 ));
15187 if let Err(error) = fs::create_dir(&root) {
15188 assert!(
15189 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15190 "required cgroup test cannot execute: {error}"
15191 );
15192 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15193 return;
15194 }
15195 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15196 let group_count = || {
15197 fs::read_dir(root.join("subc-modules"))
15198 .unwrap()
15199 .map(|entry| entry.unwrap().file_type().unwrap())
15200 .filter(|kind| kind.is_dir())
15201 .count()
15202 };
15203 let supervisor =
15204 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15205 .with_cgroup_placement(Some(placement.clone()));
15206 let runtime = supervisor.runtime_config();
15207 let mut spec = ModuleSpec {
15208 module_id: "unique-spawn".into(),
15209 program: PathBuf::from("/bin/sleep"),
15210 args: vec!["60".into()],
15211 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15212 .into_iter()
15213 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15214 .collect(),
15215 reserved: false,
15216 reserved_prefixes: vec![],
15217 protocol: ModuleProtocol::None,
15218 overlap: Default::default(),
15219 };
15220 let spawn = |spec: &ModuleSpec| {
15221 spawn_child(
15222 spec,
15223 None,
15224 None,
15225 &runtime.stderr_ring,
15226 None,
15227 &runtime.child_roster,
15228 Some(&placement),
15229 )
15230 .unwrap()
15231 };
15232 let mut live = spawn(&spec);
15233 for _ in 0..3 {
15234 let next = spawn(&spec);
15236 assert_ne!(live.module_id, next.module_id);
15237 live.start_kill().unwrap();
15238 live.wait().await.unwrap();
15239 live = next;
15240 assert!(
15241 live.child.try_wait().unwrap().is_none(),
15242 "retiring the old slot must not kill the replacement"
15243 );
15244 assert_eq!(
15245 group_count(),
15246 1,
15247 "only the live spawn's cgroup should remain"
15248 );
15249 }
15250 supervisor.begin_daemon_shutdown();
15251 let reap = tokio::spawn(async move {
15252 live.wait().await.unwrap();
15253 });
15254 supervisor
15255 .end_children_for_daemon_shutdown(false, std::future::pending())
15256 .await;
15257 reap.await.unwrap();
15258 assert_eq!(group_count(), 0);
15259 spec.program = PathBuf::from("/bin/true");
15261 spec.args.clear();
15262 let fresh_roster = ChildRoster::default();
15263 let mut short = spawn_child(
15264 &spec,
15265 None,
15266 None,
15267 &runtime.stderr_ring,
15268 None,
15269 &fresh_roster,
15270 Some(&placement),
15271 )
15272 .unwrap();
15273 short.wait().await.unwrap();
15274 assert_eq!(group_count(), 0);
15275 spec.module_id = "_".repeat(255);
15276 let mut long_id = spawn_child(
15277 &spec,
15278 None,
15279 None,
15280 &runtime.stderr_ring,
15281 None,
15282 &fresh_roster,
15283 Some(&placement),
15284 )
15285 .unwrap();
15286 long_id.wait().await.unwrap();
15287 assert_eq!(
15288 group_count(),
15289 0,
15290 "valid long module IDs must not exceed cgroup NAME_MAX"
15291 );
15292 fs::remove_dir(root.join("subc-modules")).unwrap();
15293 fs::remove_dir(root).unwrap();
15294 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15295 }
15296
15297 #[test]
15298 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15299 let path = Path::new("/definitely-missing-subc-cgroup");
15300 let mut command = Command::new("true");
15301 let error = apply_cgroup_placement(
15302 &mut command,
15303 &ModuleSpec {
15304 module_id: "broken-cgroup".to_string(),
15305 program: PathBuf::from("true"),
15306 args: Vec::new(),
15307 env: Vec::new(),
15308 reserved: false,
15309 reserved_prefixes: Vec::new(),
15310 protocol: ModuleProtocol::Subc,
15311 overlap: Default::default(),
15312 },
15313 path,
15314 )
15315 .expect_err("a parent cgroup open failure must reject the supervised spawn");
15316 let reason = error.to_string();
15317
15318 assert!(
15319 matches!(error, SuperviseError::Cgroup { .. }),
15320 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15321 );
15322 assert!(
15323 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15324 "parent cgroup open failure must name cgroup.procs: {reason}"
15325 );
15326 }
15327
15328 #[tokio::test]
15329 async fn reaping_a_child_removes_its_empty_module_cgroup() {
15330 let root = ScratchDir::new("supervisor-reap-cgroup");
15331 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15332 let placement = subc_cgroup::prepare_at(&root)
15333 .expect("prepare scratch cgroup root")
15334 .expect("scratch root has a cgroup.procs marker");
15335 let module_id = "reaped-module";
15336 let module = placement
15337 .module_path(module_id)
15338 .expect("create scratch module cgroup");
15339 let child = Command::new("true")
15340 .env("XDG_DATA_HOME", root.path())
15341 .env("XDG_RUNTIME_DIR", root.path())
15342 .env("XDG_CONFIG_HOME", root.path())
15343 .spawn()
15344 .expect("spawn short-lived child");
15345 let pid = child.id().expect("spawned child has pid");
15346 let mut child = SupervisedChild {
15347 child,
15348 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15349 protocol: ModuleProtocol::Subc,
15350 module_id: module_id.to_string(),
15351 cgroup_placement: Some(placement),
15352 stdout_pump: None,
15353 stderr_pump: None,
15354 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15355 spawned_at: tokio::time::Instant::now(),
15356 spawned_at_ms: 0,
15357 spawned_from: PathBuf::from("true"),
15358 launch_spec_digest: LaunchSpecDigest([0; 32]),
15359 spawned_file_identity: None,
15360 process_start_time: None,
15361 process_identity: None,
15362 pid,
15363 roster_guard: None,
15364 #[cfg(target_os = "macos")]
15365 privacy_exec: None,
15366 spawn_failure: None,
15367 };
15368
15369 child.wait().await.expect("reap short-lived child");
15370
15371 assert!(
15372 !module.exists(),
15373 "reaping the supervised child must remove its empty cgroup"
15374 );
15375 }
15376
15377 #[test]
15378 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15379 let root = ScratchDir::new("supervisor-non-empty-cgroup");
15380 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15381 let placement = subc_cgroup::prepare_at(&root)
15382 .expect("prepare scratch cgroup root")
15383 .expect("scratch root has a cgroup.procs marker");
15384 let module = placement
15385 .module_path("surviving-module")
15386 .expect("create scratch module cgroup");
15387 fs::write(module.join("surviving-process"), b"still present")
15388 .expect("make scratch cgroup non-empty");
15389 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15390
15391 remove_module_cgroup(&placement, "surviving-module");
15392
15393 let logs = crate::router::test_log::captured_logs(&logs);
15394 assert!(
15395 module.exists(),
15396 "failed removal must leave the cgroup intact"
15397 );
15398 assert!(
15399 logs.contains("could not remove module cgroup after process exit; continuing teardown")
15400 && logs.contains("surviving-module"),
15401 "best-effort removal must report the failure without returning it: {logs}"
15402 );
15403 }
15404
15405 #[test]
15406 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15407 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15408 let reason = SuperviseError::Spawn {
15409 program: PathBuf::from("/bin/true"),
15410 source: io::Error::from_raw_os_error(13),
15411 cgroup_path: Some(cgroup_path.clone()),
15412 }
15413 .to_string();
15414
15415 assert!(
15416 reason.contains(&cgroup_path.display().to_string()),
15417 "a pre_exec spawn failure must name the cgroup path: {reason}"
15418 );
15419 }
15420}
15421
15422#[cfg(test)]
15423mod spawn_subscriber_lag_tests {
15424 use super::*;
15425
15426 #[tokio::test]
15431 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15432 let feed = SpawnEventFeed::default();
15433 feed.configure_incarnation("lag-incarnation".to_string());
15434 let (tx, mut rx) = mpsc::channel(1);
15437 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15438 .expect("subscribe");
15439 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15440 for index in 0..emitted {
15441 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15442 tokio::task::yield_now().await;
15445 }
15446 assert_eq!(
15447 feed.subscriber_count(),
15448 0,
15449 "the lagged subscriber must be removed"
15450 );
15451
15452 let mut data = Vec::new();
15453 let mut last = None;
15454 loop {
15455 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15456 .await
15457 .expect("the forwarder must finish once the subscriber is dropped");
15458 let Some(outbound) = next else { break };
15459 let frame = outbound.frame;
15460 if frame.header.ty == FrameType::StreamData {
15461 assert!(last.is_none(), "no data may follow the terminal frame");
15462 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15463 data.push(event.cursor.seq);
15464 } else {
15465 assert!(last.is_none(), "exactly one terminal frame");
15466 last = Some(frame);
15467 }
15468 }
15469 assert!(!data.is_empty(), "queued frames drain before the terminal");
15470 for pair in data.windows(2) {
15471 assert_eq!(
15472 pair[1],
15473 pair[0] + 1,
15474 "queued frames arrive dense and in order"
15475 );
15476 }
15477 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15478 assert_eq!(terminal.header.ty, FrameType::Error);
15479 assert_eq!(terminal.header.corr, 7);
15480 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15481 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15482 let detail = body.detail.expect("lagged error carries detail");
15483 assert_eq!(
15484 detail["first_undelivered_cursor"]["seq"],
15485 data.last().unwrap() + 1,
15486 "the named cursor is the first event the subscriber did not receive"
15487 );
15488 assert_eq!(
15489 detail["first_undelivered_cursor"]["daemon_incarnation"],
15490 "lag-incarnation"
15491 );
15492 }
15493}
15494
15495#[cfg(test)]
15496mod terminal_history_read_concurrency_tests {
15497 use super::*;
15498 use crate::terminal_journal::read_pause;
15499 use cortexkit_test_support::ScratchDir;
15500 use std::sync::mpsc as std_mpsc;
15501
15502 fn journaled_ring(
15503 journal: &Arc<crate::terminal_journal::TerminalJournal>,
15504 ) -> Arc<Mutex<TerminalRing>> {
15505 Arc::new(Mutex::new(
15506 TerminalRing::new(TerminalRingConfig::default(), 1)
15507 .with_journal(Some(Arc::clone(journal))),
15508 ))
15509 }
15510
15511 fn crash(at_ms: u64) -> ExitReport {
15512 ExitReport {
15513 kind: ExitKind::Crash,
15514 code: Some(1),
15515 signal: None,
15516 at_ms,
15517 }
15518 }
15519
15520 fn record_within(
15523 module_id: &'static str,
15524 ring: &Arc<Mutex<TerminalRing>>,
15525 at_ms: u64,
15526 bound: Duration,
15527 ) -> bool {
15528 let ring = Arc::clone(ring);
15529 let (done, done_rx) = std_mpsc::channel();
15530 std::thread::spawn(move || {
15531 record_terminal(
15532 module_id,
15533 &ring,
15534 &SpawnEventFeed::default(),
15535 &crash(at_ms),
15536 TerminalDisposition::Restarting,
15537 );
15538 let _ = done.send(());
15539 });
15540 done_rx.recv_timeout(bound).is_ok()
15541 }
15542
15543 #[test]
15548 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15549 let dir = ScratchDir::new("terminal-history-concurrent-read");
15550 let path = dir.join("terminals.jsonl");
15551 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15552 path.clone(),
15553 "daemon".into(),
15554 ));
15555 let reader_ring = journaled_ring(&journal);
15556 let other_ring = journaled_ring(&journal);
15557 assert!(record_within(
15558 "reader-module",
15559 &reader_ring,
15560 10,
15561 Duration::from_secs(5)
15562 ));
15563
15564 let (started, release) = read_pause::install(&path);
15565 let reading = {
15566 let ring = Arc::clone(&reader_ring);
15567 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15568 };
15569 started
15570 .recv_timeout(Duration::from_secs(5))
15571 .expect("the history read reached its pause");
15572
15573 let bound = Duration::from_secs(1);
15574 assert!(
15575 record_within("other-module", &other_ring, 20, bound),
15576 "another module's exit waited on a history read (journal writer held)"
15577 );
15578 assert!(
15579 record_within("reader-module", &reader_ring, 30, bound),
15580 "the read module's own exit waited on its history read (ring held)"
15581 );
15582
15583 drop(release);
15584 let paused = reading.join().unwrap();
15585 assert_eq!(
15586 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15587 vec![10],
15588 "an exit recorded after the read began lands in neither half of it"
15589 );
15590 assert_eq!(paused.journal_skipped_lines, 0);
15591 assert_eq!(paused.journal_read_errors, 0);
15592
15593 let after = durable_terminal_history_of(&reader_ring, "reader-module");
15594 assert_eq!(
15595 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15596 vec![10, 30],
15597 "the next read merges ring and journal with no duplicate"
15598 );
15599 assert_eq!(after.journal_skipped_lines, 0);
15600 }
15601}
15602
15603#[cfg(test)]
15608mod stderr_settle_tests {
15609 use std::{
15610 future::Future,
15611 io,
15612 pin::Pin,
15613 sync::{Arc, Mutex},
15614 task::{Context, Poll},
15615 time::Duration,
15616 };
15617
15618 use tokio::{
15619 io::{AsyncRead, ReadBuf},
15620 sync::oneshot,
15621 time::Instant,
15622 };
15623
15624 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15625 use crate::stderr_tail::{
15626 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15627 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15628 };
15629
15630 const BOUND: Duration = Duration::from_millis(250);
15631
15632 struct HeldReader {
15636 before: Option<Vec<u8>>,
15637 gate: Option<oneshot::Receiver<()>>,
15638 after: io::Cursor<Vec<u8>>,
15639 }
15640
15641 impl AsyncRead for HeldReader {
15642 fn poll_read(
15643 mut self: Pin<&mut Self>,
15644 cx: &mut Context<'_>,
15645 buf: &mut ReadBuf<'_>,
15646 ) -> Poll<io::Result<()>> {
15647 if let Some(bytes) = self.before.take() {
15648 buf.put_slice(&bytes);
15649 return Poll::Ready(Ok(()));
15650 }
15651 if let Some(gate) = self.gate.as_mut() {
15652 match Pin::new(gate).poll(cx) {
15653 Poll::Pending => return Poll::Pending,
15654 Poll::Ready(_) => self.gate = None,
15655 }
15656 }
15657 Pin::new(&mut self.after).poll_read(cx, buf)
15658 }
15659 }
15660
15661 struct DiscardSink;
15662
15663 impl OutputSink for DiscardSink {
15664 fn write_line(&mut self, _line: &[u8]) {}
15665 }
15666
15667 fn line(text: &str) -> TailEntry {
15668 TailEntry::Line {
15669 text: text.to_string(),
15670 truncated: false,
15671 at_ms: None,
15672 }
15673 }
15674
15675 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
15676 ring.lock().unwrap()
15677 }
15678
15679 fn held_pump(
15683 ring: &Arc<Mutex<StderrRing>>,
15684 before: &str,
15685 after: &str,
15686 ) -> (StderrPump, oneshot::Sender<()>) {
15687 let generation = lock(ring).begin_process();
15688 let (release, gate) = oneshot::channel();
15689 let reader = HeldReader {
15690 before: Some(before.as_bytes().to_vec()),
15691 gate: Some(gate),
15692 after: io::Cursor::new(after.as_bytes().to_vec()),
15693 };
15694 let task = tokio::spawn(pump_stderr_to(
15695 reader,
15696 Arc::clone(ring),
15697 generation,
15698 DiscardSink,
15699 ));
15700 (StderrPump { task, generation }, release)
15701 }
15702
15703 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
15704 for _ in 0..1000 {
15705 if done(&lock(ring)) {
15706 return;
15707 }
15708 tokio::time::sleep(Duration::from_millis(1)).await;
15709 }
15710 panic!(
15711 "ring never reached the expected state: {:?}",
15712 lock(ring).snapshot(None, None)
15713 );
15714 }
15715
15716 #[tokio::test(start_paused = true)]
15717 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
15718 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15719 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
15720
15721 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
15722 let before_release = lock(&ring).snapshot(None, None);
15723 assert!(
15724 matches!(before_release.capture, CaptureState::Incomplete { .. }),
15725 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
15726 );
15727
15728 let next = lock(&ring).begin_process();
15731 lock(&ring).push_line_from(next, "next process booting");
15732 release.send(()).unwrap();
15733 wait_until(&ring, |ring| {
15734 ring.snapshot(None, None).capture == CaptureState::Captured
15735 })
15736 .await;
15737
15738 assert_eq!(
15739 untimed(lock(&ring).snapshot(None, None).entries),
15740 vec![
15741 line("booting"),
15742 line("config error: missing storage"),
15743 TailEntry::ProcessStart,
15744 line("next process booting"),
15745 ],
15746 "the crash's last line must survive a slow reader and stay in the crashed process's section"
15747 );
15748 }
15749
15750 #[tokio::test(start_paused = true)]
15751 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15752 ) {
15753 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15754 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15757
15758 let started = Instant::now();
15759 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15760 assert_eq!(
15761 started.elapsed(),
15762 BOUND,
15763 "the restart must wait exactly the bound for a pipe that stays open, no longer"
15764 );
15765
15766 let next = lock(&ring).begin_process();
15767 lock(&ring).push_line_from(next, "next process booting");
15768 tokio::time::sleep(Duration::from_secs(60)).await;
15769
15770 let snapshot = lock(&ring).snapshot(None, None);
15771 match &snapshot.capture {
15772 CaptureState::Incomplete { reason } => assert!(
15773 reason.contains("had not reached EOF") && reason.contains("250ms"),
15774 "the reason must say what is missing and after how long: {reason}"
15775 ),
15776 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15777 }
15778 assert_eq!(
15779 untimed(snapshot.entries),
15780 vec![
15781 line("parent exiting"),
15782 TailEntry::ProcessStart,
15783 line("next process booting"),
15784 ]
15785 );
15786 }
15787
15788 #[tokio::test(start_paused = true)]
15789 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15790 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15791 let (pump, release) = held_pump(&ring, "one\n", "two\n");
15792 release.send(()).unwrap();
15793
15794 settle_stderr_pump("clean", &ring, pump, BOUND).await;
15795
15796 let snapshot = lock(&ring).snapshot(None, None);
15797 assert_eq!(snapshot.capture, CaptureState::Captured);
15798 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15799 }
15800
15801 #[tokio::test(start_paused = true)]
15802 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15803 const BURST: usize = 150;
15804 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15805 let path = capture.join("two-pipe.stderr.log");
15806 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15807 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15808 let generation = lock(&ring).begin_process();
15809 let stdout = (0..BURST)
15810 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15811 .collect::<String>();
15812 let stderr = (0..BURST)
15813 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15814 .collect::<String>();
15815 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15820 let (release, gate) = oneshot::channel();
15821 let reader = HeldReader {
15822 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15823 gate: Some(gate),
15824 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15825 };
15826 let (finished, completion) = oneshot::channel();
15827 let stdout_sink = sink.clone();
15828 let pump = tokio::spawn(async move {
15829 pump_stdout_to(reader, stdout_sink).await;
15830 finished.send(()).unwrap();
15831 });
15832 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15833
15834 let started = Instant::now();
15835 settle_stdout_pump("two-pipe", pump, BOUND).await;
15836 assert_eq!(
15837 started.elapsed(),
15838 BOUND,
15839 "a late reader must not hold up restart"
15840 );
15841 assert_eq!(
15842 std::fs::read_to_string(&path).unwrap().lines().count(),
15843 BURST + 1
15844 );
15845
15846 release
15847 .send(())
15848 .expect("stdout reader must survive the exit drain bound");
15849 tokio::time::timeout(Duration::from_secs(1), completion)
15850 .await
15851 .expect("released stdout reader must reach EOF")
15852 .expect("stdout capture must finish at EOF, not be cancelled");
15853 let contents = std::fs::read_to_string(&path).unwrap();
15854 let mut lines = contents
15855 .lines()
15856 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15857 .collect::<Vec<_>>();
15858 let mut expected = (0..BURST)
15859 .flat_map(|i| {
15860 [
15861 format!("out-{i:04}-{}", "o".repeat(64)),
15862 format!("err-{i:04}-{}", "e".repeat(64)),
15863 ]
15864 })
15865 .collect::<Vec<_>>();
15866 lines.sort_unstable();
15867 expected.sort_unstable();
15868 assert_eq!(
15869 lines, expected,
15870 "both pipes must retain every complete line exactly once"
15871 );
15872 }
15873}
15874
15875#[cfg(all(test, windows))]
15889mod job_containment_tests {
15890 use super::*;
15891 use cortexkit_test_support::ScratchDir;
15892 use std::{
15893 path::{Path, PathBuf},
15894 sync::{Arc, Mutex},
15895 time::{Duration, Instant},
15896 };
15897
15898 fn stub_path() -> PathBuf {
15904 let mut path = std::env::current_exe().expect("current_exe available in tests");
15905 path.pop();
15906 path.pop();
15907 path.push("fake-aft-stub.exe");
15908 assert!(
15909 path.exists(),
15910 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15911 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15912 path.display()
15913 );
15914 path
15915 }
15916
15917 fn read_grandchild_pid(path: &Path) -> u32 {
15919 let deadline = Instant::now() + Duration::from_secs(10);
15920 loop {
15921 if let Ok(contents) = std::fs::read_to_string(path) {
15922 if let Ok(pid) = contents.trim().parse() {
15923 return pid;
15924 }
15925 }
15926 assert!(
15927 Instant::now() < deadline,
15928 "the stub never recorded a grandchild pid at {}",
15929 path.display()
15930 );
15931 std::thread::sleep(Duration::from_millis(10));
15932 }
15933 }
15934
15935 struct Fixture {
15938 _dir: ScratchDir,
15939 module_id: String,
15940 grandchild: u32,
15941 child: Option<SupervisedChild>,
15942 registry: Arc<Registry>,
15943 snapshot: Arc<Mutex<SupervisorSnapshot>>,
15944 terminal_ring: Arc<Mutex<TerminalRing>>,
15945 spawn_events: SpawnEventFeed,
15946 }
15947
15948 fn fixture(label: &str, module_id: &str) -> Fixture {
15949 let dir = ScratchDir::new(label);
15950 let pid_file = dir.join("grandchild.pid");
15951 let supervisor = Supervisor::new_for_test(
15952 Arc::new(Registry::default()),
15953 RestartPolicy::new(3, Duration::ZERO),
15954 );
15955 let runtime = supervisor.runtime_config();
15956 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15957 let spec = ModuleSpec {
15958 module_id: module_id.to_string(),
15959 program: stub_path(),
15960 args: Vec::new(),
15964 env: vec![
15965 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
15966 (
15967 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
15968 pid_file.display().to_string(),
15969 ),
15970 ],
15971 reserved: false,
15972 reserved_prefixes: Vec::new(),
15973 protocol: ModuleProtocol::Subc,
15974 overlap: Default::default(),
15975 };
15976 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
15977 .expect("spawn the supervised fixture");
15978 let grandchild = read_grandchild_pid(&pid_file);
15979 Fixture {
15980 _dir: dir,
15981 module_id: module_id.to_string(),
15982 grandchild,
15983 child: Some(child),
15984 registry: Arc::new(Registry::default()),
15985 snapshot,
15986 terminal_ring: Arc::clone(&runtime.terminal_ring),
15987 spawn_events: SpawnEventFeed::default(),
15988 }
15989 }
15990
15991 impl Fixture {
15992 async fn drain(&mut self) {
15994 let child = self
15995 .child
15996 .take()
15997 .expect("the fixture child is still present");
15998 drain_child_to_state(
15999 &self.module_id,
16000 ModuleProtocol::Subc,
16001 StopNotice::NotSent,
16004 &self.registry,
16005 None,
16006 &self.snapshot,
16007 &self.terminal_ring,
16008 &self.spawn_events,
16009 child,
16010 Duration::from_millis(500),
16011 ModuleState::Stopped,
16012 Some(false),
16013 )
16014 .await
16015 .expect("drain the supervised fixture");
16016 }
16017 }
16018
16019 #[tokio::test]
16025 async fn teardown_reaps_the_grandchild() {
16026 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16027 let grandchild = fixture.grandchild;
16028
16029 assert!(
16030 subc_jobobject::process_exists(grandchild),
16031 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16032 );
16033
16034 fixture.drain().await;
16035
16036 assert!(
16037 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16038 "grandchild {grandchild} outlived module teardown: the tree was not contained"
16039 );
16040 }
16041
16042 #[test]
16055 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16056 let dir = ScratchDir::new("teardown-uncontained");
16057 let pid_file = dir.join("grandchild.pid");
16058 let mut child = std::process::Command::new(stub_path())
16059 .env("FAKE_AFT_NEVER_CONNECT", "1")
16060 .env(
16061 "FAKE_AFT_GRANDCHILD_PID_FILE",
16062 pid_file.display().to_string(),
16063 )
16064 .stdin(std::process::Stdio::null())
16065 .stdout(std::process::Stdio::null())
16066 .stderr(std::process::Stdio::null())
16067 .spawn()
16068 .expect("spawn the uncontained fixture");
16069 let grandchild = read_grandchild_pid(&pid_file);
16070
16071 child.kill().expect("kill the direct child");
16073 let _ = child.wait();
16074
16075 assert!(
16076 subc_jobobject::process_exists(grandchild),
16077 "grandchild {grandchild} died with the direct child, so this control no longer \
16078 distinguishes contained from uncontained teardown and the regression test is \
16079 passing vacuously"
16080 );
16081
16082 kill_tree(grandchild);
16085 }
16086
16087 #[tokio::test]
16096 async fn dropping_containment_reaps_the_grandchild() {
16097 let mut fixture = fixture("drop-containment", "tree-drop");
16098 let grandchild = fixture.grandchild;
16099
16100 assert!(subc_jobobject::process_exists(grandchild));
16101
16102 fixture.child.as_mut().expect("child present").job = None;
16104
16105 assert!(
16106 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16107 "grandchild {grandchild} survived the containment handle closing, so a daemon \
16108 crash would leave the tree behind"
16109 );
16110 }
16111
16112 fn kill_tree(pid: u32) {
16114 let _ = std::process::Command::new("taskkill.exe")
16115 .args(["/PID", &pid.to_string(), "/T", "/F"])
16116 .stdin(std::process::Stdio::null())
16117 .stdout(std::process::Stdio::null())
16118 .stderr(std::process::Stdio::null())
16119 .status();
16120 assert!(
16121 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16122 "could not clean up grandchild {pid}"
16123 );
16124 }
16125}
16126
16127#[cfg(test)]
16128mod privacy_trampoline_configuration_tests {
16129 #[tokio::test]
16130 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16131 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16132 #[cfg(target_os = "macos")]
16133 {
16134 let supervisor = super::Supervisor::new(
16135 std::sync::Arc::new(crate::Registry::default()),
16136 super::RestartPolicy::default(),
16137 );
16138 let error = supervisor.spawn(spec()).unwrap_err();
16139 assert!(
16140 error
16141 .to_string()
16142 .contains("no privacy trampoline configured"),
16143 "{error}"
16144 );
16145 }
16146 }
16147
16148 #[tokio::test]
16149 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16150 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16151 #[cfg(target_os = "macos")]
16152 {
16153 let supervisor = super::Supervisor::new(
16154 std::sync::Arc::new(crate::Registry::default()),
16155 super::RestartPolicy::default(),
16156 )
16157 .with_privacy_trampoline(std::env::current_exe().unwrap());
16158 let error = supervisor.spawn(spec()).unwrap_err();
16159 assert!(
16160 error
16161 .to_string()
16162 .contains("binary does not implement the privacy trampoline protocol"),
16163 "{error}"
16164 );
16165 }
16166 }
16167
16168 #[cfg(target_os = "macos")]
16169 fn spec() -> super::ModuleSpec {
16170 super::ModuleSpec {
16171 module_id: "privacy-configuration".into(),
16172 program: "/bin/sleep".into(),
16173 args: vec!["30".into()],
16174 env: vec![],
16175 reserved: false,
16176 reserved_prefixes: vec![],
16177 protocol: subc_control::ModuleProtocol::None,
16178 overlap: super::ModuleOverlap::Exclusive,
16179 }
16180 }
16181}
16182
16183#[cfg(test)]
16184mod privacy_exec_boundary_tests {
16185 #[cfg(target_os = "macos")]
16186 use super::*;
16187 #[cfg(target_os = "macos")]
16188 use std::{
16189 io::{Read, Write},
16190 net::{TcpListener, TcpStream},
16191 };
16192
16193 #[cfg(target_os = "macos")]
16196 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16197 if let Some((_, path)) = spec
16198 .env
16199 .iter()
16200 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16201 {
16202 let mut barrier = TcpStream::connect(path).unwrap();
16203 barrier
16204 .set_read_timeout(Some(Duration::from_secs(30)))
16205 .unwrap();
16206 barrier.write_all(&pid.to_ne_bytes()).unwrap();
16207 let mut release = [0];
16208 barrier.read_exact(&mut release).unwrap();
16209 assert_eq!(&release, b"X");
16210 }
16211 }
16212
16213 #[cfg(target_os = "macos")]
16214 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16215 ModuleSpec {
16216 module_id: "privacy-boundary".into(),
16217 program: program.into(),
16218 args: args.iter().map(|arg| (*arg).into()).collect(),
16219 env: vec![],
16220 reserved: false,
16221 reserved_prefixes: vec![],
16222 protocol: ModuleProtocol::None,
16223 overlap: ModuleOverlap::Exclusive,
16224 }
16225 }
16226
16227 #[cfg(target_os = "macos")]
16228 async fn accept(listener: TcpListener) -> TcpStream {
16229 let listener = tokio::net::TcpListener::from_std({
16231 listener.set_nonblocking(true).unwrap();
16232 listener
16233 })
16234 .unwrap();
16235 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16236 .await
16237 .unwrap()
16238 .unwrap();
16239 let stream = stream.into_std().unwrap();
16240 stream.set_nonblocking(false).unwrap();
16241 stream
16242 .set_read_timeout(Some(Duration::from_secs(30)))
16243 .unwrap();
16244 stream
16245 }
16246
16247 #[tokio::test]
16248 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16249 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16250 #[cfg(target_os = "macos")]
16251 {
16252 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16253 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16256 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16257 let record = root.join("live-children.json");
16258 let supervisor =
16259 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16260 .with_live_children_record(&record);
16261 let runtime = supervisor.runtime_config();
16262 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16263 let mut spec = spec("/bin/sleep", &["30"]);
16264 spec.env = vec![
16265 (
16266 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16267 exec_listener.local_addr().unwrap().to_string(),
16268 ),
16269 (
16270 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16271 sample_listener.local_addr().unwrap().to_string(),
16272 ),
16273 ];
16274 let spawn = tokio::task::spawn_blocking(move || {
16275 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16276 });
16277 let mut sample = accept(sample_listener).await;
16278 let mut pid = [0; 4];
16279 sample.read_exact(&mut pid).unwrap();
16280 let pid = u32::from_ne_bytes(pid);
16281 let mut exec = accept(exec_listener).await;
16282 let mut ready = [0];
16283 exec.read_exact(&mut ready).unwrap();
16284 assert_eq!(&ready, b"R");
16285 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16288 assert_eq!(
16289 observe_spawned_image(pid).unwrap().executable,
16290 Some(trampoline)
16291 );
16292 sample.write_all(b"X").unwrap();
16293 let mut child = spawn.await.unwrap();
16294 let early = crate::live_children::read_record(&record).unwrap();
16295 assert_eq!(early.len(), 1);
16296 assert_eq!(early[0].pid, pid);
16297 assert_eq!(
16298 early[0].executable, None,
16299 "unconfirmed trampoline image entered the roster"
16300 );
16301 assert!(child.report_ready.get().is_none());
16302 child.privacy_exec.as_mut().unwrap().deadline =
16305 tokio::time::Instant::now() + Duration::from_secs(30);
16306 exec.write_all(b"X").unwrap();
16307 child.confirm_privacy_exec().await;
16308 assert_eq!(child.spawn_failure, None);
16309 assert!(child.report_ready.get().is_some());
16310 let confirmed = crate::live_children::read_record(&record).unwrap();
16311 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16312 assert_ne!(module, trampoline);
16313 assert_eq!(confirmed[0].executable, Some(module.into()));
16314 child.start_kill().unwrap();
16315 child.wait().await.unwrap();
16316 child.release_roster();
16317 }
16318 }
16319
16320 #[tokio::test]
16321 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16322 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16323 #[cfg(target_os = "macos")]
16324 {
16325 let registry = Arc::new(Registry::default());
16326 let policy = RestartPolicy::new(0, Duration::ZERO);
16327 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
16328 let runtime = supervisor.runtime_config();
16329 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16330 let spec = spec("/bin/sh", &["-c", "exit 121"]);
16331 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16335 let pid = child.pid;
16336 tokio::task::spawn_blocking(move || {
16337 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16338 })
16339 .await
16340 .unwrap()
16341 .unwrap();
16342 child.privacy_exec.as_mut().unwrap().deadline =
16343 tokio::time::Instant::now() + Duration::from_secs(30);
16344 let status = child.wait().await.unwrap();
16345 assert_eq!(status.code(), Some(121));
16346 assert!(child.privacy_exec.is_none());
16347 assert!(
16348 child.report_ready.get().is_none(),
16349 "an exited module must not publish a live pid"
16350 );
16351 let report = classify_reaped_child_exit(&snapshot, &child, &status);
16352 on_child_exit(
16353 &spec,
16354 policy,
16355 ®istry,
16356 &snapshot,
16357 &runtime.terminal_ring,
16358 &runtime.spawn_events,
16359 &runtime.child_roster,
16360 report,
16361 )
16362 .await;
16363 let state = lock_snapshot(&snapshot).unwrap();
16364 assert_eq!(state.state, ModuleState::Failed);
16365 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16366 assert_eq!(state.reported_pid(), None);
16367 drop(state);
16368 let history = runtime.terminal_ring.lock().unwrap().snapshot();
16369 assert_eq!(history.entries.len(), 1);
16370 let terminal = &history.entries[0];
16371 assert_eq!(terminal.exit_code, Some(121));
16372 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16373 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16374 assert_eq!(
16375 terminal.disposition_detail.as_deref(),
16376 Some(policy.budget_exhausted_detail().as_str()),
16377 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16378 );
16379 assert_eq!(child.spawn_failure, None);
16380 child.release_roster();
16381 }
16382 }
16383}
16384
16385#[cfg(all(test, unix))]
16389mod launch_nonce_descriptor_tests {
16390 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16391 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16392 use cortexkit_test_support::ScratchDir;
16393 use std::{
16394 path::PathBuf,
16395 sync::{Arc, Mutex},
16396 time::{Duration, Instant},
16397 };
16398
16399 async fn probe(role: super::SpawnRole) {
16400 let scratch = ScratchDir::new("launch-nonce-descriptor");
16401 let fd_copy = scratch.join("from-descriptor");
16402 let env_copy = scratch.join("environment");
16403 let script = format!(
16404 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16405 fd = fd_copy.display(), env = env_copy.display(),
16406 );
16407 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16408 let spec = ModuleSpec {
16409 module_id: "nonce-descriptor-probe".to_string(),
16410 program: PathBuf::from("/bin/sh"),
16411 args: vec!["-c".to_string(), script],
16412 env: vec![
16413 xdg("XDG_DATA_HOME"),
16414 xdg("XDG_RUNTIME_DIR"),
16415 xdg("XDG_CONFIG_HOME"),
16416 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16417 ],
16418 reserved: true,
16419 reserved_prefixes: Vec::new(),
16420 protocol: ModuleProtocol::Subc,
16421 overlap: Default::default(),
16422 };
16423 let handle = SupervisorHandle::new();
16424 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16425 let roster = ChildRoster::default();
16426 #[cfg(target_os = "macos")]
16427 {
16428 let path = super::test_privacy_trampoline();
16429 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16430 }
16431 let child = super::spawn_child_in_slot(
16432 &spec,
16433 None,
16434 Some(&handle),
16435 &ring,
16436 None,
16437 &roster,
16438 #[cfg(target_os = "linux")]
16439 None,
16440 role,
16441 matches!(role, super::SpawnRole::SwapCandidate),
16442 )
16443 .expect("spawn probe");
16444 let deadline = Instant::now() + Duration::from_secs(10);
16445 while !(fd_copy.exists() && env_copy.exists()) {
16446 assert!(Instant::now() < deadline, "probe never wrote its copies");
16447 tokio::time::sleep(Duration::from_millis(20)).await;
16448 }
16449 let nonce = std::fs::read_to_string(fd_copy).unwrap();
16450 assert!(!nonce.is_empty());
16451 let environment = std::fs::read_to_string(env_copy).unwrap();
16452 assert!(environment
16453 .lines()
16454 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16455 let copy = environment
16456 .lines()
16457 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16458 assert_eq!(
16459 copy, None,
16460 "Unix children must never receive the environment nonce"
16461 );
16462 if matches!(role, super::SpawnRole::Plain) {
16463 assert_eq!(
16464 handle.spawn_nonce(&spec.module_id).as_deref(),
16465 Some(nonce.as_str())
16466 );
16467 }
16468 drop(child);
16469 }
16470
16471 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16472 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16473 probe(super::SpawnRole::Plain).await;
16474 }
16475
16476 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16477 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16478 probe(super::SpawnRole::SwapCandidate).await;
16479 }
16480}
16481
16482#[cfg(all(test, target_os = "linux"))]
16483mod cgroup_containment_tests {
16484 use super::*;
16485 use cortexkit_test_support::ScratchDir;
16486
16487 fn running(pid: u32) -> bool {
16488 std::fs::read_to_string(format!("/proc/{pid}/stat"))
16490 .ok()
16491 .and_then(|stat| {
16492 stat.rsplit_once(") ")
16493 .map(|(_, rest)| rest.starts_with('Z'))
16494 })
16495 .is_some_and(|zombie| !zombie)
16496 }
16497
16498 #[tokio::test]
16499 async fn linux_teardown_reaps_the_grandchild() {
16500 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16501 }
16502
16503 #[tokio::test]
16504 async fn linux_shutdown_straggler_reaps_the_grandchild() {
16505 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16506 }
16507
16508 async fn teardown_tree(test_name: &str, shutdown: bool) {
16509 let dir = ScratchDir::new(test_name);
16510 let root = PathBuf::from(format!(
16511 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16512 std::process::id(),
16513 unix_ms_now()
16514 ));
16515 if let Err(error) = std::fs::create_dir(&root) {
16516 assert!(
16517 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16518 "required cgroup test cannot execute: {error}"
16519 );
16520 eprintln!(
16521 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16522 root.display()
16523 );
16524 return;
16525 }
16526 let placement = subc_cgroup::prepare_at(&root)
16527 .expect("prepare isolated kernel cgroup")
16528 .expect("isolated cgroup is delegated");
16529 let module_id = "tree-teardown";
16530 let module = placement
16531 .module_path(module_id)
16532 .expect("create isolated module cgroup");
16533 if !module.join("cgroup.kill").exists() {
16534 std::fs::remove_dir(&module).unwrap();
16535 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16536 std::fs::remove_dir(&root).unwrap();
16537 assert!(
16538 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16539 "required cgroup.kill interface unavailable"
16540 );
16541 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16542 return;
16543 }
16544 let supervisor = Supervisor::new_for_test(
16545 Arc::new(Registry::default()),
16546 RestartPolicy::new(3, Duration::ZERO),
16547 )
16548 .with_cgroup_placement(Some(placement));
16549 let mut runtime = supervisor.runtime_config();
16550 runtime.child_roster = runtime
16551 .child_roster
16552 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16553 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16554 let pid_file = dir.join("grandchild.pid");
16555 let spec = ModuleSpec {
16556 module_id: module_id.to_string(),
16557 program: PathBuf::from("/bin/sh"),
16558 args: vec![
16559 "-c".into(),
16560 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16561 "fixture".into(),
16562 pid_file.display().to_string(),
16563 ],
16564 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16565 .into_iter()
16566 .map(|key| (key.to_string(), dir.display().to_string()))
16567 .collect(),
16568 reserved: false,
16569 reserved_prefixes: Vec::new(),
16570 protocol: ModuleProtocol::None,
16571 overlap: Default::default(),
16572 };
16573 let child =
16574 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16575 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16576 let grandchild: u32 = loop {
16577 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16578 if let Ok(pid) = contents.trim().parse() {
16579 break pid;
16580 }
16581 }
16582 assert!(
16583 tokio::time::Instant::now() < deadline,
16584 "grandchild pid was not recorded"
16585 );
16586 tokio::time::sleep(Duration::from_millis(10)).await;
16587 };
16588 assert!(
16589 running(grandchild),
16590 "grandchild must be alive before teardown"
16591 );
16592 if shutdown {
16593 let mut child = child;
16594 crate::child_roster::end_children_for_daemon_shutdown(
16595 &runtime.child_roster,
16596 false,
16597 std::future::pending(),
16598 )
16599 .await;
16600 child.wait().await.expect("reap shutdown straggler");
16601 } else {
16602 drain_child_to_state(
16603 module_id,
16604 ModuleProtocol::None,
16605 StopNotice::NotSent,
16606 &Registry::default(),
16607 None,
16608 &snapshot,
16609 &runtime.terminal_ring,
16610 &SpawnEventFeed::default(),
16611 child,
16612 Duration::from_millis(100),
16613 ModuleState::Stopped,
16614 Some(false),
16615 )
16616 .await
16617 .expect("real supervisor teardown");
16618 }
16619 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16620 while running(grandchild) && tokio::time::Instant::now() < deadline {
16621 tokio::time::sleep(Duration::from_millis(10)).await;
16622 }
16623 let survived = running(grandchild);
16624 if survived {
16626 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16627 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16628 tokio::time::sleep(Duration::from_millis(100)).await;
16629 }
16630 if module.exists() {
16631 std::fs::remove_dir(&module).expect("remove empty module cgroup");
16632 }
16633 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16634 std::fs::remove_dir(&root).unwrap();
16635 assert!(
16636 !survived,
16637 "grandchild {grandchild} outlived module teardown"
16638 );
16639 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16640 }
16641}